mirror of
https://github.com/snobu/destreamer.git
synced 2026-02-13 01:59:42 +00:00
Compare commits
49 Commits
cea18bbf5e
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
516ca54ff1 | ||
|
|
b6a3e8f1c3 | ||
|
|
b2497865a1 | ||
|
|
a86389774d | ||
|
|
7c16da6505 | ||
|
|
52d5b227e2 | ||
|
|
a62f8ef777 | ||
|
|
d9137cc690 | ||
|
|
6ac226bda6 | ||
|
|
cb9e844d06 | ||
|
|
c7efd90f7f | ||
|
|
9fcc631236 | ||
|
|
1da56990bc | ||
|
|
7d91f32af2 | ||
|
|
918aadce5d | ||
|
|
dfab30cf46 | ||
|
|
757aab1747 | ||
|
|
ef91acaf10 | ||
|
|
cb689336d8 | ||
|
|
deadd6758c | ||
|
|
e9a0954528 | ||
|
|
eb588f74a3 | ||
|
|
22658a3706 | ||
|
|
55234af08f | ||
|
|
a129ac0240 | ||
|
|
1181290d8f | ||
|
|
21215bc9da | ||
|
|
8b2a02d0ae | ||
|
|
7efa54932f | ||
|
|
c81b16c8f3 | ||
|
|
aa569bee4d | ||
|
|
9ebd4faab3 | ||
|
|
66c018e164 | ||
|
|
3c7d61febe | ||
|
|
b5226713b6 | ||
|
|
f9bc0c7128 | ||
|
|
2c38517bcd | ||
|
|
f8207f4fd1 | ||
|
|
58122d5c4e | ||
|
|
0726bd90f4 | ||
|
|
2d0407e5c8 | ||
|
|
0da9c6fb5f | ||
|
|
f26204c38a | ||
|
|
cd1ac82fea | ||
|
|
fbe8de00de | ||
|
|
b48af65285 | ||
|
|
e9070511cf | ||
|
|
ad483f3eb7 | ||
|
|
ac0fdf5468 |
5
.github/workflows/build.yaml
vendored
5
.github/workflows/build.yaml
vendored
@@ -6,6 +6,9 @@ on:
|
|||||||
- 'README.md'
|
- 'README.md'
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
@@ -14,7 +17,7 @@ jobs:
|
|||||||
|
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
node-version: [8.x, 10.x, 12.x, 13.x]
|
node-version: [10.x, 12.x, 13.x]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- uses: actions/checkout@v1
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -9,3 +9,4 @@ node_modules
|
|||||||
videos
|
videos
|
||||||
release
|
release
|
||||||
build
|
build
|
||||||
|
yarn.lock
|
||||||
5
.vscode/settings.json
vendored
5
.vscode/settings.json
vendored
@@ -1,3 +1,4 @@
|
|||||||
{
|
{
|
||||||
"eslint.enable": true
|
"eslint.enable": true,
|
||||||
}
|
"typescript.tsdk": "node_modules\\typescript\\lib"
|
||||||
|
}
|
||||||
|
|||||||
67
README.md
67
README.md
@@ -1,15 +1,10 @@
|
|||||||
<a href="https://github.com/snobu/destreamer/actions">
|
# This project is abandoned. It will probably not work anymore against your MS Stream tenant.
|
||||||
<img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
|
|
||||||
</a>
|
|
||||||
|
|
||||||
# BREAKING
|
# A heartfelt thank you to all the contributors over the years. You are the real MVPs. 💖
|
||||||
|
|
||||||
**destreamer v3.0** is just around the corner. Download speed improvement is astonishing and we have a never before seen photo from the design sessions:<br><br>
|
## Check out kylon's Sharedown for a SharePoint-backend implementation - https://github.com/kylon/Sharedown
|
||||||

|
|
||||||
|
|
||||||
Help us pick a codename for the new release:<br><br>
|
<hr>
|
||||||
<br><br>
|
|
||||||
Comment in this thread: https://github.com/snobu/destreamer/issues/223
|
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
@@ -21,7 +16,7 @@ _(Alternative artwork proposals are welcome! Submit one through an Issue.)_
|
|||||||
|
|
||||||
This release would not have been possible without the code and time contributed by two distinguished developers: [@lukaarma](https://github.com/lukaarma) and [@kylon](https://github.com/kylon). Thank you!
|
This release would not have been possible without the code and time contributed by two distinguished developers: [@lukaarma](https://github.com/lukaarma) and [@kylon](https://github.com/kylon). Thank you!
|
||||||
|
|
||||||
### Specialized vesions
|
### Specialized versions
|
||||||
|
|
||||||
- [Politecnico di Milano][polimi]: fork over at https://github.com/SamanFekri/destreamer
|
- [Politecnico di Milano][polimi]: fork over at https://github.com/SamanFekri/destreamer
|
||||||
- [Università di Pisa][unipi]: fork over at https://github.com/Guray00/destreamer-unipi
|
- [Università di Pisa][unipi]: fork over at https://github.com/Guray00/destreamer-unipi
|
||||||
@@ -45,7 +40,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
|
|||||||
|
|
||||||
## Prereqs
|
## Prereqs
|
||||||
|
|
||||||
- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+.
|
- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
|
||||||
- **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
|
- **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
|
||||||
- [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
|
- [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
|
||||||
- [**git**][git]: one or more npm dependencies require git.
|
- [**git**][git]: one or more npm dependencies require git.
|
||||||
@@ -64,29 +59,25 @@ Note that destreamer won't run in an elevated (Administrator/root) shell. Runnin
|
|||||||
## Can i plug in my own browser?
|
## Can i plug in my own browser?
|
||||||
|
|
||||||
Yes, yes you can. This may be useful if your main browser has some authentication plugins that are required for you to logon to your Microsoft Stream tenant.
|
Yes, yes you can. This may be useful if your main browser has some authentication plugins that are required for you to logon to your Microsoft Stream tenant.
|
||||||
To use your own browser for the authentication part, locate the following snippet in `src/destreamer.ts`:
|
To use your own browser for the authentication part, locate the following snippet in `src/destreamer.ts` and `src/TokenCache.ts`:
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
const browser: puppeteer.Browser = await puppeteer.launch({
|
const browser: puppeteer.Browser = await puppeteer.launch({
|
||||||
executablePath: getPuppeteerChromiumPath(),
|
executablePath: getPuppeteerChromiumPath(),
|
||||||
headless: false,
|
// …
|
||||||
userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
|
});
|
||||||
args: [
|
|
||||||
'--disable-dev-shm-usage',
|
|
||||||
'--fast-start',
|
|
||||||
'--no-sandbox'
|
|
||||||
]
|
|
||||||
});
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Navigate to `chrome://version` in the browser you want to plug in and copy executable path from there. Use double backslash for Windows.
|
||||||
|
|
||||||
Now, change `executablePath` to reflect the path to your browser and profile (i.e. to use Microsoft Edge on Windows):
|
Now, change `executablePath` to reflect the path to your browser and profile (i.e. to use Microsoft Edge on Windows):
|
||||||
```typescript
|
```typescript
|
||||||
executablePath: "'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe' --profile-directory=Default",
|
executablePath: 'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that for Mac/Linux the path will look a little different but no other changes are necessary.
|
You can add `userDataDir` right after `executablePath` with the path to your browser profile (also shown in `chrome://version`) if you want that loaded as well.
|
||||||
|
|
||||||
You need to rebuild (`npm run build`) every time you change this configuration.
|
Remember to rebuild (`npm run build`) every time you change this configuration.
|
||||||
|
|
||||||
## How to build
|
## How to build
|
||||||
|
|
||||||
@@ -181,13 +172,13 @@ These optional lines must start with white space(s).
|
|||||||
Usage -
|
Usage -
|
||||||
```
|
```
|
||||||
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
||||||
-dir=videos/lessons/week1
|
-dir="videos/lessons/week1"
|
||||||
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
||||||
-dir=videos/lessons/week2"
|
-dir="videos/lessons/week2"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Title template
|
### Title template
|
||||||
The `-t` option allows users to input a template string for the output file names.
|
The `-t` option allows user to specify a custom filename for the videos.
|
||||||
|
|
||||||
You can use one or more of the following magic sequence which will get substituted at runtime. The magic sequence must be surrounded by curly brackets like this: `{title} {publishDate}`
|
You can use one or more of the following magic sequence which will get substituted at runtime. The magic sequence must be surrounded by curly brackets like this: `{title} {publishDate}`
|
||||||
|
|
||||||
@@ -199,8 +190,20 @@ You can use one or more of the following magic sequence which will get substitut
|
|||||||
- `authorEmail`: E-mail of video publisher
|
- `authorEmail`: E-mail of video publisher
|
||||||
- `uniqueId`: An _unique-enough_ ID generated from the video metadata
|
- `uniqueId`: An _unique-enough_ ID generated from the video metadata
|
||||||
|
|
||||||
Example -
|
Examples -
|
||||||
```
|
```
|
||||||
|
Input:
|
||||||
|
-t 'This is an example'
|
||||||
|
|
||||||
|
Expected filename:
|
||||||
|
This is an example.mkv
|
||||||
|
|
||||||
|
Input:
|
||||||
|
-t 'This is an example by {author}'
|
||||||
|
|
||||||
|
Expected filename:
|
||||||
|
This is an example by lukaarma.mkv
|
||||||
|
|
||||||
Input:
|
Input:
|
||||||
-t '{title} - {duration} - {publishDate} - {publishTime} - {author} - {authorEmail} - {uniqueId}'
|
-t '{title} - {duration} - {publishDate} - {publishTime} - {author} - {authorEmail} - {uniqueId}'
|
||||||
|
|
||||||
@@ -220,6 +223,14 @@ iTerm2 on a Mac -
|
|||||||
|
|
||||||
By default, downloads are saved under project root `Destreamer/videos/` ( Not the system media Videos folder ), unless specified by `-o` (output directory).
|
By default, downloads are saved under project root `Destreamer/videos/` ( Not the system media Videos folder ), unless specified by `-o` (output directory).
|
||||||
|
|
||||||
|
## KNOWN BUGS
|
||||||
|
|
||||||
|
If you get a
|
||||||
|
```
|
||||||
|
[FATAL ERROR] Unknown error: exit code 4
|
||||||
|
````
|
||||||
|
when running destreamer, then make sure you're running a recent (post year 2019), stable version of **ffmpeg**.
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
Contributions are welcome. Open an issue first before sending in a pull request. All pull requests require at least one code review before they are merged to master.
|
Contributions are welcome. Open an issue first before sending in a pull request. All pull requests require at least one code review before they are merged to master.
|
||||||
|
|||||||
2719
package-lock.json
generated
2719
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
42
package.json
42
package.json
@@ -17,34 +17,34 @@
|
|||||||
"author": "snobu",
|
"author": "snobu",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/mocha": "^7.0.2",
|
"@types/mocha": "^8.0.4",
|
||||||
"@types/puppeteer": "^1.20.4",
|
"@types/puppeteer": "^5.4.0",
|
||||||
"@types/readline-sync": "^1.4.3",
|
"@types/readline-sync": "^1.4.3",
|
||||||
"@types/tmp": "^0.1.0",
|
"@types/tmp": "^0.2.0",
|
||||||
"@types/yargs": "^15.0.3",
|
"@types/yargs": "^15.0.11",
|
||||||
"@typescript-eslint/eslint-plugin": "^2.25.0",
|
"@typescript-eslint/eslint-plugin": "^4.9.0",
|
||||||
"@typescript-eslint/parser": "^2.25.0",
|
"@typescript-eslint/parser": "^4.9.0",
|
||||||
"eslint": "^6.8.0",
|
"eslint": "^7.14.0",
|
||||||
"mocha": "^7.1.1",
|
"mocha": "^8.2.1",
|
||||||
"tmp": "^0.1.0"
|
"tmp": "^0.2.1"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@tedconf/fessonia": "^2.1.0",
|
"@tedconf/fessonia": "^2.1.2",
|
||||||
"@types/cli-progress": "^3.4.2",
|
"@types/cli-progress": "^3.8.0",
|
||||||
"@types/jwt-decode": "^2.2.1",
|
"@types/jwt-decode": "^2.2.1",
|
||||||
"axios": "^0.19.2",
|
"axios": "^0.21.2",
|
||||||
"axios-retry": "^3.1.8",
|
"axios-retry": "^3.1.9",
|
||||||
"cli-progress": "^3.7.0",
|
"cli-progress": "^3.8.2",
|
||||||
"colors": "^1.4.0",
|
"colors": "^1.4.0",
|
||||||
"is-elevated": "^3.0.0",
|
"is-elevated": "^3.0.0",
|
||||||
"iso8601-duration": "^1.2.0",
|
"iso8601-duration": "^1.3.0",
|
||||||
"jwt-decode": "^2.2.0",
|
"jwt-decode": "^3.1.2",
|
||||||
"puppeteer": "2.1.1",
|
"puppeteer": "5.5.0",
|
||||||
"readline-sync": "^1.4.10",
|
"readline-sync": "^1.4.10",
|
||||||
"sanitize-filename": "^1.6.3",
|
"sanitize-filename": "^1.6.3",
|
||||||
"terminal-image": "^1.0.1",
|
"terminal-image": "^1.2.1",
|
||||||
"typescript": "^3.8.3",
|
"typescript": "^4.1.2",
|
||||||
"winston": "^3.3.2",
|
"winston": "^3.3.3",
|
||||||
"yargs": "^15.0.3"
|
"yargs": "^16.1.1"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,9 @@ export class ApiClient {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
|
logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
|
||||||
logger.verbose(`Here is the error message: \n '${err.response?.data}`);
|
logger.warn('Here is the error message: ');
|
||||||
|
console.dir(err.response?.data);
|
||||||
|
logger.warn('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
|
||||||
|
|
||||||
const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
|
const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
|
||||||
|
|
||||||
|
|||||||
@@ -174,9 +174,8 @@ function checkInputConflicts(videoUrls: Array<string | number> | undefined,
|
|||||||
|
|
||||||
|
|
||||||
function isOutputTemplateValid(argv: any): boolean {
|
function isOutputTemplateValid(argv: any): boolean {
|
||||||
let finalTemplate: string = argv.outputTemplate;
|
|
||||||
const elementRegEx = RegExp(/{(.*?)}/g);
|
const elementRegEx = RegExp(/{(.*?)}/g);
|
||||||
let match = elementRegEx.exec(finalTemplate);
|
let match = elementRegEx.exec(argv.outputTemplate);
|
||||||
|
|
||||||
// if no template elements this fails
|
// if no template elements this fails
|
||||||
if (match) {
|
if (match) {
|
||||||
@@ -191,23 +190,18 @@ function isOutputTemplateValid(argv: any): boolean {
|
|||||||
|
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
match = elementRegEx.exec(finalTemplate);
|
match = elementRegEx.exec(argv.outputTemplate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// bad template from user, switching to default
|
|
||||||
else {
|
|
||||||
logger.warn('Empty output template provided, using default one \n');
|
|
||||||
finalTemplate = '{title} - {publishDate} {uniqueId}';
|
|
||||||
}
|
|
||||||
|
|
||||||
argv.outputTemplate = sanitize(finalTemplate.trim());
|
argv.outputTemplate = sanitize(argv.outputTemplate.trim());
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export function promptUser(choices: Array<string>): number {
|
export function promptUser(choices: Array<string>): number {
|
||||||
let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
|
const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
|
||||||
|
|
||||||
if (index === -1) {
|
if (index === -1) {
|
||||||
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
|
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
|
||||||
|
|||||||
@@ -3,45 +3,49 @@ export const enum ERROR_CODE {
|
|||||||
ELEVATED_SHELL,
|
ELEVATED_SHELL,
|
||||||
CANCELLED_USER_INPUT,
|
CANCELLED_USER_INPUT,
|
||||||
MISSING_FFMPEG,
|
MISSING_FFMPEG,
|
||||||
|
OUTDATED_FFMPEG,
|
||||||
UNK_FFMPEG_ERROR,
|
UNK_FFMPEG_ERROR,
|
||||||
INVALID_VIDEO_GUID,
|
INVALID_VIDEO_GUID,
|
||||||
NO_SESSION_INFO
|
NO_SESSION_INFO
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export const errors: {[key: number]: string} = {
|
export const errors: { [key: number]: string } = {
|
||||||
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
|
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
|
||||||
'Timeout or fatal error, please check your downloads directory and try again',
|
'Timeout or fatal error, please check your downloads directory and try again',
|
||||||
|
|
||||||
[ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
|
[ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
|
||||||
'Please run in a regular, non-elevated window.',
|
'Please run in a regular, non-elevated window.',
|
||||||
|
|
||||||
[ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
|
[ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
|
||||||
|
|
||||||
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
|
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
|
||||||
'Destreamer requires a fairly recent release of FFmpeg to download videos',
|
'Destreamer requires a fairly recent release of FFmpeg to download videos',
|
||||||
|
|
||||||
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
|
[ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
|
||||||
|
'Destreamer requires a fairly recent release of FFmpeg to download videos',
|
||||||
|
|
||||||
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
|
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
|
||||||
|
|
||||||
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
|
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
|
||||||
|
|
||||||
|
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
export const enum CLI_ERROR {
|
export const enum CLI_ERROR {
|
||||||
MISSING_INPUT_ARG = 'You must specify a URLs source. \n' +
|
MISSING_INPUT_ARG = 'You must specify a URLs source. \n' +
|
||||||
'Valid options are -i for one or more URLs separated by space or -f for input file. \n',
|
'Valid options are -i for one or more URLs separated by space or -f for input file. \n',
|
||||||
|
|
||||||
INPUT_ARG_CONFLICT = 'Too many URLs sources specified! \n' +
|
INPUT_ARG_CONFLICT = 'Too many URLs sources specified! \n' +
|
||||||
'Please specify a single source, either -i or -f \n',
|
'Please specify a single source, either -i or -f \n',
|
||||||
|
|
||||||
INPUTFILE_WRONG_EXTENSION = 'The specified inputFile has the wrong extension \n' +
|
INPUTFILE_WRONG_EXTENSION = 'The specified inputFile has the wrong extension \n' +
|
||||||
'Please make sure to use path/to/filename.txt when useing the -f option \n',
|
'Please make sure to use path/to/filename.txt when useing the -f option \n',
|
||||||
|
|
||||||
INPUTFILE_NOT_FOUND = 'The specified inputFile does not exists \n'+
|
INPUTFILE_NOT_FOUND = 'The specified inputFile does not exists \n' +
|
||||||
'Please check the filename and the path you provided \n',
|
'Please check the filename and the path you provided \n',
|
||||||
|
|
||||||
INVALID_OUTDIR = 'Could not create the default/specified output directory \n' +
|
INVALID_OUTDIR = 'Could not create the default/specified output directory \n' +
|
||||||
'Please check directory and permissions and try again. \n'
|
'Please check directory and permissions and try again. \n'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { AxiosResponse } from 'axios';
|
|||||||
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
|
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
|
|
||||||
let thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
|
const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
|
||||||
.then((response: AxiosResponse<any> | undefined) => response?.data);
|
.then((response: AxiosResponse<any> | undefined) => response?.data);
|
||||||
|
|
||||||
console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));
|
console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));
|
||||||
|
|||||||
@@ -19,16 +19,16 @@ export class TokenCache {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
let session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
|
const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
|
||||||
|
|
||||||
type Jwt = {
|
type Jwt = {
|
||||||
[key: string]: any
|
[key: string]: any
|
||||||
}
|
}
|
||||||
const decodedJwt: Jwt = jwtDecode(session.AccessToken);
|
const decodedJwt: Jwt = jwtDecode(session.AccessToken);
|
||||||
|
|
||||||
let now: number = Math.floor(Date.now() / 1000);
|
const now: number = Math.floor(Date.now() / 1000);
|
||||||
let exp: number = decodedJwt['exp'];
|
const exp: number = decodedJwt['exp'];
|
||||||
let timeLeft: number = exp - now;
|
const timeLeft: number = exp - now;
|
||||||
|
|
||||||
if (timeLeft < 120) {
|
if (timeLeft < 120) {
|
||||||
logger.warn('Access token has expired! \n');
|
logger.warn('Access token has expired! \n');
|
||||||
@@ -42,12 +42,13 @@ export class TokenCache {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public Write(session: Session): void {
|
public Write(session: Session): void {
|
||||||
let s: string = JSON.stringify(session, null, 4);
|
const s: string = JSON.stringify(session, null, 4);
|
||||||
fs.writeFile('.token_cache', s, (err: any) => {
|
fs.writeFile(this.tokenCacheFile, s, (err: any) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return logger.error(err);
|
return logger.error(err);
|
||||||
}
|
}
|
||||||
logger.info('Fresh access token dropped into .token_cachen \n'.green);
|
|
||||||
|
logger.info(`Fresh access token dropped into ${this.tokenCacheFile} \n`.green);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ export type Video = {
|
|||||||
outPath: string;
|
outPath: string;
|
||||||
totalChunks: number; // Abstraction of FFmpeg timemark
|
totalChunks: number; // Abstraction of FFmpeg timemark
|
||||||
playbackUrl: string;
|
playbackUrl: string;
|
||||||
posterImageUrl: string;
|
posterImageUrl: string | null;
|
||||||
captionsUrl?: string
|
captionsUrl?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
42
src/Utils.ts
42
src/Utils.ts
@@ -22,9 +22,21 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
|
|||||||
else if (groupMatch) {
|
else if (groupMatch) {
|
||||||
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
|
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
|
||||||
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
|
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
|
||||||
|
const result: Array<string> = [];
|
||||||
|
|
||||||
let result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=${videoNumber}&$orderby=publishedDate asc`, 'get')
|
// Anything above $top=100 results in 400 Bad Request
|
||||||
.then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
|
// Use $skip to skip the first 100 and get another 100 and so on
|
||||||
|
for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
|
||||||
|
const partial: Array<string> = await client.callApi(
|
||||||
|
`groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
|
||||||
|
'$top=100&$orderby=publishedDate asc', 'get')
|
||||||
|
.then(
|
||||||
|
(response: AxiosResponse<any> | undefined) =>
|
||||||
|
response?.data.value.map((item: any) => item.id)
|
||||||
|
);
|
||||||
|
|
||||||
|
result.push(...partial);
|
||||||
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -48,7 +60,7 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
|
|||||||
session: Session): Promise<Array<Array<string>>> {
|
session: Session): Promise<Array<Array<string>>> {
|
||||||
|
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
let guidList: Array<string> = [];
|
const guidList: Array<string> = [];
|
||||||
|
|
||||||
for (const url of urlList) {
|
for (const url of urlList) {
|
||||||
const guids: Array<string> | null = await extractGuids(url, apiClient);
|
const guids: Array<string> | null = await extractGuids(url, apiClient);
|
||||||
@@ -85,8 +97,8 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
|
|||||||
.split(/\r?\n/);
|
.split(/\r?\n/);
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
|
|
||||||
let guidList: Array<string> = [];
|
const guidList: Array<string> = [];
|
||||||
let outDirList: Array<string> = [];
|
const outDirList: Array<string> = [];
|
||||||
// if the last line was an url set this
|
// if the last line was an url set this
|
||||||
let foundUrl = false;
|
let foundUrl = false;
|
||||||
|
|
||||||
@@ -101,23 +113,23 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
|
|||||||
// parse if line is option
|
// parse if line is option
|
||||||
else if (line.includes('-dir')) {
|
else if (line.includes('-dir')) {
|
||||||
if (foundUrl) {
|
if (foundUrl) {
|
||||||
let outDir: string | null = parseOption('-dir', line);
|
const outDir: string | null = parseOption('-dir', line);
|
||||||
|
|
||||||
if (outDir && checkOutDir(outDir)) {
|
if (outDir && checkOutDir(outDir)) {
|
||||||
outDirList.push(...Array(guidList.length - outDirList.length)
|
outDirList.push(...Array(guidList.length - outDirList.length)
|
||||||
.fill(outDir));
|
.fill(outDir));
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
outDirList.push(...Array(guidList.length - outDirList.length)
|
outDirList.push(...Array(guidList.length - outDirList.length)
|
||||||
.fill(defaultOutDir));
|
.fill(defaultOutDir));
|
||||||
}
|
}
|
||||||
|
|
||||||
foundUrl = false;
|
foundUrl = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
|
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,7 +167,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
|
|||||||
function parseOption(optionSyntax: string, item: string): string | null {
|
function parseOption(optionSyntax: string, item: string): string | null {
|
||||||
const match: RegExpMatchArray | null = item.match(
|
const match: RegExpMatchArray | null = item.match(
|
||||||
RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
|
RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
|
||||||
);
|
);
|
||||||
|
|
||||||
return match ? match[1] : null;
|
return match ? match[1] : null;
|
||||||
}
|
}
|
||||||
@@ -168,7 +180,7 @@ export function checkOutDir(directory: string): boolean {
|
|||||||
logger.info('\nCreated directory: '.yellow + directory);
|
logger.info('\nCreated directory: '.yellow + directory);
|
||||||
}
|
}
|
||||||
catch (e) {
|
catch (e) {
|
||||||
logger.warn('Cannot create directory: '+ directory +
|
logger.warn('Cannot create directory: ' + directory +
|
||||||
'\nFalling back to default directory..');
|
'\nFalling back to default directory..');
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
@@ -181,7 +193,13 @@ export function checkOutDir(directory: string): boolean {
|
|||||||
|
|
||||||
export function checkRequirements(): void {
|
export function checkRequirements(): void {
|
||||||
try {
|
try {
|
||||||
|
const copyrightYearRe = new RegExp(/\d{4}-(\d{4})/);
|
||||||
const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
|
const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
|
||||||
|
|
||||||
|
if (parseInt(copyrightYearRe.exec(ffmpegVer)?.[1] ?? '0') <= 2019) {
|
||||||
|
process.exit(ERROR_CODE.OUTDATED_FFMPEG);
|
||||||
|
}
|
||||||
|
|
||||||
logger.verbose(`Using ${ffmpegVer}\n`);
|
logger.verbose(`Using ${ffmpegVer}\n`);
|
||||||
}
|
}
|
||||||
catch (e) {
|
catch (e) {
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ function durationToTotalChunks(duration: string): number {
|
|||||||
|
|
||||||
|
|
||||||
export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
|
export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
|
||||||
let metadata: Array<Video> = [];
|
const metadata: Array<Video> = [];
|
||||||
let title: string;
|
let title: string;
|
||||||
let duration: string;
|
let duration: string;
|
||||||
let publishDate: string;
|
let publishDate: string;
|
||||||
@@ -65,7 +65,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
|
|||||||
/* TODO: change this to a single guid at a time to ease our footprint on the
|
/* TODO: change this to a single guid at a time to ease our footprint on the
|
||||||
MSS servers or we get throttled after 10 sequential reqs */
|
MSS servers or we get throttled after 10 sequential reqs */
|
||||||
for (const guid of videoGuids) {
|
for (const guid of videoGuids) {
|
||||||
let response: AxiosResponse<any> | undefined =
|
const response: AxiosResponse<any> | undefined =
|
||||||
await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
|
await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
|
||||||
|
|
||||||
title = sanitizeWindowsName(response?.data['name']);
|
title = sanitizeWindowsName(response?.data['name']);
|
||||||
@@ -94,7 +94,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
|
|||||||
posterImageUrl = response?.data['posterImage']['medium']['url'];
|
posterImageUrl = response?.data['posterImage']['medium']['url'];
|
||||||
|
|
||||||
if (subtitles) {
|
if (subtitles) {
|
||||||
let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
|
const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
|
||||||
|
|
||||||
if (!captions?.data.value.length) {
|
if (!captions?.data.value.length) {
|
||||||
captionsUrl = undefined;
|
captionsUrl = undefined;
|
||||||
@@ -140,7 +140,7 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
|
|||||||
let match = elementRegEx.exec(template);
|
let match = elementRegEx.exec(template);
|
||||||
|
|
||||||
while (match) {
|
while (match) {
|
||||||
let value = video[match[1] as keyof Video] as string;
|
const value = video[match[1] as keyof Video] as string;
|
||||||
title = title.replace(match[0], value);
|
title = title.replace(match[0], value);
|
||||||
match = elementRegEx.exec(template);
|
match = elementRegEx.exec(template);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -185,7 +185,9 @@ async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array
|
|||||||
const headers: string = 'Authorization: Bearer ' + session.AccessToken;
|
const headers: string = 'Authorization: Bearer ' + session.AccessToken;
|
||||||
|
|
||||||
if (!argv.noExperiments) {
|
if (!argv.noExperiments) {
|
||||||
await drawThumbnail(video.posterImageUrl, session);
|
if (video.posterImageUrl) {
|
||||||
|
await drawThumbnail(video.posterImageUrl, session);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
|
const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
|
||||||
@@ -270,6 +272,7 @@ async function main(): Promise<void> {
|
|||||||
await init(); // must be first
|
await init(); // must be first
|
||||||
|
|
||||||
let session: Session;
|
let session: Session;
|
||||||
|
// eslint-disable-next-line prefer-const
|
||||||
session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
|
session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
|
||||||
|
|
||||||
logger.verbose('Session and API info \n' +
|
logger.verbose('Session and API info \n' +
|
||||||
|
|||||||
1
trigger_codeQL
Normal file
1
trigger_codeQL
Normal file
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
Reference in New Issue
Block a user