1
0
mirror of https://github.com/snobu/destreamer.git synced 2026-01-28 10:52:18 +00:00

25 Commits

Author SHA1 Message Date
lukaarma
c81b16c8f3 Merge pull request #322 from apandada1/patch-1
update README - add instructions for Linux
2021-02-09 21:33:34 +01:00
Archisman Panigrahi
aa569bee4d update README - add instructions for Linux
These instructions are valid for a lot of Linux distros, and not just limited to Raspberry Pi.
2021-02-07 12:20:18 +05:30
Adrian Calinescu
9ebd4faab3 Add Raspberry Pi note 2021-01-26 16:59:48 +02:00
dependabot[bot]
66c018e164 Bump axios from 0.21.0 to 0.21.1 (#309)
Bumps [axios](https://github.com/axios/axios) from 0.21.0 to 0.21.1.
- [Release notes](https://github.com/axios/axios/releases)
- [Changelog](https://github.com/axios/axios/blob/v0.21.1/CHANGELOG.md)
- [Commits](https://github.com/axios/axios/compare/v0.21.0...v0.21.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2021-01-26 14:52:39 +02:00
Adrian Calinescu
3c7d61febe Fix return HTTP 403 reason with or without verbose (#315) 2021-01-26 14:49:42 +02:00
Adrian Calinescu
b5226713b6 If you're using Node 8 you're on your own, pal. 2021-01-13 20:55:21 +02:00
snobu
f9bc0c7128 I've had enough of Node 8.x horse manure 2021-01-13 20:53:00 +02:00
snobu
2c38517bcd Fix some dumb npm s*** 2021-01-13 20:18:50 +02:00
lukaarma
f8207f4fd1 Group parsing fix and error out on old ffmpeg version (#298)
* fixed parsing for group with more than 100 videos

* updated all packages to latest version

* Error on old ffmpeg binaries (closes #294)
minor linting fixes

* automatic update of files

Co-authored-by: Adrian Calinescu <foo@snobu.org>
2021-01-13 20:12:12 +02:00
lukaarma
58122d5c4e Merge pull request #307 from snobu/dependabot/npm_and_yarn/axios-0.21.1
Bump axios from 0.19.2 to 0.21.1 (vulnerability fix)
2021-01-08 16:16:23 +01:00
Adrian Calinescu
0726bd90f4 Holiday cleanup 2021-01-07 18:33:57 +02:00
dependabot[bot]
2d0407e5c8 Bump axios from 0.19.2 to 0.21.1
Bumps [axios](https://github.com/axios/axios) from 0.19.2 to 0.21.1.
- [Release notes](https://github.com/axios/axios/releases)
- [Changelog](https://github.com/axios/axios/blob/v0.21.1/CHANGELOG.md)
- [Commits](https://github.com/axios/axios/compare/v0.19.2...v0.21.1)

Signed-off-by: dependabot[bot] <support@github.com>
2021-01-05 23:14:38 +00:00
Adrian Calinescu
0da9c6fb5f Fixup for the holidays 2020-12-22 21:51:07 +02:00
Adrian Calinescu
f26204c38a Revert "Fixed parsing for group with more than 100 videos (#288)" (#295)
This reverts commit cd1ac82fea.
2020-12-15 13:02:23 +02:00
lukaarma
cd1ac82fea Fixed parsing for group with more than 100 videos (#288)
* fixed parsing for group with more than 100 videos
* updated all packages to latest version
2020-12-15 12:55:10 +02:00
Adrian Calinescu
fbe8de00de Added link to aria branch in README 2020-12-03 00:07:13 +02:00
fulminemizzega
b48af65285 fix quotes in input file example in README.md (#283)
This should close #281
2020-11-22 20:54:42 +02:00
snobu
e9070511cf Srsly really fixed group download this time 2020-11-14 20:08:45 +02:00
snobu
ad483f3eb7 Fix group download, now limited to first 100 videos 2020-11-14 19:54:50 +02:00
Adrian Calinescu
ac0fdf5468 Fix MSEdge launch params 2020-10-15 16:07:20 +03:00
lukaarma
15c420333e Update report-trouble.md (#240) 2020-10-08 16:02:36 +03:00
lukaarma
22968f42ea Misc fixes (#244)
* cleared up docs

* fix 401 bug for images thumbnail
2020-10-08 16:00:28 +03:00
lukaarma
ec24ff9e1b Merge pull request #242 from rohit404404/patch-1
Update README.md
2020-10-03 21:00:46 +02:00
Rohit Devmore
389be33f74 Update README.md
made videos path clearer
2020-10-04 00:18:33 +05:30
Rohit Devmore
8848f293d7 Update README.md
made videos path clear
2020-10-04 00:07:40 +05:30
14 changed files with 1464 additions and 1527 deletions

View File

@@ -7,6 +7,24 @@ assignees: ''
---
## PLEASE NEVER PASTE YOUR ACCESS TOKEN INTO A GITHUB ISSUE AS IT MAY CONTAIN PRIVATE INFORMATION
<!--
# BEFORE OPENING A NEW ISSUE CHECK THE EXISTING ONES AND RUN DESTREAMER WITH THE -v/--verbose flag and paste down below the output
# NEVER PASTE YOUR ACCESS TOKEN INTO A GITHUB ISSUE AS IT MAY CONTAIN PRIVATE INFORMATION
When you paste in output from destreamer, locate your access token (it looks like this: `Authorization: Bearer eyJ....<a lot more base64 encoded text>.....`) and redact it.
# Please fill the form below to give us some more info.
-->
OS:
Launch command used:
<details>
<summary>Verbose log</summary>
```
PASTE VERBOSE LOG HERE
```
</details>

View File

@@ -14,7 +14,7 @@ jobs:
strategy:
matrix:
node-version: [8.x, 10.x, 12.x, 13.x]
node-version: [10.x, 12.x, 13.x]
steps:
- uses: actions/checkout@v1

View File

@@ -1,3 +1,4 @@
{
"eslint.enable": true
}
"eslint.enable": true,
"typescript.tsdk": "node_modules\\typescript\\lib"
}

View File

@@ -2,14 +2,7 @@
<img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
</a>
# BREAKING
**destreamer v3.0** is just around the corner. Download speed improvement is astonishing and we have a never before seen photo from the design sessions:<br><br>
![desilva](https://user-images.githubusercontent.com/6472374/93003437-54a7fd00-f547-11ea-8473-e4602993e69d.jpg)
Help us pick a codename for the new release:<br><br>
![codename](https://user-images.githubusercontent.com/6472374/93003896-20ced680-f54b-11ea-8be1-2c14e0bd3751.png)<br><br>
Comment in this thread: https://github.com/snobu/destreamer/issues/223
**destreamer v3.0** is just around the corner. You can try out a pre-release today by cloning [this branch](https://github.com/snobu/destreamer/tree/aria2c_forRealNow).
![destreamer](assets/logo.png)
@@ -45,7 +38,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
## Prereqs
- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+.
- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
- **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
- [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
- [**git**][git]: one or more npm dependencies require git.
@@ -81,10 +74,15 @@ const browser: puppeteer.Browser = await puppeteer.launch({
Now, change `executablePath` to reflect the path to your browser and profile (i.e. to use Microsoft Edge on Windows):
```typescript
executablePath: "'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe' --profile-directory=Default",
executablePath: 'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe',
```
In Linux for Chromium,
```typescript
executablePath: '/usr/bin/chromium-browser',
```
Depending on your distro, it may also be `/usr/bin/chromium`. You will have to change it appropriately for Google Chrome.
Note that for Mac/Linux the path will look a little different but no other changes are necessary.
Note that for Mac the path may look a little different but no other changes are necessary.
You need to rebuild (`npm run build`) every time you change this configuration.
@@ -108,31 +106,35 @@ Options:
--help Show help [boolean]
--version Show version number [boolean]
--username, -u The username used to log into Microsoft Stream (enabling this will fill in the email field for
you) [string]
--videoUrls, -i List of video urls [array]
you). [string]
--videoUrls, -i List of urls to videos or Microsoft Stream groups. [array]
--inputFile, -f Path to text file containing URLs and optionally outDirs. See the README for more on outDirs.
[string]
--outputDirectory, -o The directory where destreamer will save your downloads. [string] [default: "videos"]
--outputTemplate, -t The template for the title. See the README for more info.
[string] [default: "{title} - {publishDate} {uniqueId}"]
--outputDirectory, -o The directory where destreamer will save your downloads [string] [default: "videos"]
--keepLoginCookies, -k Let Chromium cache identity provider cookies so you can use "Remember me" during login
--keepLoginCookies, -k Let Chromium cache identity provider cookies so you can use "Remember me" during login.
Must be used every subsequent time you launch Destreamer if you want to log in automatically.
[boolean] [default: false]
--noExperiments, -x Do not attempt to render video thumbnails in the console [boolean] [default: false]
--simulate, -s Disable video download and print metadata information to the console[boolean] [default: false]
--verbose, -v Print additional information to the console (use this before opening an issue on GitHub)
--noExperiments, -x Do not attempt to render video thumbnails in the console. [boolean] [default: false]
--simulate, -s Disable video download and print metadata information to the console.
[boolean] [default: false]
--closedCaptions, --cc Check if closed captions are aviable and let the user choose which one to download (will not
ask if only one aviable) [boolean] [default: false]
--noCleanup, --nc Do not delete the downloaded video file when an FFmpeg error occurs [boolean] [default: false]
--verbose, -v Print additional information to the console (use this before opening an issue on GitHub).
[boolean] [default: false]
--closedCaptions, --cc Check if closed captions are available and let the user choose which one to download (will not
ask if only one available). [boolean] [default: false]
--noCleanup, --nc Do not delete the downloaded video file when an FFmpeg error occurs.[boolean] [default: false]
--vcodec Re-encode video track. Specify FFmpeg codec (e.g. libx265) or set to "none" to disable video.
[string] [default: "copy"]
--acodec Re-encode audio track. Specify FFmpeg codec (e.g. libopus) or set to "none" to disable audio.
[string] [default: "copy"]
--format Output container format (mkv, mp4, mov, anything that FFmpeg supports)
--format Output container format (mkv, mp4, mov, anything that FFmpeg supports).
[string] [default: "mkv"]
--skip Skip download if file already exists [boolean] [default: false]
--skip Skip download if file already exists. [boolean] [default: false]
```
- both --videoUrls and --inputFile also accept Microsoft Teams Groups url so if your Organization placed the videos you are interested in a group you can copy the link and Destreamer will download all the videos it can inside it! A group url looks like this https://web.microsoftstream.com/group/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
- Passing `--username` is optional. It's there to make logging in faster (the username field will be populated automatically on the login form).
- You can use an absolute path for `-o` (output directory), for example `/mnt/videos`.
@@ -177,9 +179,9 @@ These optional lines must start with white space(s).
Usage -
```
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
-dir=videos/lessons/week1
-dir="videos/lessons/week1"
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
-dir=videos/lessons/week2"
-dir="videos/lessons/week2"
```
### Title template
@@ -214,7 +216,7 @@ iTerm2 on a Mac -
![screenshot](assets/screenshot-mac.png)
By default, downloads are saved under `videos/` unless specified by `-o` (output directory).
By default, downloads are saved under project root `Destreamer/videos/` ( Not the system media Videos folder ), unless specified by `-o` (output directory).
## Contributing

2713
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -17,34 +17,34 @@
"author": "snobu",
"license": "MIT",
"devDependencies": {
"@types/mocha": "^7.0.2",
"@types/puppeteer": "^1.20.4",
"@types/mocha": "^8.0.4",
"@types/puppeteer": "^5.4.0",
"@types/readline-sync": "^1.4.3",
"@types/tmp": "^0.1.0",
"@types/yargs": "^15.0.3",
"@typescript-eslint/eslint-plugin": "^2.25.0",
"@typescript-eslint/parser": "^2.25.0",
"eslint": "^6.8.0",
"mocha": "^7.1.1",
"tmp": "^0.1.0"
"@types/tmp": "^0.2.0",
"@types/yargs": "^15.0.11",
"@typescript-eslint/eslint-plugin": "^4.9.0",
"@typescript-eslint/parser": "^4.9.0",
"eslint": "^7.14.0",
"mocha": "^8.2.1",
"tmp": "^0.2.1"
},
"dependencies": {
"@tedconf/fessonia": "^2.1.0",
"@types/cli-progress": "^3.4.2",
"@tedconf/fessonia": "^2.1.2",
"@types/cli-progress": "^3.8.0",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.19.2",
"axios-retry": "^3.1.8",
"cli-progress": "^3.7.0",
"axios": "^0.21.1",
"axios-retry": "^3.1.9",
"cli-progress": "^3.8.2",
"colors": "^1.4.0",
"is-elevated": "^3.0.0",
"iso8601-duration": "^1.2.0",
"jwt-decode": "^2.2.0",
"puppeteer": "2.1.1",
"iso8601-duration": "^1.3.0",
"jwt-decode": "^3.1.2",
"puppeteer": "5.5.0",
"readline-sync": "^1.4.10",
"sanitize-filename": "^1.6.3",
"terminal-image": "^1.0.1",
"typescript": "^3.8.3",
"winston": "^3.3.2",
"yargs": "^15.0.3"
"terminal-image": "^1.2.1",
"typescript": "^4.1.2",
"winston": "^3.3.3",
"yargs": "^16.1.1"
}
}

View File

@@ -34,7 +34,9 @@ export class ApiClient {
return true;
}
logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
logger.verbose(`Here is the error message: \n '${err.response?.data}`);
logger.warn('Here is the error message: ');
console.dir(err.response?.data);
logger.warn('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
@@ -43,6 +45,11 @@ export class ApiClient {
});
}
/**
* Used to initialize/retrive the active ApiClient
*
* @param session used if initializing
*/
public static getInstance(session?: Session): ApiClient {
if (!ApiClient.instance) {
ApiClient.instance = new ApiClient(session);
@@ -51,6 +58,16 @@ export class ApiClient {
return ApiClient.instance;
}
public setSession(session: Session): void {
if (!ApiClient.instance) {
logger.warn("Trying to update ApiCient session when it's not initialized!");
}
this.session = session;
return;
}
/**
* Call Microsoft Stream API. Base URL is sourced from
* the session object and prepended automatically.

View File

@@ -13,12 +13,12 @@ export const argv: any = yargs.options({
username: {
alias: 'u',
type: 'string',
describe: 'The username used to log into Microsoft Stream (enabling this will fill in the email field for you)',
describe: 'The username used to log into Microsoft Stream (enabling this will fill in the email field for you).',
demandOption: false
},
videoUrls: {
alias: 'i',
describe: 'List of video urls',
describe: 'List of urls to videos or Microsoft Stream groups.',
type: 'array',
demandOption: false
},
@@ -30,7 +30,7 @@ export const argv: any = yargs.options({
},
outputDirectory: {
alias: 'o',
describe: 'The directory where destreamer will save your downloads',
describe: 'The directory where destreamer will save your downloads.',
type: 'string',
default: 'videos',
demandOption: false
@@ -44,42 +44,43 @@ export const argv: any = yargs.options({
},
keepLoginCookies: {
alias: 'k',
describe: 'Let Chromium cache identity provider cookies so you can use "Remember me" during login',
describe: 'Let Chromium cache identity provider cookies so you can use "Remember me" during login.\n' +
'Must be used every subsequent time you launch Destreamer if you want to log in automatically.',
type: 'boolean',
default: false,
demandOption: false
},
noExperiments: {
alias: 'x',
describe: 'Do not attempt to render video thumbnails in the console',
describe: 'Do not attempt to render video thumbnails in the console.',
type: 'boolean',
default: false,
demandOption: false
},
simulate: {
alias: 's',
describe: 'Disable video download and print metadata information to the console',
describe: 'Disable video download and print metadata information to the console.',
type: 'boolean',
default: false,
demandOption: false
},
verbose: {
alias: 'v',
describe: 'Print additional information to the console (use this before opening an issue on GitHub)',
describe: 'Print additional information to the console (use this before opening an issue on GitHub).',
type: 'boolean',
default: false,
demandOption: false
},
closedCaptions: {
alias: 'cc',
describe: 'Check if closed captions are available and let the user choose which one to download (will not ask if only one available)',
describe: 'Check if closed captions are available and let the user choose which one to download (will not ask if only one available).',
type: 'boolean',
default: false,
demandOption: false
},
noCleanup: {
alias: 'nc',
describe: 'Do not delete the downloaded video file when an FFmpeg error occurs',
describe: 'Do not delete the downloaded video file when an FFmpeg error occurs.',
type: 'boolean',
default: false,
demandOption: false
@@ -97,13 +98,13 @@ export const argv: any = yargs.options({
demandOption: false
},
format: {
describe: 'Output container format (mkv, mp4, mov, anything that FFmpeg supports)',
describe: 'Output container format (mkv, mp4, mov, anything that FFmpeg supports).',
type: 'string',
default: 'mkv',
demandOption: false
},
skip: {
describe: 'Skip download if file already exists',
describe: 'Skip download if file already exists.',
type: 'boolean',
default: false,
demandOption: false
@@ -206,7 +207,7 @@ function isOutputTemplateValid(argv: any): boolean {
export function promptUser(choices: Array<string>): number {
let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
if (index === -1) {
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);

View File

@@ -3,45 +3,49 @@ export const enum ERROR_CODE {
ELEVATED_SHELL,
CANCELLED_USER_INPUT,
MISSING_FFMPEG,
OUTDATED_FFMPEG,
UNK_FFMPEG_ERROR,
INVALID_VIDEO_GUID,
NO_SESSION_INFO
}
export const errors: {[key: number]: string} = {
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
'Timeout or fatal error, please check your downloads directory and try again',
export const errors: { [key: number]: string } = {
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
'Timeout or fatal error, please check your downloads directory and try again',
[ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
'Please run in a regular, non-elevated window.',
[ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
'Please run in a regular, non-elevated window.',
[ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
[ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos',
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos',
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
[ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos',
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
};
export const enum CLI_ERROR {
MISSING_INPUT_ARG = 'You must specify a URLs source. \n' +
'Valid options are -i for one or more URLs separated by space or -f for input file. \n',
MISSING_INPUT_ARG = 'You must specify a URLs source. \n' +
'Valid options are -i for one or more URLs separated by space or -f for input file. \n',
INPUT_ARG_CONFLICT = 'Too many URLs sources specified! \n' +
'Please specify a single source, either -i or -f \n',
INPUT_ARG_CONFLICT = 'Too many URLs sources specified! \n' +
'Please specify a single source, either -i or -f \n',
INPUTFILE_WRONG_EXTENSION = 'The specified inputFile has the wrong extension \n' +
'Please make sure to use path/to/filename.txt when useing the -f option \n',
INPUTFILE_WRONG_EXTENSION = 'The specified inputFile has the wrong extension \n' +
'Please make sure to use path/to/filename.txt when useing the -f option \n',
INPUTFILE_NOT_FOUND = 'The specified inputFile does not exists \n'+
'Please check the filename and the path you provided \n',
INPUTFILE_NOT_FOUND = 'The specified inputFile does not exists \n' +
'Please check the filename and the path you provided \n',
INVALID_OUTDIR = 'Could not create the default/specified output directory \n' +
'Please check directory and permissions and try again. \n'
INVALID_OUTDIR = 'Could not create the default/specified output directory \n' +
'Please check directory and permissions and try again. \n'
}

View File

@@ -8,7 +8,7 @@ import { AxiosResponse } from 'axios';
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
const apiClient: ApiClient = ApiClient.getInstance(session);
let thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
.then((response: AxiosResponse<any> | undefined) => response?.data);
console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));

View File

@@ -19,16 +19,16 @@ export class TokenCache {
return null;
}
let session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
type Jwt = {
[key: string]: any
}
const decodedJwt: Jwt = jwtDecode(session.AccessToken);
let now: number = Math.floor(Date.now() / 1000);
let exp: number = decodedJwt['exp'];
let timeLeft: number = exp - now;
const now: number = Math.floor(Date.now() / 1000);
const exp: number = decodedJwt['exp'];
const timeLeft: number = exp - now;
if (timeLeft < 120) {
logger.warn('Access token has expired! \n');
@@ -42,7 +42,7 @@ export class TokenCache {
}
public Write(session: Session): void {
let s: string = JSON.stringify(session, null, 4);
const s: string = JSON.stringify(session, null, 4);
fs.writeFile('.token_cache', s, (err: any) => {
if (err) {
return logger.error(err);

View File

@@ -22,9 +22,21 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
else if (groupMatch) {
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
const result: Array<string> = [];
let result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=${videoNumber}&$orderby=publishedDate asc`, 'get')
.then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
// Anything above $top=100 results in 400 Bad Request
// Use $skip to skip the first 100 and get another 100 and so on
for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
const partial: Array<string> = await client.callApi(
`groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
'$top=100&$orderby=publishedDate asc', 'get')
.then(
(response: AxiosResponse<any> | undefined) =>
response?.data.value.map((item: any) => item.id)
);
result.push(...partial);
}
return result;
}
@@ -48,7 +60,7 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
session: Session): Promise<Array<Array<string>>> {
const apiClient: ApiClient = ApiClient.getInstance(session);
let guidList: Array<string> = [];
const guidList: Array<string> = [];
for (const url of urlList) {
const guids: Array<string> | null = await extractGuids(url, apiClient);
@@ -85,8 +97,8 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
.split(/\r?\n/);
const apiClient: ApiClient = ApiClient.getInstance(session);
let guidList: Array<string> = [];
let outDirList: Array<string> = [];
const guidList: Array<string> = [];
const outDirList: Array<string> = [];
// if the last line was an url set this
let foundUrl = false;
@@ -101,23 +113,23 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
// parse if line is option
else if (line.includes('-dir')) {
if (foundUrl) {
let outDir: string | null = parseOption('-dir', line);
const outDir: string | null = parseOption('-dir', line);
if (outDir && checkOutDir(outDir)) {
outDirList.push(...Array(guidList.length - outDirList.length)
.fill(outDir));
.fill(outDir));
}
else {
outDirList.push(...Array(guidList.length - outDirList.length)
.fill(defaultOutDir));
.fill(defaultOutDir));
}
foundUrl = false;
continue;
}
else {
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
continue;
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
continue;
}
}
@@ -155,7 +167,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
function parseOption(optionSyntax: string, item: string): string | null {
const match: RegExpMatchArray | null = item.match(
RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
);
);
return match ? match[1] : null;
}
@@ -168,7 +180,7 @@ export function checkOutDir(directory: string): boolean {
logger.info('\nCreated directory: '.yellow + directory);
}
catch (e) {
logger.warn('Cannot create directory: '+ directory +
logger.warn('Cannot create directory: ' + directory +
'\nFalling back to default directory..');
return false;
@@ -181,7 +193,13 @@ export function checkOutDir(directory: string): boolean {
export function checkRequirements(): void {
try {
const copyrightYearRe = new RegExp(/\d{4}-(\d{4})/);
const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
if (parseInt(copyrightYearRe.exec(ffmpegVer)?.[1] ?? '0') <= 2019) {
process.exit(ERROR_CODE.OUTDATED_FFMPEG);
}
logger.verbose(`Using ${ffmpegVer}\n`);
}
catch (e) {

View File

@@ -46,7 +46,7 @@ function durationToTotalChunks(duration: string): number {
export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
let metadata: Array<Video> = [];
const metadata: Array<Video> = [];
let title: string;
let duration: string;
let publishDate: string;
@@ -65,7 +65,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
/* TODO: change this to a single guid at a time to ease our footprint on the
MSS servers or we get throttled after 10 sequential reqs */
for (const guid of videoGuids) {
let response: AxiosResponse<any> | undefined =
const response: AxiosResponse<any> | undefined =
await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
title = sanitizeWindowsName(response?.data['name']);
@@ -94,7 +94,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
posterImageUrl = response?.data['posterImage']['medium']['url'];
if (subtitles) {
let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
if (!captions?.data.value.length) {
captionsUrl = undefined;
@@ -140,7 +140,7 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
let match = elementRegEx.exec(template);
while (match) {
let value = video[match[1] as keyof Video] as string;
const value = video[match[1] as keyof Video] as string;
title = title.replace(match[0], value);
match = elementRegEx.exec(template);
}

View File

@@ -13,6 +13,7 @@ import cliProgress from 'cli-progress';
import fs from 'fs';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import { ApiClient } from './ApiClient';
const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
@@ -151,6 +152,7 @@ async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array
if (argv.keepLoginCookies && index !== 0) {
logger.info('Trying to refresh token...');
session = await refreshSession('https://web.microsoftstream.com/video/' + videoGUIDs[index]);
ApiClient.getInstance().setSession(session);
}
const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
@@ -268,6 +270,7 @@ async function main(): Promise<void> {
await init(); // must be first
let session: Session;
// eslint-disable-next-line prefer-const
session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
logger.verbose('Session and API info \n' +