Add Raspberry Pi note

Bumps [axios](https://github.com/axios/axios) from 0.21.0 to 0.21.1.
- [Release notes](https://github.com/axios/axios/releases)
- [Changelog](https://github.com/axios/axios/blob/v0.21.1/CHANGELOG.md)
- [Commits](https://github.com/axios/axios/compare/v0.21.0...v0.21.1)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/build.yaml

@@ -14,7 +14,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [8.x, 10.x, 12.x, 13.x]
+        node-version: [10.x, 12.x, 13.x]
 
     steps:
     - uses: actions/checkout@v1

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "eslint.enable": true
+    "eslint.enable": true,
+    "typescript.tsdk": "node_modules\\typescript\\lib"
 }

README.md

@@ -2,14 +2,7 @@
   <img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
 </a>
 
-# BREAKING
+**destreamer v3.0** is just around the corner. You can try out a pre-release today by cloning [this branch](https://github.com/snobu/destreamer/tree/aria2c_forRealNow).
-
-**destreamer v3.0** is just around the corner. Download speed improvement is astonishing and we have a never before seen photo from the design sessions:<br><br>
-![desilva](https://user-images.githubusercontent.com/6472374/93003437-54a7fd00-f547-11ea-8473-e4602993e69d.jpg)
-
-Help us pick a codename for the new release:<br><br>
-![codename](https://user-images.githubusercontent.com/6472374/93003896-20ced680-f54b-11ea-8be1-2c14e0bd3751.png)<br><br>
-Comment in this thread: https://github.com/snobu/destreamer/issues/223
 
 ![destreamer](assets/logo.png)
 
@@ -45,7 +38,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
 
 ## Prereqs
 
-- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+.
+- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
 - **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
 - [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
 - [**git**][git]: one or more npm dependencies require git.
@@ -81,13 +74,15 @@ const browser: puppeteer.Browser = await puppeteer.launch({
 
 Now, change `executablePath` to reflect the path to your browser and profile (i.e. to use Microsoft Edge on Windows):
 ```typescript
-        executablePath: "'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe' --profile-directory=Default",
+        executablePath: 'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe',
 ```
 
 Note that for Mac/Linux the path will look a little different but no other changes are necessary.
 
 You need to rebuild (`npm run build`) every time you change this configuration.
 
+If you're trying to run this on a Raspberry Pi you should see [this issue](https://github.com/snobu/destreamer/issues/311).
+
 ## How to build
 
 To build destreamer clone this repository, install dependencies and run the build script -
@@ -181,9 +176,9 @@ These optional lines must start with white space(s).
 Usage -
 ```
 https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
- -dir=videos/lessons/week1
+ -dir="videos/lessons/week1"
 https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
-        -dir=videos/lessons/week2"
+ -dir="videos/lessons/week2"
 ```
 
 ### Title template

package.json

@@ -17,34 +17,34 @@
   "author": "snobu",
   "license": "MIT",
   "devDependencies": {
-    "@types/mocha": "^7.0.2",
+    "@types/mocha": "^8.0.4",
-    "@types/puppeteer": "^1.20.4",
+    "@types/puppeteer": "^5.4.0",
     "@types/readline-sync": "^1.4.3",
-    "@types/tmp": "^0.1.0",
+    "@types/tmp": "^0.2.0",
-    "@types/yargs": "^15.0.3",
+    "@types/yargs": "^15.0.11",
-    "@typescript-eslint/eslint-plugin": "^2.25.0",
+    "@typescript-eslint/eslint-plugin": "^4.9.0",
-    "@typescript-eslint/parser": "^2.25.0",
+    "@typescript-eslint/parser": "^4.9.0",
-    "eslint": "^6.8.0",
+    "eslint": "^7.14.0",
-    "mocha": "^7.1.1",
+    "mocha": "^8.2.1",
-    "tmp": "^0.1.0"
+    "tmp": "^0.2.1"
   },
   "dependencies": {
-    "@tedconf/fessonia": "^2.1.0",
+    "@tedconf/fessonia": "^2.1.2",
-    "@types/cli-progress": "^3.4.2",
+    "@types/cli-progress": "^3.8.0",
     "@types/jwt-decode": "^2.2.1",
-    "axios": "^0.19.2",
+    "axios": "^0.21.1",
-    "axios-retry": "^3.1.8",
+    "axios-retry": "^3.1.9",
-    "cli-progress": "^3.7.0",
+    "cli-progress": "^3.8.2",
     "colors": "^1.4.0",
     "is-elevated": "^3.0.0",
-    "iso8601-duration": "^1.2.0",
+    "iso8601-duration": "^1.3.0",
-    "jwt-decode": "^2.2.0",
+    "jwt-decode": "^3.1.2",
-    "puppeteer": "2.1.1",
+    "puppeteer": "5.5.0",
     "readline-sync": "^1.4.10",
     "sanitize-filename": "^1.6.3",
-    "terminal-image": "^1.0.1",
+    "terminal-image": "^1.2.1",
-    "typescript": "^3.8.3",
+    "typescript": "^4.1.2",
-    "winston": "^3.3.2",
+    "winston": "^3.3.3",
-    "yargs": "^15.0.3"
+    "yargs": "^16.1.1"
   }
 }

src/ApiClient.ts

@@ -34,7 +34,9 @@ export class ApiClient {
                     return true;
                 }
                 logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
-                logger.verbose(`Here is the error message: \n '${err.response?.data}`);
+                logger.warn('Here is the error message: ');
+                console.dir(err.response?.data);
+                logger.warn('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
 
                 const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
 

src/CommandLineParser.ts

@@ -207,7 +207,7 @@ function isOutputTemplateValid(argv: any): boolean {
 
 
 export function promptUser(choices: Array<string>): number {
-    let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
+    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
 
     if (index === -1) {
         process.exit(ERROR_CODE.CANCELLED_USER_INPUT);

src/Errors.ts

@@ -3,45 +3,49 @@ export const enum ERROR_CODE {
     ELEVATED_SHELL,
     CANCELLED_USER_INPUT,
     MISSING_FFMPEG,
+    OUTDATED_FFMPEG,
     UNK_FFMPEG_ERROR,
     INVALID_VIDEO_GUID,
     NO_SESSION_INFO
 }
 
 
-export const errors: {[key: number]: string} = {
+export const errors: { [key: number]: string } = {
-    [ERROR_CODE.UNHANDLED_ERROR]:       'Unhandled error!\n' +
+    [ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
-                                        'Timeout or fatal error, please check your downloads directory and try again',
+        'Timeout or fatal error, please check your downloads directory and try again',
 
-    [ERROR_CODE.ELEVATED_SHELL]:        'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
+    [ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
-                                        'Please run in a regular, non-elevated window.',
+        'Please run in a regular, non-elevated window.',
 
-    [ERROR_CODE.CANCELLED_USER_INPUT]:  'Input was cancelled by user',
+    [ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
 
-    [ERROR_CODE.MISSING_FFMPEG]:        'FFmpeg is missing!\n' +
+    [ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
-                                        'Destreamer requires a fairly recent release of FFmpeg to download videos',
+        'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
-    [ERROR_CODE.UNK_FFMPEG_ERROR]:      'Unknown FFmpeg error',
+    [ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
+        'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
-    [ERROR_CODE.INVALID_VIDEO_GUID]:    'Unable to get video GUID from URL',
+    [ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
 
-    [ERROR_CODE.NO_SESSION_INFO]:       'Could not evaluate sessionInfo on the page'
+    [ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
+
+    [ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
 };
 
 
 export const enum CLI_ERROR {
-    MISSING_INPUT_ARG           = 'You must specify a URLs source. \n' +
+    MISSING_INPUT_ARG = 'You must specify a URLs source. \n' +
-                                'Valid options are -i for one or more URLs separated by space or -f for input file. \n',
+    'Valid options are -i for one or more URLs separated by space or -f for input file. \n',
 
-    INPUT_ARG_CONFLICT          = 'Too many URLs sources specified! \n' +
+    INPUT_ARG_CONFLICT = 'Too many URLs sources specified! \n' +
-                                'Please specify a single source, either -i or -f \n',
+    'Please specify a single source, either -i or -f \n',
 
-    INPUTFILE_WRONG_EXTENSION   = 'The specified inputFile has the wrong extension \n' +
+    INPUTFILE_WRONG_EXTENSION = 'The specified inputFile has the wrong extension \n' +
-                                'Please make sure to use path/to/filename.txt when useing the -f option \n',
+    'Please make sure to use path/to/filename.txt when useing the -f option \n',
 
-    INPUTFILE_NOT_FOUND         = 'The specified inputFile does not exists \n'+
+    INPUTFILE_NOT_FOUND = 'The specified inputFile does not exists \n' +
-                                'Please check the filename and the path you provided \n',
+    'Please check the filename and the path you provided \n',
 
-    INVALID_OUTDIR              = 'Could not create the default/specified output directory \n' +
+    INVALID_OUTDIR = 'Could not create the default/specified output directory \n' +
-                                'Please check directory and permissions and try again. \n'
+    'Please check directory and permissions and try again. \n'
 }

src/Thumbnail.ts

@@ -8,7 +8,7 @@ import { AxiosResponse } from 'axios';
 export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
     const apiClient: ApiClient = ApiClient.getInstance(session);
 
-    let thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
+    const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
         .then((response: AxiosResponse<any> | undefined) => response?.data);
 
     console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));

src/TokenCache.ts

@@ -19,16 +19,16 @@ export class TokenCache {
             return null;
         }
 
-        let session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
+        const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
 
         type Jwt = {
             [key: string]: any
         }
         const decodedJwt: Jwt = jwtDecode(session.AccessToken);
 
-        let now: number = Math.floor(Date.now() / 1000);
+        const now: number = Math.floor(Date.now() / 1000);
-        let exp: number = decodedJwt['exp'];
+        const exp: number = decodedJwt['exp'];
-        let timeLeft: number = exp - now;
+        const timeLeft: number = exp - now;
 
         if (timeLeft < 120) {
             logger.warn('Access token has expired! \n');
@@ -42,7 +42,7 @@ export class TokenCache {
     }
 
     public Write(session: Session): void {
-        let s: string = JSON.stringify(session, null, 4);
+        const s: string = JSON.stringify(session, null, 4);
         fs.writeFile('.token_cache', s, (err: any) => {
             if (err) {
                 return logger.error(err);

src/Utils.ts

@@ -22,9 +22,21 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
     else if (groupMatch) {
         const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
             .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
+        const result: Array<string> = [];
 
-        let result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=${videoNumber}&$orderby=publishedDate asc`, 'get')
+        // Anything above $top=100 results in 400 Bad Request
-            .then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
+        // Use $skip to skip the first 100 and get another 100 and so on
+        for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
+            const partial: Array<string> = await client.callApi(
+                `groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
+                '$top=100&$orderby=publishedDate asc', 'get')
+                .then(
+                    (response: AxiosResponse<any> | undefined) =>
+                        response?.data.value.map((item: any) => item.id)
+                );
+
+            result.push(...partial);
+        }
 
         return result;
     }
@@ -48,7 +60,7 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
     session: Session): Promise<Array<Array<string>>> {
 
     const apiClient: ApiClient = ApiClient.getInstance(session);
-    let guidList: Array<string> = [];
+    const guidList: Array<string> = [];
 
     for (const url of urlList) {
         const guids: Array<string> | null = await extractGuids(url, apiClient);
@@ -85,8 +97,8 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
         .split(/\r?\n/);
     const apiClient: ApiClient = ApiClient.getInstance(session);
 
-    let guidList: Array<string> = [];
+    const guidList: Array<string> = [];
-    let outDirList: Array<string> = [];
+    const outDirList: Array<string> = [];
     // if the last line was an url set this
     let foundUrl = false;
 
@@ -101,23 +113,23 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
         // parse if line is option
         else if (line.includes('-dir')) {
             if (foundUrl) {
-                let outDir: string | null = parseOption('-dir', line);
+                const outDir: string | null = parseOption('-dir', line);
 
                 if (outDir && checkOutDir(outDir)) {
                     outDirList.push(...Array(guidList.length - outDirList.length)
-                    .fill(outDir));
+                        .fill(outDir));
                 }
                 else {
                     outDirList.push(...Array(guidList.length - outDirList.length)
-                    .fill(defaultOutDir));
+                        .fill(defaultOutDir));
                 }
 
                 foundUrl = false;
                 continue;
             }
             else {
-            logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
+                logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
-            continue;
+                continue;
             }
         }
 
@@ -155,7 +167,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
 function parseOption(optionSyntax: string, item: string): string | null {
     const match: RegExpMatchArray | null = item.match(
         RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
-        );
+    );
 
     return match ? match[1] : null;
 }
@@ -168,7 +180,7 @@ export function checkOutDir(directory: string): boolean {
             logger.info('\nCreated directory: '.yellow + directory);
         }
         catch (e) {
-            logger.warn('Cannot create directory: '+ directory +
+            logger.warn('Cannot create directory: ' + directory +
                 '\nFalling back to default directory..');
 
             return false;
@@ -181,7 +193,13 @@ export function checkOutDir(directory: string): boolean {
 
 export function checkRequirements(): void {
     try {
+        const copyrightYearRe = new RegExp(/\d{4}-(\d{4})/);
         const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
+
+        if (parseInt(copyrightYearRe.exec(ffmpegVer)?.[1] ?? '0') <= 2019) {
+            process.exit(ERROR_CODE.OUTDATED_FFMPEG);
+        }
+
         logger.verbose(`Using ${ffmpegVer}\n`);
     }
     catch (e) {

src/VideoUtils.ts

@@ -46,7 +46,7 @@ function durationToTotalChunks(duration: string): number {
 
 
 export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
-    let metadata: Array<Video> = [];
+    const metadata: Array<Video> = [];
     let title: string;
     let duration: string;
     let publishDate: string;
@@ -65,7 +65,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
     /* TODO: change this to a single guid at a time to ease our footprint on the
     MSS servers or we get throttled after 10 sequential reqs */
     for (const guid of videoGuids) {
-        let response: AxiosResponse<any> | undefined =
+        const response: AxiosResponse<any> | undefined =
             await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
 
         title = sanitizeWindowsName(response?.data['name']);
@@ -94,7 +94,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
         posterImageUrl = response?.data['posterImage']['medium']['url'];
 
         if (subtitles) {
-            let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
+            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
 
             if (!captions?.data.value.length) {
                 captionsUrl = undefined;
@@ -140,7 +140,7 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
         let match = elementRegEx.exec(template);
 
         while (match) {
-            let value = video[match[1] as keyof Video] as string;
+            const value = video[match[1] as keyof Video] as string;
             title = title.replace(match[0], value);
             match = elementRegEx.exec(template);
         }

src/destreamer.ts

@@ -270,6 +270,7 @@ async function main(): Promise<void> {
     await init(); // must be first
 
     let session: Session;
+    // eslint-disable-next-line prefer-const
     session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
 
     logger.verbose('Session and API info \n' +