Group parsing fix and error out on old ffmpeg version (#298)

* fixed parsing for group with more than 100 videos

* updated all packages to latest version

* Error on old ffmpeg binaries (closes #294)
minor linting fixes

* automatic update of files

Co-authored-by: Adrian Calinescu <foo@snobu.org>

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "eslint.enable": true
+    "eslint.enable": true,
+    "typescript.tsdk": "node_modules\\typescript\\lib"
 }

package.json

@@ -17,34 +17,34 @@
   "author": "snobu",
   "license": "MIT",
   "devDependencies": {
-    "@types/mocha": "^7.0.2",
+    "@types/mocha": "^8.0.4",
-    "@types/puppeteer": "^1.20.4",
+    "@types/puppeteer": "^5.4.0",
     "@types/readline-sync": "^1.4.3",
-    "@types/tmp": "^0.1.0",
+    "@types/tmp": "^0.2.0",
-    "@types/yargs": "^15.0.3",
+    "@types/yargs": "^15.0.11",
-    "@typescript-eslint/eslint-plugin": "^2.25.0",
+    "@typescript-eslint/eslint-plugin": "^4.9.0",
-    "@typescript-eslint/parser": "^2.25.0",
+    "@typescript-eslint/parser": "^4.9.0",
-    "eslint": "^6.8.0",
+    "eslint": "^7.14.0",
-    "mocha": "^7.1.1",
+    "mocha": "^8.2.1",
-    "tmp": "^0.1.0"
+    "tmp": "^0.2.1"
   },
   "dependencies": {
-    "@tedconf/fessonia": "^2.1.0",
+    "@tedconf/fessonia": "^2.1.2",
-    "@types/cli-progress": "^3.4.2",
+    "@types/cli-progress": "^3.8.0",
     "@types/jwt-decode": "^2.2.1",
-    "axios": "^0.21.1",
+    "axios": "^0.21.0",
-    "axios-retry": "^3.1.8",
+    "axios-retry": "^3.1.9",
-    "cli-progress": "^3.7.0",
+    "cli-progress": "^3.8.2",
     "colors": "^1.4.0",
     "is-elevated": "^3.0.0",
-    "iso8601-duration": "^1.2.0",
+    "iso8601-duration": "^1.3.0",
-    "jwt-decode": "^2.2.0",
+    "jwt-decode": "^3.1.2",
-    "puppeteer": "2.1.1",
+    "puppeteer": "5.5.0",
     "readline-sync": "^1.4.10",
     "sanitize-filename": "^1.6.3",
-    "terminal-image": "^1.0.1",
+    "terminal-image": "^1.2.1",
-    "typescript": "^3.8.3",
+    "typescript": "^4.1.2",
-    "winston": "^3.3.2",
+    "winston": "^3.3.3",
-    "yargs": "^15.0.3"
+    "yargs": "^16.1.1"
   }
 }

src/CommandLineParser.ts

@@ -207,7 +207,7 @@ function isOutputTemplateValid(argv: any): boolean {
 
 
 export function promptUser(choices: Array<string>): number {
-    let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
+    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
 
     if (index === -1) {
         process.exit(ERROR_CODE.CANCELLED_USER_INPUT);

src/Errors.ts

@@ -3,13 +3,14 @@ export const enum ERROR_CODE {
     ELEVATED_SHELL,
     CANCELLED_USER_INPUT,
     MISSING_FFMPEG,
+    OUTDATED_FFMPEG,
     UNK_FFMPEG_ERROR,
     INVALID_VIDEO_GUID,
     NO_SESSION_INFO
 }
 
 
-export const errors: {[key: number]: string} = {
+export const errors: { [key: number]: string } = {
     [ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
         'Timeout or fatal error, please check your downloads directory and try again',
 
@@ -21,6 +22,9 @@ export const errors: {[key: number]: string} = {
     [ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
         'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
+    [ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
+        'Destreamer requires a fairly recent release of FFmpeg to download videos',
+
     [ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
 
     [ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
@@ -39,7 +43,7 @@ export const enum CLI_ERROR {
     INPUTFILE_WRONG_EXTENSION = 'The specified inputFile has the wrong extension \n' +
     'Please make sure to use path/to/filename.txt when useing the -f option \n',
 
-    INPUTFILE_NOT_FOUND         = 'The specified inputFile does not exists \n'+
+    INPUTFILE_NOT_FOUND = 'The specified inputFile does not exists \n' +
     'Please check the filename and the path you provided \n',
 
     INVALID_OUTDIR = 'Could not create the default/specified output directory \n' +

src/Thumbnail.ts

@@ -8,7 +8,7 @@ import { AxiosResponse } from 'axios';
 export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
     const apiClient: ApiClient = ApiClient.getInstance(session);
 
-    let thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
+    const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
         .then((response: AxiosResponse<any> | undefined) => response?.data);
 
     console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));

src/TokenCache.ts

@@ -19,16 +19,16 @@ export class TokenCache {
             return null;
         }
 
-        let session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
+        const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
 
         type Jwt = {
             [key: string]: any
         }
         const decodedJwt: Jwt = jwtDecode(session.AccessToken);
 
-        let now: number = Math.floor(Date.now() / 1000);
+        const now: number = Math.floor(Date.now() / 1000);
-        let exp: number = decodedJwt['exp'];
+        const exp: number = decodedJwt['exp'];
-        let timeLeft: number = exp - now;
+        const timeLeft: number = exp - now;
 
         if (timeLeft < 120) {
             logger.warn('Access token has expired! \n');
@@ -42,7 +42,7 @@ export class TokenCache {
     }
 
     public Write(session: Session): void {
-        let s: string = JSON.stringify(session, null, 4);
+        const s: string = JSON.stringify(session, null, 4);
         fs.writeFile('.token_cache', s, (err: any) => {
             if (err) {
                 return logger.error(err);

src/Utils.ts

@@ -20,12 +20,23 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
         return [videoMatch[1]];
     }
     else if (groupMatch) {
-        // const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
+        const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
-        //     .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
+            .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
+        const result: Array<string> = [];
 
-        // Anything over $top=100 will return a 400 Bad Request
+        // Anything above $top=100 results in 400 Bad Request
-        let result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=100&$orderby=publishedDate asc`, 'get')
+        // Use $skip to skip the first 100 and get another 100 and so on
-            .then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
+        for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
+            const partial: Array<string> = await client.callApi(
+                `groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
+                '$top=100&$orderby=publishedDate asc', 'get')
+                .then(
+                    (response: AxiosResponse<any> | undefined) =>
+                        response?.data.value.map((item: any) => item.id)
+                );
+
+            result.push(...partial);
+        }
 
         return result;
     }
@@ -49,7 +60,7 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
     session: Session): Promise<Array<Array<string>>> {
 
     const apiClient: ApiClient = ApiClient.getInstance(session);
-    let guidList: Array<string> = [];
+    const guidList: Array<string> = [];
 
     for (const url of urlList) {
         const guids: Array<string> | null = await extractGuids(url, apiClient);
@@ -86,8 +97,8 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
         .split(/\r?\n/);
     const apiClient: ApiClient = ApiClient.getInstance(session);
 
-    let guidList: Array<string> = [];
+    const guidList: Array<string> = [];
-    let outDirList: Array<string> = [];
+    const outDirList: Array<string> = [];
     // if the last line was an url set this
     let foundUrl = false;
 
@@ -102,7 +113,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
         // parse if line is option
         else if (line.includes('-dir')) {
             if (foundUrl) {
-                let outDir: string | null = parseOption('-dir', line);
+                const outDir: string | null = parseOption('-dir', line);
 
                 if (outDir && checkOutDir(outDir)) {
                     outDirList.push(...Array(guidList.length - outDirList.length)
@@ -169,7 +180,7 @@ export function checkOutDir(directory: string): boolean {
             logger.info('\nCreated directory: '.yellow + directory);
         }
         catch (e) {
-            logger.warn('Cannot create directory: '+ directory +
+            logger.warn('Cannot create directory: ' + directory +
                 '\nFalling back to default directory..');
 
             return false;
@@ -182,7 +193,13 @@ export function checkOutDir(directory: string): boolean {
 
 export function checkRequirements(): void {
     try {
+        const copyrightYearRe = new RegExp(/\d{4}-(\d{4})/);
         const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
+
+        if (parseInt(copyrightYearRe.exec(ffmpegVer)?.[1] ?? '0') <= 2019) {
+            process.exit(ERROR_CODE.OUTDATED_FFMPEG);
+        }
+
         logger.verbose(`Using ${ffmpegVer}\n`);
     }
     catch (e) {

src/VideoUtils.ts

@@ -46,7 +46,7 @@ function durationToTotalChunks(duration: string): number {
 
 
 export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
-    let metadata: Array<Video> = [];
+    const metadata: Array<Video> = [];
     let title: string;
     let duration: string;
     let publishDate: string;
@@ -65,7 +65,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
     /* TODO: change this to a single guid at a time to ease our footprint on the
     MSS servers or we get throttled after 10 sequential reqs */
     for (const guid of videoGuids) {
-        let response: AxiosResponse<any> | undefined =
+        const response: AxiosResponse<any> | undefined =
             await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
 
         title = sanitizeWindowsName(response?.data['name']);
@@ -94,7 +94,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
         posterImageUrl = response?.data['posterImage']['medium']['url'];
 
         if (subtitles) {
-            let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
+            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
 
             if (!captions?.data.value.length) {
                 captionsUrl = undefined;
@@ -140,7 +140,7 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
         let match = elementRegEx.exec(template);
 
         while (match) {
-            let value = video[match[1] as keyof Video] as string;
+            const value = video[match[1] as keyof Video] as string;
             title = title.replace(match[0], value);
             match = elementRegEx.exec(template);
         }

src/destreamer.ts

@@ -270,6 +270,7 @@ async function main(): Promise<void> {
     await init(); // must be first
 
     let session: Session;
+    // eslint-disable-next-line prefer-const
     session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
 
     logger.verbose('Session and API info \n' +