fixed parsing for group with more than 100 videos

fixed error logging

src/ApiClient.ts

@@ -34,9 +34,10 @@ export class ApiClient {
                     return true;
                 }
                 logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
-                logger.verbose('Here is the error message: ');
+                logger.verbose('Here is the error message: \n' +
-                console.dir(err.response?.data);
+                    JSON.stringify(err.response?.data ?? undefined) +
-                logger.verbose('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
+                    '\nRetrying request...');
+                logger.verbose(`We called this URL: ${err.response?.config.baseURL}${err.response?.config.url}`);
 
                 const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
 

src/Utils.ts

@@ -20,12 +20,25 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
         return [videoMatch[1]];
     }
     else if (groupMatch) {
-        // const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
+        const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
-        //    .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
+            .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
+        const result: Array<string> = [];
+
+        logger.error(videoNumber);
 
         // Anything above $top=100 results in 400 Bad Request
-        const result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=100&$orderby=publishedDate asc`, 'get')
+        // Use $skip to skip the first 100 and get another 100 and so on
-            .then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
+        for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
+            const partial: Array<string> = await client.callApi(
+                `groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
+                '$top=100&$orderby=publishedDate asc', 'get')
+                .then(
+                    (response: AxiosResponse<any> | undefined) =>
+                        response?.data.value.map((item: any) => item.id)
+                );
+
+        result.push(...partial);
+        }
 
         return result;
     }
@@ -106,19 +119,19 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
 
                 if (outDir && checkOutDir(outDir)) {
                     outDirList.push(...Array(guidList.length - outDirList.length)
-                    .fill(outDir));
+                        .fill(outDir));
                 }
                 else {
                     outDirList.push(...Array(guidList.length - outDirList.length)
-                    .fill(defaultOutDir));
+                        .fill(defaultOutDir));
                 }
 
                 foundUrl = false;
                 continue;
             }
             else {
-            logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
+                logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
-            continue;
+                continue;
             }
         }
 
@@ -156,7 +169,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
 function parseOption(optionSyntax: string, item: string): string | null {
     const match: RegExpMatchArray | null = item.match(
         RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
-        );
+    );
 
     return match ? match[1] : null;
 }
@@ -169,7 +182,7 @@ export function checkOutDir(directory: string): boolean {
             logger.info('\nCreated directory: '.yellow + directory);
         }
         catch (e) {
-            logger.warn('Cannot create directory: '+ directory +
+            logger.warn('Cannot create directory: ' + directory +
                 '\nFalling back to default directory..');
 
             return false;

src/destreamer.ts

@@ -154,9 +154,8 @@ async function downloadVideo(videoGUIDs: Array<string>,
     logger.info('Trying to launch and connect to aria2c...\n');
 
 
-    /* FIXME: aria2Exec must be defined here for the scope but if it's not aslo undefined it says
+    /* FIXME: aria2Exec must be defined here for the scope but later on it's complaining that it's not
-    that later on is used without being initialized even if we exit if it's not initialized.
+    initialized even if we never reach line#361 if we fail the assignment here*/
-    Is there something that im missing? Probably since it's late but Ill leave it to you Adrian*/
     let aria2cExec: ChildProcess;
     let arai2cExited = false;
     await portfinder.getPortPromise({ port: 6800 }).then(