diff --git a/src/ApiClient.ts b/src/ApiClient.ts index b20b85a..cd14d32 100644 --- a/src/ApiClient.ts +++ b/src/ApiClient.ts @@ -34,9 +34,10 @@ export class ApiClient { return true; } logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`); - logger.verbose('Here is the error message: '); - console.dir(err.response?.data); - logger.verbose('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url); + logger.verbose('Here is the error message: \n' + + JSON.stringify(err.response?.data ?? undefined) + + '\nRetrying request...'); + logger.verbose(`We called this URL: ${err.response?.config.baseURL}${err.response?.config.url}`); const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0); diff --git a/src/Utils.ts b/src/Utils.ts index e925b89..236c243 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -20,12 +20,25 @@ async function extractGuids(url: string, client: ApiClient): Promise | undefined) => response?.data.metrics.videos); + const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get') + .then((response: AxiosResponse | undefined) => response?.data.metrics.videos); + const result: Array = []; + + logger.error(videoNumber); // Anything above $top=100 results in 400 Bad Request - const result: Array = await client.callApi(`groups/${groupMatch[1]}/videos?$top=100&$orderby=publishedDate asc`, 'get') - .then((response: AxiosResponse | undefined) => response?.data.value.map((item: any) => item.id)); + // Use $skip to skip the first 100 and get another 100 and so on + for (let index = 0; index <= Math.floor(videoNumber / 100); index++) { + const partial: Array = await client.callApi( + `groups/${groupMatch[1]}/videos?$skip=${100 * index}&` + + '$top=100&$orderby=publishedDate asc', 'get') + .then( + (response: AxiosResponse | undefined) => + response?.data.value.map((item: any) => item.id) + ); + + result.push(...partial); + } return result; } @@ -106,19 +119,19 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string, if (outDir && checkOutDir(outDir)) { outDirList.push(...Array(guidList.length - outDirList.length) - .fill(outDir)); + .fill(outDir)); } else { outDirList.push(...Array(guidList.length - outDirList.length) - .fill(defaultOutDir)); + .fill(defaultOutDir)); } foundUrl = false; continue; } else { - logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`); - continue; + logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`); + continue; } } @@ -156,7 +169,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string, function parseOption(optionSyntax: string, item: string): string | null { const match: RegExpMatchArray | null = item.match( RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`) - ); + ); return match ? match[1] : null; } @@ -169,7 +182,7 @@ export function checkOutDir(directory: string): boolean { logger.info('\nCreated directory: '.yellow + directory); } catch (e) { - logger.warn('Cannot create directory: '+ directory + + logger.warn('Cannot create directory: ' + directory + '\nFalling back to default directory..'); return false; diff --git a/src/destreamer.ts b/src/destreamer.ts index 9198f0e..7c31344 100644 --- a/src/destreamer.ts +++ b/src/destreamer.ts @@ -154,9 +154,8 @@ async function downloadVideo(videoGUIDs: Array, logger.info('Trying to launch and connect to aria2c...\n'); - /* FIXME: aria2Exec must be defined here for the scope but if it's not aslo undefined it says - that later on is used without being initialized even if we exit if it's not initialized. - Is there something that im missing? Probably since it's late but Ill leave it to you Adrian*/ + /* FIXME: aria2Exec must be defined here for the scope but later on it's complaining that it's not + initialized even if we never reach line#361 if we fail the assignment here*/ let aria2cExec: ChildProcess; let arai2cExited = false; await portfinder.getPortPromise({ port: 6800 }).then(