Introduce singleton API client with retry policy (#130)

* Add singleton http client
* Removed refresh token logic
* Further cleanup after refresh token
* Make tests faster maybe

package.json

@@ -31,14 +31,15 @@
     "@types/cli-progress": "^3.4.2",
     "@types/jwt-decode": "^2.2.1",
     "axios": "^0.19.2",
+    "axios-retry": "^3.1.8",
     "cli-progress": "^3.7.0",
     "colors": "^1.4.0",
     "is-elevated": "^3.0.0",
     "iso8601-duration": "^1.2.0",
     "jwt-decode": "^2.2.0",
-    "puppeteer": "^2.1.1",
+    "puppeteer": "^3.0.4",
     "sanitize-filename": "^1.6.3",
-    "terminal-image": "^0.2.0",
+    "terminal-image": "^1.0.1",
     "typescript": "^3.8.3",
     "yargs": "^15.0.3"
   }

src/ApiClient.ts

@@ -0,0 +1,91 @@
+import axios, { AxiosRequestConfig, AxiosResponse, AxiosInstance, AxiosError } from 'axios';
+import axiosRetry, { isNetworkOrIdempotentRequestError } from 'axios-retry';
+import { Session } from './Types';
+
+export class ApiClient {
+    private static instance: ApiClient;
+    private axiosInstance?: AxiosInstance;
+    private session?: Session;
+
+    private constructor(session?: Session) {
+        this.session = session;
+        this.axiosInstance = axios.create({
+            baseURL: session?.ApiGatewayUri,
+            timeout: 7000,
+            headers: { 'User-Agent': 'destreamer/2.0 (Hammer of Dawn)' }
+        });
+        axiosRetry(this.axiosInstance, {
+            shouldResetTimeout: true,
+            retries: 6,
+            retryDelay: (retryCount) => {
+                return retryCount * 2000;
+            },
+            retryCondition: (err: AxiosError) => {
+                const retryCodes = [429, 500, 502, 503];
+                if (isNetworkOrIdempotentRequestError(err)) {
+                    console.warn(`${err}. Retrying request...`);
+
+                    return true;
+                }   
+                console.warn(`Got HTTP ${err?.response?.status}. Retrying request...`);
+                const condition = retryCodes.includes(err?.response?.status ?? 0);
+
+                return condition;
+            }
+        });
+    }
+
+    public static getInstance(session?: Session): ApiClient {
+        if (!ApiClient.instance) {
+            ApiClient.instance = new ApiClient(session);
+        }
+
+        return ApiClient.instance;
+    }
+
+    /**
+     * Call Microsoft Stream API. Base URL is sourced from
+     * the session object and prepended automatically.
+     */
+    public async callApi(
+        path: string,
+        method: AxiosRequestConfig['method'] = 'get',
+        payload?: any): Promise<AxiosResponse | undefined> {
+
+        const delimiter = path.split('?').length === 1 ? '?' : '&';
+
+        const headers: object = {
+            'Authorization': 'Bearer ' + this.session?.AccessToken
+        };
+
+        return this.axiosInstance?.request({
+            method: method,
+            headers: headers,
+            data: payload ?? undefined,
+            url: path + delimiter + 'api-version=' + this.session?.ApiGatewayVersion
+        });
+    }
+
+    /**
+     * Call an absolute URL
+     */
+    public async callUrl(
+        url: string,
+        method: AxiosRequestConfig['method'] = 'get',
+        payload?: any,
+        responseType: AxiosRequestConfig['responseType'] = 'json'): Promise<AxiosResponse | undefined> {
+            
+        const headers: object = {
+            'Authorization': 'Bearer ' + this.session?.AccessToken
+        };
+
+        return this.axiosInstance?.request({
+            method: method,
+            headers: headers,
+            data: payload ?? undefined,
+            url: url,
+            responseType: responseType
+        });
+    }
+
+}

src/Errors.ts

@@ -13,7 +13,7 @@ export const enum ERROR_CODE {
     INVALID_VIDEO_ID,
     INVALID_VIDEO_GUID,
     UNK_FFMPEG_ERROR,
-    NO_SESSION_INFO,
+    NO_SESSION_INFO
 }
 
 // TODO: create better errors descriptions
@@ -43,7 +43,7 @@ export const Error: IError = {
     [ERROR_CODE.INVALID_VIDEO_GUID]:    'Unable to get video GUID from URL',
 
     [ERROR_CODE.NO_SESSION_INFO]:       'Could not evaluate sessionInfo on the page'
-}
+};
 
 export const enum CLI_ERROR {
     GRACEFULLY_STOP           = ' ', // gracefully stop execution, yargs way

src/Metadata.ts

@@ -1,8 +1,9 @@
 import { Metadata, Session } from './Types';
 import { forEachAsync } from './Utils';
+import { ApiClient } from './ApiClient';
 
 import { parse } from 'iso8601-duration';
-import axios from 'axios';
+
 
 function publishedDateToString(date: string) {
     const dateJs = new Date(date);
@@ -21,8 +22,7 @@ function durationToTotalChunks(duration: string) {
     return (hrs * 60) + mins + (secs / 60);
 }
 
-
+export async function getVideoMetadata(videoGuids: string[], session: Session): Promise<Metadata[]> {
-export async function getVideoMetadata(videoGuids: string[], session: Session, verbose: boolean): Promise<Metadata[]> {
     let metadata: Metadata[] = [];
     let title: string;
     let date: string;
@@ -30,28 +30,20 @@ export async function getVideoMetadata(videoGuids: string[], session: Session, v
     let playbackUrl: string;
     let posterImage: string;
 
+    const apiClient = ApiClient.getInstance(session);
+
     await forEachAsync(videoGuids, async (guid: string) => {
-        let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`;
+        let response = await apiClient.callApi('videos/' + guid, 'get');
 
-        if (verbose)
+        title = response?.data['name'];
-            console.info(`Calling ${apiUrl}`);
+        playbackUrl = response?.data['playbackUrls']
-
-        let response = await axios.get(apiUrl,
-            {
-                headers: {
-                    Authorization: `Bearer ${session.AccessToken}`
-                }
-            });
-
-        title = response.data['name'];
-        playbackUrl = response.data['playbackUrls']
             .filter((item: { [x: string]: string; }) =>
                 item['mimeType'] == 'application/vnd.apple.mpegurl')
             .map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0];
 
-        posterImage = response.data['posterImage']['medium']['url'];
+        posterImage = response?.data['posterImage']['medium']['url'];
-        date = publishedDateToString(response.data['publishedDate']);
+        date = publishedDateToString(response?.data['publishedDate']);
-        totalChunks = durationToTotalChunks(response.data.media['duration']);
+        totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
         metadata.push({
             date: date,

src/Thumbnail.ts

@@ -1,14 +1,10 @@
+import { ApiClient } from './ApiClient';
+import { Session } from './Types';
 import terminalImage from 'terminal-image';
-import axios from 'axios';
 
 
-export async function drawThumbnail(posterImage: string, accessToken: string) {
+export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
-    let thumbnail = await axios.get(posterImage,
+    const apiClient = ApiClient.getInstance(session);
-        {
+    let thumbnail = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer');
-            headers: {
+    console.log(await terminalImage.buffer(thumbnail?.data, { width: 70 } ));
-                Authorization: `Bearer ${accessToken}`
-            },
-            responseType: 'arraybuffer'
-        });
-    console.log(await terminalImage.buffer(thumbnail.data));
 }

src/TokenCache.ts

@@ -2,8 +2,6 @@ import * as fs from 'fs';
 import { Session } from './Types';
 import { bgGreen, bgYellow, green } from 'colors';
 import jwtDecode from 'jwt-decode';
-import axios from 'axios';
-import colors from 'colors';
 
 export class TokenCache {
     private tokenCacheFile: string = '.token_cache';
@@ -55,34 +53,4 @@ export class TokenCache {
             console.info(green('Fresh access token dropped into .token_cache'));
         });
     }
-
-    public async RefreshToken(session: Session, cookie?: string | null): Promise<string | null> {
-        let endpoint = `${session.ApiGatewayUri}refreshtoken?api-version=${session.ApiGatewayVersion}`;
-
-        let headers: Function = (): object => {
-            if (cookie) {
-                return {
-                    Cookie: cookie
-                };
-            }
-            else {
-                return {
-                    Authorization: 'Bearer ' + session.AccessToken
-                };
-            }
-        }
-
-        let response = await axios.get(endpoint, { headers: headers() });
-        let freshCookie: string | null = null;
-        
-        try {
-            let cookie: string = response.headers["set-cookie"].toString();
-            freshCookie = cookie.split(',Authorization_Api=')[0];
-        }
-        catch (e) {
-            console.error(colors.yellow("Error when calling /refreshtoken: Missing or unexpected set-cookie header."));
-        }
-
-        return freshCookie;
-    }
 }

src/destreamer.ts

@@ -11,8 +11,8 @@ import { Metadata, Session } from './Types';
 import { drawThumbnail } from './Thumbnail';
 import { argv } from './CommandLineParser';
 
-import isElevated from 'is-elevated';
 import puppeteer from 'puppeteer';
+import isElevated from 'is-elevated';
 import colors from 'colors';
 import path from 'path';
 import fs from 'fs';
@@ -23,10 +23,6 @@ import cliProgress from 'cli-progress';
 const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
 const tokenCache = new TokenCache();
 
-// The cookie lifetime is one hour,
-// let's refresh every 3000 seconds.
-const REFRESH_TOKEN_INTERVAL = 3000;
-
 async function init() {
     setProcessEvents(); // must be first!
 
@@ -54,7 +50,11 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
     const browser = await puppeteer.launch({
         executablePath: getPuppeteerChromiumPath(),
         headless: false,
-        args: ['--disable-dev-shm-usage']
+        args: [
+            '--disable-dev-shm-usage',
+            '--fast-start',
+            '--no-sandbox'
+        ]
     });
     const page = (await browser.pages())[0];
     console.log('Navigating to login page...');
@@ -103,6 +103,17 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
     console.log("At this point Chromium's job is done, shutting it down...\n");
 
     await browser.close();
+    // --- Ignore all this for now ---
+    // --- hopefully we won't need it ----
+    // await sleep(1000);
+    // let banner = await page.evaluate(
+    //     () => {
+    //             let topbar = document.getElementsByTagName('body')[0];
+    //             topbar.innerHTML = 
+    //                 '<h1 style="color: red">DESTREAMER NEEDS THIS WINDOW ' +
+    //                 'TO DO SOME ACCESS TOKEN MAGIC. DO NOT CLOSE IT.</h1>';
+    //         });
+    // --------------------------------
 
     return session;
 }
@@ -134,11 +145,10 @@ function extractVideoGuid(videoUrls: string[]): string[] {
 
 async function downloadVideo(videoUrls: string[], outputDirectories: string[], session: Session) {
     const videoGuids = extractVideoGuid(videoUrls);
-    let lastTokenRefresh: number;
 
     console.log('Fetching metadata...');
 
-    const metadata: Metadata[] = await getVideoMetadata(videoGuids, session, argv.verbose);
+    const metadata: Metadata[] = await getVideoMetadata(videoGuids, session);
 
     if (argv.simulate) {
         metadata.forEach(video => {
@@ -152,11 +162,10 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
         return;
     }
 
-    if (argv.verbose)
+    if (argv.verbose) {
         console.log(outputDirectories);
+    }
 
-
-    let freshCookie: string | null = null;
     const outDirsIdxInc = outputDirectories.length > 1 ? 1:0;
 
     for (let i=0, j=0, l=metadata.length; i<l; ++i, j+=outDirsIdxInc) {
@@ -175,9 +184,6 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
 
         video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, outputDirectories[j], argv.skip, argv.format);
 
-        // Very experimental inline thumbnail rendering
-        if (!argv.noExperiments)
-            await drawThumbnail(video.posterImage, session.AccessToken);
         
         console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...');
         if (!process.stdout.columns) {
@@ -187,33 +193,12 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
             'Please use PowerShell or cmd.exe to run destreamer on Windows.'));
         }
 
-        // Try to get a fresh cookie, else gracefully fall back
+        const headers = 'Authorization: Bearer ' + session.AccessToken;
-        // to our session access token (Bearer)
-        freshCookie = await tokenCache.RefreshToken(session, freshCookie);
 
-        // Don't remove the "useless" escapes otherwise ffmpeg will
+        // Very experimental inline thumbnail rendering
-        // not pick up the header
+        if (!argv.noExperiments) {
-        // eslint-disable-next-line no-useless-escape
+            await drawThumbnail(video.posterImage, session);
-        let headers = 'Authorization:\ Bearer\ ' + session.AccessToken;
-        if (freshCookie) {
-            lastTokenRefresh = Date.now();
-            if (argv.verbose) {
-                console.info(colors.green('Using a fresh cookie.'));
         }
-            // eslint-disable-next-line no-useless-escape
-            headers = 'Cookie:\ ' + freshCookie;
-        }
-
-        const RefreshTokenMaybe = async (): Promise<void> => {
-            let elapsed = Date.now() - lastTokenRefresh;
-            if (elapsed > REFRESH_TOKEN_INTERVAL * 1000) {
-                if (argv.verbose) {
-                    console.info(colors.green('\nRefreshing access token...'));
-                }
-                lastTokenRefresh = Date.now();
-                freshCookie = await tokenCache.RefreshToken(session, freshCookie);
-            }
-        };
 
         const outputPath = outputDirectories[j] + path.sep + video.title + '.' + argv.format;
         const ffmpegInpt = new FFmpegInput(video.playbackUrl, new Map([
@@ -234,8 +219,11 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
 
             try {
                 fs.unlinkSync(outputPath);
-            } catch(e) {}
             }
+            catch (e) {
+                // Future handling of an error maybe
+            }
+        };
 
         pbar.start(video.totalChunks, 0, {
             speed: '0'
@@ -247,7 +235,6 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
 
         ffmpegCmd.on('update', (data: any) => {
             const currentChunks = ffmpegTimemarkToChunk(data.out_time);
-            RefreshTokenMaybe();
 
             pbar.update(currentChunks, {
                 speed: data.bitrate
@@ -262,7 +249,7 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
         process.on('SIGINT', cleanupFn);
 
         // let the magic begin...
-        await new Promise((resolve: any, reject: any) => {
+        await new Promise((resolve: any) => {
             ffmpegCmd.on('error', (error: any) => {
                 if (argv.skip && error.message.includes('exists') && error.message.includes(outputPath)) {
                     pbar.update(video.totalChunks); // set progress bar to 100%
@@ -276,7 +263,7 @@ async function downloadVideo(videoUrls: string[], outputDirectories: string[], s
                 }
             });
 
-            ffmpegCmd.on('success', (data:any) => {
+            ffmpegCmd.on('success', () => {
                 pbar.update(video.totalChunks); // set progress bar to 100%
                 console.log(colors.green(`\nDownload finished: ${outputPath}`));
                 resolve();
@@ -304,5 +291,4 @@ async function main() {
     downloadVideo(videoUrls, outDirs, session);
 }
 
-
 main();

test/test.ts

@@ -11,14 +11,14 @@ describe('Puppeteer', () => {
     it('should grab GitHub page title', async () => {
         browser = await puppeteer.launch({
             headless: true,
-            args: ['--disable-dev-shm-usage']
+            args: ['--disable-dev-shm-usage', '--fast-start', '--no-sandbox']
         });
         page = await browser.newPage();
         await page.goto("https://github.com/", { waitUntil: 'load' });
         let pageTitle = await page.title();
         assert.equal(true, pageTitle.includes('GitHub'));
         await browser.close();
-    }).timeout(15000); // yeah, this may take a while...
+    }).timeout(25000); // yeah, this may take a while...
 });
 
 describe('Destreamer', () => {