Create trigger_codeQL

.gitignore

@@ -3,12 +3,10 @@
 *.log
 *.js
 *.zip
-*.xml
-
-yarn.lock
 
 .chrome_data
 node_modules
 videos
 release
 build
+yarn.lock

README.md

@@ -1,8 +1,10 @@
-<a href="https://github.com/snobu/destreamer/actions">
+# This project is abandoned. It will probably not work anymore against your MS Stream tenant.
-  <img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
-</a>
 
-**destreamer v3.0** is just around the corner. You can try out a pre-release today by cloning [this branch](https://github.com/snobu/destreamer/tree/aria2c_forRealNow).
+# A heartfelt thank you to all the contributors over the years. You are the real MVPs. 💖
+
+## Check out kylon's Sharedown for a SharePoint-backend implementation - https://github.com/kylon/Sharedown
+
+<hr>
 
 ![destreamer](assets/logo.png)
 
@@ -41,7 +43,6 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
 - [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
 - **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
 - [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
-- [**aria2**][aria2]: aria2 is a utility for downloading files with multiple threads, fast.
 - [**git**][git]: one or more npm dependencies require git.
 
 Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on. We've successfully tested it on Windows, macOS and Linux.
@@ -240,7 +241,6 @@ Please open an [issue](https://github.com/snobu/destreamer/issues) and we'll loo
 
 
 [ffmpeg]: https://www.ffmpeg.org/download.html
-[aria2]: https://github.com/aria2/aria2/releases
 [xming]: https://sourceforge.net/projects/xming/
 [node]: https://nodejs.org/en/download/
 [git]: https://git-scm.com/downloads

package.json

@@ -9,8 +9,7 @@
   "main": "build/src/destreamer.js",
   "bin": "build/src/destreamer.js",
   "scripts": {
-        "build": "echo Transpiling TypeScript to JavaScript... && tsc && echo Destreamer was built successfully.",
+    "build": "echo Transpiling TypeScript to JavaScript... && node node_modules/typescript/bin/tsc && echo Destreamer was built successfully.",
-        "watch": "tsc --watch",
     "test": "mocha build/test",
     "lint": "eslint src/*.ts"
   },

src/ApiClient.ts

@@ -1,18 +1,16 @@
 import { logger } from './Logger';
-import { ShareSession, StreamSession, Video } from './Types';
+import { Session } from './Types';
-import { publishedDateToString, publishedTimeToString } from './VideoUtils';
 
 import axios, { AxiosRequestConfig, AxiosResponse, AxiosInstance, AxiosError } from 'axios';
 import axiosRetry, { isNetworkOrIdempotentRequestError } from 'axios-retry';
-// import fs from 'fs';
 
 
-export class StreamApiClient {
+export class ApiClient {
-    private static instance: StreamApiClient;
+    private static instance: ApiClient;
     private axiosInstance?: AxiosInstance;
-    private session?: StreamSession;
+    private session?: Session;
 
-    private constructor(session?: StreamSession) {
+    private constructor(session?: Session) {
         this.session = session;
         this.axiosInstance = axios.create({
             baseURL: session?.ApiGatewayUri,
@@ -52,16 +50,16 @@ export class StreamApiClient {
      *
      * @param session used if initializing
      */
-    public static getInstance(session?: StreamSession): StreamApiClient {
+    public static getInstance(session?: Session): ApiClient {
-        if (!StreamApiClient.instance) {
+        if (!ApiClient.instance) {
-            StreamApiClient.instance = new StreamApiClient(session);
+            ApiClient.instance = new ApiClient(session);
         }
 
-        return StreamApiClient.instance;
+        return ApiClient.instance;
     }
 
-    public setSession(session: StreamSession): void {
+    public setSession(session: Session): void {
-        if (!StreamApiClient.instance) {
+        if (!ApiClient.instance) {
             logger.warn("Trying to update ApiCient session when it's not initialized!");
         }
 
@@ -115,134 +113,3 @@ export class StreamApiClient {
         });
     }
 }
-
-export class ShareApiClient {
-    private axiosInstance: AxiosInstance;
-    private site: string;
-
-    public constructor(domain: string, site: string, session: ShareSession) {
-        this.axiosInstance = axios.create({
-            baseURL: domain,
-            // timeout: 7000,
-            headers: {
-                'User-Agent': 'destreamer/3.0 ALPHA',
-                'Cookie': `rtFa=${session.rtFa}; FedAuth=${session.FedAuth}`
-            }
-        });
-        this.site = site;
-
-
-        // FIXME: disabled because it was messing with the direct download check
-        // axiosRetry(this.axiosInstance, {
-        //     // The following option is not working.
-        //     // We should open an issue on the relative GitHub
-        //     shouldResetTimeout: true,
-        //     retries: 6,
-        //     retryDelay: (retryCount: number) => {
-        //         return retryCount * 2000;
-        //     },
-        //     retryCondition: (err: AxiosError) => {
-        //         const retryCodes: Array<number> = [429, 500, 502, 503];
-        //         if (isNetworkOrIdempotentRequestError(err)) {
-        //             logger.warn(`${err}. Retrying request...`);
-
-        //             return true;
-        //         }
-        //         logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}.`);
-        //         logger.warn('Here is the error message: ');
-        //         console.dir(err.response?.data);
-        //         logger.warn('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
-
-        //         const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
-
-        //         return shouldRetry;
-        //     }
-        // });
-    }
-
-
-    public async getVideoInfo(filePath: string, outPath: string): Promise<Video> {
-        let playbackUrl: string;
-
-        // TODO: Ripped this straigth from chromium inspector. Don't know don't care what it is right now. Check later
-        const payload = {
-            parameters: {
-                __metadata: {
-                    type: 'SP.RenderListDataParameters'
-                },
-                ViewXml: `<View Scope="RecursiveAll"><Query><Where><Eq><FieldRef Name="FileRef" /><Value Type="Text"><![CDATA[${filePath}]]></Value></Eq></Where></Query><RowLimit Paged="TRUE">1</RowLimit></View>`,
-                RenderOptions: 12295,
-                AddRequiredFields: true
-            }
-        };
-        const url = `${this.site}/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream?@a1='${encodeURIComponent(filePath)}'`;
-
-        logger.verbose(`Requesting video info for '${url}'`);
-        const info = await this.axiosInstance.post(url, payload, {
-            headers: {
-                'Content-Type': 'application/json;odata=verbose'
-            }
-        }).then(res => res.data);
-        // fs.writeFileSync('info.json', JSON.stringify(info, null, 4));
-
-        // FIXME: very bad but usefull in alpha stage to check for edge cases
-        if (info.ListData.Row.length !== 1) {
-            logger.error('More than 1 row in SharePoint video info', { fatal: true });
-
-            process.exit(1000);
-        }
-
-        const direct = await this.canDirectDownload(filePath);
-        const b64VideoMetadata = JSON.parse(
-            info.ListData.Row[0].MediaServiceFastMetadata
-        ).video.altManifestMetadata;
-        const durationSeconds = Math.ceil(
-            (JSON.parse(
-                Buffer.from(b64VideoMetadata, 'base64').toString()
-            ).Duration100Nano) / 10 / 1000 / 1000
-        );
-
-        if (direct) {
-            playbackUrl = this.axiosInstance.defaults.baseURL + filePath;
-            // logger.verbose(playbackUrl);
-        }
-        else {
-            playbackUrl = info.ListSchema['.videoManifestUrl'];
-            playbackUrl = playbackUrl.replace('{.mediaBaseUrl}', info.ListSchema['.mediaBaseUrl']);
-            // the only filetype works I found
-            playbackUrl = playbackUrl.replace('{.fileType}', 'mp4');
-            playbackUrl = playbackUrl.replace('{.callerStack}', info.ListSchema['.callerStack']);
-            playbackUrl = playbackUrl.replace('{.spItemUrl}', info.ListData.Row[0]['.spItemUrl']);
-            playbackUrl = playbackUrl.replace('{.driveAccessToken}', info.ListSchema['.driveAccessToken']);
-            playbackUrl += '&part=index&format=dash';
-        }
-
-
-        return {
-            direct,
-            title: filePath.split('/').pop() ?? 'video.mp4',
-            duration: publishedTimeToString(durationSeconds),
-            publishDate: publishedDateToString(info.ListData.Row[0]['Modified.']),
-            publishTime: publishedTimeToString(info.ListData.Row[0]['Modified.']),
-            author: info.ListData.Row[0]['Author.title'],
-            authorEmail: info.ListData.Row[0]['Author.email'],
-            uniqueId: info.ListData.Row[0].GUID.substring(1, 9),
-            outPath,
-            playbackUrl,
-            totalChunks: durationSeconds
-        };
-    }
-
-
-    private async canDirectDownload(filePath: string): Promise<boolean> {
-        logger.verbose(`Checking direct download for '${filePath}'`);
-
-        return this.axiosInstance.head(
-            filePath, { maxRedirects: 0 }
-        ).then(
-            res => (res.status === 200)
-        ).catch(
-            () => false
-        );
-    }
-}

src/CommandLineParser.ts

@@ -1,14 +1,15 @@
-import { CLI_ERROR } from './Errors';
+import { CLI_ERROR, ERROR_CODE } from './Errors';
-import { makeOutDir } from './Utils';
+import { checkOutDir } from './Utils';
 import { logger } from './Logger';
 import { templateElements } from './Types';
 
 import fs from 'fs';
+import readlineSync from 'readline-sync';
 import sanitize from 'sanitize-filename';
 import yargs from 'yargs';
 
 
-export const argv = yargs.options({
+export const argv: any = yargs.options({
     username: {
         alias: 'u',
         type: 'string',
@@ -113,7 +114,7 @@ export const argv = yargs.options({
 .check(() => noArguments())
 .check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
 .check((argv: any) => {
-    if (makeOutDir(argv.outputDirectory)) {
+    if (checkOutDir(argv.outputDirectory)) {
         return true;
     }
     else {
@@ -197,3 +198,14 @@ function isOutputTemplateValid(argv: any): boolean {
 
     return true;
 }
+
+
+export function promptUser(choices: Array<string>): number {
+    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
+
+    if (index === -1) {
+        process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
+    }
+
+    return index;
+}

src/Downloaders.ts

@@ -1,335 +0,0 @@
-import { ShareApiClient, StreamApiClient } from './ApiClient';
-import { argv } from './CommandLineParser';
-import { ERROR_CODE } from './Errors';
-import { logger } from './Logger';
-import { doShareLogin, doStreamLogin } from './LoginModules';
-import { drawThumbnail } from './Thumbnail';
-import { refreshSession, TokenCache } from './TokenCache';
-import { Video, VideoUrl } from './Types';
-import { ffmpegTimemarkToChunk } from './Utils';
-import { createUniquePath, getStreamInfo } from './VideoUtils';
-
-import cliProgress from 'cli-progress';
-import fs from 'fs';
-import { execSync } from 'child_process';
-import path from 'path';
-
-
-const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
-const tokenCache: TokenCache = new TokenCache();
-
-
-export async function downloadStreamVideo(videoUrls: Array<VideoUrl>): Promise<void> {
-    logger.info('Downloading Microsoft Stream videos...');
-
-    let session = tokenCache.Read() ?? await doStreamLogin('https://web.microsoftstream.com/', tokenCache, argv.username);
-    logger.verbose(
-        'Session and API info \n' +
-        '\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
-        '\t API Gateway version: '.cyan + session.ApiGatewayVersion + '\n'
-    );
-
-    logger.info('Fetching videos info... \n');
-    const videos: Array<Video> = createUniquePath(
-        await getStreamInfo(videoUrls, session, argv.closedCaptions),
-        argv.outputTemplate, argv.format, argv.skip
-    );
-
-    if (argv.simulate) {
-        videos.forEach((video: Video) => {
-            logger.info(
-                '\nTitle:          '.green + video.title +
-                '\nOutPath:        '.green + video.outPath +
-                '\nPublished Date: '.green + video.publishDate +
-                '\nPlayback URL:   '.green + video.playbackUrl +
-                ((video.captionsUrl) ? ('\nCC URL:         '.green + video.captionsUrl) : '')
-            );
-        });
-
-        return;
-    }
-
-    for (const [index, video] of videos.entries()) {
-
-        if (argv.skip && fs.existsSync(video.outPath)) {
-            logger.info(`File already exists, skipping: ${video.outPath} \n`);
-            continue;
-        }
-
-        if (argv.keepLoginCookies && index !== 0) {
-            logger.info('Trying to refresh token...');
-            session = await refreshSession('https://web.microsoftstream.com/video/' + video.guid);
-            StreamApiClient.getInstance().setSession(session);
-        }
-
-        const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
-            barCompleteChar: '\u2588',
-            barIncompleteChar: '\u2591',
-            format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
-            // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
-            barsize: Math.floor((process.stdout.columns || 30) / 3),
-            stopOnComplete: true,
-            hideCursor: true,
-        });
-
-        logger.info(`\nDownloading Video: ${video.title} \n`);
-        logger.verbose('Extra video info \n' +
-            '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
-            '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
-            '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
-            '\t Video total chunks: '.cyan + video.totalChunks + '\n');
-
-        logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
-        if (!process.stdout.columns) {
-            logger.warn(
-                'Unable to get number of columns from terminal.\n' +
-                'This happens sometimes in Cygwin/MSYS.\n' +
-                'No progress bar can be rendered, however the download process should not be affected.\n\n' +
-                'Please use PowerShell or cmd.exe to run destreamer on Windows.'
-            );
-        }
-
-        const headers: string = 'Authorization: Bearer ' + session.AccessToken;
-
-        if (!argv.noExperiments) {
-            if (video.posterImageUrl) {
-                await drawThumbnail(video.posterImageUrl, session);
-            }
-        }
-
-        const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
-            ['headers', headers]
-        ]));
-        const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
-            argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
-            argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
-            ['n', null]
-        ]));
-        const ffmpegCmd: any = new FFmpegCommand();
-
-        const cleanupFn: () => void = () => {
-            pbar.stop();
-
-            if (argv.noCleanup) {
-                return;
-            }
-
-            try {
-                fs.unlinkSync(video.outPath);
-            }
-            catch (e) {
-                // Future handling of an error (maybe)
-            }
-        };
-
-        pbar.start(video.totalChunks, 0, {
-            speed: '0'
-        });
-
-        // prepare ffmpeg command line
-        ffmpegCmd.addInput(ffmpegInpt);
-        ffmpegCmd.addOutput(ffmpegOutput);
-        if (argv.closedCaptions && video.captionsUrl) {
-            const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
-                ['headers', headers]
-            ]));
-
-            ffmpegCmd.addInput(captionsInpt);
-        }
-
-        ffmpegCmd.on('update', async (data: any) => {
-            const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
-
-            pbar.update(currentChunks, {
-                speed: data.bitrate
-            });
-
-            // Graceful fallback in case we can't get columns (Cygwin/MSYS)
-            if (!process.stdout.columns) {
-                process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
-            }
-        });
-
-        process.on('SIGINT', cleanupFn);
-
-        // let the magic begin...
-        await new Promise((resolve: any) => {
-            ffmpegCmd.on('error', (error: any) => {
-                cleanupFn();
-
-                logger.error(`FFmpeg returned an error: ${error.message}`);
-                process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
-            });
-
-            ffmpegCmd.on('success', () => {
-                pbar.update(video.totalChunks); // set progress bar to 100%
-                logger.info(`\nDownload finished: ${video.outPath} \n`);
-                resolve();
-            });
-
-            ffmpegCmd.spawn();
-        });
-
-        process.removeListener('SIGINT', cleanupFn);
-    }
-}
-
-
-// TODO: complete overhaul of this function
-export async function downloadShareVideo(videoUrls: Array<VideoUrl>): Promise<void> {
-    const shareUrlRegex = new RegExp(/(?<domain>https:\/\/.+\.sharepoint\.com).*?(?<baseSite>\/(?:teams|sites|personal)\/.*?)(?:(?<filename>\/.*\.mp4)|\/.*id=(?<paramFilename>.*mp4))/);
-
-    logger.info('Downloading SharePoint videos...\n\n');
-
-    // FIXME: this may change we need a smart login system if a request fails
-    const session = await doShareLogin(videoUrls[0].url, argv.username);
-
-    for (const videoUrl of videoUrls) {
-        const match = shareUrlRegex.exec(videoUrl.url);
-        if (!match) {
-            logger.error(`Invalid url '${videoUrl.url}', skipping...`);
-
-            continue;
-        }
-
-        const shareDomain = match.groups!.domain;
-        const shareSite = match.groups!.baseSite;
-        const shareFilepath = decodeURIComponent(match.groups?.filename ? (shareSite + match.groups.filename) : match.groups!.paramFilename);
-        // FIXME: hardcoded video.mp4
-        const title = shareFilepath.split('/').pop()?.split('.')[0] ?? 'video';
-
-        const apiClient = new ShareApiClient(shareDomain, shareSite, session);
-
-        const video = await apiClient.getVideoInfo(shareFilepath, videoUrl.outDir);
-        createUniquePath(video, title, argv.format, argv.skip);
-
-        if (argv.simulate) {
-            if (argv.verbose) {
-                console.dir(video);
-            }
-            else {
-                logger.info(
-                    '\nTitle:          '.green + video.title +
-                    '\nOutPath:        '.green + video.outPath +
-                    '\nPublished Date: '.green + video.publishDate +
-                    '\nPlayback URL:   '.green + video.playbackUrl
-                );
-            }
-            continue;
-        }
-
-        if (video.direct) {
-            const headers = `Cookie: rtFa=${session.rtFa}; FedAuth=${session.FedAuth}`;
-
-            // FIXME: unstable and bad all-around
-            try {
-                execSync(
-                    'aria2c --max-connection-per-server 8 --console-log-level warn ' +
-                    `--header "${headers}" --dir "${path.dirname(video.outPath)}" --out "${path.basename(video.outPath)}" "${shareDomain + shareFilepath}"`,
-                    { stdio: 'inherit' }
-                );
-            }
-            catch (error: any) {
-                logger.error(`${error.status} \n\n${error.message} \n\n${error.stdout.toString()} \n\n${error.stderr.toString()}`);
-            }
-        }
-        else {
-            // FIXME: just a copy-paste, should move to separate function
-            const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
-                barCompleteChar: '\u2588',
-                barIncompleteChar: '\u2591',
-                format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
-                // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
-                barsize: Math.floor((process.stdout.columns || 30) / 3),
-                stopOnComplete: true,
-                hideCursor: true,
-            });
-
-            logger.info(`\nDownloading Video: ${video.title} \n`);
-            logger.verbose('Extra video info \n' +
-                '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
-                '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
-                '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
-                '\t Video total chunks: '.cyan + video.totalChunks + '\n');
-
-            logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
-            if (!process.stdout.columns) {
-                logger.warn(
-                    'Unable to get number of columns from terminal.\n' +
-                    'This happens sometimes in Cygwin/MSYS.\n' +
-                    'No progress bar can be rendered, however the download process should not be affected.\n\n' +
-                    'Please use PowerShell or cmd.exe to run destreamer on Windows.'
-                );
-            }
-
-            const ffmpegInpt: any = new FFmpegInput(video.playbackUrl);
-            const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
-                argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
-                argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
-                ['n', null]
-            ]));
-            const ffmpegCmd: any = new FFmpegCommand();
-
-            const cleanupFn: () => void = () => {
-                pbar.stop();
-
-                if (argv.noCleanup) {
-                    return;
-                }
-
-                try {
-                    fs.unlinkSync(video.outPath);
-                }
-                catch (e) {
-                    // Future handling of an error (maybe)
-                }
-            };
-
-            pbar.start(video.totalChunks, 0, {
-                speed: '0'
-            });
-
-            // prepare ffmpeg command line
-            ffmpegCmd.addInput(ffmpegInpt);
-            ffmpegCmd.addOutput(ffmpegOutput);
-
-            ffmpegCmd.on('update', async (data: any) => {
-                const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
-
-                pbar.update(currentChunks, {
-                    speed: data.bitrate
-                });
-
-                // Graceful fallback in case we can't get columns (Cygwin/MSYS)
-                if (!process.stdout.columns) {
-                    process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
-                }
-            });
-
-            process.on('SIGINT', cleanupFn);
-
-            // let the magic begin...
-            await new Promise((resolve: any) => {
-                ffmpegCmd.on('error', (error: any) => {
-                    cleanupFn();
-
-                    logger.error(`FFmpeg returned an error: ${error.message}`);
-                    process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
-                });
-
-                ffmpegCmd.on('success', () => {
-                    pbar.update(video.totalChunks); // set progress bar to 100%
-                    logger.info(`\nDownload finished: ${video.outPath} \n`);
-                    resolve();
-                });
-
-                ffmpegCmd.spawn();
-            });
-
-            process.removeListener('SIGINT', cleanupFn);
-            // logger.error('TODO: manifest download');
-
-            // continue;
-        }
-    }
-}

src/Errors.ts

@@ -1,9 +1,8 @@
 export const enum ERROR_CODE {
-    UNHANDLED_ERROR = 200,
+    UNHANDLED_ERROR,
     ELEVATED_SHELL,
     CANCELLED_USER_INPUT,
     MISSING_FFMPEG,
-    MISSING_ARIA2,
     OUTDATED_FFMPEG,
     UNK_FFMPEG_ERROR,
     INVALID_VIDEO_GUID,
@@ -23,10 +22,7 @@ export const errors: { [key: number]: string } = {
     [ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
         'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
-    [ERROR_CODE.MISSING_ARIA2]: 'Aria2 is missing!\n' +
+    [ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
-        'Destreamer requires a fairly recent release of Aria2 to download videos',
-
-    [ERROR_CODE.OUTDATED_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
         'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
     [ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',

src/LoginModules.ts

@@ -1,176 +0,0 @@
-import { logger } from './Logger';
-import puppeteer from 'puppeteer';
-import { getPuppeteerChromiumPath } from './PuppeteerHelper';
-import { chromeCacheFolder } from './destreamer';
-import { argv } from './CommandLineParser';
-import { ShareSession, StreamSession } from './Types';
-import { ERROR_CODE } from './Errors';
-import { TokenCache } from './TokenCache';
-
-
-export async function doStreamLogin(url: string, tokenCache: TokenCache, username?: string): Promise<StreamSession> {
-    logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
-
-    const browser: puppeteer.Browser = await puppeteer.launch({
-        executablePath: getPuppeteerChromiumPath(),
-        headless: false,
-        userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
-        defaultViewport: null,
-        args: [
-            '--disable-dev-shm-usage',
-            '--fast-start',
-            '--no-sandbox'
-        ]
-    });
-
-    // try-finally because we were leaving zombie processes if there was an error
-    try {
-        const page: puppeteer.Page = (await browser.pages())[0];
-
-        logger.info('Navigating to login page...');
-        await page.goto(url, { waitUntil: 'load' });
-
-        try {
-            if (username) {
-                await page.waitForSelector('input[type="email"]', { timeout: 3000 });
-                await page.keyboard.type(username);
-                await page.click('input[type="submit"]');
-            }
-            else {
-                /* If a username was not provided we let the user take actions that
-                lead up to the video page. */
-            }
-        }
-        catch (e) {
-            /* If there is no email input selector we aren't in the login module,
-            we are probably using the cache to aid the login.
-            It could finish the login on its own if the user said 'yes' when asked to
-            remember the credentials or it could still prompt the user for a password */
-        }
-
-        await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
-        logger.info('We are logged in.');
-
-        let session: StreamSession | null = null;
-        let tries = 1;
-        while (!session) {
-            try {
-                let sessionInfo: any;
-                session = await page.evaluate(
-                    () => {
-                        return {
-                            AccessToken: sessionInfo.AccessToken,
-                            ApiGatewayUri: sessionInfo.ApiGatewayUri,
-                            ApiGatewayVersion: sessionInfo.ApiGatewayVersion
-                        };
-                    }
-                );
-            }
-            catch (error) {
-                if (tries > 5) {
-                    process.exit(ERROR_CODE.NO_SESSION_INFO);
-                }
-
-                session = null;
-                tries++;
-                await page.waitForTimeout(3000);
-            }
-        }
-
-        tokenCache.Write(session);
-        logger.info('Wrote access token to token cache.');
-        logger.info("At this point Chromium's job is done, shutting it down...\n");
-
-
-        return session;
-    }
-    finally {
-        await browser.close();
-    }
-}
-
-
-
-export async function doShareLogin(url: string, username?: string): Promise<ShareSession> {
-    logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
-
-    let session: ShareSession | null = null;
-    const hostname = new URL(url).host;
-
-    const browser: puppeteer.Browser = await puppeteer.launch({
-        executablePath: getPuppeteerChromiumPath(),
-        headless: false,
-        devtools: argv.verbose,
-        userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
-        defaultViewport: null,
-        args: [
-            '--disable-dev-shm-usage',
-            '--fast-start',
-            '--no-sandbox'
-        ]
-    });
-
-    // try-finally because we were leaving zombie processes if there was an error
-    try {
-        const page: puppeteer.Page = (await browser.pages())[0];
-
-        logger.info('Navigating to login page...');
-        await page.goto(url, { waitUntil: 'load' });
-
-        try {
-            if (username) {
-                await page.waitForSelector('input[type="email"]', { timeout: 3000 });
-                await page.keyboard.type(username);
-                await page.click('input[type="submit"]');
-            }
-            else {
-                /* If a username was not provided we let the user take actions that
-                lead up to the video page. */
-            }
-        }
-        catch (e) {
-            /* If there is no email input selector we aren't in the login module,
-            we are probably using the cache to aid the login.
-            It could finish the login on its own if the user said 'yes' when asked to
-            remember the credentials or it could still prompt the user for a password */
-        }
-
-        logger.info('Waiting for target!');
-
-        await browser.waitForTarget((target: puppeteer.Target) => target.url().startsWith(`https://${hostname}`), { timeout: 150000 });
-        logger.info('We are logged in.');
-
-        let tries = 1;
-        while (!session) {
-            const cookieJar = (await page.cookies()).filter(
-                biscuit => biscuit.name == 'rtFa' || biscuit.name == 'FedAuth'
-            );
-
-            if (cookieJar.length != 2) {
-                if (tries > 5) {
-                    process.exit(ERROR_CODE.NO_SESSION_INFO);
-                }
-
-                await page.waitForTimeout(1000 * tries++);
-
-                continue;
-            }
-
-            session = {
-                rtFa: cookieJar.find(biscuit => biscuit.name == 'rtFa')!.value,
-                FedAuth: cookieJar.find(biscuit => biscuit.name == 'FedAuth')!.value
-            };
-        }
-
-        logger.info("At this point Chromium's job is done, shutting it down...\n");
-
-        // await page.waitForTimeout(1000 * 60 * 60 * 60);
-    }
-    finally {
-        logger.verbose('Stream login browser closing...');
-        await browser.close();
-        logger.verbose('Stream login browser closed');
-    }
-
-    return session;
-}

src/Thumbnail.ts

@@ -1,12 +1,12 @@
-import { StreamApiClient } from './ApiClient';
+import { ApiClient } from './ApiClient';
-import { StreamSession } from './Types';
+import { Session } from './Types';
 
 import terminalImage from 'terminal-image';
 import { AxiosResponse } from 'axios';
 
 
-export async function drawThumbnail(posterImage: string, session: StreamSession): Promise<void> {
+export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
-    const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
+    const apiClient: ApiClient = ApiClient.getInstance(session);
 
     const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
         .then((response: AxiosResponse<any> | undefined) => response?.data);

src/TokenCache.ts

@@ -2,7 +2,7 @@ import { chromeCacheFolder } from './destreamer';
 import { ERROR_CODE } from './Errors';
 import { logger } from './Logger';
 import { getPuppeteerChromiumPath } from './PuppeteerHelper';
-import { StreamSession } from './Types';
+import { Session } from './Types';
 
 import fs from 'fs';
 import jwtDecode from 'jwt-decode';
@@ -12,14 +12,14 @@ import puppeteer from 'puppeteer';
 export class TokenCache {
     private tokenCacheFile = '.token_cache';
 
-    public Read(): StreamSession | null {
+    public Read(): Session | null {
         if (!fs.existsSync(this.tokenCacheFile)) {
             logger.warn(`${this.tokenCacheFile} not found. \n`);
 
             return null;
         }
 
-        const session: StreamSession = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
+        const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
 
         type Jwt = {
             [key: string]: any
@@ -41,7 +41,7 @@ export class TokenCache {
         return session;
     }
 
-    public Write(session: StreamSession): void {
+    public Write(session: Session): void {
         const s: string = JSON.stringify(session, null, 4);
         fs.writeFile(this.tokenCacheFile, s, (err: any) => {
             if (err) {
@@ -54,7 +54,7 @@ export class TokenCache {
 }
 
 
-export async function refreshSession(url: string): Promise<StreamSession> {
+export async function refreshSession(url: string): Promise<Session> {
     const videoId: string = url.split('/').pop() ?? process.exit(ERROR_CODE.INVALID_VIDEO_GUID);
 
     const browser: puppeteer.Browser = await puppeteer.launch({
@@ -73,7 +73,7 @@ export async function refreshSession(url: string): Promise<StreamSession> {
 
     await browser.waitForTarget((target: puppeteer.Target) => target.url().includes(videoId), { timeout: 30000 });
 
-    let session: StreamSession | null = null;
+    let session: Session | null = null;
     let tries = 1;
 
     while (!session) {

src/Types.ts

@@ -1,34 +1,11 @@
-export type StreamSession = {
+export type Session = {
     AccessToken: string;
     ApiGatewayUri: string;
     ApiGatewayVersion: string;
 }
 
 
-export type ShareSession = {
-    FedAuth: string;
-    rtFa: string;
-}
-
-
-export type VideoUrl = {
-    url: string,
-    outDir: string
-}
-
-
-export type SharepointVideo = {
-    // if we can download the MP4 or we need to use DASH
-    direct: boolean;
-    playbackUrl: string;
-    title: string;
-    outPath: string
-}
-
-
 export type Video = {
-    guid?: string;
-    direct?: boolean;
     title: string;
     duration: string;
     publishDate: string;
@@ -39,7 +16,7 @@ export type Video = {
     outPath: string;
     totalChunks: number;    // Abstraction of FFmpeg timemark
     playbackUrl: string;
-    posterImageUrl?: string;
+    posterImageUrl: string | null;
     captionsUrl?: string
 }
 

src/Utils.ts

@@ -1,62 +1,47 @@
-import { StreamApiClient } from './ApiClient';
+import { ApiClient } from './ApiClient';
 import { ERROR_CODE } from './Errors';
 import { logger } from './Logger';
-import { StreamSession, VideoUrl } from './Types';
+import { Session } from './Types';
 
 import { AxiosResponse } from 'axios';
 import { execSync } from 'child_process';
 import fs from 'fs';
-import readlineSync from 'readline-sync';
 
 
-const streamUrlRegex = new RegExp(/https?:\/\/web\.microsoftstream\.com.*/);
+async function extractGuids(url: string, client: ApiClient): Promise<Array<string> | null> {
-const shareUrlRegex = new RegExp(/https?:\/\/.+\.sharepoint\.com.*/);
 
-
-/** we place the guid in the url fild in the return */
-export async function extractStreamGuids(urlList: Array<VideoUrl>, session: StreamSession): Promise<Array<VideoUrl>> {
     const videoRegex = new RegExp(/https:\/\/.*\/video\/(\w{8}-(?:\w{4}-){3}\w{12})/);
     const groupRegex = new RegExp(/https:\/\/.*\/group\/(\w{8}-(?:\w{4}-){3}\w{12})/);
 
-    const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
+    const videoMatch: RegExpExecArray | null = videoRegex.exec(url);
-    const guidList: Array<VideoUrl> = [];
+    const groupMatch: RegExpExecArray | null = groupRegex.exec(url);
-
-    for (const url of urlList) {
-        const videoMatch: RegExpExecArray | null = videoRegex.exec(url.url);
-        const groupMatch: RegExpExecArray | null = groupRegex.exec(url.url);
 
     if (videoMatch) {
-            guidList.push({
+        return [videoMatch[1]];
-                url: videoMatch[1],
-                outDir: url.outDir
-            });
     }
     else if (groupMatch) {
-            const videoNumber: number = await apiClient.callApi(`groups/${groupMatch[1]}`, 'get')
+        const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
             .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
+        const result: Array<string> = [];
 
         // Anything above $top=100 results in 400 Bad Request
         // Use $skip to skip the first 100 and get another 100 and so on
         for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
-                await apiClient.callApi(
+            const partial: Array<string> = await client.callApi(
                 `groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
-                    '$top=100&$orderby=publishedDate asc', 'get'
+                '$top=100&$orderby=publishedDate asc', 'get')
-                ).then((response: AxiosResponse<any> | undefined) => {
+                .then(
-                    response?.data.value.forEach((video: { id: string }) =>
+                    (response: AxiosResponse<any> | undefined) =>
-                        guidList.push({
+                        response?.data.value.map((item: any) => item.id)
-                            url: video.id,
-                            outDir: url.outDir
-                        })
                 );
-                });
+
-            }
+            result.push(...partial);
-        }
-        else {
-            logger.warn(`Invalid url '${url.url}', skipping...`);
-        }
         }
 
-    return guidList;
+        return result;
+    }
+
+    return null;
 }
 
 
@@ -67,32 +52,30 @@ export async function extractStreamGuids(urlList: Array<VideoUrl>, session: Stre
  *
  * @param {Array<string>} urlList       list of link to parse
  * @param {string}        defaultOutDir the directry used to save the videos
+ * @param {Session}       session       used to call the API to get the GUIDs from group links
  *
- * @returns Array of 2 elements: 1st an array of Microsoft Stream urls, 2nd an array of SharePoint urls
+ * @returns Array of 2 elements, 1st one being the GUIDs array, 2nd one the output directories array
  */
-export function parseCLIinput(urlList: Array<string>, defaultOutDir: string): Array<Array<VideoUrl>> {
+export async function parseCLIinput(urlList: Array<string>, defaultOutDir: string,
-    const stream: Array<VideoUrl> = [];
+    session: Session): Promise<Array<Array<string>>> {
-    const share: Array<VideoUrl> = [];
+
+    const apiClient: ApiClient = ApiClient.getInstance(session);
+    const guidList: Array<string> = [];
 
     for (const url of urlList) {
-        if (streamUrlRegex.test(url)) {
+        const guids: Array<string> | null = await extractGuids(url, apiClient);
-            stream.push({
+
-                url: url,
+        if (guids) {
-                outDir: defaultOutDir
+            guidList.push(...guids);
-            });
-        }
-        else if (shareUrlRegex.test(url)) {
-            share.push({
-                url: url,
-                outDir: defaultOutDir
-            });
         }
         else {
             logger.warn(`Invalid url '${url}', skipping..`);
         }
     }
 
-    return [stream, share];
+    const outDirList: Array<string> = Array(guidList.length).fill(defaultOutDir);
+
+    return [guidList, outDirList];
 }
 
 
@@ -103,84 +86,94 @@ export function parseCLIinput(urlList: Array<string>, defaultOutDir: string): Ar
  *
  * @param {string}  inputFile     path to the text file
  * @param {string}  defaultOutDir the default/fallback directory used to save the videos
+ * @param {Session} session       used to call the API to get the GUIDs from group links
  *
  * @returns Array of 2 elements, 1st one being the GUIDs array, 2nd one the output directories array
  */
-export function parseInputFile(inputFile: string, defaultOutDir: string): Array<Array<VideoUrl>> {
+export async function parseInputFile(inputFile: string, defaultOutDir: string,
+    session: Session): Promise<Array<Array<string>>> {
     // rawContent is a list of each line of the file
-    const rawContent: Array<string> = fs.readFileSync(inputFile).toString().split(/\r?\n/);
+    const rawContent: Array<string> = fs.readFileSync(inputFile).toString()
-    const stream: Array<VideoUrl> = [];
+        .split(/\r?\n/);
-    const share: Array<VideoUrl> = [];
+    const apiClient: ApiClient = ApiClient.getInstance(session);
-    let streamUrl = false;
+
+    const guidList: Array<string> = [];
+    const outDirList: Array<string> = [];
+    // if the last line was an url set this
+    let foundUrl = false;
 
     for (let i = 0; i < rawContent.length; i++) {
         const line: string = rawContent[i];
-        const nextLine: string | null = i < rawContent.length ? rawContent[i + 1] : null;
-        let outDir = defaultOutDir;
 
         // filter out lines with no content
         if (!line.match(/\S/)) {
             logger.warn(`Line ${i + 1} is empty, skipping..`);
             continue;
         }
-        // check for urls
+        // parse if line is option
-        else if (streamUrlRegex.test(line)) {
+        else if (line.includes('-dir')) {
-            streamUrl = true;
+            if (foundUrl) {
-        }
+                const outDir: string | null = parseOption('-dir', line);
-        else if (shareUrlRegex.test(line)) {
-            streamUrl = false;
-        }
-        // now invalid line since we skip ahead one line if we find dir option
-        else {
-            logger.warn(`Line ${i + 1}: '${line}' is invalid, skipping..`);
 
+                if (outDir && checkOutDir(outDir)) {
+                    outDirList.push(...Array(guidList.length - outDirList.length)
+                        .fill(outDir));
+                }
+                else {
+                    outDirList.push(...Array(guidList.length - outDirList.length)
+                        .fill(defaultOutDir));
+                }
+
+                foundUrl = false;
                 continue;
             }
-
+            else {
-        // we now have a valid url, check next line for option
+                logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
-        if (nextLine) {
+                continue;
-            const optionDir = parseOption('-dir', nextLine);
-
-            if (optionDir && makeOutDir(optionDir)) {
-                outDir = optionDir;
-                // if there was an option we skip a line
-                i++;
             }
         }
 
-        if (streamUrl) {
+        /* now line is not empty nor an option line.
-            stream.push({
+        If foundUrl is still true last line didn't have a directory option
-                url: line,
+        so we stil need to add the default outDir to outDirList to  */
-                outDir
+        if (foundUrl) {
-            });
+            outDirList.push(...Array(guidList.length - outDirList.length)
+                .fill(defaultOutDir));
+            foundUrl = false;
+        }
+
+        const guids: Array<string> | null = await extractGuids(line, apiClient);
+
+        if (guids) {
+            guidList.push(...guids);
+            foundUrl = true;
         }
         else {
-            share.push({
+            logger.warn(`Invalid url at line ${i + 1}, skipping..`);
-                url: line,
-                outDir
-            });
         }
     }
 
+    // if foundUrl is still true after the loop we have some url without an outDir
+    if (foundUrl) {
+        outDirList.push(...Array(guidList.length - outDirList.length)
+            .fill(defaultOutDir));
+    }
 
-    return [stream, share];
+    return [guidList, outDirList];
 }
 
 
 // This leaves us the option to add more options (badum tss) _Luca
 function parseOption(optionSyntax: string, item: string): string | null {
     const match: RegExpMatchArray | null = item.match(
-        RegExp(`^\\s+${optionSyntax}\\s*=\\s*['"](.*)['"]`)
+        RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
     );
 
     return match ? match[1] : null;
 }
 
-/**
+
- * @param directory path to create
+export function checkOutDir(directory: string): boolean {
- * @returns true on success, false otherwise
- */
-export function makeOutDir(directory: string): boolean {
     if (!fs.existsSync(directory)) {
         try {
             fs.mkdirSync(directory);
@@ -212,40 +205,14 @@ export function checkRequirements(): void {
     catch (e) {
         process.exit(ERROR_CODE.MISSING_FFMPEG);
     }
-
-    try {
-        const versionRegex = new RegExp(/aria2 version (.*)/);
-        const aira2Ver: string = execSync('aria2c --version').toString().split('\n')[0];
-
-        if (versionRegex.test(aira2Ver)) {
-            logger.verbose(`Using ${aira2Ver}\n`);
-        }
-        else {
-            throw new Error();
-        }
-    }
-    catch (e) {
-        process.exit(ERROR_CODE.MISSING_ARIA2);
-    }
 }
 
-// number of seconds
+
 export function ffmpegTimemarkToChunk(timemark: string): number {
     const timeVals: Array<string> = timemark.split(':');
     const hrs: number = parseInt(timeVals[0]);
     const mins: number = parseInt(timeVals[1]);
     const secs: number = parseInt(timeVals[2]);
 
-    return (hrs * 60 * 60) + (mins * 60) + secs;
+    return (hrs * 60) + mins + (secs / 60);
-}
-
-
-export function promptUser(choices: Array<string>): number {
-    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
-
-    if (index === -1) {
-        process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
-    }
-
-    return index;
 }

src/VideoUtils.ts

@@ -1,16 +1,15 @@
-import { StreamApiClient } from './ApiClient';
+import { ApiClient } from './ApiClient';
-import { promptUser } from './Utils';
+import { promptUser } from './CommandLineParser';
 import { logger } from './Logger';
-import { Video, StreamSession, VideoUrl } from './Types';
+import { Video, Session } from './Types';
 
 import { AxiosResponse } from 'axios';
 import fs from 'fs';
 import { parse as parseDuration, Duration } from 'iso8601-duration';
 import path from 'path';
 import sanitizeWindowsName from 'sanitize-filename';
-import { extractStreamGuids } from './Utils';
 
-export function publishedDateToString(date: string): string {
+function publishedDateToString(date: string): string {
     const dateJs: Date = new Date(date);
     const day: string = dateJs.getDate().toString().padStart(2, '0');
     const month: string = (dateJs.getMonth() + 1).toString(10).padStart(2, '0');
@@ -18,44 +17,35 @@ export function publishedDateToString(date: string): string {
     return `${dateJs.getFullYear()}-${month}-${day}`;
 }
 
-export function publishedTimeToString(seconds: number): string
-export function publishedTimeToString(date: string): string
-export function publishedTimeToString(date: string | number): string {
-    let dateJs: Date;
-
-    if (typeof (date) === 'number') {
-        dateJs = new Date(0, 0, 0, 0, 0, date);
-    }
-    else {
-        dateJs = new Date(date);
-    }
 
+function publishedTimeToString(date: string): string {
+    const dateJs: Date = new Date(date);
     const hours: string = dateJs.getHours().toString();
     const minutes: string = dateJs.getMinutes().toString();
     const seconds: string = dateJs.getSeconds().toString();
 
-    return `${hours}h ${minutes}m ${seconds}s`;
+    return `${hours}.${minutes}.${seconds}`;
 }
 
 
-export function isoDurationToString(time: string): string {
+function isoDurationToString(time: string): string {
     const duration: Duration = parseDuration(time);
 
     return `${duration.hours ?? '00'}.${duration.minutes ?? '00'}.${duration.seconds?.toFixed(0) ?? '00'}`;
 }
 
-// it's the number of seconds in the video
+
-export function durationToTotalChunks(duration: string,): number {
+function durationToTotalChunks(duration: string): number {
     const durationObj: any = parseDuration(duration);
     const hrs: number = durationObj.hours ?? 0;
     const mins: number = durationObj.minutes ?? 0;
     const secs: number = Math.ceil(durationObj.seconds ?? 0);
 
-    return (hrs * 60 * 60) + (mins * 60) + secs;
+    return (hrs * 60) + mins + (secs / 60);
 }
 
 
-export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamSession, subtitles?: boolean): Promise<Array<Video>> {
+export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
     const metadata: Array<Video> = [];
     let title: string;
     let duration: string;
@@ -64,23 +54,19 @@ export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamS
     let author: string;
     let authorEmail: string;
     let uniqueId: string;
+    const outPath = '';
     let totalChunks: number;
     let playbackUrl: string;
     let posterImageUrl: string;
     let captionsUrl: string | undefined;
 
-    const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
+    const apiClient: ApiClient = ApiClient.getInstance(session);
-
-
-    // we place the guid in the url field
-    const videoGUIDs = await extractStreamGuids(videoUrls, session);
-
 
     /* TODO: change this to a single guid at a time to ease our footprint on the
     MSS servers or we get throttled after 10 sequential reqs */
-    for (const guid of videoGUIDs) {
+    for (const guid of videoGuids) {
         const response: AxiosResponse<any> | undefined =
-            await apiClient.callApi('videos/' + guid.url + '?$expand=creator', 'get');
+            await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
 
         title = sanitizeWindowsName(response?.data['name']);
 
@@ -94,7 +80,7 @@ export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamS
 
         authorEmail = response?.data['creator'].mail;
 
-        uniqueId = '#' + guid.url.split('-')[0];
+        uniqueId = '#' + guid.split('-')[0];
 
         totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
@@ -108,7 +94,7 @@ export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamS
         posterImageUrl = response?.data['posterImage']['medium']['url'];
 
         if (subtitles) {
-            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid.url}/texttracks`, 'get');
+            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
 
             if (!captions?.data.value.length) {
                 captionsUrl = undefined;
@@ -126,19 +112,18 @@ export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamS
         }
 
         metadata.push({
-            guid: guid.url,
+            title: title,
-            title,
+            duration: duration,
-            duration,
+            publishDate: publishDate,
-            publishDate,
+            publishTime: publishTime,
-            publishTime,
+            author: author,
-            author,
+            authorEmail: authorEmail,
-            authorEmail,
+            uniqueId: uniqueId,
-            uniqueId,
+            outPath: outPath,
-            outPath: guid.outDir,
+            totalChunks: totalChunks,    // Abstraction of FFmpeg timemark
-            totalChunks,    // Abstraction of FFmpeg timemark
+            playbackUrl: playbackUrl,
-            playbackUrl,
+            posterImageUrl: posterImageUrl,
-            posterImageUrl,
+            captionsUrl: captionsUrl
-            captionsUrl
         });
     }
 
@@ -146,24 +131,16 @@ export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamS
 }
 
 
-export function createUniquePath(videos: Array<Video>, template: string, format: string, skip?: boolean): Array<Video>
+export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, template: string, format: string, skip?: boolean): Array<Video> {
-export function createUniquePath(videos: Video, template: string, format: string, skip?: boolean): Video
-export function createUniquePath(videos: Array<Video> | Video, template: string, format: string, skip?: boolean): Array<Video> | Video {
-    let singleInput = false;
 
-    if (!Array.isArray(videos)) {
+    videos.forEach((video: Video, index: number) => {
-        videos = [videos];
-        singleInput = true;
-    }
-
-    videos.forEach((video: Video) => {
         let title: string = template;
         let finalTitle: string;
         const elementRegEx = RegExp(/{(.*?)}/g);
         let match = elementRegEx.exec(template);
 
         while (match) {
-            const value = video[match[1] as keyof (Video)] as string;
+            const value = video[match[1] as keyof Video] as string;
             title = title.replace(match[0], value);
             match = elementRegEx.exec(template);
         }
@@ -171,7 +148,7 @@ export function createUniquePath(videos: Array<Video> | Video, template: string,
         let i = 0;
         finalTitle = title;
 
-        while (!skip && fs.existsSync(path.join(video.outPath, finalTitle + '.' + format))) {
+        while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
             finalTitle = `${title}.${++i}`;
         }
 
@@ -181,13 +158,9 @@ export function createUniquePath(videos: Array<Video> | Video, template: string,
             logger.warn(`Not a valid Windows file name: "${finalFileName}".\nReplacing invalid characters with underscores to preserve cross-platform consistency.`);
         }
 
-        video.outPath = path.join(video.outPath, finalFileName);
+        video.outPath = path.join(outDirs[index], finalFileName);
 
     });
 
-    if (singleInput) {
-        return videos[0];
-    }
-
     return videos;
 }

src/destreamer.ts

@@ -2,13 +2,22 @@ import { argv } from './CommandLineParser';
 import { ERROR_CODE } from './Errors';
 import { setProcessEvents } from './Events';
 import { logger } from './Logger';
-import { VideoUrl } from './Types';
+import { getPuppeteerChromiumPath } from './PuppeteerHelper';
-import { checkRequirements, parseInputFile, parseCLIinput } from './Utils';
+import { drawThumbnail } from './Thumbnail';
+import { TokenCache, refreshSession } from './TokenCache';
+import { Video, Session } from './Types';
+import { checkRequirements, ffmpegTimemarkToChunk, parseInputFile, parseCLIinput} from './Utils';
+import { getVideoInfo, createUniquePath } from './VideoUtils';
 
+import cliProgress from 'cli-progress';
+import fs from 'fs';
 import isElevated from 'is-elevated';
-import { downloadShareVideo, downloadStreamVideo } from './Downloaders';
+import puppeteer from 'puppeteer';
+import { ApiClient } from './ApiClient';
 
 
+const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
+const tokenCache: TokenCache = new TokenCache();
 export const chromeCacheFolder = '.chrome_data';
 
 
@@ -35,32 +44,260 @@ async function init(): Promise<void> {
 }
 
 
-async function main(): Promise<void> {
+async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
-    await init(); // must be first
-    let streamVideos: Array<VideoUrl>, shareVideos: Array<VideoUrl>;
 
-    if (argv.videoUrls) {
+    logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
-        logger.info('Parsing video/group urls');
+
-        [streamVideos, shareVideos] = await parseCLIinput(argv.videoUrls as Array<string>, argv.outputDirectory);
+    const browser: puppeteer.Browser = await puppeteer.launch({
+        executablePath: getPuppeteerChromiumPath(),
+        headless: false,
+        userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
+        args: [
+            '--disable-dev-shm-usage',
+            '--fast-start',
+            '--no-sandbox'
+        ]
+    });
+    const page: puppeteer.Page = (await browser.pages())[0];
+
+    logger.info('Navigating to login page...');
+    await page.goto(url, { waitUntil: 'load' });
+
+    try {
+        if (username) {
+            await page.waitForSelector('input[type="email"]', {timeout: 3000});
+            await page.keyboard.type(username);
+            await page.click('input[type="submit"]');
         }
         else {
-        logger.info('Parsing input file');
+            /* If a username was not provided we let the user take actions that
-        [streamVideos, shareVideos] = await parseInputFile(argv.inputFile!, argv.outputDirectory);
+            lead up to the video page. */
+        }
+    }
+    catch (e) {
+        /* If there is no email input selector we aren't in the login module,
+        we are probably using the cache to aid the login.
+        It could finish the login on its own if the user said 'yes' when asked to
+        remember the credentials or it could still prompt the user for a password */
     }
 
-    logger.verbose(
+    await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
-        'List of urls and corresponding output directory \n' +
+    logger.info('We are logged in.');
-        streamVideos.map(video => `\t${video.url} => ${video.outDir} \n`).join('') +
+
-        shareVideos.map(video => `\t${video.url} => ${video.outDir} \n`).join('')
+    let session: Session | null = null;
+    let tries = 1;
+    while (!session) {
+        try {
+            let sessionInfo: any;
+            session = await page.evaluate(
+                () => {
+                    return {
+                        AccessToken: sessionInfo.AccessToken,
+                        ApiGatewayUri: sessionInfo.ApiGatewayUri,
+                        ApiGatewayVersion: sessionInfo.ApiGatewayVersion
+                    };
+                }
+            );
+        }
+        catch (error) {
+            if (tries > 5) {
+                process.exit(ERROR_CODE.NO_SESSION_INFO);
+            }
+
+            session = null;
+            tries++;
+            await page.waitFor(3000);
+        }
+    }
+
+    tokenCache.Write(session);
+    logger.info('Wrote access token to token cache.');
+    logger.info("At this point Chromium's job is done, shutting it down...\n");
+
+    await browser.close();
+
+    return session;
+}
+
+
+async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array<string>, session: Session): Promise<void> {
+
+    logger.info('Fetching videos info... \n');
+    const videos: Array<Video> = createUniquePath (
+        await getVideoInfo(videoGUIDs, session, argv.closedCaptions),
+        outputDirectories, argv.outputTemplate, argv.format, argv.skip
         );
 
-    if (streamVideos.length) {
+    if (argv.simulate) {
-        await downloadStreamVideo(streamVideos);
+        videos.forEach((video: Video) => {
+            logger.info(
+                '\nTitle:          '.green + video.title +
+                '\nOutPath:        '.green + video.outPath +
+                '\nPublished Date: '.green + video.publishDate +
+                '\nPlayback URL:   '.green + video.playbackUrl +
+                ((video.captionsUrl) ? ('\nCC URL:         '.green + video.captionsUrl) : '')
+            );
+        });
+
+        return;
     }
-    if (shareVideos.length) {
+
-        await downloadShareVideo(shareVideos);
+    for (const [index, video] of videos.entries()) {
+
+        if (argv.skip && fs.existsSync(video.outPath)) {
+            logger.info(`File already exists, skipping: ${video.outPath} \n`);
+            continue;
+        }
+
+        if (argv.keepLoginCookies && index !== 0) {
+            logger.info('Trying to refresh token...');
+            session = await refreshSession('https://web.microsoftstream.com/video/' + videoGUIDs[index]);
+            ApiClient.getInstance().setSession(session);
+        }
+
+        const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
+            barCompleteChar: '\u2588',
+            barIncompleteChar: '\u2591',
+            format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
+            // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
+            barsize: Math.floor((process.stdout.columns || 30) / 3),
+            stopOnComplete: true,
+            hideCursor: true,
+        });
+
+        logger.info(`\nDownloading Video: ${video.title} \n`);
+        logger.verbose('Extra video info \n' +
+        '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
+        '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
+        '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
+        '\t Video total chunks: '.cyan + video.totalChunks + '\n');
+
+        logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
+        if (!process.stdout.columns) {
+            logger.warn(
+                'Unable to get number of columns from terminal.\n' +
+                'This happens sometimes in Cygwin/MSYS.\n' +
+                'No progress bar can be rendered, however the download process should not be affected.\n\n' +
+                'Please use PowerShell or cmd.exe to run destreamer on Windows.'
+            );
+        }
+
+        const headers: string = 'Authorization: Bearer ' + session.AccessToken;
+
+        if (!argv.noExperiments) {
+            if (video.posterImageUrl) {
+                await drawThumbnail(video.posterImageUrl, session);
+            }
+        }
+
+        const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
+            ['headers', headers]
+        ]));
+        const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
+            argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
+            argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
+            ['n', null]
+        ]));
+        const ffmpegCmd: any = new FFmpegCommand();
+
+        const cleanupFn: () => void = () => {
+            pbar.stop();
+
+           if (argv.noCleanup) {
+               return;
+           }
+
+            try {
+                fs.unlinkSync(video.outPath);
+            }
+            catch (e) {
+                // Future handling of an error (maybe)
+            }
+        };
+
+        pbar.start(video.totalChunks, 0, {
+            speed: '0'
+        });
+
+        // prepare ffmpeg command line
+        ffmpegCmd.addInput(ffmpegInpt);
+        ffmpegCmd.addOutput(ffmpegOutput);
+        if (argv.closedCaptions && video.captionsUrl) {
+            const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
+                ['headers', headers]
+            ]));
+
+            ffmpegCmd.addInput(captionsInpt);
+        }
+
+        ffmpegCmd.on('update', async (data: any) => {
+            const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
+
+            pbar.update(currentChunks, {
+                speed: data.bitrate
+            });
+
+            // Graceful fallback in case we can't get columns (Cygwin/MSYS)
+            if (!process.stdout.columns) {
+                process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
+            }
+        });
+
+        process.on('SIGINT', cleanupFn);
+
+        // let the magic begin...
+        await new Promise((resolve: any) => {
+            ffmpegCmd.on('error', (error: any) => {
+                cleanupFn();
+
+                logger.error(`FFmpeg returned an error: ${error.message}`);
+                process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
+            });
+
+            ffmpegCmd.on('success', () => {
+                pbar.update(video.totalChunks); // set progress bar to 100%
+                logger.info(`\nDownload finished: ${video.outPath} \n`);
+                resolve();
+            });
+
+            ffmpegCmd.spawn();
+        });
+
+        process.removeListener('SIGINT', cleanupFn);
     }
 }
 
 
+async function main(): Promise<void> {
+    await init(); // must be first
+
+    let session: Session;
+    // eslint-disable-next-line prefer-const
+    session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
+
+    logger.verbose('Session and API info \n' +
+        '\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
+        '\t API Gateway version: '.cyan + session.ApiGatewayVersion + '\n');
+
+    let videoGUIDs: Array<string>;
+    let outDirs: Array<string>;
+
+    if (argv.videoUrls) {
+        logger.info('Parsing video/group urls');
+        [videoGUIDs, outDirs] =  await parseCLIinput(argv.videoUrls as Array<string>, argv.outputDirectory, session);
+    }
+    else {
+        logger.info('Parsing input file');
+        [videoGUIDs, outDirs] =  await parseInputFile(argv.inputFile!, argv.outputDirectory, session);
+    }
+
+    logger.verbose('List of GUIDs and corresponding output directory \n' +
+        videoGUIDs.map((guid: string, i: number) =>
+            `\thttps://web.microsoftstream.com/video/${guid} => ${outDirs[i]} \n`).join(''));
+
+
+    downloadVideo(videoGUIDs, outDirs, session);
+}
+
+
 main();

test/test.ts

@@ -1,15 +1,32 @@
-import { extractStreamGuids, parseInputFile } from '../src/Utils';
+import { parseInputFile } from '../src/Utils';
+import puppeteer from 'puppeteer';
 import assert from 'assert';
 import tmp from 'tmp';
 import fs from 'fs';
-import { StreamSession, VideoUrl } from './Types';
+import { Session } from './Types';
+
+
+describe('Puppeteer', () => {
+    it('should grab GitHub page title', async () => {
+        const browser = await puppeteer.launch({
+            headless: true,
+            args: ['--disable-dev-shm-usage', '--fast-start', '--no-sandbox']
+        });
+
+        const page = await browser.newPage();
+        await page.goto('https://github.com/', { waitUntil: 'load' });
+
+        let pageTitle = await page.title();
+        assert.equal(true, pageTitle.includes('GitHub'));
+
+        await browser.close();
+    }).timeout(30000); // yeah, this may take a while...
+});
 
 
-// we cannot test groups parsing as that requires an actual session
-// TODO: add SharePoint urls
 describe('Destreamer parsing', () => {
-    it('Input file to arrays of guids', async () => {
+    it('Input file to arrays of URLs and DIRs', async () => {
-        const testSession: StreamSession = {
+        const testSession: Session = {
             AccessToken: '',
             ApiGatewayUri: '',
             ApiGatewayVersion: ''
@@ -27,42 +44,33 @@ describe('Destreamer parsing', () => {
             'https://web.microsoftstream.com/video/xxxxxx-gggg-xxxx-xxxx-xxxxxxxxxxxx',
             ''
         ];
-
+        const expectedGUIDsOut: Array<string> = [
-        const expectedStreamOut: Array<VideoUrl> = [
+            'xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx',
-            {
+            'xxxxxxxx-bbbb-xxxx-xxxx-xxxxxxxxxxxx',
-                url: 'xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx',
+            'xxxxxxxx-cccc-xxxx-xxxx-xxxxxxxxxxxx',
-                outDir: 'videos'
+            'xxxxxxxx-dddd-xxxx-xxxx-xxxxxxxxxxxx',
-            },
+            'xxxxxxxx-eeee-xxxx-xxxx-xxxxxxxxxxxx'
-            {
+        ];
-                url: 'xxxxxxxx-bbbb-xxxx-xxxx-xxxxxxxxxxxx',
+        const expectedDirOut: Array<string> = [
-                outDir: 'luca'
+            'videos',
-            },
+            'luca',
-            {
+            'videos',
-                url: 'xxxxxxxx-cccc-xxxx-xxxx-xxxxxxxxxxxx',
+            'videos',
-                outDir: 'videos'
+            'videos'
-            },
-            {
-                url: 'xxxxxxxx-dddd-xxxx-xxxx-xxxxxxxxxxxx',
-                outDir: 'videos'
-            },
-            {
-                url: 'xxxxxxxx-eeee-xxxx-xxxx-xxxxxxxxxxxx',
-                outDir: 'videos'
-            },
         ];
-
         const tmpFile = tmp.fileSync({ postfix: '.txt' });
         fs.writeFileSync(tmpFile.fd, testIn.join('\r\n'));
-
+        const [testUrlOut , testDirOut]: Array<Array<string>> = await parseInputFile(tmpFile.name, 'videos', testSession);
-        const [testStreamUrls]: Array<Array<VideoUrl>> = parseInputFile(tmpFile.name, 'videos');
+        if (testUrlOut.length !== expectedGUIDsOut.length) {
-
+            throw "Expected url list and test list don't have the same number of elements".red;
-        assert.deepStrictEqual(
+        }
-            await extractStreamGuids(testStreamUrls, testSession),
+        else if (testDirOut.length !== expectedDirOut.length) {
-            expectedStreamOut,
+            throw "Expected dir list and test list don't have the same number of elements".red;
-            'Error in parsing the URLs, missmatch between test and expected'.red
+        }
-        );
+        assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
-        // assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
+            'Error in parsing the URLs, missmatch between test and expected'.red);
-        //     'Error in parsing the DIRs, missmatch between test and expected'.red);
+        assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
+            'Error in parsing the DIRs, missmatch between test and expected'.red);
         assert.ok('Parsing of input file ok');
     });
 });

trigger_codeQL

@@ -0,0 +1 @@
+