Addressing https://github.com/snobu/destreamer/issues/439

* Add personal URL for new MS Teams recordings
* Encode URI components to avoid unescape char

.gitignore

@@ -3,10 +3,12 @@
 *.log
 *.js
 *.zip
+*.xml
+
+yarn.lock
 
 .chrome_data
 node_modules
 videos
 release
 build
-yarn.lock

README.md

@@ -1,10 +1,8 @@
-# This project is abandoned. It will probably not work anymore against your MS Stream tenant.
+<a href="https://github.com/snobu/destreamer/actions">
+  <img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
+</a>
 
-# A heartfelt thank you to all the contributors over the years. You are the real MVPs. 💖
+**destreamer v3.0** is just around the corner. You can try out a pre-release today by cloning [this branch](https://github.com/snobu/destreamer/tree/aria2c_forRealNow).
-
-## Check out kylon's Sharedown for a SharePoint-backend implementation - https://github.com/kylon/Sharedown
-
-<hr>
 
 ![destreamer](assets/logo.png)
 
@@ -43,6 +41,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
 - [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
 - **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
 - [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
+- [**aria2**][aria2]: aria2 is a utility for downloading files with multiple threads, fast.
 - [**git**][git]: one or more npm dependencies require git.
 
 Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on. We've successfully tested it on Windows, macOS and Linux.
@@ -241,6 +240,7 @@ Please open an [issue](https://github.com/snobu/destreamer/issues) and we'll loo
 
 
 [ffmpeg]: https://www.ffmpeg.org/download.html
+[aria2]: https://github.com/aria2/aria2/releases
 [xming]: https://sourceforge.net/projects/xming/
 [node]: https://nodejs.org/en/download/
 [git]: https://git-scm.com/downloads

package.json

@@ -9,7 +9,8 @@
     "main": "build/src/destreamer.js",
     "bin": "build/src/destreamer.js",
     "scripts": {
-    "build": "echo Transpiling TypeScript to JavaScript... && node node_modules/typescript/bin/tsc && echo Destreamer was built successfully.",
+        "build": "echo Transpiling TypeScript to JavaScript... && tsc && echo Destreamer was built successfully.",
+        "watch": "tsc --watch",
         "test": "mocha build/test",
         "lint": "eslint src/*.ts"
     },

src/ApiClient.ts

@@ -1,16 +1,18 @@
 import { logger } from './Logger';
-import { Session } from './Types';
+import { ShareSession, StreamSession, Video } from './Types';
+import { publishedDateToString, publishedTimeToString } from './VideoUtils';
 
 import axios, { AxiosRequestConfig, AxiosResponse, AxiosInstance, AxiosError } from 'axios';
 import axiosRetry, { isNetworkOrIdempotentRequestError } from 'axios-retry';
+// import fs from 'fs';
 
 
-export class ApiClient {
+export class StreamApiClient {
-    private static instance: ApiClient;
+    private static instance: StreamApiClient;
     private axiosInstance?: AxiosInstance;
-    private session?: Session;
+    private session?: StreamSession;
 
-    private constructor(session?: Session) {
+    private constructor(session?: StreamSession) {
         this.session = session;
         this.axiosInstance = axios.create({
             baseURL: session?.ApiGatewayUri,
@@ -50,16 +52,16 @@ export class ApiClient {
      *
      * @param session used if initializing
      */
-    public static getInstance(session?: Session): ApiClient {
+    public static getInstance(session?: StreamSession): StreamApiClient {
-        if (!ApiClient.instance) {
+        if (!StreamApiClient.instance) {
-            ApiClient.instance = new ApiClient(session);
+            StreamApiClient.instance = new StreamApiClient(session);
         }
 
-        return ApiClient.instance;
+        return StreamApiClient.instance;
     }
 
-    public setSession(session: Session): void {
+    public setSession(session: StreamSession): void {
-        if (!ApiClient.instance) {
+        if (!StreamApiClient.instance) {
             logger.warn("Trying to update ApiCient session when it's not initialized!");
         }
 
@@ -113,3 +115,134 @@ export class ApiClient {
         });
     }
 }
+
+export class ShareApiClient {
+    private axiosInstance: AxiosInstance;
+    private site: string;
+
+    public constructor(domain: string, site: string, session: ShareSession) {
+        this.axiosInstance = axios.create({
+            baseURL: domain,
+            // timeout: 7000,
+            headers: {
+                'User-Agent': 'destreamer/3.0 ALPHA',
+                'Cookie': `rtFa=${session.rtFa}; FedAuth=${session.FedAuth}`
+            }
+        });
+        this.site = site;
+
+
+        // FIXME: disabled because it was messing with the direct download check
+        // axiosRetry(this.axiosInstance, {
+        //     // The following option is not working.
+        //     // We should open an issue on the relative GitHub
+        //     shouldResetTimeout: true,
+        //     retries: 6,
+        //     retryDelay: (retryCount: number) => {
+        //         return retryCount * 2000;
+        //     },
+        //     retryCondition: (err: AxiosError) => {
+        //         const retryCodes: Array<number> = [429, 500, 502, 503];
+        //         if (isNetworkOrIdempotentRequestError(err)) {
+        //             logger.warn(`${err}. Retrying request...`);
+
+        //             return true;
+        //         }
+        //         logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}.`);
+        //         logger.warn('Here is the error message: ');
+        //         console.dir(err.response?.data);
+        //         logger.warn('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
+
+        //         const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
+
+        //         return shouldRetry;
+        //     }
+        // });
+    }
+
+
+    public async getVideoInfo(filePath: string, outPath: string): Promise<Video> {
+        let playbackUrl: string;
+
+        // TODO: Ripped this straigth from chromium inspector. Don't know don't care what it is right now. Check later
+        const payload = {
+            parameters: {
+                __metadata: {
+                    type: 'SP.RenderListDataParameters'
+                },
+                ViewXml: `<View Scope="RecursiveAll"><Query><Where><Eq><FieldRef Name="FileRef" /><Value Type="Text"><![CDATA[${filePath}]]></Value></Eq></Where></Query><RowLimit Paged="TRUE">1</RowLimit></View>`,
+                RenderOptions: 12295,
+                AddRequiredFields: true
+            }
+        };
+        const url = `${this.site}/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream?@a1='${encodeURIComponent(filePath)}'`;
+
+        logger.verbose(`Requesting video info for '${url}'`);
+        const info = await this.axiosInstance.post(url, payload, {
+            headers: {
+                'Content-Type': 'application/json;odata=verbose'
+            }
+        }).then(res => res.data);
+        // fs.writeFileSync('info.json', JSON.stringify(info, null, 4));
+
+        // FIXME: very bad but usefull in alpha stage to check for edge cases
+        if (info.ListData.Row.length !== 1) {
+            logger.error('More than 1 row in SharePoint video info', { fatal: true });
+
+            process.exit(1000);
+        }
+
+        const direct = await this.canDirectDownload(filePath);
+        const b64VideoMetadata = JSON.parse(
+            info.ListData.Row[0].MediaServiceFastMetadata
+        ).video.altManifestMetadata;
+        const durationSeconds = Math.ceil(
+            (JSON.parse(
+                Buffer.from(b64VideoMetadata, 'base64').toString()
+            ).Duration100Nano) / 10 / 1000 / 1000
+        );
+
+        if (direct) {
+            playbackUrl = this.axiosInstance.defaults.baseURL + filePath;
+            // logger.verbose(playbackUrl);
+        }
+        else {
+            playbackUrl = info.ListSchema['.videoManifestUrl'];
+            playbackUrl = playbackUrl.replace('{.mediaBaseUrl}', info.ListSchema['.mediaBaseUrl']);
+            // the only filetype works I found
+            playbackUrl = playbackUrl.replace('{.fileType}', 'mp4');
+            playbackUrl = playbackUrl.replace('{.callerStack}', info.ListSchema['.callerStack']);
+            playbackUrl = playbackUrl.replace('{.spItemUrl}', info.ListData.Row[0]['.spItemUrl']);
+            playbackUrl = playbackUrl.replace('{.driveAccessToken}', info.ListSchema['.driveAccessToken']);
+            playbackUrl += '&part=index&format=dash';
+        }
+
+
+        return {
+            direct,
+            title: filePath.split('/').pop() ?? 'video.mp4',
+            duration: publishedTimeToString(durationSeconds),
+            publishDate: publishedDateToString(info.ListData.Row[0]['Modified.']),
+            publishTime: publishedTimeToString(info.ListData.Row[0]['Modified.']),
+            author: info.ListData.Row[0]['Author.title'],
+            authorEmail: info.ListData.Row[0]['Author.email'],
+            uniqueId: info.ListData.Row[0].GUID.substring(1, 9),
+            outPath,
+            playbackUrl,
+            totalChunks: durationSeconds
+        };
+    }
+
+
+    private async canDirectDownload(filePath: string): Promise<boolean> {
+        logger.verbose(`Checking direct download for '${filePath}'`);
+
+        return this.axiosInstance.head(
+            filePath, { maxRedirects: 0 }
+        ).then(
+            res => (res.status === 200)
+        ).catch(
+            () => false
+        );
+    }
+}

src/CommandLineParser.ts

@@ -1,15 +1,14 @@
-import { CLI_ERROR, ERROR_CODE } from './Errors';
+import { CLI_ERROR } from './Errors';
-import { checkOutDir } from './Utils';
+import { makeOutDir } from './Utils';
 import { logger } from './Logger';
 import { templateElements } from './Types';
 
 import fs from 'fs';
-import readlineSync from 'readline-sync';
 import sanitize from 'sanitize-filename';
 import yargs from 'yargs';
 
 
-export const argv: any = yargs.options({
+export const argv = yargs.options({
     username: {
         alias: 'u',
         type: 'string',
@@ -114,7 +113,7 @@ export const argv: any = yargs.options({
 .check(() => noArguments())
 .check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
 .check((argv: any) => {
-    if (checkOutDir(argv.outputDirectory)) {
+    if (makeOutDir(argv.outputDirectory)) {
         return true;
     }
     else {
@@ -198,14 +197,3 @@ function isOutputTemplateValid(argv: any): boolean {
 
     return true;
 }
-
-
-export function promptUser(choices: Array<string>): number {
-    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
-
-    if (index === -1) {
-        process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
-    }
-
-    return index;
-}

src/Downloaders.ts

@@ -0,0 +1,335 @@
+import { ShareApiClient, StreamApiClient } from './ApiClient';
+import { argv } from './CommandLineParser';
+import { ERROR_CODE } from './Errors';
+import { logger } from './Logger';
+import { doShareLogin, doStreamLogin } from './LoginModules';
+import { drawThumbnail } from './Thumbnail';
+import { refreshSession, TokenCache } from './TokenCache';
+import { Video, VideoUrl } from './Types';
+import { ffmpegTimemarkToChunk } from './Utils';
+import { createUniquePath, getStreamInfo } from './VideoUtils';
+
+import cliProgress from 'cli-progress';
+import fs from 'fs';
+import { execSync } from 'child_process';
+import path from 'path';
+
+
+const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
+const tokenCache: TokenCache = new TokenCache();
+
+
+export async function downloadStreamVideo(videoUrls: Array<VideoUrl>): Promise<void> {
+    logger.info('Downloading Microsoft Stream videos...');
+
+    let session = tokenCache.Read() ?? await doStreamLogin('https://web.microsoftstream.com/', tokenCache, argv.username);
+    logger.verbose(
+        'Session and API info \n' +
+        '\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
+        '\t API Gateway version: '.cyan + session.ApiGatewayVersion + '\n'
+    );
+
+    logger.info('Fetching videos info... \n');
+    const videos: Array<Video> = createUniquePath(
+        await getStreamInfo(videoUrls, session, argv.closedCaptions),
+        argv.outputTemplate, argv.format, argv.skip
+    );
+
+    if (argv.simulate) {
+        videos.forEach((video: Video) => {
+            logger.info(
+                '\nTitle:          '.green + video.title +
+                '\nOutPath:        '.green + video.outPath +
+                '\nPublished Date: '.green + video.publishDate +
+                '\nPlayback URL:   '.green + video.playbackUrl +
+                ((video.captionsUrl) ? ('\nCC URL:         '.green + video.captionsUrl) : '')
+            );
+        });
+
+        return;
+    }
+
+    for (const [index, video] of videos.entries()) {
+
+        if (argv.skip && fs.existsSync(video.outPath)) {
+            logger.info(`File already exists, skipping: ${video.outPath} \n`);
+            continue;
+        }
+
+        if (argv.keepLoginCookies && index !== 0) {
+            logger.info('Trying to refresh token...');
+            session = await refreshSession('https://web.microsoftstream.com/video/' + video.guid);
+            StreamApiClient.getInstance().setSession(session);
+        }
+
+        const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
+            barCompleteChar: '\u2588',
+            barIncompleteChar: '\u2591',
+            format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
+            // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
+            barsize: Math.floor((process.stdout.columns || 30) / 3),
+            stopOnComplete: true,
+            hideCursor: true,
+        });
+
+        logger.info(`\nDownloading Video: ${video.title} \n`);
+        logger.verbose('Extra video info \n' +
+            '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
+            '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
+            '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
+            '\t Video total chunks: '.cyan + video.totalChunks + '\n');
+
+        logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
+        if (!process.stdout.columns) {
+            logger.warn(
+                'Unable to get number of columns from terminal.\n' +
+                'This happens sometimes in Cygwin/MSYS.\n' +
+                'No progress bar can be rendered, however the download process should not be affected.\n\n' +
+                'Please use PowerShell or cmd.exe to run destreamer on Windows.'
+            );
+        }
+
+        const headers: string = 'Authorization: Bearer ' + session.AccessToken;
+
+        if (!argv.noExperiments) {
+            if (video.posterImageUrl) {
+                await drawThumbnail(video.posterImageUrl, session);
+            }
+        }
+
+        const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
+            ['headers', headers]
+        ]));
+        const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
+            argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
+            argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
+            ['n', null]
+        ]));
+        const ffmpegCmd: any = new FFmpegCommand();
+
+        const cleanupFn: () => void = () => {
+            pbar.stop();
+
+            if (argv.noCleanup) {
+                return;
+            }
+
+            try {
+                fs.unlinkSync(video.outPath);
+            }
+            catch (e) {
+                // Future handling of an error (maybe)
+            }
+        };
+
+        pbar.start(video.totalChunks, 0, {
+            speed: '0'
+        });
+
+        // prepare ffmpeg command line
+        ffmpegCmd.addInput(ffmpegInpt);
+        ffmpegCmd.addOutput(ffmpegOutput);
+        if (argv.closedCaptions && video.captionsUrl) {
+            const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
+                ['headers', headers]
+            ]));
+
+            ffmpegCmd.addInput(captionsInpt);
+        }
+
+        ffmpegCmd.on('update', async (data: any) => {
+            const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
+
+            pbar.update(currentChunks, {
+                speed: data.bitrate
+            });
+
+            // Graceful fallback in case we can't get columns (Cygwin/MSYS)
+            if (!process.stdout.columns) {
+                process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
+            }
+        });
+
+        process.on('SIGINT', cleanupFn);
+
+        // let the magic begin...
+        await new Promise((resolve: any) => {
+            ffmpegCmd.on('error', (error: any) => {
+                cleanupFn();
+
+                logger.error(`FFmpeg returned an error: ${error.message}`);
+                process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
+            });
+
+            ffmpegCmd.on('success', () => {
+                pbar.update(video.totalChunks); // set progress bar to 100%
+                logger.info(`\nDownload finished: ${video.outPath} \n`);
+                resolve();
+            });
+
+            ffmpegCmd.spawn();
+        });
+
+        process.removeListener('SIGINT', cleanupFn);
+    }
+}
+
+
+// TODO: complete overhaul of this function
+export async function downloadShareVideo(videoUrls: Array<VideoUrl>): Promise<void> {
+    const shareUrlRegex = new RegExp(/(?<domain>https:\/\/.+\.sharepoint\.com).*?(?<baseSite>\/(?:teams|sites|personal)\/.*?)(?:(?<filename>\/.*\.mp4)|\/.*id=(?<paramFilename>.*mp4))/);
+
+    logger.info('Downloading SharePoint videos...\n\n');
+
+    // FIXME: this may change we need a smart login system if a request fails
+    const session = await doShareLogin(videoUrls[0].url, argv.username);
+
+    for (const videoUrl of videoUrls) {
+        const match = shareUrlRegex.exec(videoUrl.url);
+        if (!match) {
+            logger.error(`Invalid url '${videoUrl.url}', skipping...`);
+
+            continue;
+        }
+
+        const shareDomain = match.groups!.domain;
+        const shareSite = match.groups!.baseSite;
+        const shareFilepath = decodeURIComponent(match.groups?.filename ? (shareSite + match.groups.filename) : match.groups!.paramFilename);
+        // FIXME: hardcoded video.mp4
+        const title = shareFilepath.split('/').pop()?.split('.')[0] ?? 'video';
+
+        const apiClient = new ShareApiClient(shareDomain, shareSite, session);
+
+        const video = await apiClient.getVideoInfo(shareFilepath, videoUrl.outDir);
+        createUniquePath(video, title, argv.format, argv.skip);
+
+        if (argv.simulate) {
+            if (argv.verbose) {
+                console.dir(video);
+            }
+            else {
+                logger.info(
+                    '\nTitle:          '.green + video.title +
+                    '\nOutPath:        '.green + video.outPath +
+                    '\nPublished Date: '.green + video.publishDate +
+                    '\nPlayback URL:   '.green + video.playbackUrl
+                );
+            }
+            continue;
+        }
+
+        if (video.direct) {
+            const headers = `Cookie: rtFa=${session.rtFa}; FedAuth=${session.FedAuth}`;
+
+            // FIXME: unstable and bad all-around
+            try {
+                execSync(
+                    'aria2c --max-connection-per-server 8 --console-log-level warn ' +
+                    `--header "${headers}" --dir "${path.dirname(video.outPath)}" --out "${path.basename(video.outPath)}" "${shareDomain + shareFilepath}"`,
+                    { stdio: 'inherit' }
+                );
+            }
+            catch (error: any) {
+                logger.error(`${error.status} \n\n${error.message} \n\n${error.stdout.toString()} \n\n${error.stderr.toString()}`);
+            }
+        }
+        else {
+            // FIXME: just a copy-paste, should move to separate function
+            const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
+                barCompleteChar: '\u2588',
+                barIncompleteChar: '\u2591',
+                format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
+                // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
+                barsize: Math.floor((process.stdout.columns || 30) / 3),
+                stopOnComplete: true,
+                hideCursor: true,
+            });
+
+            logger.info(`\nDownloading Video: ${video.title} \n`);
+            logger.verbose('Extra video info \n' +
+                '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
+                '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
+                '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
+                '\t Video total chunks: '.cyan + video.totalChunks + '\n');
+
+            logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
+            if (!process.stdout.columns) {
+                logger.warn(
+                    'Unable to get number of columns from terminal.\n' +
+                    'This happens sometimes in Cygwin/MSYS.\n' +
+                    'No progress bar can be rendered, however the download process should not be affected.\n\n' +
+                    'Please use PowerShell or cmd.exe to run destreamer on Windows.'
+                );
+            }
+
+            const ffmpegInpt: any = new FFmpegInput(video.playbackUrl);
+            const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
+                argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
+                argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
+                ['n', null]
+            ]));
+            const ffmpegCmd: any = new FFmpegCommand();
+
+            const cleanupFn: () => void = () => {
+                pbar.stop();
+
+                if (argv.noCleanup) {
+                    return;
+                }
+
+                try {
+                    fs.unlinkSync(video.outPath);
+                }
+                catch (e) {
+                    // Future handling of an error (maybe)
+                }
+            };
+
+            pbar.start(video.totalChunks, 0, {
+                speed: '0'
+            });
+
+            // prepare ffmpeg command line
+            ffmpegCmd.addInput(ffmpegInpt);
+            ffmpegCmd.addOutput(ffmpegOutput);
+
+            ffmpegCmd.on('update', async (data: any) => {
+                const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
+
+                pbar.update(currentChunks, {
+                    speed: data.bitrate
+                });
+
+                // Graceful fallback in case we can't get columns (Cygwin/MSYS)
+                if (!process.stdout.columns) {
+                    process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
+                }
+            });
+
+            process.on('SIGINT', cleanupFn);
+
+            // let the magic begin...
+            await new Promise((resolve: any) => {
+                ffmpegCmd.on('error', (error: any) => {
+                    cleanupFn();
+
+                    logger.error(`FFmpeg returned an error: ${error.message}`);
+                    process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
+                });
+
+                ffmpegCmd.on('success', () => {
+                    pbar.update(video.totalChunks); // set progress bar to 100%
+                    logger.info(`\nDownload finished: ${video.outPath} \n`);
+                    resolve();
+                });
+
+                ffmpegCmd.spawn();
+            });
+
+            process.removeListener('SIGINT', cleanupFn);
+            // logger.error('TODO: manifest download');
+
+            // continue;
+        }
+    }
+}

src/Errors.ts

@@ -1,8 +1,9 @@
 export const enum ERROR_CODE {
-    UNHANDLED_ERROR,
+    UNHANDLED_ERROR = 200,
     ELEVATED_SHELL,
     CANCELLED_USER_INPUT,
     MISSING_FFMPEG,
+    MISSING_ARIA2,
     OUTDATED_FFMPEG,
     UNK_FFMPEG_ERROR,
     INVALID_VIDEO_GUID,
@@ -22,7 +23,10 @@ export const errors: { [key: number]: string } = {
     [ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
         'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
-    [ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
+    [ERROR_CODE.MISSING_ARIA2]: 'Aria2 is missing!\n' +
+        'Destreamer requires a fairly recent release of Aria2 to download videos',
+
+    [ERROR_CODE.OUTDATED_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
         'Destreamer requires a fairly recent release of FFmpeg to download videos',
 
     [ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',

src/LoginModules.ts

@@ -0,0 +1,176 @@
+import { logger } from './Logger';
+import puppeteer from 'puppeteer';
+import { getPuppeteerChromiumPath } from './PuppeteerHelper';
+import { chromeCacheFolder } from './destreamer';
+import { argv } from './CommandLineParser';
+import { ShareSession, StreamSession } from './Types';
+import { ERROR_CODE } from './Errors';
+import { TokenCache } from './TokenCache';
+
+
+export async function doStreamLogin(url: string, tokenCache: TokenCache, username?: string): Promise<StreamSession> {
+    logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
+
+    const browser: puppeteer.Browser = await puppeteer.launch({
+        executablePath: getPuppeteerChromiumPath(),
+        headless: false,
+        userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
+        defaultViewport: null,
+        args: [
+            '--disable-dev-shm-usage',
+            '--fast-start',
+            '--no-sandbox'
+        ]
+    });
+
+    // try-finally because we were leaving zombie processes if there was an error
+    try {
+        const page: puppeteer.Page = (await browser.pages())[0];
+
+        logger.info('Navigating to login page...');
+        await page.goto(url, { waitUntil: 'load' });
+
+        try {
+            if (username) {
+                await page.waitForSelector('input[type="email"]', { timeout: 3000 });
+                await page.keyboard.type(username);
+                await page.click('input[type="submit"]');
+            }
+            else {
+                /* If a username was not provided we let the user take actions that
+                lead up to the video page. */
+            }
+        }
+        catch (e) {
+            /* If there is no email input selector we aren't in the login module,
+            we are probably using the cache to aid the login.
+            It could finish the login on its own if the user said 'yes' when asked to
+            remember the credentials or it could still prompt the user for a password */
+        }
+
+        await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
+        logger.info('We are logged in.');
+
+        let session: StreamSession | null = null;
+        let tries = 1;
+        while (!session) {
+            try {
+                let sessionInfo: any;
+                session = await page.evaluate(
+                    () => {
+                        return {
+                            AccessToken: sessionInfo.AccessToken,
+                            ApiGatewayUri: sessionInfo.ApiGatewayUri,
+                            ApiGatewayVersion: sessionInfo.ApiGatewayVersion
+                        };
+                    }
+                );
+            }
+            catch (error) {
+                if (tries > 5) {
+                    process.exit(ERROR_CODE.NO_SESSION_INFO);
+                }
+
+                session = null;
+                tries++;
+                await page.waitForTimeout(3000);
+            }
+        }
+
+        tokenCache.Write(session);
+        logger.info('Wrote access token to token cache.');
+        logger.info("At this point Chromium's job is done, shutting it down...\n");
+
+
+        return session;
+    }
+    finally {
+        await browser.close();
+    }
+}
+
+
+
+export async function doShareLogin(url: string, username?: string): Promise<ShareSession> {
+    logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
+
+    let session: ShareSession | null = null;
+    const hostname = new URL(url).host;
+
+    const browser: puppeteer.Browser = await puppeteer.launch({
+        executablePath: getPuppeteerChromiumPath(),
+        headless: false,
+        devtools: argv.verbose,
+        userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
+        defaultViewport: null,
+        args: [
+            '--disable-dev-shm-usage',
+            '--fast-start',
+            '--no-sandbox'
+        ]
+    });
+
+    // try-finally because we were leaving zombie processes if there was an error
+    try {
+        const page: puppeteer.Page = (await browser.pages())[0];
+
+        logger.info('Navigating to login page...');
+        await page.goto(url, { waitUntil: 'load' });
+
+        try {
+            if (username) {
+                await page.waitForSelector('input[type="email"]', { timeout: 3000 });
+                await page.keyboard.type(username);
+                await page.click('input[type="submit"]');
+            }
+            else {
+                /* If a username was not provided we let the user take actions that
+                lead up to the video page. */
+            }
+        }
+        catch (e) {
+            /* If there is no email input selector we aren't in the login module,
+            we are probably using the cache to aid the login.
+            It could finish the login on its own if the user said 'yes' when asked to
+            remember the credentials or it could still prompt the user for a password */
+        }
+
+        logger.info('Waiting for target!');
+
+        await browser.waitForTarget((target: puppeteer.Target) => target.url().startsWith(`https://${hostname}`), { timeout: 150000 });
+        logger.info('We are logged in.');
+
+        let tries = 1;
+        while (!session) {
+            const cookieJar = (await page.cookies()).filter(
+                biscuit => biscuit.name == 'rtFa' || biscuit.name == 'FedAuth'
+            );
+
+            if (cookieJar.length != 2) {
+                if (tries > 5) {
+                    process.exit(ERROR_CODE.NO_SESSION_INFO);
+                }
+
+                await page.waitForTimeout(1000 * tries++);
+
+                continue;
+            }
+
+            session = {
+                rtFa: cookieJar.find(biscuit => biscuit.name == 'rtFa')!.value,
+                FedAuth: cookieJar.find(biscuit => biscuit.name == 'FedAuth')!.value
+            };
+        }
+
+        logger.info("At this point Chromium's job is done, shutting it down...\n");
+
+        // await page.waitForTimeout(1000 * 60 * 60 * 60);
+    }
+    finally {
+        logger.verbose('Stream login browser closing...');
+        await browser.close();
+        logger.verbose('Stream login browser closed');
+    }
+
+    return session;
+}

src/Thumbnail.ts

@@ -1,12 +1,12 @@
-import { ApiClient } from './ApiClient';
+import { StreamApiClient } from './ApiClient';
-import { Session } from './Types';
+import { StreamSession } from './Types';
 
 import terminalImage from 'terminal-image';
 import { AxiosResponse } from 'axios';
 
 
-export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
+export async function drawThumbnail(posterImage: string, session: StreamSession): Promise<void> {
-    const apiClient: ApiClient = ApiClient.getInstance(session);
+    const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
 
     const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
         .then((response: AxiosResponse<any> | undefined) => response?.data);

src/TokenCache.ts

@@ -2,7 +2,7 @@ import { chromeCacheFolder } from './destreamer';
 import { ERROR_CODE } from './Errors';
 import { logger } from './Logger';
 import { getPuppeteerChromiumPath } from './PuppeteerHelper';
-import { Session } from './Types';
+import { StreamSession } from './Types';
 
 import fs from 'fs';
 import jwtDecode from 'jwt-decode';
@@ -12,14 +12,14 @@ import puppeteer from 'puppeteer';
 export class TokenCache {
     private tokenCacheFile = '.token_cache';
 
-    public Read(): Session | null {
+    public Read(): StreamSession | null {
         if (!fs.existsSync(this.tokenCacheFile)) {
             logger.warn(`${this.tokenCacheFile} not found. \n`);
 
             return null;
         }
 
-        const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
+        const session: StreamSession = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
 
         type Jwt = {
             [key: string]: any
@@ -41,7 +41,7 @@ export class TokenCache {
         return session;
     }
 
-    public Write(session: Session): void {
+    public Write(session: StreamSession): void {
         const s: string = JSON.stringify(session, null, 4);
         fs.writeFile(this.tokenCacheFile, s, (err: any) => {
             if (err) {
@@ -54,7 +54,7 @@ export class TokenCache {
 }
 
 
-export async function refreshSession(url: string): Promise<Session> {
+export async function refreshSession(url: string): Promise<StreamSession> {
     const videoId: string = url.split('/').pop() ?? process.exit(ERROR_CODE.INVALID_VIDEO_GUID);
 
     const browser: puppeteer.Browser = await puppeteer.launch({
@@ -73,7 +73,7 @@ export async function refreshSession(url: string): Promise<Session> {
 
     await browser.waitForTarget((target: puppeteer.Target) => target.url().includes(videoId), { timeout: 30000 });
 
-    let session: Session | null = null;
+    let session: StreamSession | null = null;
     let tries = 1;
 
     while (!session) {

src/Types.ts

@@ -1,11 +1,34 @@
-export type Session = {
+export type StreamSession = {
     AccessToken: string;
     ApiGatewayUri: string;
     ApiGatewayVersion: string;
 }
 
 
+export type ShareSession = {
+    FedAuth: string;
+    rtFa: string;
+}
+
+
+export type VideoUrl = {
+    url: string,
+    outDir: string
+}
+
+
+export type SharepointVideo = {
+    // if we can download the MP4 or we need to use DASH
+    direct: boolean;
+    playbackUrl: string;
+    title: string;
+    outPath: string
+}
+
+
 export type Video = {
+    guid?: string;
+    direct?: boolean;
     title: string;
     duration: string;
     publishDate: string;
@@ -16,7 +39,7 @@ export type Video = {
     outPath: string;
     totalChunks: number;    // Abstraction of FFmpeg timemark
     playbackUrl: string;
-    posterImageUrl: string | null;
+    posterImageUrl?: string;
     captionsUrl?: string
 }
 

src/Utils.ts

@@ -1,47 +1,62 @@
-import { ApiClient } from './ApiClient';
+import { StreamApiClient } from './ApiClient';
 import { ERROR_CODE } from './Errors';
 import { logger } from './Logger';
-import { Session } from './Types';
+import { StreamSession, VideoUrl } from './Types';
 
 import { AxiosResponse } from 'axios';
 import { execSync } from 'child_process';
 import fs from 'fs';
+import readlineSync from 'readline-sync';
 
 
-async function extractGuids(url: string, client: ApiClient): Promise<Array<string> | null> {
+const streamUrlRegex = new RegExp(/https?:\/\/web\.microsoftstream\.com.*/);
+const shareUrlRegex = new RegExp(/https?:\/\/.+\.sharepoint\.com.*/);
 
+
+/** we place the guid in the url fild in the return */
+export async function extractStreamGuids(urlList: Array<VideoUrl>, session: StreamSession): Promise<Array<VideoUrl>> {
     const videoRegex = new RegExp(/https:\/\/.*\/video\/(\w{8}-(?:\w{4}-){3}\w{12})/);
     const groupRegex = new RegExp(/https:\/\/.*\/group\/(\w{8}-(?:\w{4}-){3}\w{12})/);
 
-    const videoMatch: RegExpExecArray | null = videoRegex.exec(url);
+    const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
-    const groupMatch: RegExpExecArray | null = groupRegex.exec(url);
+    const guidList: Array<VideoUrl> = [];
+
+    for (const url of urlList) {
+        const videoMatch: RegExpExecArray | null = videoRegex.exec(url.url);
+        const groupMatch: RegExpExecArray | null = groupRegex.exec(url.url);
 
         if (videoMatch) {
-        return [videoMatch[1]];
+            guidList.push({
+                url: videoMatch[1],
+                outDir: url.outDir
+            });
         }
         else if (groupMatch) {
-        const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
+            const videoNumber: number = await apiClient.callApi(`groups/${groupMatch[1]}`, 'get')
                 .then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
-        const result: Array<string> = [];
 
             // Anything above $top=100 results in 400 Bad Request
             // Use $skip to skip the first 100 and get another 100 and so on
             for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
-            const partial: Array<string> = await client.callApi(
+                await apiClient.callApi(
                     `groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
-                '$top=100&$orderby=publishedDate asc', 'get')
+                    '$top=100&$orderby=publishedDate asc', 'get'
-                .then(
+                ).then((response: AxiosResponse<any> | undefined) => {
-                    (response: AxiosResponse<any> | undefined) =>
+                    response?.data.value.forEach((video: { id: string }) =>
-                        response?.data.value.map((item: any) => item.id)
+                        guidList.push({
+                            url: video.id,
+                            outDir: url.outDir
+                        })
                     );
-
+                });
-            result.push(...partial);
+            }
+        }
+        else {
+            logger.warn(`Invalid url '${url.url}', skipping...`);
+        }
     }
 
-        return result;
+    return guidList;
-    }
-
-    return null;
 }
 
 
@@ -52,30 +67,32 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
  *
  * @param {Array<string>} urlList       list of link to parse
  * @param {string}        defaultOutDir the directry used to save the videos
- * @param {Session}       session       used to call the API to get the GUIDs from group links
  *
- * @returns Array of 2 elements, 1st one being the GUIDs array, 2nd one the output directories array
+ * @returns Array of 2 elements: 1st an array of Microsoft Stream urls, 2nd an array of SharePoint urls
  */
-export async function parseCLIinput(urlList: Array<string>, defaultOutDir: string,
+export function parseCLIinput(urlList: Array<string>, defaultOutDir: string): Array<Array<VideoUrl>> {
-    session: Session): Promise<Array<Array<string>>> {
+    const stream: Array<VideoUrl> = [];
-
+    const share: Array<VideoUrl> = [];
-    const apiClient: ApiClient = ApiClient.getInstance(session);
-    const guidList: Array<string> = [];
 
     for (const url of urlList) {
-        const guids: Array<string> | null = await extractGuids(url, apiClient);
+        if (streamUrlRegex.test(url)) {
-
+            stream.push({
-        if (guids) {
+                url: url,
-            guidList.push(...guids);
+                outDir: defaultOutDir
+            });
+        }
+        else if (shareUrlRegex.test(url)) {
+            share.push({
+                url: url,
+                outDir: defaultOutDir
+            });
         }
         else {
             logger.warn(`Invalid url '${url}', skipping..`);
         }
     }
 
-    const outDirList: Array<string> = Array(guidList.length).fill(defaultOutDir);
+    return [stream, share];
-
-    return [guidList, outDirList];
 }
 
 
@@ -86,94 +103,84 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
  *
  * @param {string}  inputFile     path to the text file
  * @param {string}  defaultOutDir the default/fallback directory used to save the videos
- * @param {Session} session       used to call the API to get the GUIDs from group links
  *
  * @returns Array of 2 elements, 1st one being the GUIDs array, 2nd one the output directories array
  */
-export async function parseInputFile(inputFile: string, defaultOutDir: string,
+export function parseInputFile(inputFile: string, defaultOutDir: string): Array<Array<VideoUrl>> {
-    session: Session): Promise<Array<Array<string>>> {
     // rawContent is a list of each line of the file
-    const rawContent: Array<string> = fs.readFileSync(inputFile).toString()
+    const rawContent: Array<string> = fs.readFileSync(inputFile).toString().split(/\r?\n/);
-        .split(/\r?\n/);
+    const stream: Array<VideoUrl> = [];
-    const apiClient: ApiClient = ApiClient.getInstance(session);
+    const share: Array<VideoUrl> = [];
-
+    let streamUrl = false;
-    const guidList: Array<string> = [];
-    const outDirList: Array<string> = [];
-    // if the last line was an url set this
-    let foundUrl = false;
 
     for (let i = 0; i < rawContent.length; i++) {
         const line: string = rawContent[i];
+        const nextLine: string | null = i < rawContent.length ? rawContent[i + 1] : null;
+        let outDir = defaultOutDir;
 
         // filter out lines with no content
         if (!line.match(/\S/)) {
             logger.warn(`Line ${i + 1} is empty, skipping..`);
             continue;
         }
-        // parse if line is option
+        // check for urls
-        else if (line.includes('-dir')) {
+        else if (streamUrlRegex.test(line)) {
-            if (foundUrl) {
+            streamUrl = true;
-                const outDir: string | null = parseOption('-dir', line);
-
-                if (outDir && checkOutDir(outDir)) {
-                    outDirList.push(...Array(guidList.length - outDirList.length)
-                        .fill(outDir));
         }
+        else if (shareUrlRegex.test(line)) {
+            streamUrl = false;
+        }
+        // now invalid line since we skip ahead one line if we find dir option
         else {
-                    outDirList.push(...Array(guidList.length - outDirList.length)
+            logger.warn(`Line ${i + 1}: '${line}' is invalid, skipping..`);
-                        .fill(defaultOutDir));
-                }
 
-                foundUrl = false;
             continue;
         }
-            else {
+
-                logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
+        // we now have a valid url, check next line for option
-                continue;
+        if (nextLine) {
+            const optionDir = parseOption('-dir', nextLine);
+
+            if (optionDir && makeOutDir(optionDir)) {
+                outDir = optionDir;
+                // if there was an option we skip a line
+                i++;
             }
         }
 
-        /* now line is not empty nor an option line.
+        if (streamUrl) {
-        If foundUrl is still true last line didn't have a directory option
+            stream.push({
-        so we stil need to add the default outDir to outDirList to  */
+                url: line,
-        if (foundUrl) {
+                outDir
-            outDirList.push(...Array(guidList.length - outDirList.length)
+            });
-                .fill(defaultOutDir));
-            foundUrl = false;
-        }
-
-        const guids: Array<string> | null = await extractGuids(line, apiClient);
-
-        if (guids) {
-            guidList.push(...guids);
-            foundUrl = true;
         }
         else {
-            logger.warn(`Invalid url at line ${i + 1}, skipping..`);
+            share.push({
+                url: line,
+                outDir
+            });
         }
     }
 
-    // if foundUrl is still true after the loop we have some url without an outDir
-    if (foundUrl) {
-        outDirList.push(...Array(guidList.length - outDirList.length)
-            .fill(defaultOutDir));
-    }
 
-    return [guidList, outDirList];
+    return [stream, share];
 }
 
 
 // This leaves us the option to add more options (badum tss) _Luca
 function parseOption(optionSyntax: string, item: string): string | null {
     const match: RegExpMatchArray | null = item.match(
-        RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
+        RegExp(`^\\s+${optionSyntax}\\s*=\\s*['"](.*)['"]`)
     );
 
     return match ? match[1] : null;
 }
 
-
+/**
-export function checkOutDir(directory: string): boolean {
+ * @param directory path to create
+ * @returns true on success, false otherwise
+ */
+export function makeOutDir(directory: string): boolean {
     if (!fs.existsSync(directory)) {
         try {
             fs.mkdirSync(directory);
@@ -205,14 +212,40 @@ export function checkRequirements(): void {
     catch (e) {
         process.exit(ERROR_CODE.MISSING_FFMPEG);
     }
+
+    try {
+        const versionRegex = new RegExp(/aria2 version (.*)/);
+        const aira2Ver: string = execSync('aria2c --version').toString().split('\n')[0];
+
+        if (versionRegex.test(aira2Ver)) {
+            logger.verbose(`Using ${aira2Ver}\n`);
+        }
+        else {
+            throw new Error();
+        }
+    }
+    catch (e) {
+        process.exit(ERROR_CODE.MISSING_ARIA2);
+    }
 }
 
-
+// number of seconds
 export function ffmpegTimemarkToChunk(timemark: string): number {
     const timeVals: Array<string> = timemark.split(':');
     const hrs: number = parseInt(timeVals[0]);
     const mins: number = parseInt(timeVals[1]);
     const secs: number = parseInt(timeVals[2]);
 
-    return (hrs * 60) + mins + (secs / 60);
+    return (hrs * 60 * 60) + (mins * 60) + secs;
+}
+
+
+export function promptUser(choices: Array<string>): number {
+    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
+
+    if (index === -1) {
+        process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
+    }
+
+    return index;
 }

src/VideoUtils.ts

@@ -1,15 +1,16 @@
-import { ApiClient } from './ApiClient';
+import { StreamApiClient } from './ApiClient';
-import { promptUser } from './CommandLineParser';
+import { promptUser } from './Utils';
 import { logger } from './Logger';
-import { Video, Session } from './Types';
+import { Video, StreamSession, VideoUrl } from './Types';
 
 import { AxiosResponse } from 'axios';
 import fs from 'fs';
 import { parse as parseDuration, Duration } from 'iso8601-duration';
 import path from 'path';
 import sanitizeWindowsName from 'sanitize-filename';
+import { extractStreamGuids } from './Utils';
 
-function publishedDateToString(date: string): string {
+export function publishedDateToString(date: string): string {
     const dateJs: Date = new Date(date);
     const day: string = dateJs.getDate().toString().padStart(2, '0');
     const month: string = (dateJs.getMonth() + 1).toString(10).padStart(2, '0');
@@ -17,35 +18,44 @@ function publishedDateToString(date: string): string {
     return `${dateJs.getFullYear()}-${month}-${day}`;
 }
 
+export function publishedTimeToString(seconds: number): string
+export function publishedTimeToString(date: string): string
+export function publishedTimeToString(date: string | number): string {
+    let dateJs: Date;
+
+    if (typeof (date) === 'number') {
+        dateJs = new Date(0, 0, 0, 0, 0, date);
+    }
+    else {
+        dateJs = new Date(date);
+    }
 
-function publishedTimeToString(date: string): string {
-    const dateJs: Date = new Date(date);
     const hours: string = dateJs.getHours().toString();
     const minutes: string = dateJs.getMinutes().toString();
     const seconds: string = dateJs.getSeconds().toString();
 
-    return `${hours}.${minutes}.${seconds}`;
+    return `${hours}h ${minutes}m ${seconds}s`;
 }
 
 
-function isoDurationToString(time: string): string {
+export function isoDurationToString(time: string): string {
     const duration: Duration = parseDuration(time);
 
     return `${duration.hours ?? '00'}.${duration.minutes ?? '00'}.${duration.seconds?.toFixed(0) ?? '00'}`;
 }
 
-
+// it's the number of seconds in the video
-function durationToTotalChunks(duration: string): number {
+export function durationToTotalChunks(duration: string,): number {
     const durationObj: any = parseDuration(duration);
     const hrs: number = durationObj.hours ?? 0;
     const mins: number = durationObj.minutes ?? 0;
     const secs: number = Math.ceil(durationObj.seconds ?? 0);
 
-    return (hrs * 60) + mins + (secs / 60);
+    return (hrs * 60 * 60) + (mins * 60) + secs;
 }
 
 
-export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
+export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamSession, subtitles?: boolean): Promise<Array<Video>> {
     const metadata: Array<Video> = [];
     let title: string;
     let duration: string;
@@ -54,19 +64,23 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
     let author: string;
     let authorEmail: string;
     let uniqueId: string;
-    const outPath = '';
     let totalChunks: number;
     let playbackUrl: string;
     let posterImageUrl: string;
     let captionsUrl: string | undefined;
 
-    const apiClient: ApiClient = ApiClient.getInstance(session);
+    const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
+
+
+    // we place the guid in the url field
+    const videoGUIDs = await extractStreamGuids(videoUrls, session);
+
 
     /* TODO: change this to a single guid at a time to ease our footprint on the
     MSS servers or we get throttled after 10 sequential reqs */
-    for (const guid of videoGuids) {
+    for (const guid of videoGUIDs) {
         const response: AxiosResponse<any> | undefined =
-            await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
+            await apiClient.callApi('videos/' + guid.url + '?$expand=creator', 'get');
 
         title = sanitizeWindowsName(response?.data['name']);
 
@@ -80,7 +94,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
 
         authorEmail = response?.data['creator'].mail;
 
-        uniqueId = '#' + guid.split('-')[0];
+        uniqueId = '#' + guid.url.split('-')[0];
 
         totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
@@ -94,7 +108,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
         posterImageUrl = response?.data['posterImage']['medium']['url'];
 
         if (subtitles) {
-            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
+            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid.url}/texttracks`, 'get');
 
             if (!captions?.data.value.length) {
                 captionsUrl = undefined;
@@ -112,18 +126,19 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
         }
 
         metadata.push({
-            title: title,
+            guid: guid.url,
-            duration: duration,
+            title,
-            publishDate: publishDate,
+            duration,
-            publishTime: publishTime,
+            publishDate,
-            author: author,
+            publishTime,
-            authorEmail: authorEmail,
+            author,
-            uniqueId: uniqueId,
+            authorEmail,
-            outPath: outPath,
+            uniqueId,
-            totalChunks: totalChunks,    // Abstraction of FFmpeg timemark
+            outPath: guid.outDir,
-            playbackUrl: playbackUrl,
+            totalChunks,    // Abstraction of FFmpeg timemark
-            posterImageUrl: posterImageUrl,
+            playbackUrl,
-            captionsUrl: captionsUrl
+            posterImageUrl,
+            captionsUrl
         });
     }
 
@@ -131,16 +146,24 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
 }
 
 
-export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, template: string, format: string, skip?: boolean): Array<Video> {
+export function createUniquePath(videos: Array<Video>, template: string, format: string, skip?: boolean): Array<Video>
+export function createUniquePath(videos: Video, template: string, format: string, skip?: boolean): Video
+export function createUniquePath(videos: Array<Video> | Video, template: string, format: string, skip?: boolean): Array<Video> | Video {
+    let singleInput = false;
 
-    videos.forEach((video: Video, index: number) => {
+    if (!Array.isArray(videos)) {
+        videos = [videos];
+        singleInput = true;
+    }
+
+    videos.forEach((video: Video) => {
         let title: string = template;
         let finalTitle: string;
         const elementRegEx = RegExp(/{(.*?)}/g);
         let match = elementRegEx.exec(template);
 
         while (match) {
-            const value = video[match[1] as keyof Video] as string;
+            const value = video[match[1] as keyof (Video)] as string;
             title = title.replace(match[0], value);
             match = elementRegEx.exec(template);
         }
@@ -148,7 +171,7 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
         let i = 0;
         finalTitle = title;
 
-        while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
+        while (!skip && fs.existsSync(path.join(video.outPath, finalTitle + '.' + format))) {
             finalTitle = `${title}.${++i}`;
         }
 
@@ -158,9 +181,13 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
             logger.warn(`Not a valid Windows file name: "${finalFileName}".\nReplacing invalid characters with underscores to preserve cross-platform consistency.`);
         }
 
-        video.outPath = path.join(outDirs[index], finalFileName);
+        video.outPath = path.join(video.outPath, finalFileName);
 
     });
 
+    if (singleInput) {
+        return videos[0];
+    }
+
     return videos;
 }

src/destreamer.ts

@@ -2,22 +2,13 @@ import { argv } from './CommandLineParser';
 import { ERROR_CODE } from './Errors';
 import { setProcessEvents } from './Events';
 import { logger } from './Logger';
-import { getPuppeteerChromiumPath } from './PuppeteerHelper';
+import { VideoUrl } from './Types';
-import { drawThumbnail } from './Thumbnail';
+import { checkRequirements, parseInputFile, parseCLIinput } from './Utils';
-import { TokenCache, refreshSession } from './TokenCache';
-import { Video, Session } from './Types';
-import { checkRequirements, ffmpegTimemarkToChunk, parseInputFile, parseCLIinput} from './Utils';
-import { getVideoInfo, createUniquePath } from './VideoUtils';
 
-import cliProgress from 'cli-progress';
-import fs from 'fs';
 import isElevated from 'is-elevated';
-import puppeteer from 'puppeteer';
+import { downloadShareVideo, downloadStreamVideo } from './Downloaders';
-import { ApiClient } from './ApiClient';
 
 
-const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
-const tokenCache: TokenCache = new TokenCache();
 export const chromeCacheFolder = '.chrome_data';
 
 
@@ -44,259 +35,31 @@ async function init(): Promise<void> {
 }
 
 
-async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
-
-    logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
-
-    const browser: puppeteer.Browser = await puppeteer.launch({
-        executablePath: getPuppeteerChromiumPath(),
-        headless: false,
-        userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
-        args: [
-            '--disable-dev-shm-usage',
-            '--fast-start',
-            '--no-sandbox'
-        ]
-    });
-    const page: puppeteer.Page = (await browser.pages())[0];
-
-    logger.info('Navigating to login page...');
-    await page.goto(url, { waitUntil: 'load' });
-
-    try {
-        if (username) {
-            await page.waitForSelector('input[type="email"]', {timeout: 3000});
-            await page.keyboard.type(username);
-            await page.click('input[type="submit"]');
-        }
-        else {
-            /* If a username was not provided we let the user take actions that
-            lead up to the video page. */
-        }
-    }
-    catch (e) {
-        /* If there is no email input selector we aren't in the login module,
-        we are probably using the cache to aid the login.
-        It could finish the login on its own if the user said 'yes' when asked to
-        remember the credentials or it could still prompt the user for a password */
-    }
-
-    await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
-    logger.info('We are logged in.');
-
-    let session: Session | null = null;
-    let tries = 1;
-    while (!session) {
-        try {
-            let sessionInfo: any;
-            session = await page.evaluate(
-                () => {
-                    return {
-                        AccessToken: sessionInfo.AccessToken,
-                        ApiGatewayUri: sessionInfo.ApiGatewayUri,
-                        ApiGatewayVersion: sessionInfo.ApiGatewayVersion
-                    };
-                }
-            );
-        }
-        catch (error) {
-            if (tries > 5) {
-                process.exit(ERROR_CODE.NO_SESSION_INFO);
-            }
-
-            session = null;
-            tries++;
-            await page.waitFor(3000);
-        }
-    }
-
-    tokenCache.Write(session);
-    logger.info('Wrote access token to token cache.');
-    logger.info("At this point Chromium's job is done, shutting it down...\n");
-
-    await browser.close();
-
-    return session;
-}
-
-
-async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array<string>, session: Session): Promise<void> {
-
-    logger.info('Fetching videos info... \n');
-    const videos: Array<Video> = createUniquePath (
-        await getVideoInfo(videoGUIDs, session, argv.closedCaptions),
-        outputDirectories, argv.outputTemplate, argv.format, argv.skip
-        );
-
-    if (argv.simulate) {
-        videos.forEach((video: Video) => {
-            logger.info(
-                '\nTitle:          '.green + video.title +
-                '\nOutPath:        '.green + video.outPath +
-                '\nPublished Date: '.green + video.publishDate +
-                '\nPlayback URL:   '.green + video.playbackUrl +
-                ((video.captionsUrl) ? ('\nCC URL:         '.green + video.captionsUrl) : '')
-            );
-        });
-
-        return;
-    }
-
-    for (const [index, video] of videos.entries()) {
-
-        if (argv.skip && fs.existsSync(video.outPath)) {
-            logger.info(`File already exists, skipping: ${video.outPath} \n`);
-            continue;
-        }
-
-        if (argv.keepLoginCookies && index !== 0) {
-            logger.info('Trying to refresh token...');
-            session = await refreshSession('https://web.microsoftstream.com/video/' + videoGUIDs[index]);
-            ApiClient.getInstance().setSession(session);
-        }
-
-        const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
-            barCompleteChar: '\u2588',
-            barIncompleteChar: '\u2591',
-            format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
-            // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
-            barsize: Math.floor((process.stdout.columns || 30) / 3),
-            stopOnComplete: true,
-            hideCursor: true,
-        });
-
-        logger.info(`\nDownloading Video: ${video.title} \n`);
-        logger.verbose('Extra video info \n' +
-        '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
-        '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
-        '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
-        '\t Video total chunks: '.cyan + video.totalChunks + '\n');
-
-        logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
-        if (!process.stdout.columns) {
-            logger.warn(
-                'Unable to get number of columns from terminal.\n' +
-                'This happens sometimes in Cygwin/MSYS.\n' +
-                'No progress bar can be rendered, however the download process should not be affected.\n\n' +
-                'Please use PowerShell or cmd.exe to run destreamer on Windows.'
-            );
-        }
-
-        const headers: string = 'Authorization: Bearer ' + session.AccessToken;
-
-        if (!argv.noExperiments) {
-            if (video.posterImageUrl) {
-                await drawThumbnail(video.posterImageUrl, session);
-            }
-        }
-
-        const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
-            ['headers', headers]
-        ]));
-        const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
-            argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
-            argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
-            ['n', null]
-        ]));
-        const ffmpegCmd: any = new FFmpegCommand();
-
-        const cleanupFn: () => void = () => {
-            pbar.stop();
-
-           if (argv.noCleanup) {
-               return;
-           }
-
-            try {
-                fs.unlinkSync(video.outPath);
-            }
-            catch (e) {
-                // Future handling of an error (maybe)
-            }
-        };
-
-        pbar.start(video.totalChunks, 0, {
-            speed: '0'
-        });
-
-        // prepare ffmpeg command line
-        ffmpegCmd.addInput(ffmpegInpt);
-        ffmpegCmd.addOutput(ffmpegOutput);
-        if (argv.closedCaptions && video.captionsUrl) {
-            const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
-                ['headers', headers]
-            ]));
-
-            ffmpegCmd.addInput(captionsInpt);
-        }
-
-        ffmpegCmd.on('update', async (data: any) => {
-            const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
-
-            pbar.update(currentChunks, {
-                speed: data.bitrate
-            });
-
-            // Graceful fallback in case we can't get columns (Cygwin/MSYS)
-            if (!process.stdout.columns) {
-                process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
-            }
-        });
-
-        process.on('SIGINT', cleanupFn);
-
-        // let the magic begin...
-        await new Promise((resolve: any) => {
-            ffmpegCmd.on('error', (error: any) => {
-                cleanupFn();
-
-                logger.error(`FFmpeg returned an error: ${error.message}`);
-                process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
-            });
-
-            ffmpegCmd.on('success', () => {
-                pbar.update(video.totalChunks); // set progress bar to 100%
-                logger.info(`\nDownload finished: ${video.outPath} \n`);
-                resolve();
-            });
-
-            ffmpegCmd.spawn();
-        });
-
-        process.removeListener('SIGINT', cleanupFn);
-    }
-}
-
-
 async function main(): Promise<void> {
     await init(); // must be first
-
+    let streamVideos: Array<VideoUrl>, shareVideos: Array<VideoUrl>;
-    let session: Session;
-    // eslint-disable-next-line prefer-const
-    session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
-
-    logger.verbose('Session and API info \n' +
-        '\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
-        '\t API Gateway version: '.cyan + session.ApiGatewayVersion + '\n');
-
-    let videoGUIDs: Array<string>;
-    let outDirs: Array<string>;
 
     if (argv.videoUrls) {
         logger.info('Parsing video/group urls');
-        [videoGUIDs, outDirs] =  await parseCLIinput(argv.videoUrls as Array<string>, argv.outputDirectory, session);
+        [streamVideos, shareVideos] = await parseCLIinput(argv.videoUrls as Array<string>, argv.outputDirectory);
     }
     else {
         logger.info('Parsing input file');
-        [videoGUIDs, outDirs] =  await parseInputFile(argv.inputFile!, argv.outputDirectory, session);
+        [streamVideos, shareVideos] = await parseInputFile(argv.inputFile!, argv.outputDirectory);
     }
 
-    logger.verbose('List of GUIDs and corresponding output directory \n' +
+    logger.verbose(
-        videoGUIDs.map((guid: string, i: number) =>
+        'List of urls and corresponding output directory \n' +
-            `\thttps://web.microsoftstream.com/video/${guid} => ${outDirs[i]} \n`).join(''));
+        streamVideos.map(video => `\t${video.url} => ${video.outDir} \n`).join('') +
+        shareVideos.map(video => `\t${video.url} => ${video.outDir} \n`).join('')
+    );
 
-
+    if (streamVideos.length) {
-    downloadVideo(videoGUIDs, outDirs, session);
+        await downloadStreamVideo(streamVideos);
+    }
+    if (shareVideos.length) {
+        await downloadShareVideo(shareVideos);
+    }
 }
 
 

test/test.ts

@@ -1,32 +1,15 @@
-import { parseInputFile } from '../src/Utils';
+import { extractStreamGuids, parseInputFile } from '../src/Utils';
-import puppeteer from 'puppeteer';
 import assert from 'assert';
 import tmp from 'tmp';
 import fs from 'fs';
-import { Session } from './Types';
+import { StreamSession, VideoUrl } from './Types';
-
-
-describe('Puppeteer', () => {
-    it('should grab GitHub page title', async () => {
-        const browser = await puppeteer.launch({
-            headless: true,
-            args: ['--disable-dev-shm-usage', '--fast-start', '--no-sandbox']
-        });
-
-        const page = await browser.newPage();
-        await page.goto('https://github.com/', { waitUntil: 'load' });
-
-        let pageTitle = await page.title();
-        assert.equal(true, pageTitle.includes('GitHub'));
-
-        await browser.close();
-    }).timeout(30000); // yeah, this may take a while...
-});
 
 
+// we cannot test groups parsing as that requires an actual session
+// TODO: add SharePoint urls
 describe('Destreamer parsing', () => {
-    it('Input file to arrays of URLs and DIRs', async () => {
+    it('Input file to arrays of guids', async () => {
-        const testSession: Session = {
+        const testSession: StreamSession = {
             AccessToken: '',
             ApiGatewayUri: '',
             ApiGatewayVersion: ''
@@ -44,33 +27,42 @@ describe('Destreamer parsing', () => {
             'https://web.microsoftstream.com/video/xxxxxx-gggg-xxxx-xxxx-xxxxxxxxxxxx',
             ''
         ];
-        const expectedGUIDsOut: Array<string> = [
+
-            'xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx',
+        const expectedStreamOut: Array<VideoUrl> = [
-            'xxxxxxxx-bbbb-xxxx-xxxx-xxxxxxxxxxxx',
+            {
-            'xxxxxxxx-cccc-xxxx-xxxx-xxxxxxxxxxxx',
+                url: 'xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx',
-            'xxxxxxxx-dddd-xxxx-xxxx-xxxxxxxxxxxx',
+                outDir: 'videos'
-            'xxxxxxxx-eeee-xxxx-xxxx-xxxxxxxxxxxx'
+            },
-        ];
+            {
-        const expectedDirOut: Array<string> = [
+                url: 'xxxxxxxx-bbbb-xxxx-xxxx-xxxxxxxxxxxx',
-            'videos',
+                outDir: 'luca'
-            'luca',
+            },
-            'videos',
+            {
-            'videos',
+                url: 'xxxxxxxx-cccc-xxxx-xxxx-xxxxxxxxxxxx',
-            'videos'
+                outDir: 'videos'
+            },
+            {
+                url: 'xxxxxxxx-dddd-xxxx-xxxx-xxxxxxxxxxxx',
+                outDir: 'videos'
+            },
+            {
+                url: 'xxxxxxxx-eeee-xxxx-xxxx-xxxxxxxxxxxx',
+                outDir: 'videos'
+            },
         ];
+
         const tmpFile = tmp.fileSync({ postfix: '.txt' });
         fs.writeFileSync(tmpFile.fd, testIn.join('\r\n'));
-        const [testUrlOut , testDirOut]: Array<Array<string>> = await parseInputFile(tmpFile.name, 'videos', testSession);
+
-        if (testUrlOut.length !== expectedGUIDsOut.length) {
+        const [testStreamUrls]: Array<Array<VideoUrl>> = parseInputFile(tmpFile.name, 'videos');
-            throw "Expected url list and test list don't have the same number of elements".red;
+
-        }
+        assert.deepStrictEqual(
-        else if (testDirOut.length !== expectedDirOut.length) {
+            await extractStreamGuids(testStreamUrls, testSession),
-            throw "Expected dir list and test list don't have the same number of elements".red;
+            expectedStreamOut,
-        }
+            'Error in parsing the URLs, missmatch between test and expected'.red
-        assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
+        );
-            'Error in parsing the URLs, missmatch between test and expected'.red);
+        // assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
-        assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
+        //     'Error in parsing the DIRs, missmatch between test and expected'.red);
-            'Error in parsing the DIRs, missmatch between test and expected'.red);
         assert.ok('Parsing of input file ok');
     });
 });

trigger_codeQL

@@ -1 +0,0 @@
-