Addressing https://github.com/snobu/destreamer/issues/439

* Add personal URL for new MS Teams recordings
* Encode URI components to avoid unescape char

README.md

@@ -41,6 +41,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
 - [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
 - **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
 - [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
+- [**aria2**][aria2]: aria2 is a utility for downloading files with multiple threads, fast.
 - [**git**][git]: one or more npm dependencies require git.
 
 Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on. We've successfully tested it on Windows, macOS and Linux.
@@ -239,6 +240,7 @@ Please open an [issue](https://github.com/snobu/destreamer/issues) and we'll loo
 
 
 [ffmpeg]: https://www.ffmpeg.org/download.html
+[aria2]: https://github.com/aria2/aria2/releases
 [xming]: https://sourceforge.net/projects/xming/
 [node]: https://nodejs.org/en/download/
 [git]: https://git-scm.com/downloads

src/ApiClient.ts

@@ -161,7 +161,7 @@ export class ShareApiClient {
     }
 
 
-    public async getVideoInfo(filePath: string, outDir: string): Promise<Video> {
+    public async getVideoInfo(filePath: string, outPath: string): Promise<Video> {
         let playbackUrl: string;
 
         // TODO: Ripped this straigth from chromium inspector. Don't know don't care what it is right now. Check later
@@ -175,7 +175,7 @@ export class ShareApiClient {
                 AddRequiredFields: true
             }
         };
-        const url = `${this.site}/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream?@a1='${filePath}'`;
+        const url = `${this.site}/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream?@a1='${encodeURIComponent(filePath)}'`;
 
         logger.verbose(`Requesting video info for '${url}'`);
         const info = await this.axiosInstance.post(url, payload, {
@@ -193,27 +193,43 @@ export class ShareApiClient {
         }
 
         const direct = await this.canDirectDownload(filePath);
+        const b64VideoMetadata = JSON.parse(
+            info.ListData.Row[0].MediaServiceFastMetadata
+        ).video.altManifestMetadata;
+        const durationSeconds = Math.ceil(
+            (JSON.parse(
+                Buffer.from(b64VideoMetadata, 'base64').toString()
+            ).Duration100Nano) / 10 / 1000 / 1000
+        );
 
         if (direct) {
-            playbackUrl = this.axiosInstance.getUri({ url: filePath });
+            playbackUrl = this.axiosInstance.defaults.baseURL + filePath;
+            // logger.verbose(playbackUrl);
         }
         else {
-            playbackUrl = 'placeholder';
+            playbackUrl = info.ListSchema['.videoManifestUrl'];
+            playbackUrl = playbackUrl.replace('{.mediaBaseUrl}', info.ListSchema['.mediaBaseUrl']);
+            // the only filetype works I found
+            playbackUrl = playbackUrl.replace('{.fileType}', 'mp4');
+            playbackUrl = playbackUrl.replace('{.callerStack}', info.ListSchema['.callerStack']);
+            playbackUrl = playbackUrl.replace('{.spItemUrl}', info.ListData.Row[0]['.spItemUrl']);
+            playbackUrl = playbackUrl.replace('{.driveAccessToken}', info.ListSchema['.driveAccessToken']);
+            playbackUrl += '&part=index&format=dash';
         }
 
 
         return {
             direct,
             title: filePath.split('/').pop() ?? 'video.mp4',
-            duration: '',
+            duration: publishedTimeToString(durationSeconds),
             publishDate: publishedDateToString(info.ListData.Row[0]['Modified.']),
             publishTime: publishedTimeToString(info.ListData.Row[0]['Modified.']),
             author: info.ListData.Row[0]['Author.title'],
             authorEmail: info.ListData.Row[0]['Author.email'],
-            uniqueId: info.ListData.Row[0]['GUID'].substring(1, 9),
+            uniqueId: info.ListData.Row[0].GUID.substring(1, 9),
-            outPath: outDir,
+            outPath,
             playbackUrl,
-            totalChunks: 0
+            totalChunks: durationSeconds
         };
     }
 

src/CommandLineParser.ts

@@ -1,10 +1,9 @@
-import { CLI_ERROR, ERROR_CODE } from './Errors';
+import { CLI_ERROR } from './Errors';
 import { makeOutDir } from './Utils';
 import { logger } from './Logger';
 import { templateElements } from './Types';
 
 import fs from 'fs';
-import readlineSync from 'readline-sync';
 import sanitize from 'sanitize-filename';
 import yargs from 'yargs';
 
@@ -198,14 +197,3 @@ function isOutputTemplateValid(argv: any): boolean {
 
     return true;
 }
-
-
-export function promptUser(choices: Array<string>): number {
-    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
-
-    if (index === -1) {
-        process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
-    }
-
-    return index;
-}

src/Downloaders.ts

@@ -177,7 +177,7 @@ export async function downloadStreamVideo(videoUrls: Array<VideoUrl>): Promise<v
 
 // TODO: complete overhaul of this function
 export async function downloadShareVideo(videoUrls: Array<VideoUrl>): Promise<void> {
-    const shareUrlRegex = new RegExp(/(?<domain>https:\/\/.+\.sharepoint\.com)(?<baseSite>\/sites\/.*?)(?:(?<filename>\/.*\.mp4)|\/.*id=(?<paramFilename>.*mp4))/);
+    const shareUrlRegex = new RegExp(/(?<domain>https:\/\/.+\.sharepoint\.com).*?(?<baseSite>\/(?:teams|sites|personal)\/.*?)(?:(?<filename>\/.*\.mp4)|\/.*id=(?<paramFilename>.*mp4))/);
 
     logger.info('Downloading SharePoint videos...\n\n');
 
@@ -234,9 +234,102 @@ export async function downloadShareVideo(videoUrls: Array<VideoUrl>): Promise<vo
             }
         }
         else {
-            logger.error('TODO: manifest download');
+            // FIXME: just a copy-paste, should move to separate function
+            const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
+                barCompleteChar: '\u2588',
+                barIncompleteChar: '\u2591',
+                format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
+                // process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
+                barsize: Math.floor((process.stdout.columns || 30) / 3),
+                stopOnComplete: true,
+                hideCursor: true,
+            });
 
-            continue;
+            logger.info(`\nDownloading Video: ${video.title} \n`);
+            logger.verbose('Extra video info \n' +
+                '\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
+                '\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
+                '\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
+                '\t Video total chunks: '.cyan + video.totalChunks + '\n');
+
+            logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
+            if (!process.stdout.columns) {
+                logger.warn(
+                    'Unable to get number of columns from terminal.\n' +
+                    'This happens sometimes in Cygwin/MSYS.\n' +
+                    'No progress bar can be rendered, however the download process should not be affected.\n\n' +
+                    'Please use PowerShell or cmd.exe to run destreamer on Windows.'
+                );
+            }
+
+            const ffmpegInpt: any = new FFmpegInput(video.playbackUrl);
+            const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
+                argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
+                argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
+                ['n', null]
+            ]));
+            const ffmpegCmd: any = new FFmpegCommand();
+
+            const cleanupFn: () => void = () => {
+                pbar.stop();
+
+                if (argv.noCleanup) {
+                    return;
+                }
+
+                try {
+                    fs.unlinkSync(video.outPath);
+                }
+                catch (e) {
+                    // Future handling of an error (maybe)
+                }
+            };
+
+            pbar.start(video.totalChunks, 0, {
+                speed: '0'
+            });
+
+            // prepare ffmpeg command line
+            ffmpegCmd.addInput(ffmpegInpt);
+            ffmpegCmd.addOutput(ffmpegOutput);
+
+            ffmpegCmd.on('update', async (data: any) => {
+                const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
+
+                pbar.update(currentChunks, {
+                    speed: data.bitrate
+                });
+
+                // Graceful fallback in case we can't get columns (Cygwin/MSYS)
+                if (!process.stdout.columns) {
+                    process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
+                }
+            });
+
+            process.on('SIGINT', cleanupFn);
+
+            // let the magic begin...
+            await new Promise((resolve: any) => {
+                ffmpegCmd.on('error', (error: any) => {
+                    cleanupFn();
+
+                    logger.error(`FFmpeg returned an error: ${error.message}`);
+                    process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
+                });
+
+                ffmpegCmd.on('success', () => {
+                    pbar.update(video.totalChunks); // set progress bar to 100%
+                    logger.info(`\nDownload finished: ${video.outPath} \n`);
+                    resolve();
+                });
+
+                ffmpegCmd.spawn();
+            });
+
+            process.removeListener('SIGINT', cleanupFn);
+            // logger.error('TODO: manifest download');
+
+            // continue;
         }
     }
 }

src/Errors.ts

@@ -1,5 +1,5 @@
 export const enum ERROR_CODE {
-    UNHANDLED_ERROR,
+    UNHANDLED_ERROR = 200,
     ELEVATED_SHELL,
     CANCELLED_USER_INPUT,
     MISSING_FFMPEG,

src/LoginModules.ts

@@ -94,10 +94,8 @@ export async function doStreamLogin(url: string, tokenCache: TokenCache, usernam
 export async function doShareLogin(url: string, username?: string): Promise<ShareSession> {
     logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
 
+    let session: ShareSession | null = null;
     const hostname = new URL(url).host;
-    logger.verbose(new URL(url).host);
-    logger.verbose(new URL(url).hostname);
-
 
     const browser: puppeteer.Browser = await puppeteer.launch({
         executablePath: getPuppeteerChromiumPath(),
@@ -142,7 +140,6 @@ export async function doShareLogin(url: string, username?: string): Promise<Shar
         await browser.waitForTarget((target: puppeteer.Target) => target.url().startsWith(`https://${hostname}`), { timeout: 150000 });
         logger.info('We are logged in.');
 
-        let session: ShareSession | null = null;
         let tries = 1;
         while (!session) {
             const cookieJar = (await page.cookies()).filter(
@@ -168,10 +165,12 @@ export async function doShareLogin(url: string, username?: string): Promise<Shar
         logger.info("At this point Chromium's job is done, shutting it down...\n");
 
         // await page.waitForTimeout(1000 * 60 * 60 * 60);
-        return session;
     }
     finally {
+        logger.verbose('Stream login browser closing...');
         await browser.close();
+        logger.verbose('Stream login browser closed');
     }
 
+    return session;
 }

src/Utils.ts

@@ -6,6 +6,7 @@ import { StreamSession, VideoUrl } from './Types';
 import { AxiosResponse } from 'axios';
 import { execSync } from 'child_process';
 import fs from 'fs';
+import readlineSync from 'readline-sync';
 
 
 const streamUrlRegex = new RegExp(/https?:\/\/web\.microsoftstream\.com.*/);
@@ -228,12 +229,23 @@ export function checkRequirements(): void {
     }
 }
 
-
+// number of seconds
 export function ffmpegTimemarkToChunk(timemark: string): number {
     const timeVals: Array<string> = timemark.split(':');
     const hrs: number = parseInt(timeVals[0]);
     const mins: number = parseInt(timeVals[1]);
     const secs: number = parseInt(timeVals[2]);
 
-    return (hrs * 60) + mins + (secs / 60);
+    return (hrs * 60 * 60) + (mins * 60) + secs;
+}
+
+
+export function promptUser(choices: Array<string>): number {
+    const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
+
+    if (index === -1) {
+        process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
+    }
+
+    return index;
 }

src/VideoUtils.ts

@@ -1,5 +1,5 @@
 import { StreamApiClient } from './ApiClient';
-import { promptUser } from './CommandLineParser';
+import { promptUser } from './Utils';
 import { logger } from './Logger';
 import { Video, StreamSession, VideoUrl } from './Types';
 
@@ -18,31 +18,40 @@ export function publishedDateToString(date: string): string {
     return `${dateJs.getFullYear()}-${month}-${day}`;
 }
 
+export function publishedTimeToString(seconds: number): string
+export function publishedTimeToString(date: string): string
+export function publishedTimeToString(date: string | number): string {
+    let dateJs: Date;
+
+    if (typeof (date) === 'number') {
+        dateJs = new Date(0, 0, 0, 0, 0, date);
+    }
+    else {
+        dateJs = new Date(date);
+    }
 
-export function publishedTimeToString(date: string): string {
-    const dateJs: Date = new Date(date);
     const hours: string = dateJs.getHours().toString();
     const minutes: string = dateJs.getMinutes().toString();
     const seconds: string = dateJs.getSeconds().toString();
 
-    return `${hours}.${minutes}.${seconds}`;
+    return `${hours}h ${minutes}m ${seconds}s`;
 }
 
 
-function isoDurationToString(time: string): string {
+export function isoDurationToString(time: string): string {
     const duration: Duration = parseDuration(time);
 
     return `${duration.hours ?? '00'}.${duration.minutes ?? '00'}.${duration.seconds?.toFixed(0) ?? '00'}`;
 }
 
-
+// it's the number of seconds in the video
-function durationToTotalChunks(duration: string): number {
+export function durationToTotalChunks(duration: string,): number {
     const durationObj: any = parseDuration(duration);
     const hrs: number = durationObj.hours ?? 0;
     const mins: number = durationObj.minutes ?? 0;
     const secs: number = Math.ceil(durationObj.seconds ?? 0);
 
-    return (hrs * 60) + mins + (secs / 60);
+    return (hrs * 60 * 60) + (mins * 60) + secs;
 }
 
 
@@ -99,7 +108,7 @@ export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamS
         posterImageUrl = response?.data['posterImage']['medium']['url'];
 
         if (subtitles) {
-            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
+            const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid.url}/texttracks`, 'get');
 
             if (!captions?.data.value.length) {
                 captionsUrl = undefined;

test/test.ts

@@ -6,6 +6,7 @@ import { StreamSession, VideoUrl } from './Types';
 
 
 // we cannot test groups parsing as that requires an actual session
+// TODO: add SharePoint urls
 describe('Destreamer parsing', () => {
     it('Input file to arrays of guids', async () => {
         const testSession: StreamSession = {