Added ffmpeg progress bar via fluent-ffmpeg and progress libs (#57)

* Add fluent-ffmpeg back and cross-platform progress bar

* Repo clean up

Move ts files to src, build and output js files to build folder

* Do not print messages when exit code is 0

this is triggered by signal events

Co-authored-by: kylon <kylonux@gmail.com>

src/Metadata.ts

@@ -0,0 +1,65 @@
+import { Metadata, Session } from './Types';
+
+import { parse } from 'iso8601-duration';
+import axios from 'axios';
+
+function publishedDateToString(date: string) {
+    const dateJs = new Date(date);
+    const day = dateJs.getDate().toString().padStart(2, '0');
+    const month = (dateJs.getMonth() + 1).toString(10).padStart(2, '0');
+
+    return day+'-'+month+'-'+dateJs.getFullYear();
+}
+
+function durationToTotalChuncks(duration: string) {
+    const durationObj = parse(duration);
+    const hrs = durationObj['hours'] ?? 0;
+    const mins = durationObj['minutes'] ?? 0;
+    const secs = Math.ceil(durationObj['seconds'] ?? 0);
+
+    return hrs * 1000 + mins * 100 + secs;
+}
+
+
+export async function getVideoMetadata(videoGuids: string[], session: Session, verbose: boolean): Promise<Metadata[]> {
+    let metadata: Metadata[] = [];
+    let title: string;
+    let date: string;
+    let duration: number;
+    let playbackUrl: string;
+    let posterImage: string;
+
+    await Promise.all(videoGuids.map(async guid => {
+        let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`;
+
+        if (verbose)
+            console.info(`Calling ${apiUrl}`);
+
+        let response = await axios.get(apiUrl,
+            {
+                headers: {
+                    Authorization: `Bearer ${session.AccessToken}`
+                }
+            });
+
+        title = response.data['name'];
+        playbackUrl = response.data['playbackUrls']
+            .filter((item: { [x: string]: string; }) =>
+                item['mimeType'] == 'application/vnd.apple.mpegurl')
+            .map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0];
+
+        posterImage = response.data['posterImage']['medium']['url'];
+        date = publishedDateToString(response.data['publishedDate']);
+        duration = durationToTotalChuncks(response.data.media['duration']);
+
+        metadata.push({
+            date: date,
+            duration: duration,
+            title: title,
+            playbackUrl: playbackUrl,
+            posterImage: posterImage
+        });
+    }));
+
+    return metadata;
+}

src/Thumbnail.ts

@@ -0,0 +1,14 @@
+import terminalImage from 'terminal-image';
+import axios from 'axios';
+
+
+export async function drawThumbnail(posterImage: string, accessToken: string) {
+    let thumbnail = await axios.get(posterImage,
+        {
+            headers: {
+                Authorization: `Bearer ${accessToken}`
+            },
+            responseType: 'arraybuffer'
+        });
+    console.log(await terminalImage.buffer(thumbnail.data));
+}

src/TokenCache.ts

@@ -0,0 +1,56 @@
+import * as fs from 'fs';
+import { Session } from './Types';
+import { bgGreen, bgYellow, green } from 'colors';
+import jwtDecode from 'jwt-decode';
+
+export class TokenCache {
+    private tokenCacheFile: string = '.token_cache';
+
+    public Read(): Session | null {
+        let j = null;
+        if(!fs.existsSync(this.tokenCacheFile)) {
+            console.warn(bgYellow.black(`${this.tokenCacheFile} not found.\n`));
+
+            return null;
+        }
+        let f = fs.readFileSync(this.tokenCacheFile, 'utf8');
+        j = JSON.parse(f);
+
+        interface Jwt {
+            [key: string]: any
+        }
+
+        const decodedJwt: Jwt = jwtDecode(j.AccessToken);
+
+        let now = Math.floor(Date.now() / 1000);
+        let exp = decodedJwt['exp'];
+        let timeLeft = exp - now;
+
+        let timeLeftInMinutes = Math.floor(timeLeft / 60);
+        if (timeLeft < 120) {
+            console.warn(bgYellow.black('\nAccess token has expired.'));
+
+            return null;
+        }
+
+        console.info(bgGreen.black(`\nAccess token still good for ${timeLeftInMinutes} minutes.\n`));
+
+        let session: Session = {
+            AccessToken: j.AccessToken,
+            ApiGatewayUri: j.ApiGatewayUri,
+            ApiGatewayVersion: j.ApiGatewayVersion
+        };
+
+        return session;
+    }
+
+    public Write(session: Session): void {
+        let s = JSON.stringify(session, null, 4);
+        fs.writeFile('.token_cache', s, (err: any) => {
+            if (err) {
+                return console.error(err);
+            }
+            console.info(green('Fresh access token dropped into .token_cache'));
+        });
+    }
+}

src/Types.ts

@@ -0,0 +1,36 @@
+export type Session = {
+    AccessToken: string;
+    ApiGatewayUri: string;
+    ApiGatewayVersion: string;
+}
+
+
+export type Metadata = {
+    date: string;
+    duration: number;
+    title: string;
+    playbackUrl: string;
+    posterImage: string;
+}
+
+
+interface Errors {
+    [key: number]: string
+}
+
+// I didn't use an enum because there is no real advantage that i can find and
+// we can't use multiline string for long errors
+// TODO: create better errors descriptions
+export const Errors: Errors = {
+    22: 'FFmpeg is missing. \n' +
+        'Destreamer requires a fairly recent release of FFmpeg to work properly. \n' +
+        'Please install it with your preferred package manager or copy FFmpeg binary in destreamer root directory. \n',
+
+    33: 'cannot split videoID from videUrl \n',
+
+    44: 'couldn\'t evaluate sessionInfo in the page \n',
+
+    55: 'running in an elevated shell \n',
+
+    66: 'no valid URL in the input \n'
+}

src/destreamer.ts

@@ -0,0 +1,275 @@
+import { sleep, parseVideoUrls, checkRequirements, makeUniqueTitle, ffmpegTimemarkToChunk } from './utils';
+import { TokenCache } from './TokenCache';
+import { getVideoMetadata } from './Metadata';
+import { Metadata, Session, Errors } from './Types';
+import { drawThumbnail } from './Thumbnail';
+
+import isElevated from 'is-elevated';
+import puppeteer from 'puppeteer';
+import colors from 'colors';
+import fs from 'fs';
+import path from 'path';
+import yargs from 'yargs';
+import sanitize from 'sanitize-filename';
+import ffmpeg from 'fluent-ffmpeg';
+import cliProgress from 'cli-progress';
+
+let tokenCache = new TokenCache();
+
+const argv = yargs.options({
+    username: {
+        alias: 'u',
+        type: 'string',
+        demandOption: false
+    },
+    outputDirectory: {
+        alias: 'o',
+        type: 'string',
+        default: 'videos',
+        demandOption: false
+    },
+    videoUrls: {
+        alias: 'V',
+        describe: 'List of video urls or path to txt file containing the urls',
+        type: 'array',
+        demandOption: true
+    },
+    simulate: {
+        alias: 's',
+        describe: `Disable video download and print metadata information to the console`,
+        type: 'boolean',
+        default: false,
+        demandOption: false
+    },
+    noThumbnails: {
+        alias: 'nthumb',
+        describe: `Do not display video thumbnails`,
+        type: 'boolean',
+        default: false,
+        demandOption: false
+    },
+    verbose: {
+        alias: 'v',
+        describe: `Print additional information to the console (use this before opening an issue on GitHub)`,
+        type: 'boolean',
+        default: false,
+        demandOption: false
+    }
+}).argv;
+
+async function init() {
+
+    process.on('unhandledRejection', (reason) => {
+        console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
+        console.error(colors.red(reason as string));
+    });
+
+    process.on('exit', (code) => {
+        if (code in Errors)
+            console.error(colors.bgRed(`\n\nError: ${Errors[code]} \n`))
+        else
+            console.error(colors.bgRed(`\n\nUnknown exit code ${code} \n`))
+    });
+
+    if (await isElevated())
+        process.exit(55);
+
+    // create output directory
+    if (!fs.existsSync(argv.outputDirectory)) {
+        console.log('Creating output directory: ' +
+            process.cwd() + path.sep + argv.outputDirectory);
+        fs.mkdirSync(argv.outputDirectory);
+    }
+
+    console.info('Output Directory: %s', argv.outputDirectory);
+
+    if (argv.username)
+        console.info('Username: %s', argv.username);
+
+    if (argv.simulate)
+        console.info(colors.yellow('Simulate mode, there will be no video download.\n'));
+
+    if (argv.verbose) {
+        console.info('Video URLs:');
+        console.info(argv.videoUrls);
+    }
+}
+
+async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
+
+    let videoId = url.split("/").pop() ?? process.exit(33)
+
+    console.log('Launching headless Chrome to perform the OpenID Connect dance...');
+    const browser = await puppeteer.launch({
+        headless: false,
+        args: ['--disable-dev-shm-usage']
+    });
+    const page = (await browser.pages())[0];
+    console.log('Navigating to login page...');
+
+    await page.goto(url, { waitUntil: 'load' });
+    await page.waitForSelector('input[type="email"]');
+
+    if (username) {
+        await page.keyboard.type(username);
+        await page.click('input[type="submit"]');
+    }
+
+    await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 });
+    console.info('We are logged in.');
+
+    let session = null;
+    let tries: number = 0;
+
+    while (!session) {
+        try {
+            let sessionInfo: any;
+            session = await page.evaluate(
+                () => {
+                    return {
+                        AccessToken: sessionInfo.AccessToken,
+                        ApiGatewayUri: sessionInfo.ApiGatewayUri,
+                        ApiGatewayVersion: sessionInfo.ApiGatewayVersion
+                    };
+                }
+            );
+        } catch (error) {
+            if (tries < 5){
+                session = null;
+                tries++;
+                await sleep(3000);
+            } else {
+                process.exit(44)
+            }
+        }
+    }
+
+    tokenCache.Write(session);
+    console.log('Wrote access token to token cache.');
+    console.log("At this point Chromium's job is done, shutting it down...\n");
+
+    await browser.close();
+
+    return session;
+}
+
+function extractVideoGuid(videoUrls: string[]): string[] {
+    let videoGuids: string[] = [];
+    let guid: string | undefined = '';
+
+    for (const url of videoUrls) {
+        try {
+            guid = url.split('/').pop();
+
+        } catch (e) {
+            console.error(`Could not split the video GUID from URL: ${e.message}`);
+            process.exit(33);
+        }
+
+        if (guid)
+            videoGuids.push(guid);
+    }
+
+    if (argv.verbose) {
+        console.info('Video GUIDs:');
+        console.info(videoGuids);
+    }
+
+    return videoGuids;
+}
+
+async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
+    const videoGuids = extractVideoGuid(videoUrls);
+    const pbar = new cliProgress.SingleBar({
+        barCompleteChar: '\u2588',
+        barIncompleteChar: '\u2591',
+        format: 'progress [{bar}] {percentage}% {speed}Kbps {eta_formatted}',
+        barsize: Math.floor(process.stdout.columns / 3),
+        stopOnComplete: true,
+        etaBuffer: 20
+    });
+
+    console.log('Fetching metadata...');
+
+    const metadata: Metadata[] = await getVideoMetadata(videoGuids, session, argv.verbose);
+
+    if (argv.simulate) {
+        metadata.forEach(video => {
+            console.log(
+                colors.yellow('\n\nTitle: ') + colors.green(video.title) +
+                colors.yellow('\nPublished Date: ') + colors.green(video.date) +
+                colors.yellow('\nPlayback URL: ') + colors.green(video.playbackUrl)
+            );
+        });
+
+        return;
+    }
+
+    await Promise.all(metadata.map(async video => {
+        console.log(colors.blue(`\nDownloading Video: ${video.title}\n`));
+
+        video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, argv.outputDirectory);
+
+        const outputPath = outputDirectory + path.sep + video.title + '.mp4';
+
+        // Very experimental inline thumbnail rendering
+        if (!argv.noThumbnails)
+            await drawThumbnail(video.posterImage, session.AccessToken);
+
+        console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n');
+
+        ffmpeg()
+            .input(video.playbackUrl)
+            .inputOption([
+                // Never remove those "useless" escapes or ffmpeg will not
+                // pick up the header correctly
+                // eslint-disable-next-line no-useless-escape
+                '-headers', `Authorization:\ Bearer\ ${session.AccessToken}`,
+            ])
+            .format('mp4')
+            .saveToFile(outputPath)
+            .on('codecData', data => {
+                console.log(`Input is ${data.video} with ${data.audio} audio.\n`);
+
+                pbar.start(video.duration, 0, {
+                    speed: '0'
+                });
+
+                process.on('SIGINT', () => {
+                    pbar.stop();
+                });
+            })
+            .on('progress', progress => {
+                const currentChuncks = ffmpegTimemarkToChunk(progress.timemark);
+
+                pbar.update(currentChuncks, {
+                    speed: progress.currentKbps
+                });
+            })
+            .on('error', err => {
+                pbar.stop();
+                console.log(`ffmpeg returned an error: ${err.message}`);
+            })
+            .on('end', () => {
+                console.log(colors.green(`\nDownload finished: ${outputPath}`));
+            });
+    }));
+}
+
+async function main() {
+    checkRequirements() ?? process.exit(22);
+    await init();
+
+    const videoUrls: string[] = parseVideoUrls(argv.videoUrls) ?? process.exit(66);
+
+    let session = tokenCache.Read();
+
+    if (session == null) {
+        session = await DoInteractiveLogin(videoUrls[0], argv.username);
+    }
+
+    downloadVideo(videoUrls, argv.outputDirectory, session);
+}
+
+
+main();

src/utils.ts

@@ -0,0 +1,83 @@
+import { execSync } from 'child_process';
+import colors from 'colors';
+import fs from 'fs';
+import path from 'path';
+
+
+function sanitizeUrls(urls: string[]) {
+    const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i');
+    const sanitized: string[] = [];
+
+    for (let i=0, l=urls.length; i<l; ++i) {
+        const urlAr = urls[i].split('?');
+        const query = urlAr.length === 2 && urlAr[1] !== '' ? '?'+urlAr[1] : '';
+        let url = urlAr[0];
+
+        if (!rex.test(url)) {
+            if (url !== '')
+                console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..'));
+
+            continue;
+        }
+
+        if (url.substring(0, 8) !== 'https://')
+            url = 'https://'+url;
+
+        sanitized.push(url+query);
+    }
+
+    return sanitized.length ? sanitized : null;
+}
+
+
+export function parseVideoUrls(videoUrls: any) {
+    const t = videoUrls[0] as string;
+    const isPath = t.substring(t.length-4) === '.txt';
+    let urls: string[];
+
+    if (isPath)
+        urls = fs.readFileSync(t).toString('utf-8').split(/[\r\n]/);
+    else
+        urls = videoUrls as string[];
+
+    return sanitizeUrls(urls);
+}
+
+
+export function sleep(ms: number) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+
+export function checkRequirements() {
+    try {
+        const ffmpegVer = execSync('ffmpeg -version').toString().split('\n')[0];
+        console.info(colors.green(`Using ${ffmpegVer}\n`));
+
+    } catch (e) {
+        return null;
+    }
+
+    return true;
+}
+
+
+export function makeUniqueTitle(title: string, outDir: string) {
+    let ntitle = title;
+    let k = 0;
+
+    while (fs.existsSync(outDir + path.sep + ntitle + '.mp4'))
+        ntitle = title + ' - ' + (++k).toString();
+
+    return ntitle;
+}
+
+
+export function ffmpegTimemarkToChunk(timemark: string) {
+    const timeVals: string[] = timemark.split(':');
+    const hrs = parseInt(timeVals[0]);
+    const mins = parseInt(timeVals[1]);
+    const secs = parseInt(timeVals[2]);
+
+    return hrs * 1000 + mins * 100 + secs;
+}