diff --git a/Metadata.ts b/Metadata.ts index 58e7c0e..658a1c7 100644 --- a/Metadata.ts +++ b/Metadata.ts @@ -1,16 +1,28 @@ -import axios from 'axios'; import { Metadata, Session } from './Types'; +import axios from 'axios'; -export async function getVideoMetadata(videoGuids: string[], session: Session): Promise { +function publishedDateToString(date: string) { + const dateJs = new Date(date); + const day = dateJs.getDate().toString().padStart(2, '0'); + const month = (dateJs.getMonth() + 1).toString(10).padStart(2, '0'); + + return day+'-'+month+'-'+dateJs.getFullYear(); +} + +export async function getVideoMetadata(videoGuids: string[], session: Session, verbose: boolean): Promise { let metadata: Metadata[] = []; let title: string; + let date: string; let playbackUrl: string; let posterImage: string; await Promise.all(videoGuids.map(async guid => { let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`; - console.log(`Calling ${apiUrl}`); + + if (verbose) + console.info(`Calling ${apiUrl}`); + let response = await axios.get(apiUrl, { headers: { @@ -25,8 +37,10 @@ export async function getVideoMetadata(videoGuids: string[], session: Session): .map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0]; posterImage = response.data['posterImage']['medium']['url']; + date = publishedDateToString(response.data['publishedDate']); metadata.push({ + date: date, title: title, playbackUrl: playbackUrl, posterImage: posterImage diff --git a/README.md b/README.md index 271e043..3db6c10 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,20 @@ Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) ap $ node ./destreamer.js Options: - --help Show help [boolean] - --version Show version number [boolean] - --videoUrls, -V List of video urls or path to txt file containing the urls - [array] [required] - --username, -u [string] - --outputDirectory, -o [string] [default: "videos"] - --verbose, -v Print additional information to the console - (use this before opening an issue on GitHub) - [boolean] [default: false] + --help Show help [boolean] + --version Show version number [boolean] + --username, -u [string] + --outputDirectory, -o [string] [default: "videos"] + --videoUrls, -V List of video urls or path to txt file containing the urls + [array] [required] + --simulate, -s Disable video download and print metadata + information to the console + [boolean] [default: false] + --noThumbnails, --nthumb Do not display video thumbnails + [boolean] [default: false] + --verbose, -v Print additional information to the console + (use this before opening an issue on GitHub) + [boolean] [default: false] ``` Make sure you use the right escape char for your shell if using line breaks (as this example shows). diff --git a/TokenCache.ts b/TokenCache.ts index 8ede85f..fe2ab2a 100644 --- a/TokenCache.ts +++ b/TokenCache.ts @@ -1,21 +1,19 @@ import * as fs from 'fs'; import { Session } from './Types'; import { bgGreen, bgYellow, green } from 'colors'; -const jwtDecode = require('jwt-decode'); - - -const tokenCacheFile = '.token_cache'; +import jwtDecode from 'jwt-decode'; export class TokenCache { + private tokenCacheFile: string = '.token_cache'; public Read(): Session | null { let j = null; - if(!fs.existsSync(tokenCacheFile)) { - console.warn(bgYellow.black(`${tokenCacheFile} not found.\n`)); + if(!fs.existsSync(this.tokenCacheFile)) { + console.warn(bgYellow.black(`${this.tokenCacheFile} not found.\n`)); return null; } - let f = fs.readFileSync(tokenCacheFile, 'utf8'); + let f = fs.readFileSync(this.tokenCacheFile, 'utf8'); j = JSON.parse(f); interface Jwt { diff --git a/Types.ts b/Types.ts index adec86f..26dadb9 100644 --- a/Types.ts +++ b/Types.ts @@ -5,6 +5,7 @@ export type Session = { } export type Metadata = { + date: string; title: string; playbackUrl: string; posterImage: string; diff --git a/destreamer.ts b/destreamer.ts index a700d4d..33f630e 100644 --- a/destreamer.ts +++ b/destreamer.ts @@ -1,245 +1,268 @@ -import { sleep, parseVideoUrls, checkRequirements } from './utils'; -import { TokenCache } from './TokenCache'; -import { getVideoMetadata } from './Metadata'; -import { Metadata, Session } from './Types'; -import { drawThumbnail } from './Thumbnail'; - -import isElevated from 'is-elevated'; -import puppeteer from 'puppeteer'; -import { execSync } from 'child_process'; -import colors from 'colors'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import yargs from 'yargs'; -import sanitize from 'sanitize-filename'; -import ffmpeg from 'fluent-ffmpeg'; - -/** - * exitCode 22 = ffmpeg not found in $PATH - * exitCode 25 = cannot split videoID from videUrl - * exitCode 27 = no hlsUrl in the API response - * exitCode 29 = invalid response from API - * exitCode 88 = error extracting cookies - */ - -let tokenCache = new TokenCache(); - -const argv = yargs.options({ - username: { - alias: 'u', - type: 'string', - demandOption: false - }, - outputDirectory: { - alias: 'o', - type: 'string', - default: 'videos', - demandOption: false - }, - videoUrls: { - alias: 'V', - describe: 'List of video urls or path to txt file containing the urls', - type: 'array', - demandOption: true - }, - simulate: { - alias: 's', - describe: `If this is set to true no video will be downloaded and the script - will log the video info (default: false)`, - type: 'boolean', - default: false, - demandOption: false - }, - verbose: { - alias: 'v', - describe: `Print additional information to the console - (use this before opening an issue on GitHub)`, - type: 'boolean', - default: false, - demandOption: false - } -}).argv; - -function init() { - // create output directory - if (!fs.existsSync(argv.outputDirectory)) { - console.log('Creating output directory: ' + - process.cwd() + path.sep + argv.outputDirectory); - fs.mkdirSync(argv.outputDirectory); - } - - console.info('Video URLs: %s', argv.videoUrls); - console.info('Username: %s', argv.username); - console.info('Output Directory: %s', argv.outputDirectory); - - if (argv.simulate) - console.info(colors.blue("There will be no video downloaded, it's only a simulation\n")); -} - -async function DoInteractiveLogin(url: string, username?: string): Promise { - - let videoId = url.split("/").pop() ?? ( - console.log('Couldn\'t split the video Id from the first videoUrl'), process.exit(25) - ); - - console.log('Launching headless Chrome to perform the OpenID Connect dance...'); - const browser = await puppeteer.launch({ - headless: false, - args: ['--disable-dev-shm-usage'] - }); - const page = (await browser.pages())[0]; - console.log('Navigating to login page...'); - - await page.goto(url, { waitUntil: 'load' }); - await page.waitForSelector('input[type="email"]'); - - if (username) { - await page.keyboard.type(username); - await page.click('input[type="submit"]'); - } - - await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 }); - console.info('We are logged in.'); - - let sessionInfo: any; - let session = await page.evaluate( - () => { - return { - AccessToken: sessionInfo.AccessToken, - ApiGatewayUri: sessionInfo.ApiGatewayUri, - ApiGatewayVersion: sessionInfo.ApiGatewayVersion - }; - } - ); - - tokenCache.Write(session); - console.log('Wrote access token to token cache.'); - console.log("At this point Chromium's job is done, shutting it down...\n"); - - await browser.close(); - - return session; -} - -function extractVideoGuid(videoUrls: string[]): string[] { - const first = videoUrls[0] as string; - const isPath = first.substring(first.length - 4) === '.txt'; - let urls: string[]; - - if (isPath) - urls = fs.readFileSync(first).toString('utf-8').split(/[\r\n]/); - else - urls = videoUrls as string[]; - let videoGuids: string[] = []; - let guid: string | undefined = ''; - for (let url of urls) { - console.log(url); - try { - guid = url.split('/').pop(); - - } catch (e) { - console.error(`Could not split the video GUID from URL: ${e.message}`); - process.exit(25); - } - if (guid) { - videoGuids.push(guid); - } - } - - console.log(videoGuids); - return videoGuids; -} - -async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) { - console.log(videoUrls); - const videoGuids = extractVideoGuid(videoUrls); - - console.log('Fetching title and HLS URL...'); - let metadata: Metadata[] = await getVideoMetadata(videoGuids, session); - await Promise.all(metadata.map(async video => { - video.title = sanitize(video.title); - console.log(colors.blue(`\nDownloading Video: ${video.title}\n`)); - - // Very experimental inline thumbnail rendering - await drawThumbnail(video.posterImage, session.AccessToken); - - console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n'); - - const outputPath = outputDirectory + path.sep + video.title + '.mp4'; - - // TODO: Remove this mess and it's fluent-ffmpeg dependency - // - // ffmpeg() - // .input(video.playbackUrl) - // .inputOption([ - // // Never remove those "useless" escapes or ffmpeg will not - // // pick up the header correctly - // // eslint-disable-next-line no-useless-escape - // '-headers', `Authorization:\ Bearer\ ${session.AccessToken}` - // ]) - // .format('mp4') - // .saveToFile(outputPath) - // .on('codecData', data => { - // console.log(`Input is ${data.video} with ${data.audio} audio.`); - // }) - // .on('progress', progress => { - // console.log(progress); - // }) - // .on('error', err => { - // console.log(`ffmpeg returned an error: ${err.message}`); - // }) - // .on('end', () => { - // console.log(`Download finished: ${outputPath}`); - // }); - - - // We probably need a way to be deterministic about - // how we locate that ffmpeg-bar wrapper, npx maybe? - // Do not remove those "useless" escapes or ffmpeg will - // not pick up the header correctly. - // eslint-disable-next-line no-useless-escape - let cmd = `node_modules/.bin/ffmpeg-bar -headers "Authorization:\ Bearer\ ${session.AccessToken}" -i "${video.playbackUrl}" -y "${outputPath}"`; - execSync(cmd, {stdio: 'inherit'}); - console.info(`Download finished: ${outputPath}`); - })); -} - -// FIXME -process.on('unhandledRejection', (reason) => { - console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n')); - console.error(colors.red(reason as string)); - throw new Error('Killing process..\n'); -}); - -async function main() { - const isValidUser = !(await isElevated()); - let videoUrls: string[]; - - if (!isValidUser) { - const usrName = os.platform() === 'win32' ? 'Admin':'root'; - - console.error(colors.red('\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n')); - process.exit(-1); - } - - videoUrls = parseVideoUrls(argv.videoUrls); - if (videoUrls.length === 0) { - console.error(colors.red('\nERROR: No valid URL has been found!\n')); - process.exit(-1); - } - - checkRequirements(); - - let session = tokenCache.Read(); - if (session == null) { - session = await DoInteractiveLogin(videoUrls[0], argv.username); - } - - - init(); - downloadVideo(videoUrls, argv.outputDirectory, session); -} - -// run -main(); +import { sleep, parseVideoUrls, checkRequirements, makeUniqueTitle } from './utils'; +import { TokenCache } from './TokenCache'; +import { getVideoMetadata } from './Metadata'; +import { Metadata, Session } from './Types'; +import { drawThumbnail } from './Thumbnail'; + +import isElevated from 'is-elevated'; +import puppeteer from 'puppeteer'; +import { execSync } from 'child_process'; +import colors from 'colors'; +import fs from 'fs'; +import path from 'path'; +import yargs from 'yargs'; +import sanitize from 'sanitize-filename'; +import ffmpeg from 'fluent-ffmpeg'; + +/** + * exitCode 22 = ffmpeg not found in $PATH + * exitCode 25 = cannot split videoID from videUrl + * exitCode 27 = no hlsUrl in the API response + * exitCode 29 = invalid response from API + * exitCode 88 = error extracting cookies + */ + +let tokenCache = new TokenCache(); + +const argv = yargs.options({ + username: { + alias: 'u', + type: 'string', + demandOption: false + }, + outputDirectory: { + alias: 'o', + type: 'string', + default: 'videos', + demandOption: false + }, + videoUrls: { + alias: 'V', + describe: 'List of video urls or path to txt file containing the urls', + type: 'array', + demandOption: true + }, + simulate: { + alias: 's', + describe: `Disable video download and print metadata information to the console`, + type: 'boolean', + default: false, + demandOption: false + }, + noThumbnails: { + alias: 'nthumb', + describe: `Do not display video thumbnails`, + type: 'boolean', + default: false, + demandOption: false + }, + verbose: { + alias: 'v', + describe: `Print additional information to the console (use this before opening an issue on GitHub)`, + type: 'boolean', + default: false, + demandOption: false + } +}).argv; + +async function init() { + const isValidUser = !(await isElevated()); + + if (!isValidUser) { + const usrName = process.platform === 'win32' ? 'Admin':'root'; + + console.error(colors.red( + '\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n' + )); + process.exit(-1); + } + + // create output directory + if (!fs.existsSync(argv.outputDirectory)) { + console.log('Creating output directory: ' + + process.cwd() + path.sep + argv.outputDirectory); + fs.mkdirSync(argv.outputDirectory); + } + + console.info('Output Directory: %s', argv.outputDirectory); + + if (argv.username) + console.info('Username: %s', argv.username); + + if (argv.simulate) + console.info(colors.yellow('Simulate mode, there will be no video download.\n')); + + if (argv.verbose) { + console.info('Video URLs:'); + console.info(argv.videoUrls); + } +} + +async function DoInteractiveLogin(url: string, username?: string): Promise { + + let videoId = url.split("/").pop() ?? ( + console.log('Couldn\'t split the video Id from the first videoUrl'), process.exit(25) + ); + + console.log('Launching headless Chrome to perform the OpenID Connect dance...'); + const browser = await puppeteer.launch({ + headless: false, + args: ['--disable-dev-shm-usage'] + }); + const page = (await browser.pages())[0]; + console.log('Navigating to login page...'); + + await page.goto(url, { waitUntil: 'load' }); + await page.waitForSelector('input[type="email"]'); + + if (username) { + await page.keyboard.type(username); + await page.click('input[type="submit"]'); + } + + await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 }); + console.info('We are logged in.'); + + let sessionInfo: any; + let session = await page.evaluate( + () => { + return { + AccessToken: sessionInfo.AccessToken, + ApiGatewayUri: sessionInfo.ApiGatewayUri, + ApiGatewayVersion: sessionInfo.ApiGatewayVersion + }; + } + ); + + tokenCache.Write(session); + console.log('Wrote access token to token cache.'); + console.log("At this point Chromium's job is done, shutting it down...\n"); + + await browser.close(); + + return session; +} + +function extractVideoGuid(videoUrls: string[]): string[] { + let videoGuids: string[] = []; + let guid: string | undefined = ''; + + for (const url of videoUrls) { + try { + guid = url.split('/').pop(); + + } catch (e) { + console.error(`Could not split the video GUID from URL: ${e.message}`); + process.exit(25); + } + + if (guid) + videoGuids.push(guid); + } + + if (argv.verbose) { + console.info('Video GUIDs:'); + console.info(videoGuids); + } + + return videoGuids; +} + +async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) { + const videoGuids = extractVideoGuid(videoUrls); + + console.log('Fetching metadata...'); + + const metadata: Metadata[] = await getVideoMetadata(videoGuids, session, argv.verbose); + + if (argv.simulate) { + metadata.forEach(video => { + console.log( + colors.yellow('\n\nTitle: ') + colors.green(video.title) + + colors.yellow('\nPublished Date: ') + colors.green(video.date) + + colors.yellow('\nPlayback URL: ') + colors.green(video.playbackUrl) + ); + }); + + return; + } + + await Promise.all(metadata.map(async video => { + console.log(colors.blue(`\nDownloading Video: ${video.title}\n`)); + + video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, argv.outputDirectory); + + // Very experimental inline thumbnail rendering + if (!argv.noThumbnails) + await drawThumbnail(video.posterImage, session.AccessToken); + + console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n'); + + const outputPath = outputDirectory + path.sep + video.title + '.mp4'; + + // TODO: Remove this mess and it's fluent-ffmpeg dependency + // + // ffmpeg() + // .input(video.playbackUrl) + // .inputOption([ + // // Never remove those "useless" escapes or ffmpeg will not + // // pick up the header correctly + // // eslint-disable-next-line no-useless-escape + // '-headers', `Authorization:\ Bearer\ ${session.AccessToken}` + // ]) + // .format('mp4') + // .saveToFile(outputPath) + // .on('codecData', data => { + // console.log(`Input is ${data.video} with ${data.audio} audio.`); + // }) + // .on('progress', progress => { + // console.log(progress); + // }) + // .on('error', err => { + // console.log(`ffmpeg returned an error: ${err.message}`); + // }) + // .on('end', () => { + // console.log(`Download finished: ${outputPath}`); + // }); + + + // We probably need a way to be deterministic about + // how we locate that ffmpeg-bar wrapper, npx maybe? + // Do not remove those "useless" escapes or ffmpeg will + // not pick up the header correctly. + // eslint-disable-next-line no-useless-escape + let cmd = `node_modules/.bin/ffmpeg-bar -headers "Authorization:\ Bearer\ ${session.AccessToken}" -i "${video.playbackUrl}" -y "${outputPath}"`; + execSync(cmd, {stdio: 'inherit'}); + console.info(`Download finished: ${outputPath}`); + })); +} + +// FIXME +process.on('unhandledRejection', (reason) => { + console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n')); + console.error(colors.red(reason as string)); + throw new Error('Killing process..\n'); +}); + +async function main() { + checkRequirements(); + await init(); + + const videoUrls: string[] = parseVideoUrls(argv.videoUrls); + + if (videoUrls.length === 0) { + console.error(colors.red('\nERROR: No valid URL has been found!\n')); + process.exit(-1); + } + + let session = tokenCache.Read(); + + if (session == null) { + session = await DoInteractiveLogin(videoUrls[0], argv.username); + } + + downloadVideo(videoUrls, argv.outputDirectory, session); +} + +// run +main(); diff --git a/package-lock.json b/package-lock.json index 17f6dda..65e0bd2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -133,6 +133,11 @@ "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==", "dev": true }, + "@types/jwt-decode": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@types/jwt-decode/-/jwt-decode-2.2.1.tgz", + "integrity": "sha512-aWw2YTtAdT7CskFyxEX2K21/zSDStuf/ikI3yBqmwpwJF0pS+/IX5DWv+1UFffZIbruP6cnT9/LAJV1gFwAT1A==" + }, "@types/mime-types": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz", diff --git a/package.json b/package.json index ce5132f..f48b136 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ }, "dependencies": { "@types/fluent-ffmpeg": "^2.1.14", + "@types/jwt-decode": "^2.2.1", "axios": "^0.19.2", "colors": "^1.4.0", "ffmpeg-progressbar-cli": "^1.5.0", diff --git a/utils.ts b/utils.ts index 4eed2bd..bb60ce8 100644 --- a/utils.ts +++ b/utils.ts @@ -1,6 +1,7 @@ import { execSync } from 'child_process'; import colors from 'colors'; import fs from 'fs'; +import path from 'path'; function sanitizeUrls(urls: string[]) { const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i'); @@ -13,7 +14,7 @@ function sanitizeUrls(urls: string[]) { if (!rex.test(url)) { if (url !== '') - console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..\n')); + console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..')); continue; } @@ -57,3 +58,13 @@ export function checkRequirements() { process.exit(22); } } + +export function makeUniqueTitle(title: string, outDir: string) { + let ntitle = title; + let k = 0; + + while (fs.existsSync(outDir + path.sep + ntitle + '.mp4')) + ntitle = title + ' - ' + (++k).toString(); + + return ntitle; +} \ No newline at end of file