arona/destreamer-mirror
1
0
Fork 0
mirror of https://github.com/snobu/destreamer.git synced 2026-01-17 05:22:18 +00:00
Code Issues Projects Releases Wiki Activity

Fixes and clean up (#51)

Browse Source 
* Simplify main

* Fix init

* Cleaner output for the end user

* Fix extractVideoGuid after sync with dev

* TokenCache: Make variable private

nit: switch to import

* Add option to disable video thumbnails

* Create a unique file name to avoid overwrite

* Remove os dependency

* Reimplement simulate

* Update README

Co-authored-by: @kylon
Co-authored-by: @snobu
This commit is contained in:
kylon
2020-04-10 18:35:57 +02:00
committed by GitHub
parent 177c3dcf71
commit 83fecf2894
8 changed files with 323 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
Metadata.ts
View File

@@ -1,16 +1,28 @@
import axios from 'axios';
import { Metadata, Session } from './Types'; import { Metadata, Session } from './Types';
import axios from 'axios';
export async function getVideoMetadata(videoGuids: string[], session: Session): Promise<Metadata[]> { function publishedDateToString(date: string) {
const dateJs = new Date(date);
const day = dateJs.getDate().toString().padStart(2, '0');
const month = (dateJs.getMonth() + 1).toString(10).padStart(2, '0');
return day+'-'+month+'-'+dateJs.getFullYear();
}
export async function getVideoMetadata(videoGuids: string[], session: Session, verbose: boolean): Promise<Metadata[]> {
let metadata: Metadata[] = []; let metadata: Metadata[] = [];
let title: string; let title: string;
let date: string;
let playbackUrl: string; let playbackUrl: string;
let posterImage: string; let posterImage: string;
await Promise.all(videoGuids.map(async guid => { await Promise.all(videoGuids.map(async guid => {
let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`; let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`;
console.log(`Calling ${apiUrl}`);
if (verbose)
console.info(`Calling ${apiUrl}`);
let response = await axios.get(apiUrl, let response = await axios.get(apiUrl,
{ {
headers: { headers: {
@@ -25,8 +37,10 @@ export async function getVideoMetadata(videoGuids: string[], session: Session):
.map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0]; .map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0];
posterImage = response.data['posterImage']['medium']['url']; posterImage = response.data['posterImage']['medium']['url'];
date = publishedDateToString(response.data['publishedDate']);
metadata.push({ metadata.push({
date: date,
title: title, title: title,
playbackUrl: playbackUrl, playbackUrl: playbackUrl,
posterImage: posterImage posterImage: posterImage

23
README.md
View File

@@ -47,15 +47,20 @@ Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) ap
$ node ./destreamer.js $ node ./destreamer.js
Options: Options:
--help Show help [boolean] --help Show help [boolean]
--version Show version number [boolean] --version Show version number [boolean]
--videoUrls, -V List of video urls or path to txt file containing the urls --username, -u [string]
[array] [required] --outputDirectory, -o [string] [default: "videos"]
--username, -u [string] --videoUrls, -V List of video urls or path to txt file containing the urls
--outputDirectory, -o [string] [default: "videos"] [array] [required]
--verbose, -v Print additional information to the console --simulate, -s Disable video download and print metadata
(use this before opening an issue on GitHub) information to the console
[boolean] [default: false] [boolean] [default: false]
--noThumbnails, --nthumb Do not display video thumbnails
[boolean] [default: false]
--verbose, -v Print additional information to the console
(use this before opening an issue on GitHub)
[boolean] [default: false]
``` ```
Make sure you use the right escape char for your shell if using line breaks (as this example shows). Make sure you use the right escape char for your shell if using line breaks (as this example shows).

12
TokenCache.ts
View File

@@ -1,21 +1,19 @@
import * as fs from 'fs'; import * as fs from 'fs';
import { Session } from './Types'; import { Session } from './Types';
import { bgGreen, bgYellow, green } from 'colors'; import { bgGreen, bgYellow, green } from 'colors';
const jwtDecode = require('jwt-decode'); import jwtDecode from 'jwt-decode';
const tokenCacheFile = '.token_cache';
export class TokenCache { export class TokenCache {
private tokenCacheFile: string = '.token_cache';
public Read(): Session | null { public Read(): Session | null {
let j = null; let j = null;
if(!fs.existsSync(tokenCacheFile)) { if(!fs.existsSync(this.tokenCacheFile)) {
console.warn(bgYellow.black(`${tokenCacheFile} not found.\n`)); console.warn(bgYellow.black(`${this.tokenCacheFile} not found.\n`));
return null; return null;
} }
let f = fs.readFileSync(tokenCacheFile, 'utf8'); let f = fs.readFileSync(this.tokenCacheFile, 'utf8');
j = JSON.parse(f); j = JSON.parse(f);
interface Jwt { interface Jwt {

1
Types.ts
View File

@@ -5,6 +5,7 @@ export type Session = {
} }
export type Metadata = { export type Metadata = {
date: string;
title: string; title: string;
playbackUrl: string; playbackUrl: string;
posterImage: string; posterImage: string;

513
destreamer.ts
View File

@@ -1,245 +1,268 @@
import { sleep, parseVideoUrls, checkRequirements } from './utils'; import { sleep, parseVideoUrls, checkRequirements, makeUniqueTitle } from './utils';
import { TokenCache } from './TokenCache'; import { TokenCache } from './TokenCache';
import { getVideoMetadata } from './Metadata'; import { getVideoMetadata } from './Metadata';
import { Metadata, Session } from './Types'; import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail'; import { drawThumbnail } from './Thumbnail';
import isElevated from 'is-elevated'; import isElevated from 'is-elevated';
import puppeteer from 'puppeteer'; import puppeteer from 'puppeteer';
import { execSync } from 'child_process'; import { execSync } from 'child_process';
import colors from 'colors'; import colors from 'colors';
import fs from 'fs'; import fs from 'fs';
import os from 'os'; import path from 'path';
import path from 'path'; import yargs from 'yargs';
import yargs from 'yargs'; import sanitize from 'sanitize-filename';
import sanitize from 'sanitize-filename'; import ffmpeg from 'fluent-ffmpeg';
import ffmpeg from 'fluent-ffmpeg';
/**
/** * exitCode 22 = ffmpeg not found in $PATH
* exitCode 22 = ffmpeg not found in $PATH * exitCode 25 = cannot split videoID from videUrl
* exitCode 25 = cannot split videoID from videUrl * exitCode 27 = no hlsUrl in the API response
* exitCode 27 = no hlsUrl in the API response * exitCode 29 = invalid response from API
* exitCode 29 = invalid response from API * exitCode 88 = error extracting cookies
* exitCode 88 = error extracting cookies */
*/
let tokenCache = new TokenCache();
let tokenCache = new TokenCache();
const argv = yargs.options({
const argv = yargs.options({ username: {
username: { alias: 'u',
alias: 'u', type: 'string',
type: 'string', demandOption: false
demandOption: false },
}, outputDirectory: {
outputDirectory: { alias: 'o',
alias: 'o', type: 'string',
type: 'string', default: 'videos',
default: 'videos', demandOption: false
demandOption: false },
}, videoUrls: {
videoUrls: { alias: 'V',
alias: 'V', describe: 'List of video urls or path to txt file containing the urls',
describe: 'List of video urls or path to txt file containing the urls', type: 'array',
type: 'array', demandOption: true
demandOption: true },
}, simulate: {
simulate: { alias: 's',
alias: 's', describe: `Disable video download and print metadata information to the console`,
describe: `If this is set to true no video will be downloaded and the script type: 'boolean',
will log the video info (default: false)`, default: false,
type: 'boolean', demandOption: false
default: false, },
demandOption: false noThumbnails: {
}, alias: 'nthumb',
verbose: { describe: `Do not display video thumbnails`,
alias: 'v', type: 'boolean',
describe: `Print additional information to the console default: false,
(use this before opening an issue on GitHub)`, demandOption: false
type: 'boolean', },
default: false, verbose: {
demandOption: false alias: 'v',
} describe: `Print additional information to the console (use this before opening an issue on GitHub)`,
}).argv; type: 'boolean',
default: false,
function init() { demandOption: false
// create output directory }
if (!fs.existsSync(argv.outputDirectory)) { }).argv;
console.log('Creating output directory: ' +
process.cwd() + path.sep + argv.outputDirectory); async function init() {
fs.mkdirSync(argv.outputDirectory); const isValidUser = !(await isElevated());
}
if (!isValidUser) {
console.info('Video URLs: %s', argv.videoUrls); const usrName = process.platform === 'win32' ? 'Admin':'root';
console.info('Username: %s', argv.username);
console.info('Output Directory: %s', argv.outputDirectory); console.error(colors.red(
'\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n'
if (argv.simulate) ));
console.info(colors.blue("There will be no video downloaded, it's only a simulation\n")); process.exit(-1);
} }
async function DoInteractiveLogin(url: string, username?: string): Promise<Session> { // create output directory
if (!fs.existsSync(argv.outputDirectory)) {
let videoId = url.split("/").pop() ?? ( console.log('Creating output directory: ' +
console.log('Couldn\'t split the video Id from the first videoUrl'), process.exit(25) process.cwd() + path.sep + argv.outputDirectory);
); fs.mkdirSync(argv.outputDirectory);
}
console.log('Launching headless Chrome to perform the OpenID Connect dance...');
const browser = await puppeteer.launch({ console.info('Output Directory: %s', argv.outputDirectory);
headless: false,
args: ['--disable-dev-shm-usage'] if (argv.username)
}); console.info('Username: %s', argv.username);
const page = (await browser.pages())[0];
console.log('Navigating to login page...'); if (argv.simulate)
console.info(colors.yellow('Simulate mode, there will be no video download.\n'));
await page.goto(url, { waitUntil: 'load' });
await page.waitForSelector('input[type="email"]'); if (argv.verbose) {
console.info('Video URLs:');
if (username) { console.info(argv.videoUrls);
await page.keyboard.type(username); }
await page.click('input[type="submit"]'); }
}
async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 });
console.info('We are logged in.'); let videoId = url.split("/").pop() ?? (
console.log('Couldn\'t split the video Id from the first videoUrl'), process.exit(25)
let sessionInfo: any; );
let session = await page.evaluate(
() => { console.log('Launching headless Chrome to perform the OpenID Connect dance...');
return { const browser = await puppeteer.launch({
AccessToken: sessionInfo.AccessToken, headless: false,
ApiGatewayUri: sessionInfo.ApiGatewayUri, args: ['--disable-dev-shm-usage']
ApiGatewayVersion: sessionInfo.ApiGatewayVersion });
}; const page = (await browser.pages())[0];
} console.log('Navigating to login page...');
);
await page.goto(url, { waitUntil: 'load' });
tokenCache.Write(session); await page.waitForSelector('input[type="email"]');
console.log('Wrote access token to token cache.');
console.log("At this point Chromium's job is done, shutting it down...\n"); if (username) {
await page.keyboard.type(username);
await browser.close(); await page.click('input[type="submit"]');
}
return session;
} await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 });
console.info('We are logged in.');
function extractVideoGuid(videoUrls: string[]): string[] {
const first = videoUrls[0] as string; let sessionInfo: any;
const isPath = first.substring(first.length - 4) === '.txt'; let session = await page.evaluate(
let urls: string[]; () => {
return {
if (isPath) AccessToken: sessionInfo.AccessToken,
urls = fs.readFileSync(first).toString('utf-8').split(/[\r\n]/); ApiGatewayUri: sessionInfo.ApiGatewayUri,
else ApiGatewayVersion: sessionInfo.ApiGatewayVersion
urls = videoUrls as string[]; };
let videoGuids: string[] = []; }
let guid: string | undefined = ''; );
for (let url of urls) {
console.log(url); tokenCache.Write(session);
try { console.log('Wrote access token to token cache.');
guid = url.split('/').pop(); console.log("At this point Chromium's job is done, shutting it down...\n");
} catch (e) { await browser.close();
console.error(`Could not split the video GUID from URL: ${e.message}`);
process.exit(25); return session;
} }
if (guid) {
videoGuids.push(guid); function extractVideoGuid(videoUrls: string[]): string[] {
} let videoGuids: string[] = [];
} let guid: string | undefined = '';
console.log(videoGuids); for (const url of videoUrls) {
return videoGuids; try {
} guid = url.split('/').pop();
async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) { } catch (e) {
console.log(videoUrls); console.error(`Could not split the video GUID from URL: ${e.message}`);
const videoGuids = extractVideoGuid(videoUrls); process.exit(25);
}
console.log('Fetching title and HLS URL...');
let metadata: Metadata[] = await getVideoMetadata(videoGuids, session); if (guid)
await Promise.all(metadata.map(async video => { videoGuids.push(guid);
video.title = sanitize(video.title); }
console.log(colors.blue(`\nDownloading Video: ${video.title}\n`));
if (argv.verbose) {
// Very experimental inline thumbnail rendering console.info('Video GUIDs:');
await drawThumbnail(video.posterImage, session.AccessToken); console.info(videoGuids);
}
console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n');
return videoGuids;
const outputPath = outputDirectory + path.sep + video.title + '.mp4'; }
// TODO: Remove this mess and it's fluent-ffmpeg dependency async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
// const videoGuids = extractVideoGuid(videoUrls);
// ffmpeg()
// .input(video.playbackUrl) console.log('Fetching metadata...');
// .inputOption([
// // Never remove those "useless" escapes or ffmpeg will not const metadata: Metadata[] = await getVideoMetadata(videoGuids, session, argv.verbose);
// // pick up the header correctly
// // eslint-disable-next-line no-useless-escape if (argv.simulate) {
// '-headers', `Authorization:\ Bearer\ ${session.AccessToken}` metadata.forEach(video => {
// ]) console.log(
// .format('mp4') colors.yellow('\n\nTitle: ') + colors.green(video.title) +
// .saveToFile(outputPath) colors.yellow('\nPublished Date: ') + colors.green(video.date) +
// .on('codecData', data => { colors.yellow('\nPlayback URL: ') + colors.green(video.playbackUrl)
// console.log(`Input is ${data.video} with ${data.audio} audio.`); );
// }) });
// .on('progress', progress => {
// console.log(progress); return;
// }) }
// .on('error', err => {
// console.log(`ffmpeg returned an error: ${err.message}`); await Promise.all(metadata.map(async video => {
// }) console.log(colors.blue(`\nDownloading Video: ${video.title}\n`));
// .on('end', () => {
// console.log(`Download finished: ${outputPath}`); video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, argv.outputDirectory);
// });
// Very experimental inline thumbnail rendering
if (!argv.noThumbnails)
// We probably need a way to be deterministic about await drawThumbnail(video.posterImage, session.AccessToken);
// how we locate that ffmpeg-bar wrapper, npx maybe?
// Do not remove those "useless" escapes or ffmpeg will console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n');
// not pick up the header correctly.
// eslint-disable-next-line no-useless-escape const outputPath = outputDirectory + path.sep + video.title + '.mp4';
let cmd = `node_modules/.bin/ffmpeg-bar -headers "Authorization:\ Bearer\ ${session.AccessToken}" -i "${video.playbackUrl}" -y "${outputPath}"`;
execSync(cmd, {stdio: 'inherit'}); // TODO: Remove this mess and it's fluent-ffmpeg dependency
console.info(`Download finished: ${outputPath}`); //
})); // ffmpeg()
} // .input(video.playbackUrl)
// .inputOption([
// FIXME // // Never remove those "useless" escapes or ffmpeg will not
process.on('unhandledRejection', (reason) => { // // pick up the header correctly
console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n')); // // eslint-disable-next-line no-useless-escape
console.error(colors.red(reason as string)); // '-headers', `Authorization:\ Bearer\ ${session.AccessToken}`
throw new Error('Killing process..\n'); // ])
}); // .format('mp4')
// .saveToFile(outputPath)
async function main() { // .on('codecData', data => {
const isValidUser = !(await isElevated()); // console.log(`Input is ${data.video} with ${data.audio} audio.`);
let videoUrls: string[]; // })
// .on('progress', progress => {
if (!isValidUser) { // console.log(progress);
const usrName = os.platform() === 'win32' ? 'Admin':'root'; // })
// .on('error', err => {
console.error(colors.red('\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n')); // console.log(`ffmpeg returned an error: ${err.message}`);
process.exit(-1); // })
} // .on('end', () => {
// console.log(`Download finished: ${outputPath}`);
videoUrls = parseVideoUrls(argv.videoUrls); // });
if (videoUrls.length === 0) {
console.error(colors.red('\nERROR: No valid URL has been found!\n'));
process.exit(-1); // We probably need a way to be deterministic about
} // how we locate that ffmpeg-bar wrapper, npx maybe?
// Do not remove those "useless" escapes or ffmpeg will
checkRequirements(); // not pick up the header correctly.
// eslint-disable-next-line no-useless-escape
let session = tokenCache.Read(); let cmd = `node_modules/.bin/ffmpeg-bar -headers "Authorization:\ Bearer\ ${session.AccessToken}" -i "${video.playbackUrl}" -y "${outputPath}"`;
if (session == null) { execSync(cmd, {stdio: 'inherit'});
session = await DoInteractiveLogin(videoUrls[0], argv.username); console.info(`Download finished: ${outputPath}`);
} }));
}
init(); // FIXME
downloadVideo(videoUrls, argv.outputDirectory, session); process.on('unhandledRejection', (reason) => {
} console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
console.error(colors.red(reason as string));
// run throw new Error('Killing process..\n');
main(); });
async function main() {
checkRequirements();
await init();
const videoUrls: string[] = parseVideoUrls(argv.videoUrls);
if (videoUrls.length === 0) {
console.error(colors.red('\nERROR: No valid URL has been found!\n'));
process.exit(-1);
}
let session = tokenCache.Read();
if (session == null) {
session = await DoInteractiveLogin(videoUrls[0], argv.username);
}
downloadVideo(videoUrls, argv.outputDirectory, session);
}
// run
main();

5
package-lock.json generated
View File

@@ -133,6 +133,11 @@
"integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==", "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==",
"dev": true "dev": true
}, },
"@types/jwt-decode": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/@types/jwt-decode/-/jwt-decode-2.2.1.tgz",
"integrity": "sha512-aWw2YTtAdT7CskFyxEX2K21/zSDStuf/ikI3yBqmwpwJF0pS+/IX5DWv+1UFffZIbruP6cnT9/LAJV1gFwAT1A=="
},
"@types/mime-types": { "@types/mime-types": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz", "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz",

1
package.json
View File

@@ -29,6 +29,7 @@
}, },
"dependencies": { "dependencies": {
"@types/fluent-ffmpeg": "^2.1.14", "@types/fluent-ffmpeg": "^2.1.14",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.19.2", "axios": "^0.19.2",
"colors": "^1.4.0", "colors": "^1.4.0",
"ffmpeg-progressbar-cli": "^1.5.0", "ffmpeg-progressbar-cli": "^1.5.0",

13
utils.ts
View File

@@ -1,6 +1,7 @@
import { execSync } from 'child_process'; import { execSync } from 'child_process';
import colors from 'colors'; import colors from 'colors';
import fs from 'fs'; import fs from 'fs';
import path from 'path';
function sanitizeUrls(urls: string[]) { function sanitizeUrls(urls: string[]) {
const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i'); const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i');
@@ -13,7 +14,7 @@ function sanitizeUrls(urls: string[]) {
if (!rex.test(url)) { if (!rex.test(url)) {
if (url !== '') if (url !== '')
console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..\n')); console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..'));
continue; continue;
} }
@@ -57,3 +58,13 @@ export function checkRequirements() {
process.exit(22); process.exit(22);
} }
} }
export function makeUniqueTitle(title: string, outDir: string) {
let ntitle = title;
let k = 0;
while (fs.existsSync(outDir + path.sep + ntitle + '.mp4'))
ntitle = title + ' - ' + (++k).toString();
return ntitle;
}