arona/destreamer-mirror
1
0
Fork 0
mirror of https://github.com/snobu/destreamer.git synced 2026-01-17 05:22:18 +00:00
Code Issues Projects Releases Wiki Activity

Fixes and clean up (#51)

Browse Source 
* Simplify main

* Fix init

* Cleaner output for the end user

* Fix extractVideoGuid after sync with dev

* TokenCache: Make variable private

nit: switch to import

* Add option to disable video thumbnails

* Create a unique file name to avoid overwrite

* Remove os dependency

* Reimplement simulate

* Update README

Co-authored-by: @kylon
Co-authored-by: @snobu
This commit is contained in:
kylon
2020-04-10 18:35:57 +02:00
committed by GitHub
parent 177c3dcf71
commit 83fecf2894
8 changed files with 323 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
Metadata.ts
View File

@@ -1,16 +1,28 @@
import axios from 'axios';
import { Metadata, Session } from './Types';
import axios from 'axios';
export async function getVideoMetadata(videoGuids: string[], session: Session): Promise<Metadata[]> {
function publishedDateToString(date: string) {
const dateJs = new Date(date);
const day = dateJs.getDate().toString().padStart(2, '0');
const month = (dateJs.getMonth() + 1).toString(10).padStart(2, '0');
return day+'-'+month+'-'+dateJs.getFullYear();
}
export async function getVideoMetadata(videoGuids: string[], session: Session, verbose: boolean): Promise<Metadata[]> {
let metadata: Metadata[] = [];
let title: string;
let date: string;
let playbackUrl: string;
let posterImage: string;
await Promise.all(videoGuids.map(async guid => {
let apiUrl = `${session.ApiGatewayUri}videos/${guid}?api-version=${session.ApiGatewayVersion}`;
console.log(`Calling ${apiUrl}`);
if (verbose)
console.info(`Calling ${apiUrl}`);
let response = await axios.get(apiUrl,
{
headers: {
@@ -25,8 +37,10 @@ export async function getVideoMetadata(videoGuids: string[], session: Session):
.map((item: { [x: string]: string }) => { return item['playbackUrl']; })[0];
posterImage = response.data['posterImage']['medium']['url'];
date = publishedDateToString(response.data['publishedDate']);
metadata.push({
date: date,
title: title,
playbackUrl: playbackUrl,
posterImage: posterImage

23
README.md
View File

@@ -47,15 +47,20 @@ Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) ap
$ node ./destreamer.js
Options:
--help Show help [boolean]
--version Show version number [boolean]
--videoUrls, -V List of video urls or path to txt file containing the urls
[array] [required]
--username, -u [string]
--outputDirectory, -o [string] [default: "videos"]
--verbose, -v Print additional information to the console
(use this before opening an issue on GitHub)
[boolean] [default: false]
--help Show help [boolean]
--version Show version number [boolean]
--username, -u [string]
--outputDirectory, -o [string] [default: "videos"]
--videoUrls, -V List of video urls or path to txt file containing the urls
[array] [required]
--simulate, -s Disable video download and print metadata
information to the console
[boolean] [default: false]
--noThumbnails, --nthumb Do not display video thumbnails
[boolean] [default: false]
--verbose, -v Print additional information to the console
(use this before opening an issue on GitHub)
[boolean] [default: false]
```
Make sure you use the right escape char for your shell if using line breaks (as this example shows).

12
TokenCache.ts
View File

@@ -1,21 +1,19 @@
import * as fs from 'fs';
import { Session } from './Types';
import { bgGreen, bgYellow, green } from 'colors';
const jwtDecode = require('jwt-decode');
const tokenCacheFile = '.token_cache';
import jwtDecode from 'jwt-decode';
export class TokenCache {
private tokenCacheFile: string = '.token_cache';
public Read(): Session | null {
let j = null;
if(!fs.existsSync(tokenCacheFile)) {
console.warn(bgYellow.black(`${tokenCacheFile} not found.\n`));
if(!fs.existsSync(this.tokenCacheFile)) {
console.warn(bgYellow.black(`${this.tokenCacheFile} not found.\n`));
return null;
}
let f = fs.readFileSync(tokenCacheFile, 'utf8');
let f = fs.readFileSync(this.tokenCacheFile, 'utf8');
j = JSON.parse(f);
interface Jwt {

1
Types.ts
View File

@@ -5,6 +5,7 @@ export type Session = {
}
export type Metadata = {
date: string;
title: string;
playbackUrl: string;
posterImage: string;

513
destreamer.ts
View File

@@ -1,245 +1,268 @@
import { sleep, parseVideoUrls, checkRequirements } from './utils';
import { TokenCache } from './TokenCache';
import { getVideoMetadata } from './Metadata';
import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import { execSync } from 'child_process';
import colors from 'colors';
import fs from 'fs';
import os from 'os';
import path from 'path';
import yargs from 'yargs';
import sanitize from 'sanitize-filename';
import ffmpeg from 'fluent-ffmpeg';
/**
* exitCode 22 = ffmpeg not found in $PATH
* exitCode 25 = cannot split videoID from videUrl
* exitCode 27 = no hlsUrl in the API response
* exitCode 29 = invalid response from API
* exitCode 88 = error extracting cookies
*/
let tokenCache = new TokenCache();
const argv = yargs.options({
username: {
alias: 'u',
type: 'string',
demandOption: false
},
outputDirectory: {
alias: 'o',
type: 'string',
default: 'videos',
demandOption: false
},
videoUrls: {
alias: 'V',
describe: 'List of video urls or path to txt file containing the urls',
type: 'array',
demandOption: true
},
simulate: {
alias: 's',
describe: `If this is set to true no video will be downloaded and the script
will log the video info (default: false)`,
type: 'boolean',
default: false,
demandOption: false
},
verbose: {
alias: 'v',
describe: `Print additional information to the console
(use this before opening an issue on GitHub)`,
type: 'boolean',
default: false,
demandOption: false
}
}).argv;
function init() {
// create output directory
if (!fs.existsSync(argv.outputDirectory)) {
console.log('Creating output directory: ' +
process.cwd() + path.sep + argv.outputDirectory);
fs.mkdirSync(argv.outputDirectory);
}
console.info('Video URLs: %s', argv.videoUrls);
console.info('Username: %s', argv.username);
console.info('Output Directory: %s', argv.outputDirectory);
if (argv.simulate)
console.info(colors.blue("There will be no video downloaded, it's only a simulation\n"));
}
async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
let videoId = url.split("/").pop() ?? (
console.log('Couldn\'t split the video Id from the first videoUrl'), process.exit(25)
);
console.log('Launching headless Chrome to perform the OpenID Connect dance...');
const browser = await puppeteer.launch({
headless: false,
args: ['--disable-dev-shm-usage']
});
const page = (await browser.pages())[0];
console.log('Navigating to login page...');
await page.goto(url, { waitUntil: 'load' });
await page.waitForSelector('input[type="email"]');
if (username) {
await page.keyboard.type(username);
await page.click('input[type="submit"]');
}
await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 });
console.info('We are logged in.');
let sessionInfo: any;
let session = await page.evaluate(
() => {
return {
AccessToken: sessionInfo.AccessToken,
ApiGatewayUri: sessionInfo.ApiGatewayUri,
ApiGatewayVersion: sessionInfo.ApiGatewayVersion
};
}
);
tokenCache.Write(session);
console.log('Wrote access token to token cache.');
console.log("At this point Chromium's job is done, shutting it down...\n");
await browser.close();
return session;
}
function extractVideoGuid(videoUrls: string[]): string[] {
const first = videoUrls[0] as string;
const isPath = first.substring(first.length - 4) === '.txt';
let urls: string[];
if (isPath)
urls = fs.readFileSync(first).toString('utf-8').split(/[\r\n]/);
else
urls = videoUrls as string[];
let videoGuids: string[] = [];
let guid: string | undefined = '';
for (let url of urls) {
console.log(url);
try {
guid = url.split('/').pop();
} catch (e) {
console.error(`Could not split the video GUID from URL: ${e.message}`);
process.exit(25);
}
if (guid) {
videoGuids.push(guid);
}
}
console.log(videoGuids);
return videoGuids;
}
async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
console.log(videoUrls);
const videoGuids = extractVideoGuid(videoUrls);
console.log('Fetching title and HLS URL...');
let metadata: Metadata[] = await getVideoMetadata(videoGuids, session);
await Promise.all(metadata.map(async video => {
video.title = sanitize(video.title);
console.log(colors.blue(`\nDownloading Video: ${video.title}\n`));
// Very experimental inline thumbnail rendering
await drawThumbnail(video.posterImage, session.AccessToken);
console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n');
const outputPath = outputDirectory + path.sep + video.title + '.mp4';
// TODO: Remove this mess and it's fluent-ffmpeg dependency
//
// ffmpeg()
// .input(video.playbackUrl)
// .inputOption([
// // Never remove those "useless" escapes or ffmpeg will not
// // pick up the header correctly
// // eslint-disable-next-line no-useless-escape
// '-headers', `Authorization:\ Bearer\ ${session.AccessToken}`
// ])
// .format('mp4')
// .saveToFile(outputPath)
// .on('codecData', data => {
// console.log(`Input is ${data.video} with ${data.audio} audio.`);
// })
// .on('progress', progress => {
// console.log(progress);
// })
// .on('error', err => {
// console.log(`ffmpeg returned an error: ${err.message}`);
// })
// .on('end', () => {
// console.log(`Download finished: ${outputPath}`);
// });
// We probably need a way to be deterministic about
// how we locate that ffmpeg-bar wrapper, npx maybe?
// Do not remove those "useless" escapes or ffmpeg will
// not pick up the header correctly.
// eslint-disable-next-line no-useless-escape
let cmd = `node_modules/.bin/ffmpeg-bar -headers "Authorization:\ Bearer\ ${session.AccessToken}" -i "${video.playbackUrl}" -y "${outputPath}"`;
execSync(cmd, {stdio: 'inherit'});
console.info(`Download finished: ${outputPath}`);
}));
}
// FIXME
process.on('unhandledRejection', (reason) => {
console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
console.error(colors.red(reason as string));
throw new Error('Killing process..\n');
});
async function main() {
const isValidUser = !(await isElevated());
let videoUrls: string[];
if (!isValidUser) {
const usrName = os.platform() === 'win32' ? 'Admin':'root';
console.error(colors.red('\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n'));
process.exit(-1);
}
videoUrls = parseVideoUrls(argv.videoUrls);
if (videoUrls.length === 0) {
console.error(colors.red('\nERROR: No valid URL has been found!\n'));
process.exit(-1);
}
checkRequirements();
let session = tokenCache.Read();
if (session == null) {
session = await DoInteractiveLogin(videoUrls[0], argv.username);
}
init();
downloadVideo(videoUrls, argv.outputDirectory, session);
}
// run
main();
import { sleep, parseVideoUrls, checkRequirements, makeUniqueTitle } from './utils';
import { TokenCache } from './TokenCache';
import { getVideoMetadata } from './Metadata';
import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import { execSync } from 'child_process';
import colors from 'colors';
import fs from 'fs';
import path from 'path';
import yargs from 'yargs';
import sanitize from 'sanitize-filename';
import ffmpeg from 'fluent-ffmpeg';
/**
* exitCode 22 = ffmpeg not found in $PATH
* exitCode 25 = cannot split videoID from videUrl
* exitCode 27 = no hlsUrl in the API response
* exitCode 29 = invalid response from API
* exitCode 88 = error extracting cookies
*/
let tokenCache = new TokenCache();
const argv = yargs.options({
username: {
alias: 'u',
type: 'string',
demandOption: false
},
outputDirectory: {
alias: 'o',
type: 'string',
default: 'videos',
demandOption: false
},
videoUrls: {
alias: 'V',
describe: 'List of video urls or path to txt file containing the urls',
type: 'array',
demandOption: true
},
simulate: {
alias: 's',
describe: `Disable video download and print metadata information to the console`,
type: 'boolean',
default: false,
demandOption: false
},
noThumbnails: {
alias: 'nthumb',
describe: `Do not display video thumbnails`,
type: 'boolean',
default: false,
demandOption: false
},
verbose: {
alias: 'v',
describe: `Print additional information to the console (use this before opening an issue on GitHub)`,
type: 'boolean',
default: false,
demandOption: false
}
}).argv;
async function init() {
const isValidUser = !(await isElevated());
if (!isValidUser) {
const usrName = process.platform === 'win32' ? 'Admin':'root';
console.error(colors.red(
'\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n'
));
process.exit(-1);
}
// create output directory
if (!fs.existsSync(argv.outputDirectory)) {
console.log('Creating output directory: ' +
process.cwd() + path.sep + argv.outputDirectory);
fs.mkdirSync(argv.outputDirectory);
}
console.info('Output Directory: %s', argv.outputDirectory);
if (argv.username)
console.info('Username: %s', argv.username);
if (argv.simulate)
console.info(colors.yellow('Simulate mode, there will be no video download.\n'));
if (argv.verbose) {
console.info('Video URLs:');
console.info(argv.videoUrls);
}
}
async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
let videoId = url.split("/").pop() ?? (
console.log('Couldn\'t split the video Id from the first videoUrl'), process.exit(25)
);
console.log('Launching headless Chrome to perform the OpenID Connect dance...');
const browser = await puppeteer.launch({
headless: false,
args: ['--disable-dev-shm-usage']
});
const page = (await browser.pages())[0];
console.log('Navigating to login page...');
await page.goto(url, { waitUntil: 'load' });
await page.waitForSelector('input[type="email"]');
if (username) {
await page.keyboard.type(username);
await page.click('input[type="submit"]');
}
await browser.waitForTarget(target => target.url().includes(videoId), { timeout: 150000 });
console.info('We are logged in.');
let sessionInfo: any;
let session = await page.evaluate(
() => {
return {
AccessToken: sessionInfo.AccessToken,
ApiGatewayUri: sessionInfo.ApiGatewayUri,
ApiGatewayVersion: sessionInfo.ApiGatewayVersion
};
}
);
tokenCache.Write(session);
console.log('Wrote access token to token cache.');
console.log("At this point Chromium's job is done, shutting it down...\n");
await browser.close();
return session;
}
function extractVideoGuid(videoUrls: string[]): string[] {
let videoGuids: string[] = [];
let guid: string | undefined = '';
for (const url of videoUrls) {
try {
guid = url.split('/').pop();
} catch (e) {
console.error(`Could not split the video GUID from URL: ${e.message}`);
process.exit(25);
}
if (guid)
videoGuids.push(guid);
}
if (argv.verbose) {
console.info('Video GUIDs:');
console.info(videoGuids);
}
return videoGuids;
}
async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
const videoGuids = extractVideoGuid(videoUrls);
console.log('Fetching metadata...');
const metadata: Metadata[] = await getVideoMetadata(videoGuids, session, argv.verbose);
if (argv.simulate) {
metadata.forEach(video => {
console.log(
colors.yellow('\n\nTitle: ') + colors.green(video.title) +
colors.yellow('\nPublished Date: ') + colors.green(video.date) +
colors.yellow('\nPlayback URL: ') + colors.green(video.playbackUrl)
);
});
return;
}
await Promise.all(metadata.map(async video => {
console.log(colors.blue(`\nDownloading Video: ${video.title}\n`));
video.title = makeUniqueTitle(sanitize(video.title) + ' - ' + video.date, argv.outputDirectory);
// Very experimental inline thumbnail rendering
if (!argv.noThumbnails)
await drawThumbnail(video.posterImage, session.AccessToken);
console.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n');
const outputPath = outputDirectory + path.sep + video.title + '.mp4';
// TODO: Remove this mess and it's fluent-ffmpeg dependency
//
// ffmpeg()
// .input(video.playbackUrl)
// .inputOption([
// // Never remove those "useless" escapes or ffmpeg will not
// // pick up the header correctly
// // eslint-disable-next-line no-useless-escape
// '-headers', `Authorization:\ Bearer\ ${session.AccessToken}`
// ])
// .format('mp4')
// .saveToFile(outputPath)
// .on('codecData', data => {
// console.log(`Input is ${data.video} with ${data.audio} audio.`);
// })
// .on('progress', progress => {
// console.log(progress);
// })
// .on('error', err => {
// console.log(`ffmpeg returned an error: ${err.message}`);
// })
// .on('end', () => {
// console.log(`Download finished: ${outputPath}`);
// });
// We probably need a way to be deterministic about
// how we locate that ffmpeg-bar wrapper, npx maybe?
// Do not remove those "useless" escapes or ffmpeg will
// not pick up the header correctly.
// eslint-disable-next-line no-useless-escape
let cmd = `node_modules/.bin/ffmpeg-bar -headers "Authorization:\ Bearer\ ${session.AccessToken}" -i "${video.playbackUrl}" -y "${outputPath}"`;
execSync(cmd, {stdio: 'inherit'});
console.info(`Download finished: ${outputPath}`);
}));
}
// FIXME
process.on('unhandledRejection', (reason) => {
console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
console.error(colors.red(reason as string));
throw new Error('Killing process..\n');
});
async function main() {
checkRequirements();
await init();
const videoUrls: string[] = parseVideoUrls(argv.videoUrls);
if (videoUrls.length === 0) {
console.error(colors.red('\nERROR: No valid URL has been found!\n'));
process.exit(-1);
}
let session = tokenCache.Read();
if (session == null) {
session = await DoInteractiveLogin(videoUrls[0], argv.username);
}
downloadVideo(videoUrls, argv.outputDirectory, session);
}
// run
main();

5
package-lock.json generated
View File

@@ -133,6 +133,11 @@
"integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==",
"dev": true
},
"@types/jwt-decode": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/@types/jwt-decode/-/jwt-decode-2.2.1.tgz",
"integrity": "sha512-aWw2YTtAdT7CskFyxEX2K21/zSDStuf/ikI3yBqmwpwJF0pS+/IX5DWv+1UFffZIbruP6cnT9/LAJV1gFwAT1A=="
},
"@types/mime-types": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz",

1
package.json
View File

@@ -29,6 +29,7 @@
},
"dependencies": {
"@types/fluent-ffmpeg": "^2.1.14",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.19.2",
"colors": "^1.4.0",
"ffmpeg-progressbar-cli": "^1.5.0",

13
utils.ts
View File

@@ -1,6 +1,7 @@
import { execSync } from 'child_process';
import colors from 'colors';
import fs from 'fs';
import path from 'path';
function sanitizeUrls(urls: string[]) {
const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i');
@@ -13,7 +14,7 @@ function sanitizeUrls(urls: string[]) {
if (!rex.test(url)) {
if (url !== '')
console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..\n'));
console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..'));
continue;
}
@@ -57,3 +58,13 @@ export function checkRequirements() {
process.exit(22);
}
}
export function makeUniqueTitle(title: string, outDir: string) {
let ntitle = title;
let k = 0;
while (fs.existsSync(outDir + path.sep + ntitle + '.mp4'))
ntitle = title + ' - ' + (++k).toString();
return ntitle;
}