arona/destreamer-mirror
1
0
Fork 0
mirror of https://github.com/snobu/destreamer.git synced 2026-01-17 05:22:18 +00:00
Code Issues Projects Releases Wiki Activity

Sync tokencache and dev branches

Browse Source
This commit is contained in:
kylon
2020-04-10 12:21:41 +02:00
parent f534ca160b
commit 48f12f873f
6 changed files with 701 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
README.md
View File

@@ -6,7 +6,7 @@
![](logo.png) ![](logo.png)
## Saves Microsoft Stream videos for offline enjoyment. ## Saves Microsoft Stream videos for offline enjoyment
Alpha-quality, don't expect much. It does work though, so that's a neat feature. Alpha-quality, don't expect much. It does work though, so that's a neat feature.
@@ -36,7 +36,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
* **Node.js**: anything above v8.0 seems to work. A GitHub Action runs tests on all major Node versions on every commit. * **Node.js**: anything above v8.0 seems to work. A GitHub Action runs tests on all major Node versions on every commit.
* **ffmpeg**: a recent version (year 2019 or above), in `$PATH` or in the same directory as `destreamer.ts`. * **ffmpeg**: a recent version (year 2019 or above), in `$PATH` or in the same directory as `destreamer.ts`.
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on, tested on Windows, results may vary, feel free to open an issue if trouble arise. Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on, tested on Windows, macOS and Linux, results may vary, feel free to open an issue if trouble arise.
## USAGE ## USAGE
@@ -49,27 +49,39 @@ $ node ./destreamer.js
Options: Options:
--help Show help [boolean] --help Show help [boolean]
--version Show version number [boolean] --version Show version number [boolean]
--videoUrls [array] [required] --videoUrls, -V List of video urls or path to txt file containing the urls
--username [string] [array] [required]
--outputDirectory [string] [default: "videos"] --username, -u [string]
--outputDirectory, -o [string] [default: "videos"]
--verbose, -v Print additional information to the console --verbose, -v Print additional information to the console
(use this before opening an issue on GitHub) (use this before opening an issue on GitHub)
[boolean] [default: false] [boolean] [default: false]
```
Make sure you use the right escape char for your shell if using line breaks (as this example shows).
For PowerShell your escape char is the backtick (`) instead of backslash (\\), for cmd.exe use caret (^).
```
$ node destreamer.js --username username@example.com --outputDirectory "videos" \ $ node destreamer.js --username username@example.com --outputDirectory "videos" \
--videoUrls "https://web.microsoftstream.com/video/VIDEO-1" \ --videoUrls "https://web.microsoftstream.com/video/VIDEO-1" \
"https://web.microsoftstream.com/video/VIDEO-2" \ "https://web.microsoftstream.com/video/VIDEO-2" \
"https://web.microsoftstream.com/video/VIDEO-3" "https://web.microsoftstream.com/video/VIDEO-3"
``` ```
You can create a `.txt` file containing your video URLs, one video per line. The text file can have any name, followed by the `.txt` extension. Run destreamer as follows:
```
$ node destreamer.js --username username@example.com --outputDirectory "videos" \
--videoUrls list.txt
```
Passing `--username` is optional. It's there to make logging in faster (the username field will be populated automatically on the login form). Passing `--username` is optional. It's there to make logging in faster (the username field will be populated automatically on the login form).
You can use an absolute path for `--outputDirectory`, for example `/mnt/videos`. You can use an absolute path for `--outputDirectory`, for example `/mnt/videos`.
Your video URLs **must** include the URL schema (the leading `https://`).
## RANDOM NOTE ## RANDOM NOTE
## IMPORTANT NOTE
Just ignore this error, we already have what we need to start the download, no time to deal with collaterals - Just ignore this error, we already have what we need to start the download, no time to deal with collaterals -
![image](https://user-images.githubusercontent.com/6472374/77905069-4c585000-728e-11ea-914e-26f1ce5e595b.png) ![image](https://user-images.githubusercontent.com/6472374/77905069-4c585000-728e-11ea-914e-26f1ce5e595b.png)
@@ -81,4 +93,4 @@ Just ignore this error, we already have what we need to start the download, no t
<<<< OUTPUT >>>> <<<< OUTPUT >>>>
``` ```
The video is now saved under `videos/`, or the path from `--outputDirectory`. The video is now saved under `videos/`, or whatever the `outputDirectory` const points to.

140
destreamer.ts
View File

@@ -1,19 +1,21 @@
import { sleep, getVideoUrls, checkRequirements } from './utils';
import { TokenCache } from './TokenCache'; import { TokenCache } from './TokenCache';
import { getVideoMetadata } from './Metadata'; import { getVideoMetadata } from './Metadata';
import { Metadata, Session } from './Types'; import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail'; import { drawThumbnail } from './Thumbnail';
import { execSync } from 'child_process'; import isElevated from 'is-elevated';
import puppeteer from 'puppeteer'; import puppeteer from 'puppeteer';
import colors from 'colors'; import colors from 'colors';
import fs from 'fs'; import fs from 'fs';
import os from 'os';
import path from 'path'; import path from 'path';
import yargs from 'yargs'; import yargs from 'yargs';
import sanitize from 'sanitize-filename'; import sanitize from 'sanitize-filename';
import ffmpeg from 'fluent-ffmpeg'; import ffmpeg from 'fluent-ffmpeg';
/** /**
* exitCode 22 = ffmpeg not found in $PATH
* exitCode 25 = cannot split videoID from videUrl * exitCode 25 = cannot split videoID from videUrl
* exitCode 27 = no hlsUrl in the API response * exitCode 27 = no hlsUrl in the API response
* exitCode 29 = invalid response from API * exitCode 29 = invalid response from API
@@ -23,9 +25,23 @@ import ffmpeg from 'fluent-ffmpeg';
let tokenCache = new TokenCache(); let tokenCache = new TokenCache();
const argv = yargs.options({ const argv = yargs.options({
videoUrls: { type: 'array', alias: 'videourls', demandOption: true }, username: {
username: { type: 'string', demandOption: false }, alias: 'u',
outputDirectory: { type: 'string', alias: 'outputdirectory', default: 'videos' }, type: 'string',
demandOption: false
},
outputDirectory: {
alias: 'o',
type: 'string',
default: 'videos',
demandOption: false
},
videoUrls: {
alias: 'V',
describe: 'List of video urls or path to txt file containing the urls',
type: 'array',
demandOption: true
},
simulate: { simulate: {
alias: 's', alias: 's',
describe: `If this is set to true no video will be downloaded and the script describe: `If this is set to true no video will be downloaded and the script
@@ -34,37 +50,31 @@ const argv = yargs.options({
default: false, default: false,
demandOption: false demandOption: false
}, },
verbose: {
alias: 'v',
describe: `Print additional information to the console
(use this before opening an issue on GitHub)`,
type: 'boolean',
default: false,
demandOption: false
}
}).argv; }).argv;
if (argv.simulate) { function init() {
console.info('Video URLs: %s', argv.videoUrls); // create output directory
console.info('Username: %s', argv.username);
console.info(colors.green('There will be no video downloaded, it\'s only a simulation\n'));
} else {
console.info('Video URLs: %s', argv.videoUrls);
console.info('Username: %s', argv.username);
console.info('Output Directory: %s', argv.outputDirectory);
console.info('Video/Audio Quality: %s', argv.format);
}
function sanityChecks() {
try {
const ffmpegVer = execSync('ffmpeg -version')
.toString().split('\n')[0];
console.info(colors.green(`Using ${ffmpegVer}\n`));
}
catch (e) {
console.error('FFmpeg is missing. You need a fairly recent release of FFmpeg in $PATH.');
}
if (!fs.existsSync(argv.outputDirectory)) { if (!fs.existsSync(argv.outputDirectory)) {
console.log('Creating output directory: ' + console.log('Creating output directory: ' +
process.cwd() + path.sep + argv.outputDirectory); process.cwd() + path.sep + argv.outputDirectory);
fs.mkdirSync(argv.outputDirectory); fs.mkdirSync(argv.outputDirectory);
} }
}
console.info('Video URLs: %s', argv.videoUrls);
console.info('Username: %s', argv.username);
console.info('Output Directory: %s', argv.outputDirectory);
if (argv.simulate)
console.info(colors.blue("There will be no video downloaded, it's only a simulation\n"));
}
async function DoInteractiveLogin(username?: string): Promise<Session> { async function DoInteractiveLogin(username?: string): Promise<Session> {
console.log('Launching headless Chrome to perform the OpenID Connect dance...'); console.log('Launching headless Chrome to perform the OpenID Connect dance...');
@@ -118,7 +128,6 @@ async function DoInteractiveLogin(username?: string): Promise<Session> {
return session; return session;
} }
function extractVideoGuid(videoUrls: string[]): string[] { function extractVideoGuid(videoUrls: string[]): string[] {
const first = videoUrls[0] as string; const first = videoUrls[0] as string;
const isPath = first.substring(first.length - 4) === '.txt'; const isPath = first.substring(first.length - 4) === '.txt';
@@ -134,8 +143,8 @@ function extractVideoGuid(videoUrls: string[]): string[] {
console.log(url); console.log(url);
try { try {
guid = url.split('/').pop(); guid = url.split('/').pop();
}
catch (e) { } catch (e) {
console.error(`Could not split the video GUID from URL: ${e.message}`); console.error(`Could not split the video GUID from URL: ${e.message}`);
process.exit(25); process.exit(25);
} }
@@ -148,7 +157,6 @@ function extractVideoGuid(videoUrls: string[]): string[] {
return videoGuids; return videoGuids;
} }
async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) { async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
console.log(videoUrls); console.log(videoUrls);
const videoGuids = extractVideoGuid(videoUrls); const videoGuids = extractVideoGuid(videoUrls);
@@ -173,39 +181,59 @@ async function downloadVideo(videoUrls: string[], outputDirectory: string, sessi
// pick up the header correctly // pick up the header correctly
// eslint-disable-next-line no-useless-escape // eslint-disable-next-line no-useless-escape
'-headers', `Authorization:\ Bearer\ ${session.AccessToken}` '-headers', `Authorization:\ Bearer\ ${session.AccessToken}`
]) ])
.format('mp4') .format('mp4')
.saveToFile(outputPath) .saveToFile(outputPath)
.on('codecData', data => { .on('codecData', data => {
console.log(`Input is ${data.video} with ${data.audio} audio.`); console.log(`Input is ${data.video} with ${data.audio} audio.`);
}) })
.on('progress', progress => { .on('progress', progress => {
console.log(progress); console.log(progress);
}) })
.on('error', err => { .on('error', err => {
console.log(`ffmpeg returned an error: ${err.message}`); console.log(`ffmpeg returned an error: ${err.message}`);
}) })
.on('end', () => { .on('end', () => {
console.log(`Download finished: ${outputPath}`); console.log(`Download finished: ${outputPath}`);
}); });
})); }));
} }
// FIXME
function sleep(ms: number) { process.on('unhandledRejection', (reason) => {
return new Promise(resolve => setTimeout(resolve, ms)); console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
} console.error(colors.red(reason as string));
throw new Error('Killing process..\n');
});
async function main() { async function main() {
sanityChecks(); const isValidUser = !(await isElevated());
let videoUrls: string[];
if (!isValidUser) {
const usrName = os.platform() === 'win32' ? 'Admin':'root';
console.error(colors.red('\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n'));
process.exit(-1);
}
videoUrls = getVideoUrls(argv.videoUrls);
if (videoUrls.length === 0) {
console.error(colors.red('\nERROR: No valid URL has been found!\n'));
process.exit(-1);
}
checkRequirements();
let session = tokenCache.Read(); let session = tokenCache.Read();
if (session == null) { if (session == null) {
session = await DoInteractiveLogin(argv.username); session = await DoInteractiveLogin(argv.username);
} }
downloadVideo(argv.videoUrls as string[], argv.outputDirectory, session);
init();
downloadVideo(videoUrls, argv.outputDirectory, session);
} }
main(); // run
main();

753
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

14
package.json
View File

@@ -17,21 +17,23 @@
"author": "snobu", "author": "snobu",
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
"@types/fluent-ffmpeg": "^2.1.14", "@types/mocha": "^7.0.2",
"@types/jwt-decode": "^2.2.1", "@types/puppeteer": "^1.20.4",
"@types/puppeteer": "^1.20.0", "@types/tmp": "^0.1.0",
"@types/yargs": "^15.0.3", "@types/yargs": "^15.0.3",
"@typescript-eslint/eslint-plugin": "^2.25.0", "@typescript-eslint/eslint-plugin": "^2.25.0",
"@typescript-eslint/parser": "^2.25.0", "@typescript-eslint/parser": "^2.25.0",
"eslint": "^6.8.0" "eslint": "^6.8.0",
"mocha": "^7.1.1",
"tmp": "^0.1.0"
}, },
"dependencies": { "dependencies": {
"@types/mocha": "^7.0.2", "@types/fluent-ffmpeg": "^2.1.14",
"axios": "^0.19.2", "axios": "^0.19.2",
"colors": "^1.4.0", "colors": "^1.4.0",
"fluent-ffmpeg": "^2.1.2", "fluent-ffmpeg": "^2.1.2",
"is-elevated": "^3.0.0",
"jwt-decode": "^2.2.0", "jwt-decode": "^2.2.0",
"mocha": "^7.1.1",
"puppeteer": "^2.1.1", "puppeteer": "^2.1.1",
"sanitize-filename": "^1.6.3", "sanitize-filename": "^1.6.3",
"terminal-image": "^0.2.0", "terminal-image": "^0.2.0",

41
test/test.ts
View File

@@ -1,5 +1,8 @@
import { getVideoUrls } from '../utils';
import puppeteer from 'puppeteer'; import puppeteer from 'puppeteer';
import assert from 'assert'; import assert from 'assert';
import tmp from 'tmp';
import fs from 'fs';
let browser: any; let browser: any;
let page: any; let page: any;
@@ -23,4 +26,42 @@ describe('Puppeteer', () => {
after(async () => { after(async () => {
await browser.close(); await browser.close();
});
describe('Destreamer', () => {
it('should parse and sanitize URL list from file', () => {
const testIn: string[] = [
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd&",
"",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a=b&c",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddd",
"https://web.microsoftstream.com/video/xxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
""
];
const expectedOut: string[] = [
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a=b&c",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a"
];
const tmpFile = tmp.fileSync({ postfix: '.txt' });
let testOut: string[];
fs.writeFileSync(tmpFile.fd, testIn.join('\r\n'));
testOut = getVideoUrls([tmpFile.name]);
if (testOut.length !== expectedOut.length)
assert.strictEqual(testOut, expectedOut, "URL list not sanitized");
for (let i=0, l=testOut.length; i<l; ++i) {
if (testOut[i] !== expectedOut[i])
assert.strictEqual(testOut[i], expectedOut[i], "URL not sanitized");
}
assert.ok("sanitizeUrls ok");
});
}); });

59
utils.ts Normal file
View File

@@ -0,0 +1,59 @@
import { execSync } from 'child_process';
import colors from 'colors';
import fs from 'fs';
function sanitizeUrls(urls: string[]) {
const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i');
const sanitized: string[] = [];
for (let i=0, l=urls.length; i<l; ++i) {
const urlAr = urls[i].split('?');
const query = urlAr.length === 2 && urlAr[1] !== '' ? '?'+urlAr[1] : '';
let url = urlAr[0];
if (!rex.test(url)) {
if (url !== '')
console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..\n'));
continue;
}
if (url.substring(0, 8) !== 'https://')
url = 'https://'+url;
sanitized.push(url+query);
}
return sanitized;
}
export function getVideoUrls(videoUrls: any) {
const t = videoUrls[0] as string;
const isPath = t.substring(t.length-4) === '.txt';
let urls: string[];
if (isPath)
urls = fs.readFileSync(t).toString('utf-8').split(/[\r\n]/);
else
urls = videoUrls as string[];
return sanitizeUrls(urls);
}
export function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
export function checkRequirements() {
try {
const ffmpegVer = execSync('ffmpeg -version').toString().split('\n')[0];
console.info(colors.green(`Using ${ffmpegVer}\n`));
} catch (e) {
console.error(colors.red(
'FFmpeg is missing.\nDestreamer requires a fairly recent release of FFmpeg to work properly.\n' +
'Please install it with your preferred package manager or copy FFmpeg binary in destreamer root directory.\n'
));
process.exit(22);
}
}