arona/destreamer-mirror
1
0
Fork 0
mirror of https://github.com/snobu/destreamer.git synced 2026-01-25 01:12:18 +00:00
Code Issues Projects Releases Wiki Activity

Sync tokencache and dev branches

Browse Source
This commit is contained in:
kylon
2020-04-10 12:21:41 +02:00
parent f534ca160b
commit 48f12f873f
6 changed files with 701 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
README.md
View File

@@ -6,7 +6,7 @@
![](logo.png)
## Saves Microsoft Stream videos for offline enjoyment.
## Saves Microsoft Stream videos for offline enjoyment
Alpha-quality, don't expect much. It does work though, so that's a neat feature.
@@ -36,7 +36,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
* **Node.js**: anything above v8.0 seems to work. A GitHub Action runs tests on all major Node versions on every commit.
* **ffmpeg**: a recent version (year 2019 or above), in `$PATH` or in the same directory as `destreamer.ts`.
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on, tested on Windows, results may vary, feel free to open an issue if trouble arise.
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on, tested on Windows, macOS and Linux, results may vary, feel free to open an issue if trouble arise.
## USAGE
@@ -49,27 +49,39 @@ $ node ./destreamer.js
Options:
--help Show help [boolean]
--version Show version number [boolean]
--videoUrls [array] [required]
--username [string]
--outputDirectory [string] [default: "videos"]
--videoUrls, -V List of video urls or path to txt file containing the urls
[array] [required]
--username, -u [string]
--outputDirectory, -o [string] [default: "videos"]
--verbose, -v Print additional information to the console
(use this before opening an issue on GitHub)
[boolean] [default: false]
```
Make sure you use the right escape char for your shell if using line breaks (as this example shows).
For PowerShell your escape char is the backtick (`) instead of backslash (\\), for cmd.exe use caret (^).
```
$ node destreamer.js --username username@example.com --outputDirectory "videos" \
--videoUrls "https://web.microsoftstream.com/video/VIDEO-1" \
"https://web.microsoftstream.com/video/VIDEO-2" \
"https://web.microsoftstream.com/video/VIDEO-3"
```
You can create a `.txt` file containing your video URLs, one video per line. The text file can have any name, followed by the `.txt` extension. Run destreamer as follows:
```
$ node destreamer.js --username username@example.com --outputDirectory "videos" \
--videoUrls list.txt
```
Passing `--username` is optional. It's there to make logging in faster (the username field will be populated automatically on the login form).
You can use an absolute path for `--outputDirectory`, for example `/mnt/videos`.
Your video URLs **must** include the URL schema (the leading `https://`).
## RANDOM NOTE
## IMPORTANT NOTE
Just ignore this error, we already have what we need to start the download, no time to deal with collaterals -
![image](https://user-images.githubusercontent.com/6472374/77905069-4c585000-728e-11ea-914e-26f1ce5e595b.png)
@@ -81,4 +93,4 @@ Just ignore this error, we already have what we need to start the download, no t
<<<< OUTPUT >>>>
```
The video is now saved under `videos/`, or the path from `--outputDirectory`.
The video is now saved under `videos/`, or whatever the `outputDirectory` const points to.

140
destreamer.ts
View File

@@ -1,19 +1,21 @@
import { sleep, getVideoUrls, checkRequirements } from './utils';
import { TokenCache } from './TokenCache';
import { getVideoMetadata } from './Metadata';
import { Metadata, Session } from './Types';
import { drawThumbnail } from './Thumbnail';
import { execSync } from 'child_process';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import colors from 'colors';
import fs from 'fs';
import os from 'os';
import path from 'path';
import yargs from 'yargs';
import sanitize from 'sanitize-filename';
import ffmpeg from 'fluent-ffmpeg';
/**
* exitCode 22 = ffmpeg not found in $PATH
* exitCode 25 = cannot split videoID from videUrl
* exitCode 27 = no hlsUrl in the API response
* exitCode 29 = invalid response from API
@@ -23,9 +25,23 @@ import ffmpeg from 'fluent-ffmpeg';
let tokenCache = new TokenCache();
const argv = yargs.options({
videoUrls: { type: 'array', alias: 'videourls', demandOption: true },
username: { type: 'string', demandOption: false },
outputDirectory: { type: 'string', alias: 'outputdirectory', default: 'videos' },
username: {
alias: 'u',
type: 'string',
demandOption: false
},
outputDirectory: {
alias: 'o',
type: 'string',
default: 'videos',
demandOption: false
},
videoUrls: {
alias: 'V',
describe: 'List of video urls or path to txt file containing the urls',
type: 'array',
demandOption: true
},
simulate: {
alias: 's',
describe: `If this is set to true no video will be downloaded and the script
@@ -34,37 +50,31 @@ const argv = yargs.options({
default: false,
demandOption: false
},
verbose: {
alias: 'v',
describe: `Print additional information to the console
(use this before opening an issue on GitHub)`,
type: 'boolean',
default: false,
demandOption: false
}
}).argv;
if (argv.simulate) {
console.info('Video URLs: %s', argv.videoUrls);
console.info('Username: %s', argv.username);
console.info(colors.green('There will be no video downloaded, it\'s only a simulation\n'));
} else {
console.info('Video URLs: %s', argv.videoUrls);
console.info('Username: %s', argv.username);
console.info('Output Directory: %s', argv.outputDirectory);
console.info('Video/Audio Quality: %s', argv.format);
}
function sanityChecks() {
try {
const ffmpegVer = execSync('ffmpeg -version')
.toString().split('\n')[0];
console.info(colors.green(`Using ${ffmpegVer}\n`));
}
catch (e) {
console.error('FFmpeg is missing. You need a fairly recent release of FFmpeg in $PATH.');
}
function init() {
// create output directory
if (!fs.existsSync(argv.outputDirectory)) {
console.log('Creating output directory: ' +
process.cwd() + path.sep + argv.outputDirectory);
fs.mkdirSync(argv.outputDirectory);
}
}
console.info('Video URLs: %s', argv.videoUrls);
console.info('Username: %s', argv.username);
console.info('Output Directory: %s', argv.outputDirectory);
if (argv.simulate)
console.info(colors.blue("There will be no video downloaded, it's only a simulation\n"));
}
async function DoInteractiveLogin(username?: string): Promise<Session> {
console.log('Launching headless Chrome to perform the OpenID Connect dance...');
@@ -118,7 +128,6 @@ async function DoInteractiveLogin(username?: string): Promise<Session> {
return session;
}
function extractVideoGuid(videoUrls: string[]): string[] {
const first = videoUrls[0] as string;
const isPath = first.substring(first.length - 4) === '.txt';
@@ -134,8 +143,8 @@ function extractVideoGuid(videoUrls: string[]): string[] {
console.log(url);
try {
guid = url.split('/').pop();
}
catch (e) {
} catch (e) {
console.error(`Could not split the video GUID from URL: ${e.message}`);
process.exit(25);
}
@@ -148,7 +157,6 @@ function extractVideoGuid(videoUrls: string[]): string[] {
return videoGuids;
}
async function downloadVideo(videoUrls: string[], outputDirectory: string, session: Session) {
console.log(videoUrls);
const videoGuids = extractVideoGuid(videoUrls);
@@ -173,39 +181,59 @@ async function downloadVideo(videoUrls: string[], outputDirectory: string, sessi
// pick up the header correctly
// eslint-disable-next-line no-useless-escape
'-headers', `Authorization:\ Bearer\ ${session.AccessToken}`
])
.format('mp4')
.saveToFile(outputPath)
.on('codecData', data => {
console.log(`Input is ${data.video} with ${data.audio} audio.`);
})
.on('progress', progress => {
console.log(progress);
})
.on('error', err => {
console.log(`ffmpeg returned an error: ${err.message}`);
})
.on('end', () => {
console.log(`Download finished: ${outputPath}`);
});
])
.format('mp4')
.saveToFile(outputPath)
.on('codecData', data => {
console.log(`Input is ${data.video} with ${data.audio} audio.`);
})
.on('progress', progress => {
console.log(progress);
})
.on('error', err => {
console.log(`ffmpeg returned an error: ${err.message}`);
})
.on('end', () => {
console.log(`Download finished: ${outputPath}`);
});
}));
}
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// FIXME
process.on('unhandledRejection', (reason) => {
console.error(colors.red('Unhandled error!\nTimeout or fatal error, please check your downloads and try again if necessary.\n'));
console.error(colors.red(reason as string));
throw new Error('Killing process..\n');
});
async function main() {
sanityChecks();
const isValidUser = !(await isElevated());
let videoUrls: string[];
if (!isValidUser) {
const usrName = os.platform() === 'win32' ? 'Admin':'root';
console.error(colors.red('\nERROR: Destreamer does not run as '+usrName+'!\nPlease run destreamer with a non-privileged user.\n'));
process.exit(-1);
}
videoUrls = getVideoUrls(argv.videoUrls);
if (videoUrls.length === 0) {
console.error(colors.red('\nERROR: No valid URL has been found!\n'));
process.exit(-1);
}
checkRequirements();
let session = tokenCache.Read();
if (session == null) {
session = await DoInteractiveLogin(argv.username);
}
downloadVideo(argv.videoUrls as string[], argv.outputDirectory, session);
init();
downloadVideo(videoUrls, argv.outputDirectory, session);
}
main();
// run
main();

753
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

14
package.json
View File

@@ -17,21 +17,23 @@
"author": "snobu",
"license": "MIT",
"devDependencies": {
"@types/fluent-ffmpeg": "^2.1.14",
"@types/jwt-decode": "^2.2.1",
"@types/puppeteer": "^1.20.0",
"@types/mocha": "^7.0.2",
"@types/puppeteer": "^1.20.4",
"@types/tmp": "^0.1.0",
"@types/yargs": "^15.0.3",
"@typescript-eslint/eslint-plugin": "^2.25.0",
"@typescript-eslint/parser": "^2.25.0",
"eslint": "^6.8.0"
"eslint": "^6.8.0",
"mocha": "^7.1.1",
"tmp": "^0.1.0"
},
"dependencies": {
"@types/mocha": "^7.0.2",
"@types/fluent-ffmpeg": "^2.1.14",
"axios": "^0.19.2",
"colors": "^1.4.0",
"fluent-ffmpeg": "^2.1.2",
"is-elevated": "^3.0.0",
"jwt-decode": "^2.2.0",
"mocha": "^7.1.1",
"puppeteer": "^2.1.1",
"sanitize-filename": "^1.6.3",
"terminal-image": "^0.2.0",

41
test/test.ts
View File

@@ -1,5 +1,8 @@
import { getVideoUrls } from '../utils';
import puppeteer from 'puppeteer';
import assert from 'assert';
import tmp from 'tmp';
import fs from 'fs';
let browser: any;
let page: any;
@@ -23,4 +26,42 @@ describe('Puppeteer', () => {
after(async () => {
await browser.close();
});
describe('Destreamer', () => {
it('should parse and sanitize URL list from file', () => {
const testIn: string[] = [
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd&",
"",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a=b&c",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddd",
"https://web.microsoftstream.com/video/xxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
""
];
const expectedOut: string[] = [
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a=b&c",
"https://web.microsoftstream.com/video/xxxxxxxx-zzzz-hhhh-rrrr-dddddddddddd?a"
];
const tmpFile = tmp.fileSync({ postfix: '.txt' });
let testOut: string[];
fs.writeFileSync(tmpFile.fd, testIn.join('\r\n'));
testOut = getVideoUrls([tmpFile.name]);
if (testOut.length !== expectedOut.length)
assert.strictEqual(testOut, expectedOut, "URL list not sanitized");
for (let i=0, l=testOut.length; i<l; ++i) {
if (testOut[i] !== expectedOut[i])
assert.strictEqual(testOut[i], expectedOut[i], "URL not sanitized");
}
assert.ok("sanitizeUrls ok");
});
});

59
utils.ts Normal file
View File

@@ -0,0 +1,59 @@
import { execSync } from 'child_process';
import colors from 'colors';
import fs from 'fs';
function sanitizeUrls(urls: string[]) {
const rex = new RegExp(/(?:https:\/\/)?.*\/video\/[a-z0-9]{8}-(?:[a-z0-9]{4}\-){3}[a-z0-9]{12}$/, 'i');
const sanitized: string[] = [];
for (let i=0, l=urls.length; i<l; ++i) {
const urlAr = urls[i].split('?');
const query = urlAr.length === 2 && urlAr[1] !== '' ? '?'+urlAr[1] : '';
let url = urlAr[0];
if (!rex.test(url)) {
if (url !== '')
console.warn(colors.yellow('Invalid URL at line ' + (i+1) + ', skip..\n'));
continue;
}
if (url.substring(0, 8) !== 'https://')
url = 'https://'+url;
sanitized.push(url+query);
}
return sanitized;
}
export function getVideoUrls(videoUrls: any) {
const t = videoUrls[0] as string;
const isPath = t.substring(t.length-4) === '.txt';
let urls: string[];
if (isPath)
urls = fs.readFileSync(t).toString('utf-8').split(/[\r\n]/);
else
urls = videoUrls as string[];
return sanitizeUrls(urls);
}
export function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
export function checkRequirements() {
try {
const ffmpegVer = execSync('ffmpeg -version').toString().split('\n')[0];
console.info(colors.green(`Using ${ffmpegVer}\n`));
} catch (e) {
console.error(colors.red(
'FFmpeg is missing.\nDestreamer requires a fairly recent release of FFmpeg to work properly.\n' +
'Please install it with your preferred package manager or copy FFmpeg binary in destreamer root directory.\n'
));
process.exit(22);
}
}