arona/destreamer-mirror
1
0
Fork 0
mirror of https://github.com/snobu/destreamer.git synced 2026-01-18 22:12:16 +00:00
Code Issues Projects Releases Wiki Activity

Merge e93ad80ef6 into 516ca54ff1

Browse Source
This commit is contained in:
lukaarma
2024-08-31 04:41:28 +08:00
committed by GitHub
parent 516ca54ff1 e93ad80ef6
commit 38b77d8060
15 changed files with 1000 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -3,10 +3,12 @@
*.log
*.js
*.zip
*.xml
yarn.lock
.chrome_data
node_modules
videos
release
build
yarn.lock

2
README.md
View File

@@ -43,6 +43,7 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
- [**Node.js**][node]: You'll need Node.js version 8.0 or higher. A GitHub Action runs tests on all major Node versions on every commit. One caveat for Node 8, if you get a `Parse Error` with `code: HPE_HEADER_OVERFLOW` you're out of luck and you'll need to upgrade to Node 10+. PLEASE NOTE WE NO LONGER TEST BUILDS AGAINST NODE 8.x. YOU ARE ON YOUR OWN.
- **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
- [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
- [**aria2**][aria2]: aria2 is a utility for downloading files with multiple threads, fast.
- [**git**][git]: one or more npm dependencies require git.
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on. We've successfully tested it on Windows, macOS and Linux.
@@ -241,6 +242,7 @@ Please open an [issue](https://github.com/snobu/destreamer/issues) and we'll loo
[ffmpeg]: https://www.ffmpeg.org/download.html
[aria2]: https://github.com/aria2/aria2/releases
[xming]: https://sourceforge.net/projects/xming/
[node]: https://nodejs.org/en/download/
[git]: https://git-scm.com/downloads

97
package.json
View File

@@ -1,50 +1,51 @@
{
"name": "destreamer",
"repository": {
"type": "git",
"url": "git://github.com/snobu/destreamer.git"
},
"version": "2.1.0",
"description": "Save Microsoft Stream videos for offline enjoyment.",
"main": "build/src/destreamer.js",
"bin": "build/src/destreamer.js",
"scripts": {
"build": "echo Transpiling TypeScript to JavaScript... && node node_modules/typescript/bin/tsc && echo Destreamer was built successfully.",
"test": "mocha build/test",
"lint": "eslint src/*.ts"
},
"keywords": [],
"author": "snobu",
"license": "MIT",
"devDependencies": {
"@types/mocha": "^8.0.4",
"@types/puppeteer": "^5.4.0",
"@types/readline-sync": "^1.4.3",
"@types/tmp": "^0.2.0",
"@types/yargs": "^15.0.11",
"@typescript-eslint/eslint-plugin": "^4.9.0",
"@typescript-eslint/parser": "^4.9.0",
"eslint": "^7.14.0",
"mocha": "^8.2.1",
"tmp": "^0.2.1"
},
"dependencies": {
"@tedconf/fessonia": "^2.1.2",
"@types/cli-progress": "^3.8.0",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.21.2",
"axios-retry": "^3.1.9",
"cli-progress": "^3.8.2",
"colors": "^1.4.0",
"is-elevated": "^3.0.0",
"iso8601-duration": "^1.3.0",
"jwt-decode": "^3.1.2",
"puppeteer": "5.5.0",
"readline-sync": "^1.4.10",
"sanitize-filename": "^1.6.3",
"terminal-image": "^1.2.1",
"typescript": "^4.1.2",
"winston": "^3.3.3",
"yargs": "^16.1.1"
}
"name": "destreamer",
"repository": {
"type": "git",
"url": "git://github.com/snobu/destreamer.git"
},
"version": "2.1.0",
"description": "Save Microsoft Stream videos for offline enjoyment.",
"main": "build/src/destreamer.js",
"bin": "build/src/destreamer.js",
"scripts": {
"build": "echo Transpiling TypeScript to JavaScript... && tsc && echo Destreamer was built successfully.",
"watch": "tsc --watch",
"test": "mocha build/test",
"lint": "eslint src/*.ts"
},
"keywords": [],
"author": "snobu",
"license": "MIT",
"devDependencies": {
"@types/mocha": "^8.0.4",
"@types/puppeteer": "^5.4.0",
"@types/readline-sync": "^1.4.3",
"@types/tmp": "^0.2.0",
"@types/yargs": "^15.0.11",
"@typescript-eslint/eslint-plugin": "^4.9.0",
"@typescript-eslint/parser": "^4.9.0",
"eslint": "^7.14.0",
"mocha": "^8.2.1",
"tmp": "^0.2.1"
},
"dependencies": {
"@tedconf/fessonia": "^2.1.2",
"@types/cli-progress": "^3.8.0",
"@types/jwt-decode": "^2.2.1",
"axios": "^0.21.2",
"axios-retry": "^3.1.9",
"cli-progress": "^3.8.2",
"colors": "^1.4.0",
"is-elevated": "^3.0.0",
"iso8601-duration": "^1.3.0",
"jwt-decode": "^3.1.2",
"puppeteer": "5.5.0",
"readline-sync": "^1.4.10",
"sanitize-filename": "^1.6.3",
"terminal-image": "^1.2.1",
"typescript": "^4.1.2",
"winston": "^3.3.3",
"yargs": "^16.1.1"
}
}

155
src/ApiClient.ts
View File

@@ -1,16 +1,18 @@
import { logger } from './Logger';
import { Session } from './Types';
import { ShareSession, StreamSession, Video } from './Types';
import { publishedDateToString, publishedTimeToString } from './VideoUtils';
import axios, { AxiosRequestConfig, AxiosResponse, AxiosInstance, AxiosError } from 'axios';
import axiosRetry, { isNetworkOrIdempotentRequestError } from 'axios-retry';
// import fs from 'fs';
export class ApiClient {
private static instance: ApiClient;
export class StreamApiClient {
private static instance: StreamApiClient;
private axiosInstance?: AxiosInstance;
private session?: Session;
private session?: StreamSession;
private constructor(session?: Session) {
private constructor(session?: StreamSession) {
this.session = session;
this.axiosInstance = axios.create({
baseURL: session?.ApiGatewayUri,
@@ -50,16 +52,16 @@ export class ApiClient {
*
* @param session used if initializing
*/
public static getInstance(session?: Session): ApiClient {
if (!ApiClient.instance) {
ApiClient.instance = new ApiClient(session);
public static getInstance(session?: StreamSession): StreamApiClient {
if (!StreamApiClient.instance) {
StreamApiClient.instance = new StreamApiClient(session);
}
return ApiClient.instance;
return StreamApiClient.instance;
}
public setSession(session: Session): void {
if (!ApiClient.instance) {
public setSession(session: StreamSession): void {
if (!StreamApiClient.instance) {
logger.warn("Trying to update ApiCient session when it's not initialized!");
}
@@ -113,3 +115,134 @@ export class ApiClient {
});
}
}
export class ShareApiClient {
private axiosInstance: AxiosInstance;
private site: string;
public constructor(domain: string, site: string, session: ShareSession) {
this.axiosInstance = axios.create({
baseURL: domain,
// timeout: 7000,
headers: {
'User-Agent': 'destreamer/3.0 ALPHA',
'Cookie': `rtFa=${session.rtFa}; FedAuth=${session.FedAuth}`
}
});
this.site = site;
// FIXME: disabled because it was messing with the direct download check
// axiosRetry(this.axiosInstance, {
// // The following option is not working.
// // We should open an issue on the relative GitHub
// shouldResetTimeout: true,
// retries: 6,
// retryDelay: (retryCount: number) => {
// return retryCount * 2000;
// },
// retryCondition: (err: AxiosError) => {
// const retryCodes: Array<number> = [429, 500, 502, 503];
// if (isNetworkOrIdempotentRequestError(err)) {
// logger.warn(`${err}. Retrying request...`);
// return true;
// }
// logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}.`);
// logger.warn('Here is the error message: ');
// console.dir(err.response?.data);
// logger.warn('We called this URL: ' + err.response?.config.baseURL + err.response?.config.url);
// const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
// return shouldRetry;
// }
// });
}
public async getVideoInfo(filePath: string, outPath: string): Promise<Video> {
let playbackUrl: string;
// TODO: Ripped this straigth from chromium inspector. Don't know don't care what it is right now. Check later
const payload = {
parameters: {
__metadata: {
type: 'SP.RenderListDataParameters'
},
ViewXml: `<View Scope="RecursiveAll"><Query><Where><Eq><FieldRef Name="FileRef" /><Value Type="Text"><![CDATA[${filePath}]]></Value></Eq></Where></Query><RowLimit Paged="TRUE">1</RowLimit></View>`,
RenderOptions: 12295,
AddRequiredFields: true
}
};
const url = `${this.site}/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream?@a1='${encodeURIComponent(filePath)}'`;
logger.verbose(`Requesting video info for '${url}'`);
const info = await this.axiosInstance.post(url, payload, {
headers: {
'Content-Type': 'application/json;odata=verbose'
}
}).then(res => res.data);
// fs.writeFileSync('info.json', JSON.stringify(info, null, 4));
// FIXME: very bad but usefull in alpha stage to check for edge cases
if (info.ListData.Row.length !== 1) {
logger.error('More than 1 row in SharePoint video info', { fatal: true });
process.exit(1000);
}
const direct = await this.canDirectDownload(filePath);
const b64VideoMetadata = JSON.parse(
info.ListData.Row[0].MediaServiceFastMetadata
).video.altManifestMetadata;
const durationSeconds = Math.ceil(
(JSON.parse(
Buffer.from(b64VideoMetadata, 'base64').toString()
).Duration100Nano) / 10 / 1000 / 1000
);
if (direct) {
playbackUrl = this.axiosInstance.defaults.baseURL + filePath;
// logger.verbose(playbackUrl);
}
else {
playbackUrl = info.ListSchema['.videoManifestUrl'];
playbackUrl = playbackUrl.replace('{.mediaBaseUrl}', info.ListSchema['.mediaBaseUrl']);
// the only filetype works I found
playbackUrl = playbackUrl.replace('{.fileType}', 'mp4');
playbackUrl = playbackUrl.replace('{.callerStack}', info.ListSchema['.callerStack']);
playbackUrl = playbackUrl.replace('{.spItemUrl}', info.ListData.Row[0]['.spItemUrl']);
playbackUrl = playbackUrl.replace('{.driveAccessToken}', info.ListSchema['.driveAccessToken']);
playbackUrl += '&part=index&format=dash';
}
return {
direct,
title: filePath.split('/').pop() ?? 'video.mp4',
duration: publishedTimeToString(durationSeconds),
publishDate: publishedDateToString(info.ListData.Row[0]['Modified.']),
publishTime: publishedTimeToString(info.ListData.Row[0]['Modified.']),
author: info.ListData.Row[0]['Author.title'],
authorEmail: info.ListData.Row[0]['Author.email'],
uniqueId: info.ListData.Row[0].GUID.substring(1, 9),
outPath,
playbackUrl,
totalChunks: durationSeconds
};
}
private async canDirectDownload(filePath: string): Promise<boolean> {
logger.verbose(`Checking direct download for '${filePath}'`);
return this.axiosInstance.head(
filePath, { maxRedirects: 0 }
).then(
res => (res.status === 200)
).catch(
() => false
);
}
}

20
src/CommandLineParser.ts
View File

@@ -1,15 +1,14 @@
import { CLI_ERROR, ERROR_CODE } from './Errors';
import { checkOutDir } from './Utils';
import { CLI_ERROR } from './Errors';
import { makeOutDir } from './Utils';
import { logger } from './Logger';
import { templateElements } from './Types';
import fs from 'fs';
import readlineSync from 'readline-sync';
import sanitize from 'sanitize-filename';
import yargs from 'yargs';
export const argv: any = yargs.options({
export const argv = yargs.options({
username: {
alias: 'u',
type: 'string',
@@ -114,7 +113,7 @@ export const argv: any = yargs.options({
.check(() => noArguments())
.check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
.check((argv: any) => {
if (checkOutDir(argv.outputDirectory)) {
if (makeOutDir(argv.outputDirectory)) {
return true;
}
else {
@@ -198,14 +197,3 @@ function isOutputTemplateValid(argv: any): boolean {
return true;
}
export function promptUser(choices: Array<string>): number {
const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
if (index === -1) {
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
}
return index;
}

335
src/Downloaders.ts Normal file
View File

@@ -0,0 +1,335 @@
import { ShareApiClient, StreamApiClient } from './ApiClient';
import { argv } from './CommandLineParser';
import { ERROR_CODE } from './Errors';
import { logger } from './Logger';
import { doShareLogin, doStreamLogin } from './LoginModules';
import { drawThumbnail } from './Thumbnail';
import { refreshSession, TokenCache } from './TokenCache';
import { Video, VideoUrl } from './Types';
import { ffmpegTimemarkToChunk } from './Utils';
import { createUniquePath, getStreamInfo } from './VideoUtils';
import cliProgress from 'cli-progress';
import fs from 'fs';
import { execSync } from 'child_process';
import path from 'path';
const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
const tokenCache: TokenCache = new TokenCache();
export async function downloadStreamVideo(videoUrls: Array<VideoUrl>): Promise<void> {
logger.info('Downloading Microsoft Stream videos...');
let session = tokenCache.Read() ?? await doStreamLogin('https://web.microsoftstream.com/', tokenCache, argv.username);
logger.verbose(
'Session and API info \n' +
'\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
'\t API Gateway version: '.cyan + session.ApiGatewayVersion + '\n'
);
logger.info('Fetching videos info... \n');
const videos: Array<Video> = createUniquePath(
await getStreamInfo(videoUrls, session, argv.closedCaptions),
argv.outputTemplate, argv.format, argv.skip
);
if (argv.simulate) {
videos.forEach((video: Video) => {
logger.info(
'\nTitle: '.green + video.title +
'\nOutPath: '.green + video.outPath +
'\nPublished Date: '.green + video.publishDate +
'\nPlayback URL: '.green + video.playbackUrl +
((video.captionsUrl) ? ('\nCC URL: '.green + video.captionsUrl) : '')
);
});
return;
}
for (const [index, video] of videos.entries()) {
if (argv.skip && fs.existsSync(video.outPath)) {
logger.info(`File already exists, skipping: ${video.outPath} \n`);
continue;
}
if (argv.keepLoginCookies && index !== 0) {
logger.info('Trying to refresh token...');
session = await refreshSession('https://web.microsoftstream.com/video/' + video.guid);
StreamApiClient.getInstance().setSession(session);
}
const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
// process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
barsize: Math.floor((process.stdout.columns || 30) / 3),
stopOnComplete: true,
hideCursor: true,
});
logger.info(`\nDownloading Video: ${video.title} \n`);
logger.verbose('Extra video info \n' +
'\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
'\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
'\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
'\t Video total chunks: '.cyan + video.totalChunks + '\n');
logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
if (!process.stdout.columns) {
logger.warn(
'Unable to get number of columns from terminal.\n' +
'This happens sometimes in Cygwin/MSYS.\n' +
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
'Please use PowerShell or cmd.exe to run destreamer on Windows.'
);
}
const headers: string = 'Authorization: Bearer ' + session.AccessToken;
if (!argv.noExperiments) {
if (video.posterImageUrl) {
await drawThumbnail(video.posterImageUrl, session);
}
}
const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
['headers', headers]
]));
const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
['n', null]
]));
const ffmpegCmd: any = new FFmpegCommand();
const cleanupFn: () => void = () => {
pbar.stop();
if (argv.noCleanup) {
return;
}
try {
fs.unlinkSync(video.outPath);
}
catch (e) {
// Future handling of an error (maybe)
}
};
pbar.start(video.totalChunks, 0, {
speed: '0'
});
// prepare ffmpeg command line
ffmpegCmd.addInput(ffmpegInpt);
ffmpegCmd.addOutput(ffmpegOutput);
if (argv.closedCaptions && video.captionsUrl) {
const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
['headers', headers]
]));
ffmpegCmd.addInput(captionsInpt);
}
ffmpegCmd.on('update', async (data: any) => {
const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
pbar.update(currentChunks, {
speed: data.bitrate
});
// Graceful fallback in case we can't get columns (Cygwin/MSYS)
if (!process.stdout.columns) {
process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
}
});
process.on('SIGINT', cleanupFn);
// let the magic begin...
await new Promise((resolve: any) => {
ffmpegCmd.on('error', (error: any) => {
cleanupFn();
logger.error(`FFmpeg returned an error: ${error.message}`);
process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
});
ffmpegCmd.on('success', () => {
pbar.update(video.totalChunks); // set progress bar to 100%
logger.info(`\nDownload finished: ${video.outPath} \n`);
resolve();
});
ffmpegCmd.spawn();
});
process.removeListener('SIGINT', cleanupFn);
}
}
// TODO: complete overhaul of this function
export async function downloadShareVideo(videoUrls: Array<VideoUrl>): Promise<void> {
const shareUrlRegex = new RegExp(/(?<domain>https:\/\/.+\.sharepoint\.com).*?(?<baseSite>\/(?:teams|sites|personal)\/.*?)(?:(?<filename>\/.*\.mp4)|\/.*id=(?<paramFilename>.*mp4))/);
logger.info('Downloading SharePoint videos...\n\n');
// FIXME: this may change we need a smart login system if a request fails
const session = await doShareLogin(videoUrls[0].url, argv.username);
for (const videoUrl of videoUrls) {
const match = shareUrlRegex.exec(videoUrl.url);
if (!match) {
logger.error(`Invalid url '${videoUrl.url}', skipping...`);
continue;
}
const shareDomain = match.groups!.domain;
const shareSite = match.groups!.baseSite;
const shareFilepath = decodeURIComponent(match.groups?.filename ? (shareSite + match.groups.filename) : match.groups!.paramFilename);
// FIXME: hardcoded video.mp4
const title = shareFilepath.split('/').pop()?.split('.')[0] ?? 'video';
const apiClient = new ShareApiClient(shareDomain, shareSite, session);
const video = await apiClient.getVideoInfo(shareFilepath, videoUrl.outDir);
createUniquePath(video, title, argv.format, argv.skip);
if (argv.simulate) {
if (argv.verbose) {
console.dir(video);
}
else {
logger.info(
'\nTitle: '.green + video.title +
'\nOutPath: '.green + video.outPath +
'\nPublished Date: '.green + video.publishDate +
'\nPlayback URL: '.green + video.playbackUrl
);
}
continue;
}
if (video.direct) {
const headers = `Cookie: rtFa=${session.rtFa}; FedAuth=${session.FedAuth}`;
// FIXME: unstable and bad all-around
try {
execSync(
'aria2c --max-connection-per-server 8 --console-log-level warn ' +
`--header "${headers}" --dir "${path.dirname(video.outPath)}" --out "${path.basename(video.outPath)}" "${shareDomain + shareFilepath}"`,
{ stdio: 'inherit' }
);
}
catch (error: any) {
logger.error(`${error.status} \n\n${error.message} \n\n${error.stdout.toString()} \n\n${error.stderr.toString()}`);
}
}
else {
// FIXME: just a copy-paste, should move to separate function
const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
// process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
barsize: Math.floor((process.stdout.columns || 30) / 3),
stopOnComplete: true,
hideCursor: true,
});
logger.info(`\nDownloading Video: ${video.title} \n`);
logger.verbose('Extra video info \n' +
'\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
'\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
'\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
'\t Video total chunks: '.cyan + video.totalChunks + '\n');
logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
if (!process.stdout.columns) {
logger.warn(
'Unable to get number of columns from terminal.\n' +
'This happens sometimes in Cygwin/MSYS.\n' +
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
'Please use PowerShell or cmd.exe to run destreamer on Windows.'
);
}
const ffmpegInpt: any = new FFmpegInput(video.playbackUrl);
const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
['n', null]
]));
const ffmpegCmd: any = new FFmpegCommand();
const cleanupFn: () => void = () => {
pbar.stop();
if (argv.noCleanup) {
return;
}
try {
fs.unlinkSync(video.outPath);
}
catch (e) {
// Future handling of an error (maybe)
}
};
pbar.start(video.totalChunks, 0, {
speed: '0'
});
// prepare ffmpeg command line
ffmpegCmd.addInput(ffmpegInpt);
ffmpegCmd.addOutput(ffmpegOutput);
ffmpegCmd.on('update', async (data: any) => {
const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
pbar.update(currentChunks, {
speed: data.bitrate
});
// Graceful fallback in case we can't get columns (Cygwin/MSYS)
if (!process.stdout.columns) {
process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
}
});
process.on('SIGINT', cleanupFn);
// let the magic begin...
await new Promise((resolve: any) => {
ffmpegCmd.on('error', (error: any) => {
cleanupFn();
logger.error(`FFmpeg returned an error: ${error.message}`);
process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
});
ffmpegCmd.on('success', () => {
pbar.update(video.totalChunks); // set progress bar to 100%
logger.info(`\nDownload finished: ${video.outPath} \n`);
resolve();
});
ffmpegCmd.spawn();
});
process.removeListener('SIGINT', cleanupFn);
// logger.error('TODO: manifest download');
// continue;
}
}
}

8
src/Errors.ts
View File

@@ -1,8 +1,9 @@
export const enum ERROR_CODE {
UNHANDLED_ERROR,
UNHANDLED_ERROR = 200,
ELEVATED_SHELL,
CANCELLED_USER_INPUT,
MISSING_FFMPEG,
MISSING_ARIA2,
OUTDATED_FFMPEG,
UNK_FFMPEG_ERROR,
INVALID_VIDEO_GUID,
@@ -22,7 +23,10 @@ export const errors: { [key: number]: string } = {
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos',
[ERROR_CODE.MISSING_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
[ERROR_CODE.MISSING_ARIA2]: 'Aria2 is missing!\n' +
'Destreamer requires a fairly recent release of Aria2 to download videos',
[ERROR_CODE.OUTDATED_FFMPEG]: 'The FFmpeg version currently installed is too old!\n' +
'Destreamer requires a fairly recent release of FFmpeg to download videos',
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',

176
src/LoginModules.ts Normal file
View File

@@ -0,0 +1,176 @@
import { logger } from './Logger';
import puppeteer from 'puppeteer';
import { getPuppeteerChromiumPath } from './PuppeteerHelper';
import { chromeCacheFolder } from './destreamer';
import { argv } from './CommandLineParser';
import { ShareSession, StreamSession } from './Types';
import { ERROR_CODE } from './Errors';
import { TokenCache } from './TokenCache';
export async function doStreamLogin(url: string, tokenCache: TokenCache, username?: string): Promise<StreamSession> {
logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
const browser: puppeteer.Browser = await puppeteer.launch({
executablePath: getPuppeteerChromiumPath(),
headless: false,
userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
defaultViewport: null,
args: [
'--disable-dev-shm-usage',
'--fast-start',
'--no-sandbox'
]
});
// try-finally because we were leaving zombie processes if there was an error
try {
const page: puppeteer.Page = (await browser.pages())[0];
logger.info('Navigating to login page...');
await page.goto(url, { waitUntil: 'load' });
try {
if (username) {
await page.waitForSelector('input[type="email"]', { timeout: 3000 });
await page.keyboard.type(username);
await page.click('input[type="submit"]');
}
else {
/* If a username was not provided we let the user take actions that
lead up to the video page. */
}
}
catch (e) {
/* If there is no email input selector we aren't in the login module,
we are probably using the cache to aid the login.
It could finish the login on its own if the user said 'yes' when asked to
remember the credentials or it could still prompt the user for a password */
}
await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
logger.info('We are logged in.');
let session: StreamSession | null = null;
let tries = 1;
while (!session) {
try {
let sessionInfo: any;
session = await page.evaluate(
() => {
return {
AccessToken: sessionInfo.AccessToken,
ApiGatewayUri: sessionInfo.ApiGatewayUri,
ApiGatewayVersion: sessionInfo.ApiGatewayVersion
};
}
);
}
catch (error) {
if (tries > 5) {
process.exit(ERROR_CODE.NO_SESSION_INFO);
}
session = null;
tries++;
await page.waitForTimeout(3000);
}
}
tokenCache.Write(session);
logger.info('Wrote access token to token cache.');
logger.info("At this point Chromium's job is done, shutting it down...\n");
return session;
}
finally {
await browser.close();
}
}
export async function doShareLogin(url: string, username?: string): Promise<ShareSession> {
logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
let session: ShareSession | null = null;
const hostname = new URL(url).host;
const browser: puppeteer.Browser = await puppeteer.launch({
executablePath: getPuppeteerChromiumPath(),
headless: false,
devtools: argv.verbose,
userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
defaultViewport: null,
args: [
'--disable-dev-shm-usage',
'--fast-start',
'--no-sandbox'
]
});
// try-finally because we were leaving zombie processes if there was an error
try {
const page: puppeteer.Page = (await browser.pages())[0];
logger.info('Navigating to login page...');
await page.goto(url, { waitUntil: 'load' });
try {
if (username) {
await page.waitForSelector('input[type="email"]', { timeout: 3000 });
await page.keyboard.type(username);
await page.click('input[type="submit"]');
}
else {
/* If a username was not provided we let the user take actions that
lead up to the video page. */
}
}
catch (e) {
/* If there is no email input selector we aren't in the login module,
we are probably using the cache to aid the login.
It could finish the login on its own if the user said 'yes' when asked to
remember the credentials or it could still prompt the user for a password */
}
logger.info('Waiting for target!');
await browser.waitForTarget((target: puppeteer.Target) => target.url().startsWith(`https://${hostname}`), { timeout: 150000 });
logger.info('We are logged in.');
let tries = 1;
while (!session) {
const cookieJar = (await page.cookies()).filter(
biscuit => biscuit.name == 'rtFa' || biscuit.name == 'FedAuth'
);
if (cookieJar.length != 2) {
if (tries > 5) {
process.exit(ERROR_CODE.NO_SESSION_INFO);
}
await page.waitForTimeout(1000 * tries++);
continue;
}
session = {
rtFa: cookieJar.find(biscuit => biscuit.name == 'rtFa')!.value,
FedAuth: cookieJar.find(biscuit => biscuit.name == 'FedAuth')!.value
};
}
logger.info("At this point Chromium's job is done, shutting it down...\n");
// await page.waitForTimeout(1000 * 60 * 60 * 60);
}
finally {
logger.verbose('Stream login browser closing...');
await browser.close();
logger.verbose('Stream login browser closed');
}
return session;
}

8
src/Thumbnail.ts
View File

@@ -1,12 +1,12 @@
import { ApiClient } from './ApiClient';
import { Session } from './Types';
import { StreamApiClient } from './ApiClient';
import { StreamSession } from './Types';
import terminalImage from 'terminal-image';
import { AxiosResponse } from 'axios';
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
const apiClient: ApiClient = ApiClient.getInstance(session);
export async function drawThumbnail(posterImage: string, session: StreamSession): Promise<void> {
const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
.then((response: AxiosResponse<any> | undefined) => response?.data);

12
src/TokenCache.ts
View File

@@ -2,7 +2,7 @@ import { chromeCacheFolder } from './destreamer';
import { ERROR_CODE } from './Errors';
import { logger } from './Logger';
import { getPuppeteerChromiumPath } from './PuppeteerHelper';
import { Session } from './Types';
import { StreamSession } from './Types';
import fs from 'fs';
import jwtDecode from 'jwt-decode';
@@ -12,14 +12,14 @@ import puppeteer from 'puppeteer';
export class TokenCache {
private tokenCacheFile = '.token_cache';
public Read(): Session | null {
public Read(): StreamSession | null {
if (!fs.existsSync(this.tokenCacheFile)) {
logger.warn(`${this.tokenCacheFile} not found. \n`);
return null;
}
const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
const session: StreamSession = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
type Jwt = {
[key: string]: any
@@ -41,7 +41,7 @@ export class TokenCache {
return session;
}
public Write(session: Session): void {
public Write(session: StreamSession): void {
const s: string = JSON.stringify(session, null, 4);
fs.writeFile(this.tokenCacheFile, s, (err: any) => {
if (err) {
@@ -54,7 +54,7 @@ export class TokenCache {
}
export async function refreshSession(url: string): Promise<Session> {
export async function refreshSession(url: string): Promise<StreamSession> {
const videoId: string = url.split('/').pop() ?? process.exit(ERROR_CODE.INVALID_VIDEO_GUID);
const browser: puppeteer.Browser = await puppeteer.launch({
@@ -73,7 +73,7 @@ export async function refreshSession(url: string): Promise<Session> {
await browser.waitForTarget((target: puppeteer.Target) => target.url().includes(videoId), { timeout: 30000 });
let session: Session | null = null;
let session: StreamSession | null = null;
let tries = 1;
while (!session) {

27
src/Types.ts
View File

@@ -1,11 +1,34 @@
export type Session = {
export type StreamSession = {
AccessToken: string;
ApiGatewayUri: string;
ApiGatewayVersion: string;
}
export type ShareSession = {
FedAuth: string;
rtFa: string;
}
export type VideoUrl = {
url: string,
outDir: string
}
export type SharepointVideo = {
// if we can download the MP4 or we need to use DASH
direct: boolean;
playbackUrl: string;
title: string;
outPath: string
}
export type Video = {
guid?: string;
direct?: boolean;
title: string;
duration: string;
publishDate: string;
@@ -16,7 +39,7 @@ export type Video = {
outPath: string;
totalChunks: number; // Abstraction of FFmpeg timemark
playbackUrl: string;
posterImageUrl: string | null;
posterImageUrl?: string;
captionsUrl?: string
}

223
src/Utils.ts
View File

@@ -1,47 +1,62 @@
import { ApiClient } from './ApiClient';
import { StreamApiClient } from './ApiClient';
import { ERROR_CODE } from './Errors';
import { logger } from './Logger';
import { Session } from './Types';
import { StreamSession, VideoUrl } from './Types';
import { AxiosResponse } from 'axios';
import { execSync } from 'child_process';
import fs from 'fs';
import readlineSync from 'readline-sync';
async function extractGuids(url: string, client: ApiClient): Promise<Array<string> | null> {
const streamUrlRegex = new RegExp(/https?:\/\/web\.microsoftstream\.com.*/);
const shareUrlRegex = new RegExp(/https?:\/\/.+\.sharepoint\.com.*/);
/** we place the guid in the url fild in the return */
export async function extractStreamGuids(urlList: Array<VideoUrl>, session: StreamSession): Promise<Array<VideoUrl>> {
const videoRegex = new RegExp(/https:\/\/.*\/video\/(\w{8}-(?:\w{4}-){3}\w{12})/);
const groupRegex = new RegExp(/https:\/\/.*\/group\/(\w{8}-(?:\w{4}-){3}\w{12})/);
const videoMatch: RegExpExecArray | null = videoRegex.exec(url);
const groupMatch: RegExpExecArray | null = groupRegex.exec(url);
const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
const guidList: Array<VideoUrl> = [];
if (videoMatch) {
return [videoMatch[1]];
}
else if (groupMatch) {
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
const result: Array<string> = [];
for (const url of urlList) {
const videoMatch: RegExpExecArray | null = videoRegex.exec(url.url);
const groupMatch: RegExpExecArray | null = groupRegex.exec(url.url);
// Anything above $top=100 results in 400 Bad Request
// Use $skip to skip the first 100 and get another 100 and so on
for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
const partial: Array<string> = await client.callApi(
`groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
'$top=100&$orderby=publishedDate asc', 'get')
.then(
(response: AxiosResponse<any> | undefined) =>
response?.data.value.map((item: any) => item.id)
);
result.push(...partial);
if (videoMatch) {
guidList.push({
url: videoMatch[1],
outDir: url.outDir
});
}
else if (groupMatch) {
const videoNumber: number = await apiClient.callApi(`groups/${groupMatch[1]}`, 'get')
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
return result;
// Anything above $top=100 results in 400 Bad Request
// Use $skip to skip the first 100 and get another 100 and so on
for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
await apiClient.callApi(
`groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
'$top=100&$orderby=publishedDate asc', 'get'
).then((response: AxiosResponse<any> | undefined) => {
response?.data.value.forEach((video: { id: string }) =>
guidList.push({
url: video.id,
outDir: url.outDir
})
);
});
}
}
else {
logger.warn(`Invalid url '${url.url}', skipping...`);
}
}
return null;
return guidList;
}
@@ -52,30 +67,32 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
*
* @param {Array<string>} urlList list of link to parse
* @param {string} defaultOutDir the directry used to save the videos
* @param {Session} session used to call the API to get the GUIDs from group links
*
* @returns Array of 2 elements, 1st one being the GUIDs array, 2nd one the output directories array
* @returns Array of 2 elements: 1st an array of Microsoft Stream urls, 2nd an array of SharePoint urls
*/
export async function parseCLIinput(urlList: Array<string>, defaultOutDir: string,
session: Session): Promise<Array<Array<string>>> {
const apiClient: ApiClient = ApiClient.getInstance(session);
const guidList: Array<string> = [];
export function parseCLIinput(urlList: Array<string>, defaultOutDir: string): Array<Array<VideoUrl>> {
const stream: Array<VideoUrl> = [];
const share: Array<VideoUrl> = [];
for (const url of urlList) {
const guids: Array<string> | null = await extractGuids(url, apiClient);
if (guids) {
guidList.push(...guids);
if (streamUrlRegex.test(url)) {
stream.push({
url: url,
outDir: defaultOutDir
});
}
else if (shareUrlRegex.test(url)) {
share.push({
url: url,
outDir: defaultOutDir
});
}
else {
logger.warn(`Invalid url '${url}', skipping..`);
}
}
const outDirList: Array<string> = Array(guidList.length).fill(defaultOutDir);
return [guidList, outDirList];
return [stream, share];
}
@@ -86,94 +103,84 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
*
* @param {string} inputFile path to the text file
* @param {string} defaultOutDir the default/fallback directory used to save the videos
* @param {Session} session used to call the API to get the GUIDs from group links
*
* @returns Array of 2 elements, 1st one being the GUIDs array, 2nd one the output directories array
*/
export async function parseInputFile(inputFile: string, defaultOutDir: string,
session: Session): Promise<Array<Array<string>>> {
export function parseInputFile(inputFile: string, defaultOutDir: string): Array<Array<VideoUrl>> {
// rawContent is a list of each line of the file
const rawContent: Array<string> = fs.readFileSync(inputFile).toString()
.split(/\r?\n/);
const apiClient: ApiClient = ApiClient.getInstance(session);
const guidList: Array<string> = [];
const outDirList: Array<string> = [];
// if the last line was an url set this
let foundUrl = false;
const rawContent: Array<string> = fs.readFileSync(inputFile).toString().split(/\r?\n/);
const stream: Array<VideoUrl> = [];
const share: Array<VideoUrl> = [];
let streamUrl = false;
for (let i = 0; i < rawContent.length; i++) {
const line: string = rawContent[i];
const nextLine: string | null = i < rawContent.length ? rawContent[i + 1] : null;
let outDir = defaultOutDir;
// filter out lines with no content
if (!line.match(/\S/)) {
logger.warn(`Line ${i + 1} is empty, skipping..`);
continue;
}
// parse if line is option
else if (line.includes('-dir')) {
if (foundUrl) {
const outDir: string | null = parseOption('-dir', line);
// check for urls
else if (streamUrlRegex.test(line)) {
streamUrl = true;
}
else if (shareUrlRegex.test(line)) {
streamUrl = false;
}
// now invalid line since we skip ahead one line if we find dir option
else {
logger.warn(`Line ${i + 1}: '${line}' is invalid, skipping..`);
if (outDir && checkOutDir(outDir)) {
outDirList.push(...Array(guidList.length - outDirList.length)
.fill(outDir));
}
else {
outDirList.push(...Array(guidList.length - outDirList.length)
.fill(defaultOutDir));
}
continue;
}
foundUrl = false;
continue;
}
else {
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
continue;
// we now have a valid url, check next line for option
if (nextLine) {
const optionDir = parseOption('-dir', nextLine);
if (optionDir && makeOutDir(optionDir)) {
outDir = optionDir;
// if there was an option we skip a line
i++;
}
}
/* now line is not empty nor an option line.
If foundUrl is still true last line didn't have a directory option
so we stil need to add the default outDir to outDirList to */
if (foundUrl) {
outDirList.push(...Array(guidList.length - outDirList.length)
.fill(defaultOutDir));
foundUrl = false;
}
const guids: Array<string> | null = await extractGuids(line, apiClient);
if (guids) {
guidList.push(...guids);
foundUrl = true;
if (streamUrl) {
stream.push({
url: line,
outDir
});
}
else {
logger.warn(`Invalid url at line ${i + 1}, skipping..`);
share.push({
url: line,
outDir
});
}
}
// if foundUrl is still true after the loop we have some url without an outDir
if (foundUrl) {
outDirList.push(...Array(guidList.length - outDirList.length)
.fill(defaultOutDir));
}
return [guidList, outDirList];
return [stream, share];
}
// This leaves us the option to add more options (badum tss) _Luca
function parseOption(optionSyntax: string, item: string): string | null {
const match: RegExpMatchArray | null = item.match(
RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
RegExp(`^\\s+${optionSyntax}\\s*=\\s*['"](.*)['"]`)
);
return match ? match[1] : null;
}
export function checkOutDir(directory: string): boolean {
/**
* @param directory path to create
* @returns true on success, false otherwise
*/
export function makeOutDir(directory: string): boolean {
if (!fs.existsSync(directory)) {
try {
fs.mkdirSync(directory);
@@ -205,14 +212,40 @@ export function checkRequirements(): void {
catch (e) {
process.exit(ERROR_CODE.MISSING_FFMPEG);
}
try {
const versionRegex = new RegExp(/aria2 version (.*)/);
const aira2Ver: string = execSync('aria2c --version').toString().split('\n')[0];
if (versionRegex.test(aira2Ver)) {
logger.verbose(`Using ${aira2Ver}\n`);
}
else {
throw new Error();
}
}
catch (e) {
process.exit(ERROR_CODE.MISSING_ARIA2);
}
}
// number of seconds
export function ffmpegTimemarkToChunk(timemark: string): number {
const timeVals: Array<string> = timemark.split(':');
const hrs: number = parseInt(timeVals[0]);
const mins: number = parseInt(timeVals[1]);
const secs: number = parseInt(timeVals[2]);
return (hrs * 60) + mins + (secs / 60);
return (hrs * 60 * 60) + (mins * 60) + secs;
}
export function promptUser(choices: Array<string>): number {
const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
if (index === -1) {
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
}
return index;
}

97
src/VideoUtils.ts
View File

@@ -1,15 +1,16 @@
import { ApiClient } from './ApiClient';
import { promptUser } from './CommandLineParser';
import { StreamApiClient } from './ApiClient';
import { promptUser } from './Utils';
import { logger } from './Logger';
import { Video, Session } from './Types';
import { Video, StreamSession, VideoUrl } from './Types';
import { AxiosResponse } from 'axios';
import fs from 'fs';
import { parse as parseDuration, Duration } from 'iso8601-duration';
import path from 'path';
import sanitizeWindowsName from 'sanitize-filename';
import { extractStreamGuids } from './Utils';
function publishedDateToString(date: string): string {
export function publishedDateToString(date: string): string {
const dateJs: Date = new Date(date);
const day: string = dateJs.getDate().toString().padStart(2, '0');
const month: string = (dateJs.getMonth() + 1).toString(10).padStart(2, '0');
@@ -17,35 +18,44 @@ function publishedDateToString(date: string): string {
return `${dateJs.getFullYear()}-${month}-${day}`;
}
export function publishedTimeToString(seconds: number): string
export function publishedTimeToString(date: string): string
export function publishedTimeToString(date: string | number): string {
let dateJs: Date;
if (typeof (date) === 'number') {
dateJs = new Date(0, 0, 0, 0, 0, date);
}
else {
dateJs = new Date(date);
}
function publishedTimeToString(date: string): string {
const dateJs: Date = new Date(date);
const hours: string = dateJs.getHours().toString();
const minutes: string = dateJs.getMinutes().toString();
const seconds: string = dateJs.getSeconds().toString();
return `${hours}.${minutes}.${seconds}`;
return `${hours}h ${minutes}m ${seconds}s`;
}
function isoDurationToString(time: string): string {
export function isoDurationToString(time: string): string {
const duration: Duration = parseDuration(time);
return `${duration.hours ?? '00'}.${duration.minutes ?? '00'}.${duration.seconds?.toFixed(0) ?? '00'}`;
}
function durationToTotalChunks(duration: string): number {
// it's the number of seconds in the video
export function durationToTotalChunks(duration: string,): number {
const durationObj: any = parseDuration(duration);
const hrs: number = durationObj.hours ?? 0;
const mins: number = durationObj.minutes ?? 0;
const secs: number = Math.ceil(durationObj.seconds ?? 0);
return (hrs * 60) + mins + (secs / 60);
return (hrs * 60 * 60) + (mins * 60) + secs;
}
export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
export async function getStreamInfo(videoUrls: Array<VideoUrl>, session: StreamSession, subtitles?: boolean): Promise<Array<Video>> {
const metadata: Array<Video> = [];
let title: string;
let duration: string;
@@ -54,19 +64,23 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
let author: string;
let authorEmail: string;
let uniqueId: string;
const outPath = '';
let totalChunks: number;
let playbackUrl: string;
let posterImageUrl: string;
let captionsUrl: string | undefined;
const apiClient: ApiClient = ApiClient.getInstance(session);
const apiClient: StreamApiClient = StreamApiClient.getInstance(session);
// we place the guid in the url field
const videoGUIDs = await extractStreamGuids(videoUrls, session);
/* TODO: change this to a single guid at a time to ease our footprint on the
MSS servers or we get throttled after 10 sequential reqs */
for (const guid of videoGuids) {
for (const guid of videoGUIDs) {
const response: AxiosResponse<any> | undefined =
await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
await apiClient.callApi('videos/' + guid.url + '?$expand=creator', 'get');
title = sanitizeWindowsName(response?.data['name']);
@@ -80,7 +94,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
authorEmail = response?.data['creator'].mail;
uniqueId = '#' + guid.split('-')[0];
uniqueId = '#' + guid.url.split('-')[0];
totalChunks = durationToTotalChunks(response?.data.media['duration']);
@@ -94,7 +108,7 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
posterImageUrl = response?.data['posterImage']['medium']['url'];
if (subtitles) {
const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid.url}/texttracks`, 'get');
if (!captions?.data.value.length) {
captionsUrl = undefined;
@@ -112,18 +126,19 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
}
metadata.push({
title: title,
duration: duration,
publishDate: publishDate,
publishTime: publishTime,
author: author,
authorEmail: authorEmail,
uniqueId: uniqueId,
outPath: outPath,
totalChunks: totalChunks, // Abstraction of FFmpeg timemark
playbackUrl: playbackUrl,
posterImageUrl: posterImageUrl,
captionsUrl: captionsUrl
guid: guid.url,
title,
duration,
publishDate,
publishTime,
author,
authorEmail,
uniqueId,
outPath: guid.outDir,
totalChunks, // Abstraction of FFmpeg timemark
playbackUrl,
posterImageUrl,
captionsUrl
});
}
@@ -131,16 +146,24 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
}
export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, template: string, format: string, skip?: boolean): Array<Video> {
export function createUniquePath(videos: Array<Video>, template: string, format: string, skip?: boolean): Array<Video>
export function createUniquePath(videos: Video, template: string, format: string, skip?: boolean): Video
export function createUniquePath(videos: Array<Video> | Video, template: string, format: string, skip?: boolean): Array<Video> | Video {
let singleInput = false;
videos.forEach((video: Video, index: number) => {
if (!Array.isArray(videos)) {
videos = [videos];
singleInput = true;
}
videos.forEach((video: Video) => {
let title: string = template;
let finalTitle: string;
const elementRegEx = RegExp(/{(.*?)}/g);
let match = elementRegEx.exec(template);
while (match) {
const value = video[match[1] as keyof Video] as string;
const value = video[match[1] as keyof (Video)] as string;
title = title.replace(match[0], value);
match = elementRegEx.exec(template);
}
@@ -148,7 +171,7 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
let i = 0;
finalTitle = title;
while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
while (!skip && fs.existsSync(path.join(video.outPath, finalTitle + '.' + format))) {
finalTitle = `${title}.${++i}`;
}
@@ -158,9 +181,13 @@ export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, t
logger.warn(`Not a valid Windows file name: "${finalFileName}".\nReplacing invalid characters with underscores to preserve cross-platform consistency.`);
}
video.outPath = path.join(outDirs[index], finalFileName);
video.outPath = path.join(video.outPath, finalFileName);
});
if (singleInput) {
return videos[0];
}
return videos;
}

271
src/destreamer.ts
View File

@@ -2,22 +2,13 @@ import { argv } from './CommandLineParser';
import { ERROR_CODE } from './Errors';
import { setProcessEvents } from './Events';
import { logger } from './Logger';
import { getPuppeteerChromiumPath } from './PuppeteerHelper';
import { drawThumbnail } from './Thumbnail';
import { TokenCache, refreshSession } from './TokenCache';
import { Video, Session } from './Types';
import { checkRequirements, ffmpegTimemarkToChunk, parseInputFile, parseCLIinput} from './Utils';
import { getVideoInfo, createUniquePath } from './VideoUtils';
import { VideoUrl } from './Types';
import { checkRequirements, parseInputFile, parseCLIinput } from './Utils';
import cliProgress from 'cli-progress';
import fs from 'fs';
import isElevated from 'is-elevated';
import puppeteer from 'puppeteer';
import { ApiClient } from './ApiClient';
import { downloadShareVideo, downloadStreamVideo } from './Downloaders';
const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
const tokenCache: TokenCache = new TokenCache();
export const chromeCacheFolder = '.chrome_data';
@@ -44,259 +35,31 @@ async function init(): Promise<void> {
}
async function DoInteractiveLogin(url: string, username?: string): Promise<Session> {
logger.info('Launching headless Chrome to perform the OpenID Connect dance...');
const browser: puppeteer.Browser = await puppeteer.launch({
executablePath: getPuppeteerChromiumPath(),
headless: false,
userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
args: [
'--disable-dev-shm-usage',
'--fast-start',
'--no-sandbox'
]
});
const page: puppeteer.Page = (await browser.pages())[0];
logger.info('Navigating to login page...');
await page.goto(url, { waitUntil: 'load' });
try {
if (username) {
await page.waitForSelector('input[type="email"]', {timeout: 3000});
await page.keyboard.type(username);
await page.click('input[type="submit"]');
}
else {
/* If a username was not provided we let the user take actions that
lead up to the video page. */
}
}
catch (e) {
/* If there is no email input selector we aren't in the login module,
we are probably using the cache to aid the login.
It could finish the login on its own if the user said 'yes' when asked to
remember the credentials or it could still prompt the user for a password */
}
await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
logger.info('We are logged in.');
let session: Session | null = null;
let tries = 1;
while (!session) {
try {
let sessionInfo: any;
session = await page.evaluate(
() => {
return {
AccessToken: sessionInfo.AccessToken,
ApiGatewayUri: sessionInfo.ApiGatewayUri,
ApiGatewayVersion: sessionInfo.ApiGatewayVersion
};
}
);
}
catch (error) {
if (tries > 5) {
process.exit(ERROR_CODE.NO_SESSION_INFO);
}
session = null;
tries++;
await page.waitFor(3000);
}
}
tokenCache.Write(session);
logger.info('Wrote access token to token cache.');
logger.info("At this point Chromium's job is done, shutting it down...\n");
await browser.close();
return session;
}
async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array<string>, session: Session): Promise<void> {
logger.info('Fetching videos info... \n');
const videos: Array<Video> = createUniquePath (
await getVideoInfo(videoGUIDs, session, argv.closedCaptions),
outputDirectories, argv.outputTemplate, argv.format, argv.skip
);
if (argv.simulate) {
videos.forEach((video: Video) => {
logger.info(
'\nTitle: '.green + video.title +
'\nOutPath: '.green + video.outPath +
'\nPublished Date: '.green + video.publishDate +
'\nPlayback URL: '.green + video.playbackUrl +
((video.captionsUrl) ? ('\nCC URL: '.green + video.captionsUrl) : '')
);
});
return;
}
for (const [index, video] of videos.entries()) {
if (argv.skip && fs.existsSync(video.outPath)) {
logger.info(`File already exists, skipping: ${video.outPath} \n`);
continue;
}
if (argv.keepLoginCookies && index !== 0) {
logger.info('Trying to refresh token...');
session = await refreshSession('https://web.microsoftstream.com/video/' + videoGUIDs[index]);
ApiClient.getInstance().setSession(session);
}
const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
barCompleteChar: '\u2588',
barIncompleteChar: '\u2591',
format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
// process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
barsize: Math.floor((process.stdout.columns || 30) / 3),
stopOnComplete: true,
hideCursor: true,
});
logger.info(`\nDownloading Video: ${video.title} \n`);
logger.verbose('Extra video info \n' +
'\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
'\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
'\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
'\t Video total chunks: '.cyan + video.totalChunks + '\n');
logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
if (!process.stdout.columns) {
logger.warn(
'Unable to get number of columns from terminal.\n' +
'This happens sometimes in Cygwin/MSYS.\n' +
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
'Please use PowerShell or cmd.exe to run destreamer on Windows.'
);
}
const headers: string = 'Authorization: Bearer ' + session.AccessToken;
if (!argv.noExperiments) {
if (video.posterImageUrl) {
await drawThumbnail(video.posterImageUrl, session);
}
}
const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
['headers', headers]
]));
const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
['n', null]
]));
const ffmpegCmd: any = new FFmpegCommand();
const cleanupFn: () => void = () => {
pbar.stop();
if (argv.noCleanup) {
return;
}
try {
fs.unlinkSync(video.outPath);
}
catch (e) {
// Future handling of an error (maybe)
}
};
pbar.start(video.totalChunks, 0, {
speed: '0'
});
// prepare ffmpeg command line
ffmpegCmd.addInput(ffmpegInpt);
ffmpegCmd.addOutput(ffmpegOutput);
if (argv.closedCaptions && video.captionsUrl) {
const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
['headers', headers]
]));
ffmpegCmd.addInput(captionsInpt);
}
ffmpegCmd.on('update', async (data: any) => {
const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
pbar.update(currentChunks, {
speed: data.bitrate
});
// Graceful fallback in case we can't get columns (Cygwin/MSYS)
if (!process.stdout.columns) {
process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
}
});
process.on('SIGINT', cleanupFn);
// let the magic begin...
await new Promise((resolve: any) => {
ffmpegCmd.on('error', (error: any) => {
cleanupFn();
logger.error(`FFmpeg returned an error: ${error.message}`);
process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
});
ffmpegCmd.on('success', () => {
pbar.update(video.totalChunks); // set progress bar to 100%
logger.info(`\nDownload finished: ${video.outPath} \n`);
resolve();
});
ffmpegCmd.spawn();
});
process.removeListener('SIGINT', cleanupFn);
}
}
async function main(): Promise<void> {
await init(); // must be first
let session: Session;
// eslint-disable-next-line prefer-const
session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
logger.verbose('Session and API info \n' +
'\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
'\t API Gateway version: '.cyan + session.ApiGatewayVersion + '\n');
let videoGUIDs: Array<string>;
let outDirs: Array<string>;
let streamVideos: Array<VideoUrl>, shareVideos: Array<VideoUrl>;
if (argv.videoUrls) {
logger.info('Parsing video/group urls');
[videoGUIDs, outDirs] = await parseCLIinput(argv.videoUrls as Array<string>, argv.outputDirectory, session);
[streamVideos, shareVideos] = await parseCLIinput(argv.videoUrls as Array<string>, argv.outputDirectory);
}
else {
logger.info('Parsing input file');
[videoGUIDs, outDirs] = await parseInputFile(argv.inputFile!, argv.outputDirectory, session);
[streamVideos, shareVideos] = await parseInputFile(argv.inputFile!, argv.outputDirectory);
}
logger.verbose('List of GUIDs and corresponding output directory \n' +
videoGUIDs.map((guid: string, i: number) =>
`\thttps://web.microsoftstream.com/video/${guid} => ${outDirs[i]} \n`).join(''));
logger.verbose(
'List of urls and corresponding output directory \n' +
streamVideos.map(video => `\t${video.url} => ${video.outDir} \n`).join('') +
shareVideos.map(video => `\t${video.url} => ${video.outDir} \n`).join('')
);
downloadVideo(videoGUIDs, outDirs, session);
if (streamVideos.length) {
await downloadStreamVideo(streamVideos);
}
if (shareVideos.length) {
await downloadShareVideo(shareVideos);
}
}

86
test/test.ts
View File

@@ -1,32 +1,15 @@
import { parseInputFile } from '../src/Utils';
import puppeteer from 'puppeteer';
import { extractStreamGuids, parseInputFile } from '../src/Utils';
import assert from 'assert';
import tmp from 'tmp';
import fs from 'fs';
import { Session } from './Types';
describe('Puppeteer', () => {
it('should grab GitHub page title', async () => {
const browser = await puppeteer.launch({
headless: true,
args: ['--disable-dev-shm-usage', '--fast-start', '--no-sandbox']
});
const page = await browser.newPage();
await page.goto('https://github.com/', { waitUntil: 'load' });
let pageTitle = await page.title();
assert.equal(true, pageTitle.includes('GitHub'));
await browser.close();
}).timeout(30000); // yeah, this may take a while...
});
import { StreamSession, VideoUrl } from './Types';
// we cannot test groups parsing as that requires an actual session
// TODO: add SharePoint urls
describe('Destreamer parsing', () => {
it('Input file to arrays of URLs and DIRs', async () => {
const testSession: Session = {
it('Input file to arrays of guids', async () => {
const testSession: StreamSession = {
AccessToken: '',
ApiGatewayUri: '',
ApiGatewayVersion: ''
@@ -44,33 +27,42 @@ describe('Destreamer parsing', () => {
'https://web.microsoftstream.com/video/xxxxxx-gggg-xxxx-xxxx-xxxxxxxxxxxx',
''
];
const expectedGUIDsOut: Array<string> = [
'xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx',
'xxxxxxxx-bbbb-xxxx-xxxx-xxxxxxxxxxxx',
'xxxxxxxx-cccc-xxxx-xxxx-xxxxxxxxxxxx',
'xxxxxxxx-dddd-xxxx-xxxx-xxxxxxxxxxxx',
'xxxxxxxx-eeee-xxxx-xxxx-xxxxxxxxxxxx'
];
const expectedDirOut: Array<string> = [
'videos',
'luca',
'videos',
'videos',
'videos'
const expectedStreamOut: Array<VideoUrl> = [
{
url: 'xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx',
outDir: 'videos'
},
{
url: 'xxxxxxxx-bbbb-xxxx-xxxx-xxxxxxxxxxxx',
outDir: 'luca'
},
{
url: 'xxxxxxxx-cccc-xxxx-xxxx-xxxxxxxxxxxx',
outDir: 'videos'
},
{
url: 'xxxxxxxx-dddd-xxxx-xxxx-xxxxxxxxxxxx',
outDir: 'videos'
},
{
url: 'xxxxxxxx-eeee-xxxx-xxxx-xxxxxxxxxxxx',
outDir: 'videos'
},
];
const tmpFile = tmp.fileSync({ postfix: '.txt' });
fs.writeFileSync(tmpFile.fd, testIn.join('\r\n'));
const [testUrlOut , testDirOut]: Array<Array<string>> = await parseInputFile(tmpFile.name, 'videos', testSession);
if (testUrlOut.length !== expectedGUIDsOut.length) {
throw "Expected url list and test list don't have the same number of elements".red;
}
else if (testDirOut.length !== expectedDirOut.length) {
throw "Expected dir list and test list don't have the same number of elements".red;
}
assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
'Error in parsing the URLs, missmatch between test and expected'.red);
assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
'Error in parsing the DIRs, missmatch between test and expected'.red);
const [testStreamUrls]: Array<Array<VideoUrl>> = parseInputFile(tmpFile.name, 'videos');
assert.deepStrictEqual(
await extractStreamGuids(testStreamUrls, testSession),
expectedStreamOut,
'Error in parsing the URLs, missmatch between test and expected'.red
);
// assert.deepStrictEqual(testUrlOut, expectedGUIDsOut,
// 'Error in parsing the DIRs, missmatch between test and expected'.red);
assert.ok('Parsing of input file ok');
});
});