From ee9ec1aa5f80831569ee6162e7bf8d22a1e7dd74 Mon Sep 17 00:00:00 2001 From: shirj Date: Sat, 15 Feb 2020 12:06:31 -0500 Subject: [PATCH 1/2] Download a list of videos --- destreamer.ts | 84 ++++++++++++++++++++++++++++----------------------- package.json | 6 ++-- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/destreamer.ts b/destreamer.ts index 9b0acbb..753fd05 100644 --- a/destreamer.ts +++ b/destreamer.ts @@ -4,13 +4,21 @@ import { terminal as term } from 'terminal-kit'; import fs from 'fs'; import path from 'path'; import { BrowserTests } from './BrowserTests'; +import yargs = require('yargs'); // Type in your username here (the one you use to // login to Microsoft Stream). -const username: string = 'somebody@example.com'; -const args: string[] = process.argv.slice(2); -const videoUrl: string = args[0]; -const outputDirectory: string = 'videos'; +const args: string[] = process.argv.slice(2); // TODO: Remove this + +const argv = yargs.options({ + videoUrls: { type: 'array', demandOption: true }, + username: { type: 'string', demandOption: true }, + outputDirectory: { type: 'string', default: 'videos' } +}).argv; + +console.info('Video URLs: %s', argv.videoUrls); +console.info('Username: %s', argv.username); +console.info('Output Directory: %s', argv.outputDirectory); function sanityChecks() { try { @@ -31,10 +39,10 @@ function sanityChecks() { console.error('FFmpeg is missing. You need a fairly recent release of FFmpeg in $PATH.'); } - if (!fs.existsSync(outputDirectory)){ + if (!fs.existsSync(argv.outputDirectory)){ console.log('Creating output directory: ' + - process.cwd() + path.sep + outputDirectory); - fs.mkdirSync(outputDirectory); + process.cwd() + path.sep + argv.outputDirectory); + fs.mkdirSync(argv.outputDirectory); } if (args[0] == null || args[0].length < 10) { @@ -44,7 +52,7 @@ function sanityChecks() { } } -async function rentVideoForLater() { +async function rentVideoForLater(videoUrls: string[], username: string, outputDirectory: string) { console.log('Launching headless Chrome to perform the OpenID Connect dance...'); const browser = await puppeteer.launch({ // Switch to false if you need to login interactively @@ -56,7 +64,7 @@ async function rentVideoForLater() { // This breaks on slow connections, needs more reliable logic //const oidcUrl = "https://login.microsoftonline.com/common/oauth2/authorize?client_id=cf53fce8-def6-4aeb-8d30-b158e7b1cf83&response_mode=form_post&response_type=code+id_token&scope=openid+profile&state=OpenIdConnect.AuthenticationProperties%3d1VtrsKV5QUHtzn8cDWL4wJmacu-VHH_DfpPxMQBhnfbar-_e8X016GGJDPfqfvcyUK3F3vBoiFwUpahR2ANfrzHE469vcw7Mk86wcAqBGXCvAUmv59MDU_OZFHpSL360oVRBo84GfVXAKYdhCjhPtelRHLHEM_ADiARXeMdVTAO3SaTiVQMhw3c9vLWuXqrKKevpI7E5esCQy5V_dhr2Q7kKrlW3gHX0232b8UWAnSDpc-94&nonce=636832485747560726.NzMyOWIyYWQtM2I3NC00MmIyLTg1NTMtODBkNDIwZTI1YjAxNDJiN2JkNDMtMmU5Ni00OTc3LWFkYTQtNTNlNmUwZmM1NTVl&nonceKey=OpenIdConnect.nonce.F1tPks6em0M%2fWMwvatuGWfFM9Gj83LwRKLvbx9rYs5M%3d&site_id=500453&redirect_uri=https%3a%2f%2fmsit.microsoftstream.com%2f&post_logout_redirect_uri=https%3a%2f%2fproducts.office.com%2fmicrosoft-stream&msafed=0"; - await page.goto(videoUrl, { waitUntil: 'networkidle2' }); + await page.goto(videoUrls[0], { waitUntil: 'networkidle2' }); await page.waitForSelector('input[type="email"]'); await page.keyboard.type(username); await page.click('input[type="submit"]'); @@ -66,40 +74,42 @@ async function rentVideoForLater() { await sleep(1500); console.log('Sorry, i mean "you".'); - await page.goto(videoUrl, { waitUntil: 'networkidle2' }); - await sleep(2000); - // try this instead of hardcoding sleep - // https://github.com/GoogleChrome/puppeteer/issues/3649 + for (let videoUrl of videoUrls) { + await page.goto(videoUrl, { waitUntil: 'networkidle2' }); + await sleep(2000); + // try this instead of hardcoding sleep + // https://github.com/GoogleChrome/puppeteer/issues/3649 - const cookie = await exfiltrateCookie(page); - console.log('Got cookie. Consuming cookie...'); + const cookie = await exfiltrateCookie(page); + console.log('Got cookie. Consuming cookie...'); - await sleep(4000); - console.log('Looking up AMS stream locator...'); - let amp: any; - const amsUrl = await page.evaluate( - () => { return amp.Player.players["vjs_video_3"].cache_.src } - ); + await sleep(4000); + console.log('Looking up AMS stream locator...'); + let amp: any; + const amsUrl = await page.evaluate( + () => { return amp.Player.players["vjs_video_3"].cache_.src } + ); - const title = await page.evaluate( - // Clear abuse of null assertion operator, - // someone fix this please - () => { return document!.querySelector(".title")!.textContent!.trim() } - ); + const title = await page.evaluate( + // Clear abuse of null assertion operator, + // someone fix this please + () => { return document!.querySelector(".title")!.textContent!.trim() } + ); - console.log(`Video title is: ${title}`); + console.log(`Video title is: ${title}`); + + console.log('Constructing HLS URL...'); + const hlsUrl = amsUrl.substring(0, amsUrl.lastIndexOf('/')) + '/manifest(format=m3u8-aapl)'; + + console.log('Spawning youtube-dl with cookie and HLS URL...'); + const youtubedlCmd = 'youtube-dl --no-call-home --no-warnings ' + + `--output "${outputDirectory}/${title}.mp4" --add-header Cookie:"${cookie}" "${hlsUrl}"`; + // console.log(`\n\n[DEBUG] Invoking youtube-dl: ${youtubedlCmd}\n\n`); + var result = execSync(youtubedlCmd, { stdio: 'inherit' }); + } console.log("At this point Chrome's job is done, shutting it down..."); await browser.close(); - - console.log('Constructing HLS URL...'); - const hlsUrl = amsUrl.substring(0, amsUrl.lastIndexOf('/')) + '/manifest(format=m3u8-aapl)'; - - console.log('Spawning youtube-dl with cookie and HLS URL...'); - const youtubedlCmd = 'youtube-dl --no-call-home --no-progress --no-warnings ' + - `--output "${outputDirectory}/${title}.mp4" --add-header Cookie:"${cookie}" "${hlsUrl}"`; - // console.log(`\n\n[DEBUG] Invoking youtube-dl: ${youtubedlCmd}\n\n`); - var result = execSync(youtubedlCmd, { stdio: 'inherit' }); } function sleep(ms: number) { @@ -134,5 +144,5 @@ if (args[0] === 'test') else { sanityChecks(); - rentVideoForLater(); + rentVideoForLater(argv.videoUrls as string[], argv.username, argv.outputDirectory); } \ No newline at end of file diff --git a/package.json b/package.json index 2b0788a..2f88e8e 100644 --- a/package.json +++ b/package.json @@ -18,11 +18,13 @@ "license": "MIT", "devDependencies": { "@types/puppeteer": "^1.20.0", - "@types/terminal-kit": "^1.28.0" + "@types/terminal-kit": "^1.28.0", + "@types/yargs": "^15.0.3" }, "dependencies": { "puppeteer": "^1.20.0", "terminal-kit": "^1.26.10", - "typescript": "^3.6.3" + "typescript": "^3.6.3", + "yargs": "^15.0.3" } } From ec0cd2f948585edf93c6386d28a370c1fd217dbe Mon Sep 17 00:00:00 2001 From: Bill Date: Sat, 15 Feb 2020 12:11:10 -0500 Subject: [PATCH 2/2] Update README.md --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d3d7d75..99b6e08 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,23 @@ Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) ap * Edit `destreamer.ts` and replace the username const with your own, you may still need to enter your password or go through 2FA if you don't have the STS cookie saved in Chrome. If you do (i.e. you usually log in to Microsoft Stream with Chrome), then you may try turning `headless: false` to `true` for a truly headless experience) ======= * Edit `destreamer.ts` (`.js` if using the vanilla JS master branch) and replace the username const with your own, you may still need to enter your password or go through 2FA if you don't have the STS cookie saved in Chrome. If you do (i.e. you usually log in to Microsoft Stream with Chrome), then you may try turning `headless: false` to `true` for a truly headless experience) -* `npm install` to restore packages -* `npm start ` +* `npm install` to restore packages* `npm install` to restore packages +* `npm run -s build` to transpile TypeScript to JavaScript +* `node ./destreamer.js` +``` +Options: + --help Show help [boolean] + --version Show version number [boolean] + --videoUrls [array] [required] + --username [string] [required] + --outputDirectory [string] [default: "videos"] +``` +* `node destreamer.js --username username@example.com --outputDirectory "C:\videos" --videoUrls "https://web.microsoftstream.com/video/VIDEO-1" "https://web.microsoftstream.com/video/VIDEO-2" "https://web.microsoftstream.com/video/VIDEO-3"` ### To download a list of videos -There's no implementation that does that (yet). There's some work happening to support this, give it some time. +~~There's no implementation that does that (yet). There's some work happening to support this, give it some time.~~ +See usage above. ## EXPECTED OUTPUT