mirror of
https://github.com/snobu/destreamer.git
synced 2026-02-12 17:49:42 +00:00
Compare commits
1 Commits
cea18bbf5e
...
api_thrott
| Author | SHA1 | Date | |
|---|---|---|---|
|
f4a9934efd |
20
.github/ISSUE_TEMPLATE/report-trouble.md
vendored
20
.github/ISSUE_TEMPLATE/report-trouble.md
vendored
@@ -7,24 +7,6 @@ assignees: ''
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
<!--
|
## PLEASE NEVER PASTE YOUR ACCESS TOKEN INTO A GITHUB ISSUE AS IT MAY CONTAIN PRIVATE INFORMATION
|
||||||
# BEFORE OPENING A NEW ISSUE CHECK THE EXISTING ONES AND RUN DESTREAMER WITH THE -v/--verbose flag and paste down below the output
|
|
||||||
|
|
||||||
# NEVER PASTE YOUR ACCESS TOKEN INTO A GITHUB ISSUE AS IT MAY CONTAIN PRIVATE INFORMATION
|
|
||||||
When you paste in output from destreamer, locate your access token (it looks like this: `Authorization: Bearer eyJ....<a lot more base64 encoded text>.....`) and redact it.
|
When you paste in output from destreamer, locate your access token (it looks like this: `Authorization: Bearer eyJ....<a lot more base64 encoded text>.....`) and redact it.
|
||||||
|
|
||||||
# Please fill the form below to give us some more info.
|
|
||||||
-->
|
|
||||||
|
|
||||||
OS:
|
|
||||||
|
|
||||||
Launch command used:
|
|
||||||
|
|
||||||
|
|
||||||
<details>
|
|
||||||
<summary>Verbose log</summary>
|
|
||||||
|
|
||||||
```
|
|
||||||
PASTE VERBOSE LOG HERE
|
|
||||||
```
|
|
||||||
</details>
|
|
||||||
|
|||||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -4,8 +4,6 @@
|
|||||||
*.js
|
*.js
|
||||||
*.zip
|
*.zip
|
||||||
|
|
||||||
.vscode\launch.json
|
|
||||||
|
|
||||||
.chrome_data
|
.chrome_data
|
||||||
node_modules
|
node_modules
|
||||||
videos
|
videos
|
||||||
|
|||||||
20
.vscode/launch.json
vendored
Normal file
20
.vscode/launch.json
vendored
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
// Use IntelliSense to learn about possible attributes.
|
||||||
|
// Hover to view descriptions of existing attributes.
|
||||||
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
|
||||||
|
{
|
||||||
|
"type": "node",
|
||||||
|
"runtimeArgs": ["--max-http-header-size", "32768"],
|
||||||
|
"request": "launch",
|
||||||
|
"name": "Launch Program",
|
||||||
|
"program": "${workspaceFolder}/build/src/destreamer.js",
|
||||||
|
"args": [
|
||||||
|
"-i",
|
||||||
|
"https://web.microsoftstream.com/video/ce4da1ff-0400-86ec-2ad6-f1ea83412074" // Bacon and Eggs
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
72
README.md
72
README.md
@@ -2,17 +2,13 @@
|
|||||||
<img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
|
<img src="https://github.com/snobu/destreamer/workflows/Node%20CI/badge.svg" alt="CI build status" />
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
# destreamer v3.0 (aria2c as download manager)
|
|
||||||
|
|
||||||
## This is a pre-release branch so don't expect stability, do expect speed improvements. Tons of it.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
_(Alternative artwork proposals are welcome! Submit one through an Issue.)_
|
_(Alternative artwork proposals are welcome! Submit one through an Issue.)_
|
||||||
|
|
||||||
# Saves Microsoft Stream videos for offline enjoyment
|
# Saves Microsoft Stream videos for offline enjoyment
|
||||||
|
|
||||||
### v2 Release, codename _Hammer of Dawn<sup>TM</sup>_
|
### v2.1 Release, codename _Hammer of Dawn<sup>TM</sup>_
|
||||||
|
|
||||||
This release would not have been possible without the code and time contributed by two distinguished developers: [@lukaarma](https://github.com/lukaarma) and [@kylon](https://github.com/kylon). Thank you!
|
This release would not have been possible without the code and time contributed by two distinguished developers: [@lukaarma](https://github.com/lukaarma) and [@kylon](https://github.com/kylon). Thank you!
|
||||||
|
|
||||||
@@ -21,7 +17,6 @@ This release would not have been possible without the code and time contributed
|
|||||||
- [Politecnico di Milano][polimi]: fork over at https://github.com/SamanFekri/destreamer
|
- [Politecnico di Milano][polimi]: fork over at https://github.com/SamanFekri/destreamer
|
||||||
- [Università di Pisa][unipi]: fork over at https://github.com/Guray00/destreamer-unipi
|
- [Università di Pisa][unipi]: fork over at https://github.com/Guray00/destreamer-unipi
|
||||||
- [Università della Calabria][unical]: fork over at https://github.com/peppelongo96/UnicalDown
|
- [Università della Calabria][unical]: fork over at https://github.com/peppelongo96/UnicalDown
|
||||||
- [Università degli Studi di Parma][unipr]: fork over at https://github.com/vRuslan/destreamer-unipr
|
|
||||||
|
|
||||||
## What's new
|
## What's new
|
||||||
### v2.2
|
### v2.2
|
||||||
@@ -44,8 +39,6 @@ Hopefully this doesn't break the end user agreement for Microsoft Stream. Since
|
|||||||
- **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
|
- **npm**: usually comes with Node.js, type `npm` in your terminal to check for its presence
|
||||||
- [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
|
- [**ffmpeg**][ffmpeg]: a recent version (year 2019 or above), in `$PATH` or in the same directory as this README file (project root).
|
||||||
- [**git**][git]: one or more npm dependencies require git.
|
- [**git**][git]: one or more npm dependencies require git.
|
||||||
- [**aria2**][aria2]: present in your `$PATH`, on Linux you can install via `sudo apt install aria2`.
|
|
||||||
|
|
||||||
|
|
||||||
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on. We've successfully tested it on Windows, macOS and Linux.
|
Destreamer takes a [honeybadger](https://www.youtube.com/watch?v=4r7wHMg5Yjg) approach towards the OS it's running on. We've successfully tested it on Windows, macOS and Linux.
|
||||||
|
|
||||||
@@ -58,33 +51,6 @@ Note that destreamer won't run in an elevated (Administrator/root) shell. Runnin
|
|||||||
|
|
||||||
**WSL** (Windows Subsystem for Linux) is not supported as it can't easily pop up a browser window. It *may* work by installing an X Window server (like [Xming][xming]) and exporting the default display to it (`export DISPLAY=:0`) before running destreamer. See [this issue for more on WSL v1 and v2][wsl].
|
**WSL** (Windows Subsystem for Linux) is not supported as it can't easily pop up a browser window. It *may* work by installing an X Window server (like [Xming][xming]) and exporting the default display to it (`export DISPLAY=:0`) before running destreamer. See [this issue for more on WSL v1 and v2][wsl].
|
||||||
|
|
||||||
## Can i plug in my own browser?
|
|
||||||
|
|
||||||
Yes, yes you can. This may be useful if your main browser has some authentication plugins that are required for you to logon to your Microsoft Stream tenant.
|
|
||||||
To use your own browser for the authentication part, locate the following snippet in `src/destreamer.ts`:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const browser: puppeteer.Browser = await puppeteer.launch({
|
|
||||||
executablePath: getPuppeteerChromiumPath(),
|
|
||||||
headless: false,
|
|
||||||
userDataDir: (argv.keepLoginCookies) ? chromeCacheFolder : undefined,
|
|
||||||
args: [
|
|
||||||
'--disable-dev-shm-usage',
|
|
||||||
'--fast-start',
|
|
||||||
'--no-sandbox'
|
|
||||||
]
|
|
||||||
});
|
|
||||||
```
|
|
||||||
|
|
||||||
Now, change `executablePath` to reflect the path to your browser and profile (i.e. to use Microsoft Edge on Windows):
|
|
||||||
```typescript
|
|
||||||
executablePath: "'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe' --profile-directory=Default",
|
|
||||||
```
|
|
||||||
|
|
||||||
Note that for Mac/Linux the path will look a little different but no other changes are necessary.
|
|
||||||
|
|
||||||
You need to rebuild (`npm run build`) every time you change this configuration.
|
|
||||||
|
|
||||||
## How to build
|
## How to build
|
||||||
|
|
||||||
To build destreamer clone this repository, install dependencies and run the build script -
|
To build destreamer clone this repository, install dependencies and run the build script -
|
||||||
@@ -105,35 +71,31 @@ Options:
|
|||||||
--help Show help [boolean]
|
--help Show help [boolean]
|
||||||
--version Show version number [boolean]
|
--version Show version number [boolean]
|
||||||
--username, -u The username used to log into Microsoft Stream (enabling this will fill in the email field for
|
--username, -u The username used to log into Microsoft Stream (enabling this will fill in the email field for
|
||||||
you). [string]
|
you) [string]
|
||||||
--videoUrls, -i List of urls to videos or Microsoft Stream groups. [array]
|
--videoUrls, -i List of video urls [array]
|
||||||
--inputFile, -f Path to text file containing URLs and optionally outDirs. See the README for more on outDirs.
|
--inputFile, -f Path to text file containing URLs and optionally outDirs. See the README for more on outDirs.
|
||||||
[string]
|
[string]
|
||||||
--outputDirectory, -o The directory where destreamer will save your downloads. [string] [default: "videos"]
|
|
||||||
--outputTemplate, -t The template for the title. See the README for more info.
|
--outputTemplate, -t The template for the title. See the README for more info.
|
||||||
[string] [default: "{title} - {publishDate} {uniqueId}"]
|
[string] [default: "{title} - {publishDate} {uniqueId}"]
|
||||||
--keepLoginCookies, -k Let Chromium cache identity provider cookies so you can use "Remember me" during login.
|
--outputDirectory, -o The directory where destreamer will save your downloads [string] [default: "videos"]
|
||||||
Must be used every subsequent time you launch Destreamer if you want to log in automatically.
|
--keepLoginCookies, -k Let Chromium cache identity provider cookies so you can use "Remember me" during login
|
||||||
[boolean] [default: false]
|
[boolean] [default: false]
|
||||||
--noExperiments, -x Do not attempt to render video thumbnails in the console. [boolean] [default: false]
|
--noExperiments, -x Do not attempt to render video thumbnails in the console [boolean] [default: false]
|
||||||
--simulate, -s Disable video download and print metadata information to the console.
|
--simulate, -s Disable video download and print metadata information to the console[boolean] [default: false]
|
||||||
|
--verbose, -v Print additional information to the console (use this before opening an issue on GitHub)
|
||||||
[boolean] [default: false]
|
[boolean] [default: false]
|
||||||
--verbose, -v Print additional information to the console (use this before opening an issue on GitHub).
|
--closedCaptions, --cc Check if closed captions are aviable and let the user choose which one to download (will not
|
||||||
[boolean] [default: false]
|
ask if only one aviable) [boolean] [default: false]
|
||||||
--closedCaptions, --cc Check if closed captions are available and let the user choose which one to download (will not
|
--noCleanup, --nc Do not delete the downloaded video file when an FFmpeg error occurs [boolean] [default: false]
|
||||||
ask if only one available). [boolean] [default: false]
|
|
||||||
--noCleanup, --nc Do not delete the downloaded video file when an FFmpeg error occurs.[boolean] [default: false]
|
|
||||||
--vcodec Re-encode video track. Specify FFmpeg codec (e.g. libx265) or set to "none" to disable video.
|
--vcodec Re-encode video track. Specify FFmpeg codec (e.g. libx265) or set to "none" to disable video.
|
||||||
[string] [default: "copy"]
|
[string] [default: "copy"]
|
||||||
--acodec Re-encode audio track. Specify FFmpeg codec (e.g. libopus) or set to "none" to disable audio.
|
--acodec Re-encode audio track. Specify FFmpeg codec (e.g. libopus) or set to "none" to disable audio.
|
||||||
[string] [default: "copy"]
|
[string] [default: "copy"]
|
||||||
--format Output container format (mkv, mp4, mov, anything that FFmpeg supports).
|
--format Output container format (mkv, mp4, mov, anything that FFmpeg supports)
|
||||||
[string] [default: "mkv"]
|
[string] [default: "mkv"]
|
||||||
--skip Skip download if file already exists. [boolean] [default: false]
|
--skip Skip download if file already exists [boolean] [default: false]
|
||||||
```
|
```
|
||||||
|
|
||||||
- both --videoUrls and --inputFile also accept Microsoft Teams Groups url so if your Organization placed the videos you are interested in a group you can copy the link and Destreamer will download all the videos it can inside it! A group url looks like this https://web.microsoftstream.com/group/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
|
||||||
|
|
||||||
- Passing `--username` is optional. It's there to make logging in faster (the username field will be populated automatically on the login form).
|
- Passing `--username` is optional. It's there to make logging in faster (the username field will be populated automatically on the login form).
|
||||||
|
|
||||||
- You can use an absolute path for `-o` (output directory), for example `/mnt/videos`.
|
- You can use an absolute path for `-o` (output directory), for example `/mnt/videos`.
|
||||||
@@ -178,9 +140,9 @@ These optional lines must start with white space(s).
|
|||||||
Usage -
|
Usage -
|
||||||
```
|
```
|
||||||
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
||||||
-dir="videos/lessons/week1"
|
-dir=videos/lessons/week1
|
||||||
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
|
||||||
-dir="videos/lessons/week2"
|
-dir=videos/lessons/week2"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Title template
|
### Title template
|
||||||
@@ -215,7 +177,7 @@ iTerm2 on a Mac -
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
By default, downloads are saved under project root `Destreamer/videos/` ( Not the system media Videos folder ), unless specified by `-o` (output directory).
|
By default, downloads are saved under `videos/` unless specified by `-o` (output directory).
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
@@ -230,9 +192,7 @@ Please open an [issue](https://github.com/snobu/destreamer/issues) and we'll loo
|
|||||||
[xming]: https://sourceforge.net/projects/xming/
|
[xming]: https://sourceforge.net/projects/xming/
|
||||||
[node]: https://nodejs.org/en/download/
|
[node]: https://nodejs.org/en/download/
|
||||||
[git]: https://git-scm.com/downloads
|
[git]: https://git-scm.com/downloads
|
||||||
[aria2]: https://aria2.github.io
|
|
||||||
[wsl]: https://github.com/snobu/destreamer/issues/90#issuecomment-619377950
|
[wsl]: https://github.com/snobu/destreamer/issues/90#issuecomment-619377950
|
||||||
[polimi]: https://www.polimi.it
|
[polimi]: https://www.polimi.it
|
||||||
[unipi]: https://www.unipi.it/
|
[unipi]: https://www.unipi.it/
|
||||||
[unical]: https://www.unical.it/portale/
|
[unical]: https://www.unical.it/portale/
|
||||||
[unipr]: https://www.unipr.it/
|
|
||||||
|
|||||||
7234
package-lock.json
generated
7234
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
51
package.json
51
package.json
@@ -7,9 +7,7 @@
|
|||||||
"version": "2.1.0",
|
"version": "2.1.0",
|
||||||
"description": "Save Microsoft Stream videos for offline enjoyment.",
|
"description": "Save Microsoft Stream videos for offline enjoyment.",
|
||||||
"main": "build/src/destreamer.js",
|
"main": "build/src/destreamer.js",
|
||||||
"bin": {
|
"bin": "build/src/destreamer.js",
|
||||||
"destreamer": "build/src/destreamer.js"
|
|
||||||
},
|
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "echo Transpiling TypeScript to JavaScript... && node node_modules/typescript/bin/tsc && echo Destreamer was built successfully.",
|
"build": "echo Transpiling TypeScript to JavaScript... && node node_modules/typescript/bin/tsc && echo Destreamer was built successfully.",
|
||||||
"test": "mocha build/test",
|
"test": "mocha build/test",
|
||||||
@@ -19,37 +17,34 @@
|
|||||||
"author": "snobu",
|
"author": "snobu",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/cli-progress": "^3.8.0",
|
"@types/mocha": "^7.0.2",
|
||||||
"@types/jwt-decode": "^2.2.1",
|
"@types/puppeteer": "^1.20.4",
|
||||||
"@types/mocha": "^8.0.4",
|
|
||||||
"@types/puppeteer": "^5.4.0",
|
|
||||||
"@types/readline-sync": "^1.4.3",
|
"@types/readline-sync": "^1.4.3",
|
||||||
"@types/tmp": "^0.2.0",
|
"@types/tmp": "^0.1.0",
|
||||||
"@types/ws": "^7.4.0",
|
"@types/yargs": "^15.0.3",
|
||||||
"@types/yargs": "^15.0.11",
|
"@typescript-eslint/eslint-plugin": "^2.25.0",
|
||||||
"@typescript-eslint/eslint-plugin": "^4.9.0",
|
"@typescript-eslint/parser": "^2.25.0",
|
||||||
"@typescript-eslint/parser": "^4.9.0",
|
"eslint": "^6.8.0",
|
||||||
"eslint": "^7.14.0",
|
"mocha": "^7.1.1",
|
||||||
"mocha": "^8.2.1",
|
"tmp": "^0.1.0"
|
||||||
"typescript": "^4.1.2"
|
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"axios": "^0.21.0",
|
"@tedconf/fessonia": "^2.1.0",
|
||||||
"axios-retry": "^3.1.9",
|
"@types/cli-progress": "^3.4.2",
|
||||||
"cli-progress": "^3.8.2",
|
"@types/jwt-decode": "^2.2.1",
|
||||||
|
"axios": "^0.19.2",
|
||||||
|
"axios-retry": "^3.1.8",
|
||||||
|
"cli-progress": "^3.7.0",
|
||||||
"colors": "^1.4.0",
|
"colors": "^1.4.0",
|
||||||
"is-elevated": "^3.0.0",
|
"is-elevated": "^3.0.0",
|
||||||
"iso8601-duration": "^1.3.0",
|
"iso8601-duration": "^1.2.0",
|
||||||
"jwt-decode": "^3.1.2",
|
"jwt-decode": "^2.2.0",
|
||||||
"m3u8-parser": "^4.5.0",
|
"puppeteer": "2.1.1",
|
||||||
"portfinder": "^1.0.28",
|
|
||||||
"puppeteer": "5.5.0",
|
|
||||||
"readline-sync": "^1.4.10",
|
"readline-sync": "^1.4.10",
|
||||||
"sanitize-filename": "^1.6.3",
|
"sanitize-filename": "^1.6.3",
|
||||||
"terminal-image": "^1.2.1",
|
"terminal-image": "^1.0.1",
|
||||||
"tmp": "^0.2.1",
|
"typescript": "^3.8.3",
|
||||||
"winston": "^3.3.3",
|
"winston": "^3.3.2",
|
||||||
"ws": "^7.4.0",
|
"yargs": "^15.0.3"
|
||||||
"yargs": "^16.1.1"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ export class ApiClient {
|
|||||||
this.axiosInstance = axios.create({
|
this.axiosInstance = axios.create({
|
||||||
baseURL: session?.ApiGatewayUri,
|
baseURL: session?.ApiGatewayUri,
|
||||||
// timeout: 7000,
|
// timeout: 7000,
|
||||||
headers: { 'User-Agent': 'destreamer/3.0 (Preview)' }
|
headers: { 'User-Agent': 'destreamer/2.0 (Hammer of Dawn)' }
|
||||||
});
|
});
|
||||||
|
|
||||||
axiosRetry(this.axiosInstance, {
|
axiosRetry(this.axiosInstance, {
|
||||||
@@ -34,10 +34,6 @@ export class ApiClient {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
|
logger.warn(`Got HTTP code ${err?.response?.status ?? undefined}. Retrying request...`);
|
||||||
logger.warn('Here is the error message: \n' +
|
|
||||||
JSON.stringify(err.response?.data ?? undefined) +
|
|
||||||
'\nRetrying request...');
|
|
||||||
logger.warn(`We called this URL: ${err.response?.config.baseURL}${err.response?.config.url}`);
|
|
||||||
|
|
||||||
const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
|
const shouldRetry: boolean = retryCodes.includes(err?.response?.status ?? 0);
|
||||||
|
|
||||||
@@ -46,11 +42,6 @@ export class ApiClient {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Used to initialize/retrive the active ApiClient
|
|
||||||
*
|
|
||||||
* @param session used if initializing
|
|
||||||
*/
|
|
||||||
public static getInstance(session?: Session): ApiClient {
|
public static getInstance(session?: Session): ApiClient {
|
||||||
if (!ApiClient.instance) {
|
if (!ApiClient.instance) {
|
||||||
ApiClient.instance = new ApiClient(session);
|
ApiClient.instance = new ApiClient(session);
|
||||||
@@ -59,16 +50,6 @@ export class ApiClient {
|
|||||||
return ApiClient.instance;
|
return ApiClient.instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
public setSession(session: Session): void {
|
|
||||||
if (!ApiClient.instance) {
|
|
||||||
logger.warn("Trying to update ApiCient session when it's not initialized!");
|
|
||||||
}
|
|
||||||
|
|
||||||
this.session = session;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Call Microsoft Stream API. Base URL is sourced from
|
* Call Microsoft Stream API. Base URL is sourced from
|
||||||
* the session object and prepended automatically.
|
* the session object and prepended automatically.
|
||||||
@@ -84,13 +65,6 @@ export class ApiClient {
|
|||||||
'Authorization': 'Bearer ' + this.session?.AccessToken
|
'Authorization': 'Bearer ' + this.session?.AccessToken
|
||||||
};
|
};
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
'[ApiClient.callApi]\n' +
|
|
||||||
'path: ' + path + '\n' +
|
|
||||||
'method: ' + method + '\n' +
|
|
||||||
'payload: ' + payload + '\n'
|
|
||||||
);
|
|
||||||
|
|
||||||
return this.axiosInstance?.request({
|
return this.axiosInstance?.request({
|
||||||
method: method,
|
method: method,
|
||||||
headers: headers,
|
headers: headers,
|
||||||
@@ -112,14 +86,6 @@ export class ApiClient {
|
|||||||
'Authorization': 'Bearer ' + this.session?.AccessToken
|
'Authorization': 'Bearer ' + this.session?.AccessToken
|
||||||
};
|
};
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
'[ApiClient.callUrl]\n' +
|
|
||||||
'url: ' + url + '\n' +
|
|
||||||
'method: ' + method + '\n' +
|
|
||||||
'payload: ' + payload + '\n' +
|
|
||||||
'responseType: ' + responseType + '\n'
|
|
||||||
);
|
|
||||||
|
|
||||||
return this.axiosInstance?.request({
|
return this.axiosInstance?.request({
|
||||||
method: method,
|
method: method,
|
||||||
headers: headers,
|
headers: headers,
|
||||||
|
|||||||
@@ -13,12 +13,12 @@ export const argv: any = yargs.options({
|
|||||||
username: {
|
username: {
|
||||||
alias: 'u',
|
alias: 'u',
|
||||||
type: 'string',
|
type: 'string',
|
||||||
describe: 'The username used to log into Microsoft Stream (enabling this will fill in the email field for you).',
|
describe: 'The username used to log into Microsoft Stream (enabling this will fill in the email field for you)',
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
videoUrls: {
|
videoUrls: {
|
||||||
alias: 'i',
|
alias: 'i',
|
||||||
describe: 'List of urls to videos or Microsoft Stream groups.',
|
describe: 'List of video urls',
|
||||||
type: 'array',
|
type: 'array',
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
@@ -30,7 +30,7 @@ export const argv: any = yargs.options({
|
|||||||
},
|
},
|
||||||
outputDirectory: {
|
outputDirectory: {
|
||||||
alias: 'o',
|
alias: 'o',
|
||||||
describe: 'The directory where destreamer will save your downloads.',
|
describe: 'The directory where destreamer will save your downloads',
|
||||||
type: 'string',
|
type: 'string',
|
||||||
default: 'videos',
|
default: 'videos',
|
||||||
demandOption: false
|
demandOption: false
|
||||||
@@ -44,62 +44,66 @@ export const argv: any = yargs.options({
|
|||||||
},
|
},
|
||||||
keepLoginCookies: {
|
keepLoginCookies: {
|
||||||
alias: 'k',
|
alias: 'k',
|
||||||
describe: 'Let Chromium cache identity provider cookies so you can use "Remember me" during login.\n' +
|
describe: 'Let Chromium cache identity provider cookies so you can use "Remember me" during login',
|
||||||
'Must be used every subsequent time you launch Destreamer if you want to log in automatically.',
|
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
default: false,
|
default: false,
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
noExperiments: {
|
noExperiments: {
|
||||||
alias: 'x',
|
alias: 'x',
|
||||||
describe: 'Do not attempt to render video thumbnails in the console.',
|
describe: 'Do not attempt to render video thumbnails in the console',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
default: false,
|
default: false,
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
simulate: {
|
simulate: {
|
||||||
alias: 's',
|
alias: 's',
|
||||||
describe: 'Disable video download and print metadata information to the console.',
|
describe: 'Disable video download and print metadata information to the console',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
default: false,
|
default: false,
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
verbose: {
|
verbose: {
|
||||||
alias: 'v',
|
alias: 'v',
|
||||||
describe: 'Print additional information to the console (use this before opening an issue on GitHub).',
|
describe: 'Print additional information to the console (use this before opening an issue on GitHub)',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
default: false,
|
default: false,
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
debug: {
|
|
||||||
alias: 'd',
|
|
||||||
describe: 'Set logging level to debug (only use this if you know what are doing)',
|
|
||||||
type: 'boolean',
|
|
||||||
default: false,
|
|
||||||
demandOption: false
|
|
||||||
},
|
|
||||||
selectQuality: {
|
|
||||||
alias: 'q',
|
|
||||||
describe: 'Select the quality with a number 1 (worst) trough 10 (best), 0 prompt the user for each video',
|
|
||||||
default: 10,
|
|
||||||
type: 'number',
|
|
||||||
demandOption: false
|
|
||||||
},
|
|
||||||
closedCaptions: {
|
closedCaptions: {
|
||||||
alias: 'cc',
|
alias: 'cc',
|
||||||
describe: 'Check if closed captions are available and let the user choose which one to download (will not ask if only one available)',
|
describe: 'Check if closed captions are aviable and let the user choose which one to download (will not ask if only one aviable)',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
default: false,
|
default: false,
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
|
noCleanup: {
|
||||||
|
alias: 'nc',
|
||||||
|
describe: 'Do not delete the downloaded video file when an FFmpeg error occurs',
|
||||||
|
type: 'boolean',
|
||||||
|
default: false,
|
||||||
|
demandOption: false
|
||||||
|
},
|
||||||
|
vcodec: {
|
||||||
|
describe: 'Re-encode video track. Specify FFmpeg codec (e.g. libx265) or set to "none" to disable video.',
|
||||||
|
type: 'string',
|
||||||
|
default: 'copy',
|
||||||
|
demandOption: false
|
||||||
|
},
|
||||||
|
acodec: {
|
||||||
|
describe: 'Re-encode audio track. Specify FFmpeg codec (e.g. libopus) or set to "none" to disable audio.',
|
||||||
|
type: 'string',
|
||||||
|
default: 'copy',
|
||||||
|
demandOption: false
|
||||||
|
},
|
||||||
format: {
|
format: {
|
||||||
describe: 'Output container format (mkv, mp4, mov, anything that FFmpeg supports).',
|
describe: 'Output container format (mkv, mp4, mov, anything that FFmpeg supports)',
|
||||||
type: 'string',
|
type: 'string',
|
||||||
default: 'mkv',
|
default: 'mkv',
|
||||||
demandOption: false
|
demandOption: false
|
||||||
},
|
},
|
||||||
skip: {
|
skip: {
|
||||||
describe: 'Skip download if file already exists.',
|
describe: 'Skip download if file already exists',
|
||||||
type: 'boolean',
|
type: 'boolean',
|
||||||
default: false,
|
default: false,
|
||||||
demandOption: false
|
demandOption: false
|
||||||
@@ -108,9 +112,17 @@ export const argv: any = yargs.options({
|
|||||||
.wrap(120)
|
.wrap(120)
|
||||||
.check(() => noArguments())
|
.check(() => noArguments())
|
||||||
.check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
|
.check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
|
||||||
.check((argv: any) => checkOutputDirectoryExistance(argv.outputDirectory))
|
.check((argv: any) => {
|
||||||
|
if (checkOutDir(argv.outputDirectory)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
logger.error(CLI_ERROR.INVALID_OUTDIR);
|
||||||
|
|
||||||
|
throw new Error(' ');
|
||||||
|
}
|
||||||
|
})
|
||||||
.check((argv: any) => isOutputTemplateValid(argv))
|
.check((argv: any) => isOutputTemplateValid(argv))
|
||||||
.check((argv: any) => checkQualityValue(argv))
|
|
||||||
.argv;
|
.argv;
|
||||||
|
|
||||||
|
|
||||||
@@ -160,18 +172,6 @@ function checkInputConflicts(videoUrls: Array<string | number> | undefined,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function checkOutputDirectoryExistance(dir: string): boolean {
|
|
||||||
if (checkOutDir(dir)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
logger.error(CLI_ERROR.INVALID_OUTDIR, { fatal: true });
|
|
||||||
|
|
||||||
throw new Error(' ');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function isOutputTemplateValid(argv: any): boolean {
|
function isOutputTemplateValid(argv: any): boolean {
|
||||||
let finalTemplate: string = argv.outputTemplate;
|
let finalTemplate: string = argv.outputTemplate;
|
||||||
const elementRegEx = RegExp(/{(.*?)}/g);
|
const elementRegEx = RegExp(/{(.*?)}/g);
|
||||||
@@ -183,8 +183,8 @@ function isOutputTemplateValid(argv: any): boolean {
|
|||||||
while (match) {
|
while (match) {
|
||||||
if (!templateElements.includes(match[1])) {
|
if (!templateElements.includes(match[1])) {
|
||||||
logger.error(
|
logger.error(
|
||||||
`'${match[0]}' is not available as a template element \n` +
|
`'${match[0]}' is not aviable as a template element \n` +
|
||||||
`Available templates elements: '${templateElements.join("', '")}' \n`,
|
`Aviable templates elements: '${templateElements.join("', '")}' \n`,
|
||||||
{ fatal: true }
|
{ fatal: true }
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -205,27 +205,8 @@ function isOutputTemplateValid(argv: any): boolean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function checkQualityValue(argv: any): boolean {
|
|
||||||
if (isNaN(argv.selectQuality)) {
|
|
||||||
logger.error('The quality value provided was not a number, switching to default');
|
|
||||||
argv.selectQuality = 10;
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else if (argv.selectQuality < 0 || argv.selectQuality > 10) {
|
|
||||||
logger.error('The quality value provided was outside the valid range, switching to default');
|
|
||||||
argv.selectQuality = 10;
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
export function promptUser(choices: Array<string>): number {
|
export function promptUser(choices: Array<string>): number {
|
||||||
const index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
|
let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
|
||||||
|
|
||||||
if (index === -1) {
|
if (index === -1) {
|
||||||
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
|
process.exit(ERROR_CODE.CANCELLED_USER_INPUT);
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
import { ApiClient } from './ApiClient';
|
|
||||||
import { logger } from './Logger';
|
|
||||||
import { Session } from './Types';
|
|
||||||
|
|
||||||
import crypto from 'crypto';
|
|
||||||
|
|
||||||
|
|
||||||
export async function getDecrypter(playlistUrl: string, session: Session): Promise<crypto.Decipher> {
|
|
||||||
const apiClient = ApiClient.getInstance(session);
|
|
||||||
|
|
||||||
const keyOption = await apiClient.callUrl(playlistUrl, 'get', null, 'text')
|
|
||||||
.then(res => (res?.data as string).split(/\r?\n/)
|
|
||||||
.find(line => line.startsWith('#EXT-X-KEY')));
|
|
||||||
|
|
||||||
if (keyOption) {
|
|
||||||
logger.debug('[Decrypter] CRIPTO LINE IN M3U8: ' + keyOption);
|
|
||||||
|
|
||||||
const match = RegExp(/#EXT-X-KEY:METHOD=(.*?),URI="(.*?)",IV=0X(.*)/).exec(keyOption);
|
|
||||||
|
|
||||||
if (!match) {
|
|
||||||
throw new Error('No match for regex');
|
|
||||||
}
|
|
||||||
|
|
||||||
const algorithm = match[1].toLowerCase().replace('-', '');
|
|
||||||
|
|
||||||
const key: Buffer = await apiClient.callUrl(match[2], 'post', null, 'arraybuffer')
|
|
||||||
.then(res => res?.data);
|
|
||||||
|
|
||||||
const iv = Buffer.from(match[3], 'hex');
|
|
||||||
|
|
||||||
return crypto.createDecipheriv(algorithm, key, iv);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
process.exit(555);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,310 +0,0 @@
|
|||||||
import { ERROR_CODE } from './Errors';
|
|
||||||
import { logger } from './Logger';
|
|
||||||
|
|
||||||
import cliProgress from 'cli-progress';
|
|
||||||
import WebSocket from 'ws';
|
|
||||||
|
|
||||||
|
|
||||||
export class DownloadManager {
|
|
||||||
// it's initalized in this.init()
|
|
||||||
private webSocket!: WebSocket;
|
|
||||||
private connected: boolean;
|
|
||||||
// NOTE: is there a way to fix the ETA? Can't get size nor ETA from aria that I can see
|
|
||||||
// we initialize this for each download
|
|
||||||
private progresBar!: cliProgress.Bar;
|
|
||||||
private completed: number;
|
|
||||||
private queue: Set<string>;
|
|
||||||
private index: number;
|
|
||||||
|
|
||||||
public constructor() {
|
|
||||||
this.connected = false;
|
|
||||||
this.completed = 0;
|
|
||||||
this.queue = new Set<string>();
|
|
||||||
this.index = 1;
|
|
||||||
|
|
||||||
if (!process.stdout.columns) {
|
|
||||||
logger.warn(
|
|
||||||
'Unable to get number of columns from terminal.\n' +
|
|
||||||
'This happens sometimes in Cygwin/MSYS.\n' +
|
|
||||||
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
|
|
||||||
'Please use PowerShell or cmd.exe to run destreamer on Windows.'
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* MUST BE CALLED BEFORE ANY OTHER OPERATION
|
|
||||||
*
|
|
||||||
* Wait for an established connection between the webSocket
|
|
||||||
* and Aria2c with a 10s timeout.
|
|
||||||
* Then send aria2c the global config option if specified.
|
|
||||||
*/
|
|
||||||
public async init(port: number, options?: { [option: string]: string }): Promise<void> {
|
|
||||||
let socTries = 0;
|
|
||||||
const maxTries = 10;
|
|
||||||
let timer = 0;
|
|
||||||
const waitTime = 20;
|
|
||||||
|
|
||||||
const errorHanlder = async (err: WebSocket.ErrorEvent): Promise<void> => {
|
|
||||||
// we try for 10 sec to initialize a socket on the specified port
|
|
||||||
if (err.error.code === 'ECONNREFUSED' && socTries < maxTries) {
|
|
||||||
logger.debug(`[DownloadMangaer] trying webSocket init ${socTries}/${maxTries}`);
|
|
||||||
await new Promise(r => setTimeout(r, 1000));
|
|
||||||
|
|
||||||
this.webSocket = new WebSocket(`http://localhost:${port}/jsonrpc`);
|
|
||||||
this.webSocket.onerror = errorHanlder;
|
|
||||||
this.webSocket.onopen = openHandler;
|
|
||||||
socTries++;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
logger.error(err);
|
|
||||||
process.exit(ERROR_CODE.NO_CONNECT_ARIA2C);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const openHandler = (event: WebSocket.OpenEvent): void => {
|
|
||||||
this.connected = true;
|
|
||||||
logger.debug(`[DownloadMangaer] open event recived ${event}`);
|
|
||||||
logger.info('Connected to aria2 daemon!');
|
|
||||||
};
|
|
||||||
|
|
||||||
// create webSocket
|
|
||||||
this.webSocket = new WebSocket(`http://localhost:${port}/jsonrpc`);
|
|
||||||
this.webSocket.onerror = errorHanlder;
|
|
||||||
this.webSocket.onopen = openHandler;
|
|
||||||
|
|
||||||
|
|
||||||
// wait for socket connection
|
|
||||||
while (!this.connected) {
|
|
||||||
if (timer < waitTime) {
|
|
||||||
timer++;
|
|
||||||
await new Promise(r => setTimeout(r, 1000));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
process.exit(ERROR_CODE.NO_CONNECT_ARIA2C);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setup messages handling
|
|
||||||
this.webSocket.on('message', (data: WebSocket.Data) => {
|
|
||||||
const parsed = JSON.parse(data.toString());
|
|
||||||
|
|
||||||
// print only messaged not handled during download
|
|
||||||
// NOTE: maybe we could remove this and re-add when the downloads are done
|
|
||||||
if (parsed.method !== 'aria2.onDownloadComplete' &&
|
|
||||||
parsed.method !== 'aria2.onDownloadStart' &&
|
|
||||||
parsed.method !== 'aria2.onDownloadError' &&
|
|
||||||
parsed.id !== 'getSpeed' &&
|
|
||||||
parsed.id !== 'addUrl' &&
|
|
||||||
parsed.id !== 'shutdown' &&
|
|
||||||
parsed.id !== 'getUrlForRetry') {
|
|
||||||
logger.info('[INCOMING] \n' + JSON.stringify(parsed, null, 4) + '\n\n');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
if (options) {
|
|
||||||
logger.info('Now trying to send configs...');
|
|
||||||
this.setOptions(options);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.webSocket.send(JSON.stringify({
|
|
||||||
jsonrpc: '2.0',
|
|
||||||
id: 'Destreamer',
|
|
||||||
method: 'aria2.getGlobalOption'
|
|
||||||
}));
|
|
||||||
|
|
||||||
logger.debug('[DownloadMangaer] Setup listener count on "message": ' + this.webSocket.listenerCount('message'));
|
|
||||||
}
|
|
||||||
|
|
||||||
public async close(): Promise<void> {
|
|
||||||
let exited = false;
|
|
||||||
let timer = 0;
|
|
||||||
const waitTime = 10;
|
|
||||||
|
|
||||||
this.webSocket.on('message', (data: WebSocket.Data) => {
|
|
||||||
const parsed = JSON.parse(data.toString());
|
|
||||||
|
|
||||||
if (parsed.result === 'OK') {
|
|
||||||
exited = true;
|
|
||||||
logger.verbose('Aria2c shutdown complete');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
this.webSocket.send(this.createMessage('aria2.shutdown', null, 'shutdown'));
|
|
||||||
this.webSocket.close();
|
|
||||||
|
|
||||||
while ((this.webSocket.readyState !== this.webSocket.CLOSED) || !exited) {
|
|
||||||
if (timer < waitTime) {
|
|
||||||
timer++;
|
|
||||||
await new Promise(r => setTimeout(r, 1000));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
throw new Error();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private initProgresBar(): void {
|
|
||||||
this.progresBar = new cliProgress.SingleBar({
|
|
||||||
barCompleteChar: '\u2588',
|
|
||||||
barIncompleteChar: '\u2591',
|
|
||||||
format: 'progress [{bar}] {percentage}% {speed} MB/s {eta_formatted}',
|
|
||||||
noTTYOutput: true,
|
|
||||||
notTTYSchedule: 3000,
|
|
||||||
// process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
|
|
||||||
barsize: Math.floor((process.stdout.columns || 30) / 3),
|
|
||||||
stopOnComplete: true,
|
|
||||||
hideCursor: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private createMessage(method: 'aria2.addUri', params: [[string]] | [[string], object], id?: string): string;
|
|
||||||
private createMessage(method: 'aria2.tellStatus', params: [[string]] | [string, object], id?: string): string;
|
|
||||||
private createMessage(method: 'aria2.changeOption', params: [string, object], id?: string): string;
|
|
||||||
private createMessage(method: 'aria2.changeGlobalOption', params: [{ [option: string]: string }], id?: string): string;
|
|
||||||
private createMessage(method: 'system.multicall', params: [Array<object>], id?: string): string;
|
|
||||||
// FIXME: I don't know how to properly implement this one that doesn't require params..
|
|
||||||
private createMessage(method: 'aria2.getGlobalStat', params?: null, id?: string): string;
|
|
||||||
private createMessage(method: 'aria2.shutdown', params?: null, id?: string): string;
|
|
||||||
private createMessage(method: string, params?: any, id?: string): string {
|
|
||||||
return JSON.stringify({
|
|
||||||
jsonrpc: '2.0',
|
|
||||||
id: id ?? 'Destreamer',
|
|
||||||
method: method,
|
|
||||||
// This took 40 mins just because I didn't want to use an if...so smart -_-
|
|
||||||
...(!!params && { params: params })
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private createMulticallElement(method: string, params?: any): any {
|
|
||||||
return {
|
|
||||||
methodName: method,
|
|
||||||
// This took 40 mins just because I didn't want to use an if...so smart -_-
|
|
||||||
...(!!params && { params: params })
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* For general options see
|
|
||||||
* {@link https://aria2.github.io/manual/en/html/aria2c.html#aria2.changeOption here}.
|
|
||||||
* For single download options see
|
|
||||||
* {@link https://aria2.github.io/manual/en/html/aria2c.html#aria2.changeGlobalOption here}
|
|
||||||
*
|
|
||||||
* @param options object with key: value pairs
|
|
||||||
*/
|
|
||||||
private setOptions(options: { [option: string]: string }, guid?: string): void {
|
|
||||||
const message: string = guid ?
|
|
||||||
this.createMessage('aria2.changeOption', [guid, options]) :
|
|
||||||
this.createMessage('aria2.changeGlobalOption', [options]);
|
|
||||||
|
|
||||||
this.webSocket.send(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public downloadUrls(urls: Array<string>, directory: string): Promise<void> {
|
|
||||||
return new Promise(resolve => {
|
|
||||||
|
|
||||||
this.index = 1;
|
|
||||||
this.completed = 0;
|
|
||||||
// initialize the bar as a new one
|
|
||||||
this.initProgresBar();
|
|
||||||
let barStarted = false;
|
|
||||||
|
|
||||||
const handleResponse = (data: WebSocket.Data): void => {
|
|
||||||
const parsed = JSON.parse(data.toString());
|
|
||||||
|
|
||||||
/* I ordered them in order of (probable) times called so
|
|
||||||
that we don't check useless ifs (even if we aren't caring about efficency) */
|
|
||||||
|
|
||||||
// handle download completions
|
|
||||||
if (parsed.method === 'aria2.onDownloadComplete') {
|
|
||||||
this.queue.delete(parsed.params.pop().gid.toString());
|
|
||||||
this.progresBar.update(++this.completed);
|
|
||||||
|
|
||||||
/* NOTE: probably we could use setIntervall because reling on
|
|
||||||
a completed download is good in most cases (since the segments
|
|
||||||
are small and a lot, somany and frequent updates) BUT if the user
|
|
||||||
internet speed is really low the completed downalods come in
|
|
||||||
less frequently and we have less updates */
|
|
||||||
this.webSocket.send(this.createMessage('aria2.getGlobalStat', null, 'getSpeed'));
|
|
||||||
|
|
||||||
if (this.queue.size === 0) {
|
|
||||||
this.webSocket.off('message', handleResponse);
|
|
||||||
logger.debug('[DownloadMangaer] End download listener count on "message": ' + this.webSocket.listenerCount('message'));
|
|
||||||
resolve();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handle speed update packages
|
|
||||||
else if (parsed.id === 'getSpeed') {
|
|
||||||
this.progresBar.update(this.completed,
|
|
||||||
{ speed: ((parsed.result.downloadSpeed as number) / 1000000).toFixed(2) });
|
|
||||||
}
|
|
||||||
|
|
||||||
// handle download errors
|
|
||||||
else if (parsed.method === 'aria2.onDownloadError') {
|
|
||||||
logger.error('Error while downloading, retrying...');
|
|
||||||
|
|
||||||
const errorGid: string = parsed.params.pop().gid.toString();
|
|
||||||
this.queue.delete(errorGid);
|
|
||||||
|
|
||||||
// FIXME: I don't know if it's fixed, I was not able to reproduce a fail reliably
|
|
||||||
this.webSocket.send(this.createMessage('aria2.tellStatus', [errorGid, ['files']], 'getUrlForRetry'));
|
|
||||||
}
|
|
||||||
|
|
||||||
else if (parsed.id === 'getUrlForRetry') {
|
|
||||||
const retryUrl = parsed.result.files[0].uris[0].uri;
|
|
||||||
const retryTitle = parsed.result.files[0].path;
|
|
||||||
this.webSocket.send(this.createMessage('aria2.addUri', [[retryUrl], { out: retryTitle }], 'addUrl'));
|
|
||||||
}
|
|
||||||
|
|
||||||
// handle url added to download list in aria
|
|
||||||
else if (parsed.id === 'addUrl') {
|
|
||||||
// if we recive array it's the starting list of downloads
|
|
||||||
// if it's a single string it's an error download being re-added
|
|
||||||
if (typeof parsed.result === 'string') {
|
|
||||||
this.queue.add(parsed.result.gid.toString());
|
|
||||||
}
|
|
||||||
else if (Array.isArray(parsed.result)) {
|
|
||||||
parsed.result.forEach((gid: string) =>
|
|
||||||
this.queue.add(gid.toString())
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!barStarted) {
|
|
||||||
barStarted = true;
|
|
||||||
logger.debug(`[DownloadMangaer] Starting download queue size: ${this.queue.size}`);
|
|
||||||
this.progresBar.start(this.queue.size, 0, { speed: 0 });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// FIXME: terrible workaround for 'https://github.com/snobu/destreamer/issues/232#issuecomment-699642770' :/
|
|
||||||
this.webSocket.removeAllListeners('message');
|
|
||||||
this.webSocket.on('message', (data: WebSocket.Data) => {
|
|
||||||
const parsed = JSON.parse(data.toString());
|
|
||||||
if (parsed.method !== 'aria2.onDownloadComplete' &&
|
|
||||||
parsed.method !== 'aria2.onDownloadStart' &&
|
|
||||||
parsed.method !== 'aria2.onDownloadError' &&
|
|
||||||
parsed.id !== 'getSpeed' &&
|
|
||||||
parsed.id !== 'addUrl' &&
|
|
||||||
parsed.id !== 'shutdown' &&
|
|
||||||
parsed.id !== 'getUrlForRetry') {
|
|
||||||
logger.info('[INCOMING] \n' + JSON.stringify(parsed, null, 4) + '\n\n');
|
|
||||||
}
|
|
||||||
});
|
|
||||||
logger.debug('[DownloadMangaer] Start download listener count on "message": ' + this.webSocket.listenerCount('message'));
|
|
||||||
this.webSocket.on('message', data => handleResponse(data));
|
|
||||||
|
|
||||||
const paramsForDownload: Array<any> = urls.map(url => {
|
|
||||||
const title: string = (this.index++).toString().padStart(16, '0') + '.encr';
|
|
||||||
|
|
||||||
return this.createMulticallElement(
|
|
||||||
'aria2.addUri', [[url], { out: title, dir: directory }]);
|
|
||||||
});
|
|
||||||
|
|
||||||
this.webSocket.send(
|
|
||||||
this.createMessage('system.multicall', [paramsForDownload], 'addUrl')
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,48 +1,31 @@
|
|||||||
/* let's start our error codes up high so we
|
|
||||||
don't exit with the wrong message if other modules exit with some code */
|
|
||||||
export const enum ERROR_CODE {
|
export const enum ERROR_CODE {
|
||||||
UNHANDLED_ERROR = 1000,
|
UNHANDLED_ERROR,
|
||||||
ELEVATED_SHELL,
|
ELEVATED_SHELL,
|
||||||
CANCELLED_USER_INPUT,
|
CANCELLED_USER_INPUT,
|
||||||
MISSING_FFMPEG,
|
MISSING_FFMPEG,
|
||||||
UNK_FFMPEG_ERROR,
|
UNK_FFMPEG_ERROR,
|
||||||
INVALID_VIDEO_GUID,
|
INVALID_VIDEO_GUID,
|
||||||
NO_SESSION_INFO,
|
NO_SESSION_INFO
|
||||||
NO_ENCRYPTION,
|
|
||||||
ARIA2C_CRASH,
|
|
||||||
NO_CONNECT_ARIA2C,
|
|
||||||
NO_DAEMON_PORT,
|
|
||||||
MISSING_ARIA2
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export const errors: {[key: number]: string} = {
|
export const errors: {[key: number]: string} = {
|
||||||
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error or uncaught exception! \n' +
|
[ERROR_CODE.UNHANDLED_ERROR]: 'Unhandled error!\n' +
|
||||||
'Please check your download directory/directories and try again. \n' +
|
'Timeout or fatal error, please check your downloads directory and try again',
|
||||||
'If this keep happening please report it on github "https://github.com/snobu/destreamer/issues"',
|
|
||||||
|
|
||||||
[ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell. \n' +
|
[ERROR_CODE.ELEVATED_SHELL]: 'Destreamer cannot run in an elevated (Administrator/root) shell.\n' +
|
||||||
'Please run in a regular, non-elevated window.',
|
'Please run in a regular, non-elevated window.',
|
||||||
|
|
||||||
[ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
|
[ERROR_CODE.CANCELLED_USER_INPUT]: 'Input was cancelled by user',
|
||||||
|
|
||||||
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing! Destreamer requires FFmpeg to merge videos',
|
[ERROR_CODE.MISSING_FFMPEG]: 'FFmpeg is missing!\n' +
|
||||||
|
'Destreamer requires a fairly recent release of FFmpeg to download videos',
|
||||||
[ERROR_CODE.MISSING_ARIA2]: 'Aria2c is missing! Destreamer requires Aria2c to download videos',
|
|
||||||
|
|
||||||
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
|
[ERROR_CODE.UNK_FFMPEG_ERROR]: 'Unknown FFmpeg error',
|
||||||
|
|
||||||
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
|
[ERROR_CODE.INVALID_VIDEO_GUID]: 'Unable to get video GUID from URL',
|
||||||
|
|
||||||
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page',
|
[ERROR_CODE.NO_SESSION_INFO]: 'Could not evaluate sessionInfo on the page'
|
||||||
|
|
||||||
[ERROR_CODE.NO_ENCRYPTION]: 'Could not extract the encryption info from the playlist',
|
|
||||||
|
|
||||||
[ERROR_CODE.ARIA2C_CRASH]: 'The Aria2c RPC server crashed with the previous message',
|
|
||||||
|
|
||||||
[ERROR_CODE.NO_CONNECT_ARIA2C]: 'Could not connect to Aria2c JSON-RPC WebSocket before timeout!',
|
|
||||||
|
|
||||||
[ERROR_CODE.NO_DAEMON_PORT]: 'Could not get a free port to use'
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import { logger } from './Logger';
|
|||||||
/**
|
/**
|
||||||
* This file contains global destreamer process events
|
* This file contains global destreamer process events
|
||||||
*
|
*
|
||||||
|
* @note SIGINT event is overridden in downloadVideo function
|
||||||
|
*
|
||||||
* @note function is required for non-packaged destreamer, so we can't do better
|
* @note function is required for non-packaged destreamer, so we can't do better
|
||||||
*/
|
*/
|
||||||
export function setProcessEvents(): void {
|
export function setProcessEvents(): void {
|
||||||
@@ -19,16 +21,6 @@ export function setProcessEvents(): void {
|
|||||||
logger.error({ message: msg, fatal: true });
|
logger.error({ message: msg, fatal: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
process.on('SIGINT', signal => {
|
|
||||||
logger.error(signal);
|
|
||||||
process.exit(777);
|
|
||||||
});
|
|
||||||
|
|
||||||
process.on('uncaughtException', (err: Error) => {
|
|
||||||
logger.error(err);
|
|
||||||
process.exit(ERROR_CODE.UNHANDLED_ERROR);
|
|
||||||
});
|
|
||||||
|
|
||||||
process.on('unhandledRejection', (reason: {} | null | undefined) => {
|
process.on('unhandledRejection', (reason: {} | null | undefined) => {
|
||||||
if (reason instanceof Error) {
|
if (reason instanceof Error) {
|
||||||
logger.error({ message: (reason as Error) });
|
logger.error({ message: (reason as Error) });
|
||||||
|
|||||||
@@ -35,9 +35,6 @@ function customPrint (info: winston.Logform.TransformableInfo): string {
|
|||||||
else if (info.level === 'verbose') {
|
else if (info.level === 'verbose') {
|
||||||
return colors.cyan('\n[VERBOSE] ') + info.message;
|
return colors.cyan('\n[VERBOSE] ') + info.message;
|
||||||
}
|
}
|
||||||
else if (info.level === 'debug') {
|
|
||||||
return colors.magenta('\n[debug] ') + info.message;
|
|
||||||
}
|
|
||||||
|
|
||||||
return `${info.level}: ${info.message} - ${info.timestamp}`;
|
return `${info.level}: ${info.message} - ${info.timestamp}`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { AxiosResponse } from 'axios';
|
|||||||
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
|
export async function drawThumbnail(posterImage: string, session: Session): Promise<void> {
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
|
|
||||||
const thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
|
let thumbnail: Buffer = await apiClient.callUrl(posterImage, 'get', null, 'arraybuffer')
|
||||||
.then((response: AxiosResponse<any> | undefined) => response?.data);
|
.then((response: AxiosResponse<any> | undefined) => response?.data);
|
||||||
|
|
||||||
console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));
|
console.log(await terminalImage.buffer(thumbnail, { width: 70 } ));
|
||||||
|
|||||||
@@ -9,10 +9,6 @@ import jwtDecode from 'jwt-decode';
|
|||||||
import puppeteer from 'puppeteer';
|
import puppeteer from 'puppeteer';
|
||||||
|
|
||||||
|
|
||||||
type Jwt = {
|
|
||||||
[key: string]: any
|
|
||||||
}
|
|
||||||
|
|
||||||
export class TokenCache {
|
export class TokenCache {
|
||||||
private tokenCacheFile = '.token_cache';
|
private tokenCacheFile = '.token_cache';
|
||||||
|
|
||||||
@@ -23,24 +19,30 @@ export class TokenCache {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
|
let session: Session = JSON.parse(fs.readFileSync(this.tokenCacheFile, 'utf8'));
|
||||||
|
|
||||||
const [isExpiring, timeLeft] = this.isExpiring(session);
|
type Jwt = {
|
||||||
|
[key: string]: any
|
||||||
|
}
|
||||||
|
const decodedJwt: Jwt = jwtDecode(session.AccessToken);
|
||||||
|
|
||||||
if (isExpiring) {
|
let now: number = Math.floor(Date.now() / 1000);
|
||||||
|
let exp: number = decodedJwt['exp'];
|
||||||
|
let timeLeft: number = exp - now;
|
||||||
|
|
||||||
|
if (timeLeft < 120) {
|
||||||
logger.warn('Access token has expired! \n');
|
logger.warn('Access token has expired! \n');
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
else {
|
|
||||||
logger.info(`Access token still good for ${Math.floor(timeLeft / 60)} minutes.\n`.green);
|
|
||||||
|
|
||||||
return session;
|
logger.info(`Access token still good for ${Math.floor(timeLeft / 60)} minutes.\n`.green);
|
||||||
}
|
|
||||||
|
return session;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Write(session: Session): void {
|
public Write(session: Session): void {
|
||||||
const s: string = JSON.stringify(session, null, 4);
|
let s: string = JSON.stringify(session, null, 4);
|
||||||
fs.writeFile('.token_cache', s, (err: any) => {
|
fs.writeFile('.token_cache', s, (err: any) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
return logger.error(err);
|
return logger.error(err);
|
||||||
@@ -48,23 +50,11 @@ export class TokenCache {
|
|||||||
logger.info('Fresh access token dropped into .token_cachen \n'.green);
|
logger.info('Fresh access token dropped into .token_cachen \n'.green);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public isExpiring(session: Session): [boolean, number] {
|
|
||||||
const decodedJwt: Jwt = jwtDecode(session.AccessToken);
|
|
||||||
|
|
||||||
const timeLeft: number = decodedJwt['exp'] - Math.floor(Date.now() / 1000);
|
|
||||||
|
|
||||||
if (timeLeft < (5 * 60)) {
|
|
||||||
return [true, 0];
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
return [false, timeLeft];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export async function refreshSession(url: string): Promise<Session> {
|
export async function refreshSession(url: string): Promise<Session> {
|
||||||
|
const videoId: string = url.split('/').pop() ?? process.exit(ERROR_CODE.INVALID_VIDEO_GUID);
|
||||||
|
|
||||||
const browser: puppeteer.Browser = await puppeteer.launch({
|
const browser: puppeteer.Browser = await puppeteer.launch({
|
||||||
executablePath: getPuppeteerChromiumPath(),
|
executablePath: getPuppeteerChromiumPath(),
|
||||||
@@ -80,7 +70,7 @@ export async function refreshSession(url: string): Promise<Session> {
|
|||||||
const page: puppeteer.Page = (await browser.pages())[0];
|
const page: puppeteer.Page = (await browser.pages())[0];
|
||||||
await page.goto(url, { waitUntil: 'load' });
|
await page.goto(url, { waitUntil: 'load' });
|
||||||
|
|
||||||
await browser.waitForTarget((target: puppeteer.Target) => target.url().endsWith('microsoftstream.com/'), { timeout: 150000 });
|
await browser.waitForTarget((target: puppeteer.Target) => target.url().includes(videoId), { timeout: 30000 });
|
||||||
|
|
||||||
let session: Session | null = null;
|
let session: Session | null = null;
|
||||||
let tries = 1;
|
let tries = 1;
|
||||||
|
|||||||
12
src/Types.ts
12
src/Types.ts
@@ -6,7 +6,6 @@ export type Session = {
|
|||||||
|
|
||||||
|
|
||||||
export type Video = {
|
export type Video = {
|
||||||
// the following properties are all for the title template
|
|
||||||
title: string;
|
title: string;
|
||||||
duration: string;
|
duration: string;
|
||||||
publishDate: string;
|
publishDate: string;
|
||||||
@@ -14,20 +13,15 @@ export type Video = {
|
|||||||
author: string;
|
author: string;
|
||||||
authorEmail: string;
|
authorEmail: string;
|
||||||
uniqueId: string;
|
uniqueId: string;
|
||||||
|
outPath: string;
|
||||||
// the following properties are all the urls neede for the download
|
totalChunks: number; // Abstraction of FFmpeg timemark
|
||||||
playbackUrl: string;
|
playbackUrl: string;
|
||||||
posterImageUrl: string;
|
posterImageUrl: string;
|
||||||
captionsUrl?: string
|
captionsUrl?: string
|
||||||
|
|
||||||
// final filename, already sanitized and unique
|
|
||||||
filename: string;
|
|
||||||
// complete path to save the video
|
|
||||||
outPath: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/* NOTE: expand this template once we are all on board with a list
|
/* TODO: expand this template once we are all on board with a list
|
||||||
see https://github.com/snobu/destreamer/issues/190#issuecomment-663718010 for list*/
|
see https://github.com/snobu/destreamer/issues/190#issuecomment-663718010 for list*/
|
||||||
export const templateElements: Array<string> = [
|
export const templateElements: Array<string> = [
|
||||||
'title',
|
'title',
|
||||||
|
|||||||
53
src/Utils.ts
53
src/Utils.ts
@@ -22,23 +22,9 @@ async function extractGuids(url: string, client: ApiClient): Promise<Array<strin
|
|||||||
else if (groupMatch) {
|
else if (groupMatch) {
|
||||||
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
|
const videoNumber: number = await client.callApi(`groups/${groupMatch[1]}`, 'get')
|
||||||
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
|
.then((response: AxiosResponse<any> | undefined) => response?.data.metrics.videos);
|
||||||
const result: Array<string> = [];
|
|
||||||
|
|
||||||
logger.error(videoNumber);
|
let result: Array<string> = await client.callApi(`groups/${groupMatch[1]}/videos?$top=${videoNumber}&$orderby=publishedDate asc`, 'get')
|
||||||
|
.then((response: AxiosResponse<any> | undefined) => response?.data.value.map((item: any) => item.id));
|
||||||
// Anything above $top=100 results in 400 Bad Request
|
|
||||||
// Use $skip to skip the first 100 and get another 100 and so on
|
|
||||||
for (let index = 0; index <= Math.floor(videoNumber / 100); index++) {
|
|
||||||
const partial: Array<string> = await client.callApi(
|
|
||||||
`groups/${groupMatch[1]}/videos?$skip=${100 * index}&` +
|
|
||||||
'$top=100&$orderby=publishedDate asc', 'get')
|
|
||||||
.then(
|
|
||||||
(response: AxiosResponse<any> | undefined) =>
|
|
||||||
response?.data.value.map((item: any) => item.id)
|
|
||||||
);
|
|
||||||
|
|
||||||
result.push(...partial);
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -62,7 +48,7 @@ export async function parseCLIinput(urlList: Array<string>, defaultOutDir: strin
|
|||||||
session: Session): Promise<Array<Array<string>>> {
|
session: Session): Promise<Array<Array<string>>> {
|
||||||
|
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
const guidList: Array<string> = [];
|
let guidList: Array<string> = [];
|
||||||
|
|
||||||
for (const url of urlList) {
|
for (const url of urlList) {
|
||||||
const guids: Array<string> | null = await extractGuids(url, apiClient);
|
const guids: Array<string> | null = await extractGuids(url, apiClient);
|
||||||
@@ -99,8 +85,8 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
|
|||||||
.split(/\r?\n/);
|
.split(/\r?\n/);
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
|
|
||||||
const guidList: Array<string> = [];
|
let guidList: Array<string> = [];
|
||||||
const outDirList: Array<string> = [];
|
let outDirList: Array<string> = [];
|
||||||
// if the last line was an url set this
|
// if the last line was an url set this
|
||||||
let foundUrl = false;
|
let foundUrl = false;
|
||||||
|
|
||||||
@@ -115,23 +101,23 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
|
|||||||
// parse if line is option
|
// parse if line is option
|
||||||
else if (line.includes('-dir')) {
|
else if (line.includes('-dir')) {
|
||||||
if (foundUrl) {
|
if (foundUrl) {
|
||||||
const outDir: string | null = parseOption('-dir', line);
|
let outDir: string | null = parseOption('-dir', line);
|
||||||
|
|
||||||
if (outDir && checkOutDir(outDir)) {
|
if (outDir && checkOutDir(outDir)) {
|
||||||
outDirList.push(...Array(guidList.length - outDirList.length)
|
outDirList.push(...Array(guidList.length - outDirList.length)
|
||||||
.fill(outDir));
|
.fill(outDir));
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
outDirList.push(...Array(guidList.length - outDirList.length)
|
outDirList.push(...Array(guidList.length - outDirList.length)
|
||||||
.fill(defaultOutDir));
|
.fill(defaultOutDir));
|
||||||
}
|
}
|
||||||
|
|
||||||
foundUrl = false;
|
foundUrl = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
|
logger.warn(`Found options without preceding url at line ${i + 1}, skipping..`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,7 +155,7 @@ export async function parseInputFile(inputFile: string, defaultOutDir: string,
|
|||||||
function parseOption(optionSyntax: string, item: string): string | null {
|
function parseOption(optionSyntax: string, item: string): string | null {
|
||||||
const match: RegExpMatchArray | null = item.match(
|
const match: RegExpMatchArray | null = item.match(
|
||||||
RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
|
RegExp(`^\\s*${optionSyntax}\\s?=\\s?['"](.*)['"]`)
|
||||||
);
|
);
|
||||||
|
|
||||||
return match ? match[1] : null;
|
return match ? match[1] : null;
|
||||||
}
|
}
|
||||||
@@ -182,7 +168,7 @@ export function checkOutDir(directory: string): boolean {
|
|||||||
logger.info('\nCreated directory: '.yellow + directory);
|
logger.info('\nCreated directory: '.yellow + directory);
|
||||||
}
|
}
|
||||||
catch (e) {
|
catch (e) {
|
||||||
logger.warn('Cannot create directory: ' + directory +
|
logger.warn('Cannot create directory: '+ directory +
|
||||||
'\nFalling back to default directory..');
|
'\nFalling back to default directory..');
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
@@ -193,13 +179,6 @@ export function checkOutDir(directory: string): boolean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export async function getUrlsFromPlaylist(playlistUrl: string, session: Session): Promise<Array<string>> {
|
|
||||||
return await ApiClient.getInstance(session).callUrl(playlistUrl, 'get', null, 'text')
|
|
||||||
.then(res => (res?.data as string).split(/\r?\n/)
|
|
||||||
.filter(line => !(line.startsWith('#') || line === '')));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
export function checkRequirements(): void {
|
export function checkRequirements(): void {
|
||||||
try {
|
try {
|
||||||
const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
|
const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
|
||||||
@@ -208,14 +187,6 @@ export function checkRequirements(): void {
|
|||||||
catch (e) {
|
catch (e) {
|
||||||
process.exit(ERROR_CODE.MISSING_FFMPEG);
|
process.exit(ERROR_CODE.MISSING_FFMPEG);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
|
||||||
const aria2Ver: string = execSync('aria2c --version').toString().split('\n')[0];
|
|
||||||
logger.verbose(`Using ${aria2Ver}\n`);
|
|
||||||
}
|
|
||||||
catch (e) {
|
|
||||||
process.exit(ERROR_CODE.MISSING_ARIA2);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import { parse as parseDuration, Duration } from 'iso8601-duration';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
import sanitizeWindowsName from 'sanitize-filename';
|
import sanitizeWindowsName from 'sanitize-filename';
|
||||||
|
|
||||||
|
|
||||||
function publishedDateToString(date: string): string {
|
function publishedDateToString(date: string): string {
|
||||||
const dateJs: Date = new Date(date);
|
const dateJs: Date = new Date(date);
|
||||||
const day: string = dateJs.getDate().toString().padStart(2, '0');
|
const day: string = dateJs.getDate().toString().padStart(2, '0');
|
||||||
@@ -25,22 +24,29 @@ function publishedTimeToString(date: string): string {
|
|||||||
const minutes: string = dateJs.getMinutes().toString();
|
const minutes: string = dateJs.getMinutes().toString();
|
||||||
const seconds: string = dateJs.getSeconds().toString();
|
const seconds: string = dateJs.getSeconds().toString();
|
||||||
|
|
||||||
return `${hours}.${minutes}.${seconds}`;
|
return `${hours}:${minutes}:${seconds}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function isoDurationToString(time: string): string {
|
function isoDurationToString(time: string): string {
|
||||||
const duration: Duration = parseDuration(time);
|
const duration: Duration = parseDuration(time);
|
||||||
|
|
||||||
return `${duration.hours ?? '00'}.${duration.minutes ?? '00'}.${duration.seconds?.toFixed(0) ?? '00'}`;
|
return `${duration.hours ?? '00'}:${duration.minutes ?? '00'}:${duration.seconds?.toFixed(0) ?? '00'}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export async function getVideosInfo(videoGuids: Array<string>,
|
function durationToTotalChunks(duration: string): number {
|
||||||
session: Session, subtitles?: boolean): Promise<Array<Video>> {
|
const durationObj: any = parseDuration(duration);
|
||||||
|
const hrs: number = durationObj.hours ?? 0;
|
||||||
|
const mins: number = durationObj.minutes ?? 0;
|
||||||
|
const secs: number = Math.ceil(durationObj.seconds ?? 0);
|
||||||
|
|
||||||
const metadata: Array<Video> = [];
|
return (hrs * 60) + mins + (secs / 60);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export async function getVideoInfo(videoGuid: string, session: Session, subtitles?: boolean): Promise<Video> {
|
||||||
|
// template elements
|
||||||
let title: string;
|
let title: string;
|
||||||
let duration: string;
|
let duration: string;
|
||||||
let publishDate: string;
|
let publishDate: string;
|
||||||
@@ -48,120 +54,101 @@ export async function getVideosInfo(videoGuids: Array<string>,
|
|||||||
let author: string;
|
let author: string;
|
||||||
let authorEmail: string;
|
let authorEmail: string;
|
||||||
let uniqueId: string;
|
let uniqueId: string;
|
||||||
|
// final video path (here for consistency with typedef)
|
||||||
|
const outPath = '';
|
||||||
|
// ffmpeg magic (abstraction of FFmpeg timemark)
|
||||||
|
let totalChunks: number;
|
||||||
|
// various sources
|
||||||
let playbackUrl: string;
|
let playbackUrl: string;
|
||||||
let posterImageUrl: string;
|
let posterImageUrl: string;
|
||||||
let captionsUrl: string | undefined;
|
let captionsUrl: string | undefined;
|
||||||
|
|
||||||
const apiClient: ApiClient = ApiClient.getInstance(session);
|
const apiClient: ApiClient = ApiClient.getInstance(session);
|
||||||
|
|
||||||
|
let response: AxiosResponse<any> | undefined =
|
||||||
|
await apiClient.callApi('videos/' + videoGuid + '?$expand=creator', 'get');
|
||||||
|
|
||||||
/* See 'https://github.com/snobu/destreamer/pull/203' for API throttling mitigation */
|
title = sanitizeWindowsName(response?.data['name']);
|
||||||
for (const guid of videoGuids) {
|
|
||||||
|
|
||||||
const response: AxiosResponse<any> | undefined =
|
duration = isoDurationToString(response?.data.media['duration']);
|
||||||
await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
|
|
||||||
|
|
||||||
title = sanitizeWindowsName(response?.data['name']);
|
publishDate = publishedDateToString(response?.data['publishedDate']);
|
||||||
|
|
||||||
duration = isoDurationToString(response?.data.media['duration']);
|
publishTime = publishedTimeToString(response?.data['publishedDate']);
|
||||||
|
|
||||||
publishDate = publishedDateToString(response?.data['publishedDate']);
|
author = response?.data['creator'].name;
|
||||||
|
|
||||||
publishTime = publishedTimeToString(response?.data['publishedDate']);
|
authorEmail = response?.data['creator'].mail;
|
||||||
|
|
||||||
author = response?.data['creator'].name;
|
uniqueId = '#' + videoGuid.split('-')[0];
|
||||||
|
|
||||||
authorEmail = response?.data['creator'].mail;
|
totalChunks = durationToTotalChunks(response?.data.media['duration']);
|
||||||
|
|
||||||
uniqueId = '#' + guid.split('-')[0];
|
playbackUrl = response?.data['playbackUrls']
|
||||||
|
.filter((item: { [x: string]: string; }) =>
|
||||||
|
item['mimeType'] == 'application/vnd.apple.mpegurl')
|
||||||
|
.map((item: { [x: string]: string }) => {
|
||||||
|
return item['playbackUrl'];
|
||||||
|
})[0];
|
||||||
|
|
||||||
playbackUrl = response?.data['playbackUrls']
|
posterImageUrl = response?.data['posterImage']['medium']['url'];
|
||||||
.filter((item: { [x: string]: string; }) =>
|
|
||||||
item['mimeType'] == 'application/vnd.apple.mpegurl')
|
|
||||||
.map((item: { [x: string]: string }) => {
|
|
||||||
return item['playbackUrl'];
|
|
||||||
})[0];
|
|
||||||
|
|
||||||
posterImageUrl = response?.data['posterImage']['medium']['url'];
|
if (subtitles) {
|
||||||
|
let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${videoGuid}/texttracks`, 'get');
|
||||||
|
|
||||||
if (subtitles) {
|
if (!captions?.data.value.length) {
|
||||||
const captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
|
captionsUrl = undefined;
|
||||||
|
}
|
||||||
if (!captions?.data.value.length) {
|
else if (captions?.data.value.length === 1) {
|
||||||
captionsUrl = undefined;
|
logger.info(`Found subtitles for ${title}. \n`);
|
||||||
}
|
captionsUrl = captions?.data.value.pop().url;
|
||||||
else if (captions?.data.value.length === 1) {
|
}
|
||||||
logger.info(`Found subtitles for ${title}. \n`);
|
else {
|
||||||
captionsUrl = captions?.data.value.pop().url;
|
const index: number = promptUser(captions.data.value.map((item: { language: string; autoGenerated: string; }) => {
|
||||||
}
|
return `[${item.language}] autogenerated: ${item.autoGenerated}`;
|
||||||
else {
|
}));
|
||||||
const index: number = promptUser(captions.data.value.map((item: { language: string; autoGenerated: string; }) => {
|
captionsUrl = captions.data.value[index].url;
|
||||||
return `[${item.language}] autogenerated: ${item.autoGenerated}`;
|
|
||||||
}));
|
|
||||||
captionsUrl = captions.data.value[index].url;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata.push({
|
|
||||||
title: title,
|
|
||||||
duration: duration,
|
|
||||||
publishDate: publishDate,
|
|
||||||
publishTime: publishTime,
|
|
||||||
author: author,
|
|
||||||
authorEmail: authorEmail,
|
|
||||||
uniqueId: uniqueId,
|
|
||||||
|
|
||||||
// totalChunks: totalChunks, // Abstraction of FFmpeg timemark
|
|
||||||
playbackUrl: playbackUrl,
|
|
||||||
posterImageUrl: posterImageUrl,
|
|
||||||
captionsUrl: captionsUrl,
|
|
||||||
|
|
||||||
filename: '',
|
|
||||||
outPath: '',
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return metadata;
|
return {
|
||||||
|
title: title,
|
||||||
|
duration: duration,
|
||||||
|
publishDate: publishDate,
|
||||||
|
publishTime: publishTime,
|
||||||
|
author: author,
|
||||||
|
authorEmail: authorEmail,
|
||||||
|
uniqueId: uniqueId,
|
||||||
|
outPath: outPath,
|
||||||
|
totalChunks: totalChunks,
|
||||||
|
playbackUrl: playbackUrl,
|
||||||
|
posterImageUrl: posterImageUrl,
|
||||||
|
captionsUrl: captionsUrl
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export function createUniquePaths(videos: Array<Video>, outDirs: Array<string>,
|
export function createUniquePath(video: Video, outDir: string, template: string, format: string, skip?: boolean): Video {
|
||||||
template: string, format: string, skip?: boolean): Array<Video> {
|
|
||||||
|
|
||||||
videos.forEach((video: Video, index: number) => {
|
let title: string = template;
|
||||||
let title: string = template;
|
let finalTitle: string;
|
||||||
let finalTitle: string;
|
const elementRegEx = RegExp(/{(.*?)}/g);
|
||||||
const elementRegEx = RegExp(/{(.*?)}/g);
|
let match = elementRegEx.exec(template);
|
||||||
let match = elementRegEx.exec(template);
|
|
||||||
|
|
||||||
while (match) {
|
while (match) {
|
||||||
const value = video[match[1] as keyof Video] as string;
|
let value = video[match[1] as keyof Video] as string;
|
||||||
title = title.replace(match[0], value);
|
title = title.replace(match[0], value);
|
||||||
match = elementRegEx.exec(template);
|
match = elementRegEx.exec(template);
|
||||||
}
|
}
|
||||||
|
|
||||||
let i = 0;
|
let i = 0;
|
||||||
finalTitle = title;
|
finalTitle = title;
|
||||||
|
|
||||||
while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
|
while (!skip && fs.existsSync(path.join(outDir, finalTitle + '.' + format))) {
|
||||||
finalTitle = `${title}.${++i}`;
|
finalTitle = `${title}.${++i}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const finalFileName = `${finalTitle}.${format}`;
|
video.outPath = path.join(outDir, finalTitle + '.' + format);
|
||||||
const cleanFileName = sanitizeWindowsName(finalFileName, { replacement: '_' });
|
|
||||||
if (finalFileName !== cleanFileName) {
|
|
||||||
logger.warn(
|
|
||||||
`Not a valid Windows file name: "${finalFileName}"` +
|
|
||||||
'\nReplacing invalid characters with underscores to ' +
|
|
||||||
'preserve cross-platform consistency.');
|
|
||||||
}
|
|
||||||
|
|
||||||
|
return video;
|
||||||
video.filename = finalFileName;
|
|
||||||
video.outPath = path.join(outDirs[index], finalFileName);
|
|
||||||
|
|
||||||
});
|
|
||||||
|
|
||||||
return videos;
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,38 +1,31 @@
|
|||||||
import { ApiClient } from './ApiClient';
|
import { argv } from './CommandLineParser';
|
||||||
import { argv, promptUser } from './CommandLineParser';
|
|
||||||
import { getDecrypter } from './Decrypter';
|
|
||||||
import { DownloadManager } from './DownloadManager';
|
|
||||||
import { ERROR_CODE } from './Errors';
|
import { ERROR_CODE } from './Errors';
|
||||||
import { setProcessEvents } from './Events';
|
import { setProcessEvents } from './Events';
|
||||||
import { logger } from './Logger';
|
import { logger } from './Logger';
|
||||||
import { getPuppeteerChromiumPath } from './PuppeteerHelper';
|
import { getPuppeteerChromiumPath } from './PuppeteerHelper';
|
||||||
import { drawThumbnail } from './Thumbnail';
|
import { drawThumbnail } from './Thumbnail';
|
||||||
import { TokenCache, refreshSession} from './TokenCache';
|
import { TokenCache, refreshSession } from './TokenCache';
|
||||||
import { Video, Session } from './Types';
|
import { Video, Session } from './Types';
|
||||||
import { checkRequirements, parseInputFile, parseCLIinput, getUrlsFromPlaylist} from './Utils';
|
import { checkRequirements, ffmpegTimemarkToChunk, parseInputFile, parseCLIinput} from './Utils';
|
||||||
import { getVideosInfo, createUniquePaths } from './VideoUtils';
|
import { getVideoInfo, createUniquePath } from './VideoUtils';
|
||||||
|
|
||||||
import { spawn, execSync, ChildProcess } from 'child_process';
|
import cliProgress from 'cli-progress';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import isElevated from 'is-elevated';
|
import isElevated from 'is-elevated';
|
||||||
import portfinder from 'portfinder';
|
|
||||||
import puppeteer from 'puppeteer';
|
import puppeteer from 'puppeteer';
|
||||||
import path from 'path';
|
|
||||||
import tmp from 'tmp';
|
|
||||||
|
|
||||||
|
|
||||||
// TODO: can we create an export or something for this?
|
const { FFmpegCommand, FFmpegInput, FFmpegOutput } = require('@tedconf/fessonia')();
|
||||||
const m3u8Parser: any = require('m3u8-parser');
|
|
||||||
const tokenCache: TokenCache = new TokenCache();
|
const tokenCache: TokenCache = new TokenCache();
|
||||||
const downloadManager = new DownloadManager();
|
|
||||||
export const chromeCacheFolder = '.chrome_data';
|
export const chromeCacheFolder = '.chrome_data';
|
||||||
tmp.setGracefulCleanup();
|
|
||||||
|
|
||||||
|
|
||||||
async function init(): Promise<void> {
|
async function init(): Promise<void> {
|
||||||
setProcessEvents(); // must be first!
|
setProcessEvents(); // must be first!
|
||||||
|
|
||||||
logger.level = argv.debug ? 'debug' : (argv.verbose ? 'verbose' : 'info');
|
if (argv.verbose) {
|
||||||
|
logger.level = 'verbose';
|
||||||
|
}
|
||||||
|
|
||||||
if (await isElevated()) {
|
if (await isElevated()) {
|
||||||
process.exit(ERROR_CODE.ELEVATED_SHELL);
|
process.exit(ERROR_CODE.ELEVATED_SHELL);
|
||||||
@@ -118,261 +111,163 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
|
|||||||
|
|
||||||
tokenCache.Write(session);
|
tokenCache.Write(session);
|
||||||
logger.info('Wrote access token to token cache.');
|
logger.info('Wrote access token to token cache.');
|
||||||
logger.info("At this point Chromium's job is done, shutting it down... \n\n");
|
logger.info("At this point Chromium's job is done, shutting it down...\n");
|
||||||
|
|
||||||
await browser.close();
|
await browser.close();
|
||||||
|
|
||||||
return session;
|
return session;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function downloadVideo(videoGUIDs: Array<string>,
|
|
||||||
outputDirectories: Array<string>, session: Session): Promise<void> {
|
|
||||||
|
|
||||||
const apiClient = ApiClient.getInstance(session);
|
async function downloadVideo(videoGuidArray: Array<string>, outputDirectoryArray: Array<string>, session: Session): Promise<void> {
|
||||||
|
|
||||||
logger.info('Downloading video info, this might take a while...');
|
for (const [index, videoGuid] of videoGuidArray.entries()) {
|
||||||
|
logger.info(`Fetching video's #${index} info... \n`);
|
||||||
|
|
||||||
const videos: Array<Video> = createUniquePaths (
|
const video: Video = createUniquePath (
|
||||||
await getVideosInfo(videoGUIDs, session, argv.closedCaptions),
|
await getVideoInfo(videoGuid, session, argv.closedCaptions),
|
||||||
outputDirectories, argv.outputTemplate ,argv.format, argv.skip
|
outputDirectoryArray[index], argv.outputTemplate, argv.format, argv.skip
|
||||||
);
|
);
|
||||||
|
|
||||||
if (argv.simulate) {
|
if (argv.simulate) {
|
||||||
videos.forEach(video => {
|
|
||||||
logger.info(
|
logger.info(
|
||||||
'\nTitle: '.green + video.title +
|
'\nTitle: '.green + video.title +
|
||||||
'\nOutPath: '.green + video.outPath +
|
'\nOutPath: '.green + video.outPath +
|
||||||
'\nPublished Date: '.green + video.publishDate + ' ' + video.publishTime +
|
'\nPublished Date: '.green + video.publishDate +
|
||||||
'\nPlayback URL: '.green + video.playbackUrl +
|
'\nPlayback URL: '.green + video.playbackUrl +
|
||||||
((video.captionsUrl) ? ('\nCC URL: '.green + video.captionsUrl) : '')
|
((video.captionsUrl) ? ('\nCC URL: '.green + video.captionsUrl) : '')
|
||||||
);
|
);
|
||||||
});
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info('Trying to launch and connect to aria2c...\n');
|
|
||||||
|
|
||||||
|
|
||||||
/* FIXME: aria2Exec must be defined here for the scope but later on it's complaining that it's not
|
|
||||||
initialized even if we never reach line#361 if we fail the assignment here*/
|
|
||||||
let aria2cExec: ChildProcess;
|
|
||||||
let arai2cExited = false;
|
|
||||||
await portfinder.getPortPromise({ port: 6800 }).then(
|
|
||||||
async (port: number) => {
|
|
||||||
logger.debug(`[DESTREAMER] Trying to use port ${port}`);
|
|
||||||
// Launch aria2c
|
|
||||||
aria2cExec = spawn(
|
|
||||||
'aria2c',
|
|
||||||
['--pause=true', '--enable-rpc', '--allow-overwrite=true', '--auto-file-renaming=false', `--rpc-listen-port=${port}`],
|
|
||||||
{stdio: 'ignore'}
|
|
||||||
);
|
|
||||||
|
|
||||||
aria2cExec.on('exit', (code: number | null, signal: string) => {
|
|
||||||
if (code === 0) {
|
|
||||||
logger.verbose('Aria2c process exited');
|
|
||||||
arai2cExited = true;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
logger.error(`aria2c exit code: ${code}` + '\n' + `aria2c exit signal: ${signal}`);
|
|
||||||
process.exit(ERROR_CODE.ARIA2C_CRASH);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
aria2cExec.on('error', (err) => {
|
|
||||||
logger.error(err as Error);
|
|
||||||
});
|
|
||||||
|
|
||||||
// init webSocket
|
|
||||||
await downloadManager.init(port, );
|
|
||||||
// We are connected
|
|
||||||
},
|
|
||||||
error => {
|
|
||||||
logger.error(error);
|
|
||||||
process.exit(ERROR_CODE.NO_DAEMON_PORT);
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
for (const video of videos) {
|
|
||||||
const masterParser = new m3u8Parser.Parser();
|
|
||||||
|
|
||||||
logger.info(`\nDownloading video no.${videos.indexOf(video) + 1} \n`);
|
|
||||||
|
|
||||||
if (argv.skip && fs.existsSync(video.outPath)) {
|
|
||||||
logger.info(`File already exists, skipping: ${video.outPath} \n`);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const [isSessionExpiring] = tokenCache.isExpiring(session);
|
if (argv.skip && fs.existsSync(video.outPath)) {
|
||||||
if (argv.keepLoginCookies && isSessionExpiring) {
|
logger.info(`File already exists, skipping: ${video.outPath} \n`);
|
||||||
logger.info('Trying to refresh access token...');
|
|
||||||
session = await refreshSession('https://web.microsoftstream.com/');
|
continue;
|
||||||
apiClient.setSession(session);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
masterParser.push(await apiClient.callUrl(video.playbackUrl).then(res => res?.data));
|
if (argv.keepLoginCookies && index !== 0) {
|
||||||
masterParser.end();
|
logger.info('Trying to refresh token...');
|
||||||
|
session = await refreshSession('https://web.microsoftstream.com/video/' + videoGuidArray[index]);
|
||||||
// video playlist url
|
|
||||||
let videoPlaylistUrl: string;
|
|
||||||
const videoPlaylists: Array<any> = (masterParser.manifest.playlists as Array<any>)
|
|
||||||
.filter(playlist =>
|
|
||||||
Object.prototype.hasOwnProperty.call(playlist.attributes, 'RESOLUTION'));
|
|
||||||
|
|
||||||
if (videoPlaylists.length === 1 || argv.selectQuality === 10) {
|
|
||||||
videoPlaylistUrl = videoPlaylists.pop().uri;
|
|
||||||
}
|
}
|
||||||
else if (argv.selectQuality === 0) {
|
|
||||||
const resolutions = videoPlaylists.map(playlist =>
|
const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
|
||||||
playlist.attributes.RESOLUTION.width + 'x' +
|
barCompleteChar: '\u2588',
|
||||||
playlist.attributes.RESOLUTION.height
|
barIncompleteChar: '\u2591',
|
||||||
|
format: 'progress [{bar}] {percentage}% {speed} {eta_formatted}',
|
||||||
|
// process.stdout.columns may return undefined in some terminals (Cygwin/MSYS)
|
||||||
|
barsize: Math.floor((process.stdout.columns || 30) / 3),
|
||||||
|
stopOnComplete: true,
|
||||||
|
hideCursor: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info(`\nDownloading Video: ${video.title} \n`);
|
||||||
|
logger.verbose('Extra video info \n' +
|
||||||
|
'\t Video m3u8 playlist URL: '.cyan + video.playbackUrl + '\n' +
|
||||||
|
'\t Video tumbnail URL: '.cyan + video.posterImageUrl + '\n' +
|
||||||
|
'\t Video subtitle URL (may not exist): '.cyan + video.captionsUrl + '\n' +
|
||||||
|
'\t Video total chunks: '.cyan + video.totalChunks + '\n');
|
||||||
|
|
||||||
|
logger.info('Spawning ffmpeg with access token and HLS URL. This may take a few seconds...\n\n');
|
||||||
|
if (!process.stdout.columns) {
|
||||||
|
logger.warn(
|
||||||
|
'Unable to get number of columns from terminal.\n' +
|
||||||
|
'This happens sometimes in Cygwin/MSYS.\n' +
|
||||||
|
'No progress bar can be rendered, however the download process should not be affected.\n\n' +
|
||||||
|
'Please use PowerShell or cmd.exe to run destreamer on Windows.'
|
||||||
);
|
);
|
||||||
|
|
||||||
videoPlaylistUrl = videoPlaylists[promptUser(resolutions)].uri;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
let choiche = Math.round((argv.selectQuality * videoPlaylists.length) / 10);
|
|
||||||
if (choiche === videoPlaylists.length) {
|
|
||||||
choiche--;
|
|
||||||
}
|
|
||||||
logger.debug(`Video quality choiche: ${choiche}`);
|
|
||||||
videoPlaylistUrl = videoPlaylists[choiche].uri;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// audio playlist url
|
const headers: string = 'Authorization: Bearer ' + session.AccessToken;
|
||||||
// TODO: better audio playlists parsing? With language maybe?
|
|
||||||
const audioPlaylists: Array<string> =
|
|
||||||
Object.keys(masterParser.manifest.mediaGroups.AUDIO.audio);
|
|
||||||
const audioPlaylistUrl: string =
|
|
||||||
masterParser.manifest.mediaGroups.AUDIO.audio[audioPlaylists[0]].uri;
|
|
||||||
// if (audioPlaylists.length === 1){
|
|
||||||
// audioPlaylistUrl = masterParser.manifest.mediaGroups.AUDIO
|
|
||||||
// .audio[audioPlaylists[0]].uri;
|
|
||||||
// }
|
|
||||||
// else {
|
|
||||||
// audioPlaylistUrl = masterParser.manifest.mediaGroups.AUDIO
|
|
||||||
// .audio[audioPlaylists[promptUser(audioPlaylists)]].uri;
|
|
||||||
// }
|
|
||||||
|
|
||||||
const videoUrls = await getUrlsFromPlaylist(videoPlaylistUrl, session);
|
|
||||||
const audioUrls = await getUrlsFromPlaylist(audioPlaylistUrl, session);
|
|
||||||
const videoDecrypter = await getDecrypter(videoPlaylistUrl, session);
|
|
||||||
const audioDecrypter = await getDecrypter(videoPlaylistUrl, session);
|
|
||||||
|
|
||||||
if (!argv.noExperiments) {
|
if (!argv.noExperiments) {
|
||||||
await drawThumbnail(video.posterImageUrl, session);
|
await drawThumbnail(video.posterImageUrl, session);
|
||||||
}
|
}
|
||||||
|
|
||||||
// video download
|
const ffmpegInpt: any = new FFmpegInput(video.playbackUrl, new Map([
|
||||||
const videoSegmentsDir = tmp.dirSync({
|
['headers', headers]
|
||||||
prefix: 'video',
|
]));
|
||||||
tmpdir: path.dirname(video.outPath),
|
const ffmpegOutput: any = new FFmpegOutput(video.outPath, new Map([
|
||||||
unsafeCleanup: true
|
argv.acodec === 'none' ? ['an', null] : ['c:a', argv.acodec],
|
||||||
|
argv.vcodec === 'none' ? ['vn', null] : ['c:v', argv.vcodec],
|
||||||
|
['n', null]
|
||||||
|
]));
|
||||||
|
const ffmpegCmd: any = new FFmpegCommand();
|
||||||
|
|
||||||
|
const cleanupFn: () => void = () => {
|
||||||
|
pbar.stop();
|
||||||
|
|
||||||
|
if (argv.noCleanup) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(video.outPath);
|
||||||
|
}
|
||||||
|
catch (e) {
|
||||||
|
// Future handling of an error (maybe)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
pbar.start(video.totalChunks, 0, {
|
||||||
|
speed: '0'
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.info('\nDownloading video segments \n');
|
// prepare ffmpeg command line
|
||||||
await downloadManager.downloadUrls(videoUrls, videoSegmentsDir.name);
|
ffmpegCmd.addInput(ffmpegInpt);
|
||||||
|
ffmpegCmd.addOutput(ffmpegOutput);
|
||||||
// audio download
|
|
||||||
const audioSegmentsDir = tmp.dirSync({
|
|
||||||
prefix: 'audio',
|
|
||||||
tmpdir: path.dirname(video.outPath),
|
|
||||||
unsafeCleanup: true
|
|
||||||
});
|
|
||||||
|
|
||||||
logger.info('\nDownloading audio segments \n');
|
|
||||||
await downloadManager.downloadUrls(audioUrls, audioSegmentsDir.name);
|
|
||||||
|
|
||||||
// subs download
|
|
||||||
if (argv.closedCaptions && video.captionsUrl) {
|
if (argv.closedCaptions && video.captionsUrl) {
|
||||||
logger.info('\nDownloading subtitles \n');
|
const captionsInpt: any = new FFmpegInput(video.captionsUrl, new Map([
|
||||||
await apiClient.callUrl(video.captionsUrl, 'get', null, 'text')
|
['headers', headers]
|
||||||
.then(res => fs.writeFileSync(
|
]));
|
||||||
path.join(videoSegmentsDir.name, 'CC.vtt'), res?.data));
|
|
||||||
|
ffmpegCmd.addInput(captionsInpt);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info('\n\nMerging and decrypting video and audio segments...\n');
|
ffmpegCmd.on('update', async (data: any) => {
|
||||||
|
const currentChunks: number = ffmpegTimemarkToChunk(data.out_time);
|
||||||
|
|
||||||
const cmd = (process.platform == 'win32') ? 'copy /b *.encr ' : 'cat *.encr > ';
|
pbar.update(currentChunks, {
|
||||||
|
speed: data.bitrate
|
||||||
|
});
|
||||||
|
|
||||||
execSync(cmd + `"${video.filename}.video.encr"`, { cwd: videoSegmentsDir.name });
|
// Graceful fallback in case we can't get columns (Cygwin/MSYS)
|
||||||
const videoDecryptInput = fs.createReadStream(
|
if (!process.stdout.columns) {
|
||||||
path.join(videoSegmentsDir.name, video.filename + '.video.encr'));
|
process.stdout.write(`--- Speed: ${data.bitrate}, Cursor: ${data.out_time}\r`);
|
||||||
const videoDecryptOutput = fs.createWriteStream(
|
}
|
||||||
path.join(videoSegmentsDir.name, video.filename + '.video'));
|
|
||||||
|
|
||||||
const decryptVideoPromise = new Promise(resolve => {
|
|
||||||
videoDecryptOutput.on('finish', resolve);
|
|
||||||
videoDecryptInput.pipe(videoDecrypter).pipe(videoDecryptOutput);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
execSync(cmd + `"${video.filename}.audio.encr"`, {cwd: audioSegmentsDir.name});
|
process.on('SIGINT', cleanupFn);
|
||||||
const audioDecryptInput = fs.createReadStream(
|
|
||||||
path.join(audioSegmentsDir.name, video.filename + '.audio.encr'));
|
|
||||||
const audioDecryptOutput = fs.createWriteStream(
|
|
||||||
path.join(audioSegmentsDir.name, video.filename + '.audio'));
|
|
||||||
|
|
||||||
const decryptAudioPromise = new Promise(resolve => {
|
// let the magic begin...
|
||||||
audioDecryptOutput.on('finish', resolve);
|
await new Promise((resolve: any) => {
|
||||||
audioDecryptInput.pipe(audioDecrypter).pipe(audioDecryptOutput);
|
ffmpegCmd.on('error', (error: any) => {
|
||||||
|
cleanupFn();
|
||||||
|
|
||||||
|
logger.error(`FFmpeg returned an error: ${error.message}`);
|
||||||
|
process.exit(ERROR_CODE.UNK_FFMPEG_ERROR);
|
||||||
|
});
|
||||||
|
|
||||||
|
ffmpegCmd.on('success', () => {
|
||||||
|
pbar.update(video.totalChunks); // set progress bar to 100%
|
||||||
|
logger.info(`\nDownload finished: ${video.outPath} \n`);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
ffmpegCmd.spawn();
|
||||||
});
|
});
|
||||||
|
|
||||||
await Promise.all([decryptVideoPromise, decryptAudioPromise]);
|
process.removeListener('SIGINT', cleanupFn);
|
||||||
|
|
||||||
logger.info('Decrypted!\n');
|
|
||||||
|
|
||||||
logger.info('Merging vdeo and audio together...\n');
|
|
||||||
const mergeCommand = (
|
|
||||||
// add video input
|
|
||||||
`ffmpeg -i "${path.join(videoSegmentsDir.name, video.filename + '.video')}" ` +
|
|
||||||
// add audio input
|
|
||||||
`-i "${path.join(audioSegmentsDir.name, video.filename + '.audio')}" ` +
|
|
||||||
// add subtitles input if present and wanted
|
|
||||||
((argv.closedCaptions && video.captionsUrl) ?
|
|
||||||
`-i "${path.join(videoSegmentsDir.name, 'CC.vtt')}" ` : '') +
|
|
||||||
// copy codec and output path
|
|
||||||
`-c copy "${video.outPath}"`
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.debug('[destreamer] ' + mergeCommand);
|
|
||||||
|
|
||||||
execSync(mergeCommand, { stdio: 'ignore' });
|
|
||||||
|
|
||||||
logger.info('Done! Removing temp files...\n');
|
|
||||||
|
|
||||||
videoSegmentsDir.removeCallback();
|
|
||||||
audioSegmentsDir.removeCallback();
|
|
||||||
|
|
||||||
logger.info(`Video no.${videos.indexOf(video) + 1} downloaded!!\n\n`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info('Exiting, this will take some seconds...');
|
|
||||||
|
|
||||||
logger.debug('[destreamer] closing downloader socket');
|
|
||||||
await downloadManager.close();
|
|
||||||
logger.debug('[destreamer] closed downloader. Waiting aria2c deamon exit');
|
|
||||||
let tries = 0;
|
|
||||||
while (!arai2cExited) {
|
|
||||||
if (tries < 10) {
|
|
||||||
tries++;
|
|
||||||
await new Promise(r => setTimeout(r, 1000));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
aria2cExec!.kill('SIGINT');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logger.debug('[destreamer] stopped aria2c');
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
async function main(): Promise<void> {
|
||||||
await init(); // must be first
|
await init(); // must be first
|
||||||
|
|
||||||
|
let session: Session;
|
||||||
const session: Session = tokenCache.Read() ??
|
session = tokenCache.Read() ?? await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
|
||||||
await DoInteractiveLogin('https://web.microsoftstream.com/', argv.username);
|
|
||||||
|
|
||||||
logger.verbose('Session and API info \n' +
|
logger.verbose('Session and API info \n' +
|
||||||
'\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
|
'\t API Gateway URL: '.cyan + session.ApiGatewayUri + '\n' +
|
||||||
@@ -390,13 +285,12 @@ async function main(): Promise<void> {
|
|||||||
[videoGUIDs, outDirs] = await parseInputFile(argv.inputFile!, argv.outputDirectory, session);
|
[videoGUIDs, outDirs] = await parseInputFile(argv.inputFile!, argv.outputDirectory, session);
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.verbose('List of videos and corresponding output directory \n' +
|
logger.verbose('List of GUIDs and corresponding output directory \n' +
|
||||||
videoGUIDs.map((guid: string, i: number) =>
|
videoGUIDs.map((guid: string, i: number) =>
|
||||||
`\thttps://web.microsoftstream.com/video/${guid} => ${outDirs[i]} \n`).join(''));
|
`\thttps://web.microsoftstream.com/video/${guid} => ${outDirs[i]} \n`).join(''));
|
||||||
|
|
||||||
|
|
||||||
// fuck you bug, I WON!!!
|
downloadVideo(videoGUIDs, outDirs, session);
|
||||||
await downloadVideo(videoGUIDs, outDirs, session);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user