Title template (#194)

* added template option and validation

* update comment link to element list

* get author info when fetching video info

* added template elements to video object

* minor function naming changes

* better exit message for template error

* changed template elements for better substitution

* implemented video title template

* removed trailing decimals on duration

* added template description

* removed hashing from uniqueId
removed debug logger.warn()

* fixed typos in default template
added elements to template fail message

* moved ffmpeg version logging to verbose

README.md

@@ -19,6 +19,11 @@ This release would not have been possible without the code and time contributed
 - [Università della Calabria][unical]: fork over at https://github.com/peppelongo96/UnicalDown
 
 ## What's new
+### v2.2
+
+- Added title template
+
+### v2.1
 
 - Major code refactoring (all credits to @lukaarma)
 - Destreamer is now able to refresh the session's access token. Use this with `-k` (keep cookies) and tick "Remember Me" on login.
@@ -138,6 +143,26 @@ https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
         -dir=videos/lessons/week2"
 ```
 
+### Title template
+The `-t` option allows users to input a template string for the output file names.
+In the template you have to use 1 or more of the following special sequence that will be substituted at runtime.
+The special sequences must be surrounded by curly brackets like this '{title} {publishDate}'
+
+- `title`: the video title
+- `duration`: the video duration in HH:MM:SS format
+- `publishDate`: the date when the video was first published in YYYY-MM-DD format
+- `publishTime`: the time when the video was first published in HH:MM:SS format
+- `author`: the video publisher's name
+- `authorEmail`: the video publisher's email
+- `uniqueId`: a (almost) unique ID generated from the video informations
+
+Example -
+```
+Input:
+-t '{title} - {duration} - {publishDate} - {publishTime} - {author} - {authorEmail} - {uniqueId}'
+Expected filename:
+This is an example - 0:16:18 - 2020-07-30 - 10:30:13 - lukaarma - example@domain.org - #3c6ca929.mkv
+```
 
 ## Expected output
 

src/CommandLineParser.ts

@@ -1,9 +1,11 @@
 import { CLI_ERROR, ERROR_CODE } from './Errors';
 import { checkOutDir } from './Utils';
 import { logger } from './Logger';
+import { templateElements } from './Types';
 
 import fs from 'fs';
 import readlineSync from 'readline-sync';
+import sanitize from 'sanitize-filename';
 import yargs from 'yargs';
 
 
@@ -33,6 +35,13 @@ export const argv: any = yargs.options({
         default: 'videos',
         demandOption: false
     },
+    outputTemplate: {
+        alias: 't',
+        describe: 'The template for the title. See the README for more info.',
+        type: 'string',
+        default: '{title} - {publishDate} {uniqueId}',
+        demandOption: false
+    },
     keepLoginCookies: {
         alias: 'k',
         describe: 'Let Chromium cache identity provider cookies so you can use "Remember me" during login',
@@ -102,7 +111,7 @@ export const argv: any = yargs.options({
 })
 .wrap(120)
 .check(() => noArguments())
-.check((argv: any) => inputConflicts(argv.videoUrls, argv.inputFile))
+.check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
 .check((argv: any) => {
     if (checkOutDir(argv.outputDirectory)) {
         return true;
@@ -113,6 +122,7 @@ export const argv: any = yargs.options({
         throw new Error(' ');
     }
 })
+.check((argv: any) => isOutputTemplateValid(argv))
 .argv;
 
 
@@ -129,7 +139,7 @@ function noArguments(): boolean {
 }
 
 
-function inputConflicts(videoUrls: Array<string | number> | undefined,
+function checkInputConflicts(videoUrls: Array<string | number> | undefined,
     inputFile: string | undefined): boolean {
     // check if both inputs are declared
     if ((videoUrls !== undefined) && (inputFile !== undefined)) {
@@ -162,6 +172,39 @@ function inputConflicts(videoUrls: Array<string | number> | undefined,
 }
 
 
+function isOutputTemplateValid(argv: any): boolean {
+    let finalTemplate: string = argv.outputTemplate;
+    const elementRegEx = RegExp(/{(.*?)}/g);
+    let match = elementRegEx.exec(finalTemplate);
+
+    // if no template elements this fails
+    if (match) {
+        // keep iterating untill we find no more elements
+        while (match) {
+            if (!templateElements.includes(match[1])) {
+                logger.error(
+                    `'${match[0]}' is not aviable as a template element \n` +
+                    `Aviable templates elements: '${templateElements.join("', '")}' \n`,
+                    { fatal: true }
+                );
+
+                process.exit(1);
+            }
+            match = elementRegEx.exec(finalTemplate);
+        }
+    }
+    // bad template from user, switching to default
+    else {
+        logger.warn('Empty output template provided, using default one \n');
+        finalTemplate = '{title} - {publishDate} {uniqueId}';
+    }
+
+    argv.outputTemplate = sanitize(finalTemplate.trim());
+
+    return true;
+}
+
+
 export function promptUser(choices: Array<string>): number {
     let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
 

src/Types.ts

@@ -6,11 +6,29 @@ export type Session = {
 
 
 export type Video = {
-    date: string;
     title: string;
+    duration: string;
+    publishDate: string;
+    publishTime: string;
+    author: string;
+    authorEmail: string;
+    uniqueId: string;
     outPath: string;
     totalChunks: number;    // Abstraction of FFmpeg timemark
     playbackUrl: string;
     posterImageUrl: string;
     captionsUrl?: string
 }
+
+
+/* TODO: expand this template once we are all on board with a list
+see https://github.com/snobu/destreamer/issues/190#issuecomment-663718010 for list*/
+export const templateElements: Array<string> = [
+    'title',
+    'duration',
+    'publishDate',
+    'publishTime',
+    'author',
+    'authorEmail',
+    'uniqueId'
+];

src/Utils.ts

@@ -182,7 +182,7 @@ export function checkOutDir(directory: string): boolean {
 export function checkRequirements(): void {
     try {
         const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
-        logger.info(`Using ${ffmpegVer}\n`);
+        logger.verbose(`Using ${ffmpegVer}\n`);
     }
     catch (e) {
         process.exit(ERROR_CODE.MISSING_FFMPEG);

src/VideoUtils.ts

@@ -5,10 +5,9 @@ import { Video, Session } from './Types';
 
 import { AxiosResponse } from 'axios';
 import fs from 'fs';
-import { parse } from 'iso8601-duration';
+import { parse as parseDuration, Duration } from 'iso8601-duration';
 import path from 'path';
-import sanitize from 'sanitize-filename';
+import sanitizeWindowsName from 'sanitize-filename';
-
 
 function publishedDateToString(date: string): string {
     const dateJs: Date = new Date(date);
@@ -19,8 +18,25 @@ function publishedDateToString(date: string): string {
 }
 
 
+function publishedTimeToString(date: string): string {
+    const dateJs: Date = new Date(date);
+    const hours: string = dateJs.getHours().toString();
+    const minutes: string = dateJs.getMinutes().toString();
+    const seconds: string = dateJs.getSeconds().toString();
+
+    return `${hours}:${minutes}:${seconds}`;
+}
+
+
+function isoDurationToString(time: string): string {
+    const duration: Duration = parseDuration(time);
+
+    return `${duration.hours ?? '00'}:${duration.minutes ?? '00'}:${duration.seconds?.toFixed(0) ?? '00'}`;
+}
+
+
 function durationToTotalChunks(duration: string): number {
-    const durationObj: any = parse(duration);
+    const durationObj: any = parseDuration(duration);
     const hrs: number = durationObj.hours ?? 0;
     const mins: number = durationObj.minutes ?? 0;
     const secs: number = Math.ceil(durationObj.seconds ?? 0);
@@ -32,7 +48,13 @@ function durationToTotalChunks(duration: string): number {
 export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
     let metadata: Array<Video> = [];
     let title: string;
-    let date: string;
+    let duration: string;
+    let publishDate: string;
+    let publishTime: string;
+    let author: string;
+    let authorEmail: string;
+    let uniqueId: string;
+    const outPath = '';
     let totalChunks: number;
     let playbackUrl: string;
     let posterImageUrl: string;
@@ -40,10 +62,28 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
 
     const apiClient: ApiClient = ApiClient.getInstance(session);
 
-    for (const GUID of videoGuids) {
+    /* TODO: change this to a single guid at a time to ease our footprint on the
-        let response: AxiosResponse<any> | undefined= await apiClient.callApi('videos/' + GUID, 'get');
+    MSS servers or we get throttled after 10 sequential reqs */
+    for (const guid of videoGuids) {
+        let response: AxiosResponse<any> | undefined =
+            await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
+
+        title = sanitizeWindowsName(response?.data['name']);
+
+        duration = isoDurationToString(response?.data.media['duration']);
+
+        publishDate = publishedDateToString(response?.data['publishedDate']);
+
+        publishTime = publishedTimeToString(response?.data['publishedDate']);
+
+        author = response?.data['creator'].name;
+
+        authorEmail = response?.data['creator'].mail;
+
+        uniqueId = '#' + guid.split('-')[0];
+
+        totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
-        title = sanitize(response?.data['name']);
         playbackUrl = response?.data['playbackUrls']
             .filter((item: { [x: string]: string; }) =>
                 item['mimeType'] == 'application/vnd.apple.mpegurl')
@@ -52,11 +92,9 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
             })[0];
 
         posterImageUrl = response?.data['posterImage']['medium']['url'];
-        date = publishedDateToString(response?.data['publishedDate']);
-        totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
         if (subtitles) {
-            let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${GUID}/texttracks`, 'get');
+            let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
 
             if (!captions?.data.value.length) {
                 captionsUrl = undefined;
@@ -74,10 +112,15 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
         }
 
         metadata.push({
-            date: date,
-            totalChunks: totalChunks,
             title: title,
-            outPath: '',
+            duration: duration,
+            publishDate: publishDate,
+            publishTime: publishTime,
+            author: author,
+            authorEmail: authorEmail,
+            uniqueId: uniqueId,
+            outPath: outPath,
+            totalChunks: totalChunks,    // Abstraction of FFmpeg timemark
             playbackUrl: playbackUrl,
             posterImageUrl: posterImageUrl,
             captionsUrl: captionsUrl
@@ -88,18 +131,29 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
 }
 
 
-export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, format: string, skip?: boolean): Array<Video> {
+export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, template: string, format: string, skip?: boolean): Array<Video> {
 
     videos.forEach((video: Video, index: number) => {
-        let title = `${video.title} - ${video.date}`;
+        let title: string = template;
-        let i = 0;
+        let finalTitle: string;
+        const elementRegEx = RegExp(/{(.*?)}/g);
+        let match = elementRegEx.exec(template);
 
-        while (!skip && fs.existsSync(path.join(outDirs[index], title + '.' + format))) {
+        while (match) {
-            title = `${video.title} - ${video.date}_${++i}`;
+            let value = video[match[1] as keyof Video] as string;
+            title = title.replace(match[0], value);
+            match = elementRegEx.exec(template);
+        }
+
+        let i = 0;
+        finalTitle = title;
+
+        while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
+            finalTitle = `${title}.${++i}`;
         }
 
 
-        video.outPath = path.join(outDirs[index], title + '.' + format);
+        video.outPath = path.join(outDirs[index], finalTitle + '.' + format);
     });
 
     return videos;

src/destreamer.ts

@@ -124,7 +124,7 @@ async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array
     logger.info('Fetching videos info... \n');
     const videos: Array<Video> = createUniquePath (
         await getVideoInfo(videoGUIDs, session, argv.closedCaptions),
-        outputDirectories, argv.format, argv.skip
+        outputDirectories, argv.outputTemplate, argv.format, argv.skip
         );
 
     if (argv.simulate) {
@@ -132,7 +132,7 @@ async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array
             logger.info(
                 '\nTitle:          '.green + video.title +
                 '\nOutPath:        '.green + video.outPath +
-                '\nPublished Date: '.green + video.date +
+                '\nPublished Date: '.green + video.publishDate +
                 '\nPlayback URL:   '.green + video.playbackUrl +
                 ((video.captionsUrl) ? ('\nCC URL:         '.green + video.captionsUrl) : '')
             );