From 97db39b85a19627a640fc888167f24d688a21169 Mon Sep 17 00:00:00 2001 From: Inrixia Date: Fri, 19 Apr 2024 09:42:35 +1200 Subject: [PATCH] Merge VideoBase & Video --- src/lib/Subscription.ts | 4 +- src/lib/Video.ts | 249 ++++++++++++++++++++++++++++++++++-- src/lib/VideoBase.ts | 239 ---------------------------------- src/lib/prompts/settings.ts | 4 +- src/quickStart.ts | 4 +- 5 files changed, 243 insertions(+), 257 deletions(-) delete mode 100644 src/lib/VideoBase.ts diff --git a/src/lib/Subscription.ts b/src/lib/Subscription.ts index 4921d37..97de44a 100644 --- a/src/lib/Subscription.ts +++ b/src/lib/Subscription.ts @@ -7,8 +7,6 @@ import type { ChannelOptions, SubscriptionSettings } from "./types.js"; import type { ContentPost, VideoContent } from "floatplane/content"; import type { BlogPost } from "floatplane/creator"; -import { VideoBase } from "./VideoBase.js"; - import { settings } from "./helpers/index.js"; import { ItemCache } from "./Caches.js"; import { Video } from "./Video.js"; @@ -64,7 +62,7 @@ export default class Subscription { let deletedFiles = 0; let deletedVideos = 0; - for (const video of VideoBase.find((video) => video.releaseDate < ignoreBeforeTimestamp && video.videoTitle === channel.title)) { + for (const video of Video.find((video) => video.releaseDate < ignoreBeforeTimestamp && video.videoTitle === channel.title)) { deletedVideos++; const deletionResults = await Promise.allSettled([ rm(`${video.filePath}.mp4`), diff --git a/src/lib/Video.ts b/src/lib/Video.ts index ce1f12b..f57f6bd 100644 --- a/src/lib/Video.ts +++ b/src/lib/Video.ts @@ -1,20 +1,30 @@ +import { ThrottleGroup, type ThrottleOptions } from "stream-throttle"; +import { exec as execCallback, execFile } from "child_process"; import { Counter, Gauge } from "prom-client"; -import { VideoBase, type VideoInfo } from "./VideoBase.js"; +import { htmlToText } from "html-to-text"; +import { extension } from "mime-types"; import type { Progress } from "got"; - +import builder from "xmlbuilder"; +import { promisify } from "util"; import chalk from "chalk"; -import { settings, args } from "./helpers/index.js"; +import { createWriteStream } from "fs"; +import fs from "fs/promises"; -import { ProgressHeadless } from "./logging/ProgressConsole.js"; -import { ProgressBars } from "./logging/ProgressBars.js"; +import { Attachment } from "./Attachment.js"; -import { ThrottleGroup, type ThrottleOptions } from "stream-throttle"; -import { createWriteStream } from "fs"; +import { nPad } from "@inrixia/helpers/math"; -import { promisify } from "util"; +import { settings, fApi, args } from "./helpers/index.js"; import { Semaphore } from "./helpers/Semaphore.js"; +import { fileExists } from "./helpers/fileExists.js"; +import { Selector } from "./helpers/Selector.js"; import { updatePlex } from "./helpers/updatePlex.js"; + +import { ProgressHeadless } from "./logging/ProgressConsole.js"; +import { ProgressBars } from "./logging/ProgressBars.js"; + +const exec = promisify(execCallback); const sleep = promisify(setTimeout); const promQueued = new Gauge({ @@ -35,9 +45,28 @@ const promDownloadedBytes = new Counter({ help: "Video downloaded bytes", }); +enum VideoState { + Missing, + Partial, + Muxed, +} +export type VideoInfo = { + description: string; + artworkUrl?: string; + attachmentId: string; + channelTitle: string; + videoTitle: string; + releaseDate: Date; +}; + const byteToMbits = 131072; -export class Video extends VideoBase { +export class Video extends Attachment { + private readonly description: string; + private readonly artworkUrl?: string; + + public static State = VideoState; + private static readonly MaxRetries = 5; private static readonly DownloadThreads = 8; @@ -54,8 +83,12 @@ export class Video extends VideoBase { if (this.Videos[videoInfo.attachmentId] !== undefined) return this.Videos[videoInfo.attachmentId]; return (this.Videos[videoInfo.attachmentId] = new this(videoInfo)); } + private constructor(videoInfo: VideoInfo) { super(videoInfo); + + this.description = videoInfo.description; + this.artworkUrl = videoInfo.artworkUrl; } public async download() { @@ -67,7 +100,7 @@ export class Video extends VideoBase { for (let retries = 1; retries < Video.MaxRetries + 1; retries++) { try { switch (await this.getState()) { - case VideoBase.State.Missing: { + case Video.State.Missing: { logger.log("Waiting on delivery cdn..."); const downloadRequest = await this.getVideoStream(settings.floatplane.videoResolution); @@ -112,7 +145,7 @@ export class Video extends VideoBase { } } // eslint-disable-next-line no-fallthrough - case VideoBase.State.Partial: { + case Video.State.Partial: { logger.log("Muxing ffmpeg metadata..."); await this.muxffmpegMetadata(); @@ -157,4 +190,198 @@ export class Video extends VideoBase { } return message; } + + private static async pathBytes(path: string) { + const { size } = await fs.stat(path).catch(() => ({ size: -1 })); + return size; + } + + public async getState() { + const attrStore = await this.attachmentInfo(); + + const muxedBytes = await Video.pathBytes(this.muxedPath); + // If considerAllNonPartialDownloaded is true, return true if the file exists. Otherwise check if the file is the correct size + if (settings.extras.considerAllNonPartialDownloaded && muxedBytes !== -1) attrStore.muxedBytes = muxedBytes; + if (attrStore.muxedBytes === muxedBytes) return VideoState.Muxed; + + const partialBytes = await Video.pathBytes(this.partialPath); + if (attrStore.partialBytes === partialBytes) return VideoState.Partial; + + return VideoState.Missing; + } + + public async saveNfo() { + if (await fileExists(this.nfoPath)) return; + + // Make sure the folder for the video exists + await fs.mkdir(this.folderPath, { recursive: true }); + + let season = ""; + let episode = ""; + const match = /S(\d+)E(\d+)/i.exec(this.nfoPath); + if (match !== null) { + season = match[1]; + episode = match[2]; + } + const nfo = builder + .create("episodedetails") + .ele("title") + .text(this.videoTitle) + .up() + .ele("showtitle") + .text(this.channelTitle) + .up() + .ele("description") + .text(htmlToText(this.description)) + .up() + .ele("plot") // Kodi/Plex NFO format uses `plot` as the episode description + .text(htmlToText(this.description)) + .up() + .ele("aired") // format: yyyy-mm-dd required for Kodi/Plex + .text(this.releaseDate.getFullYear().toString() + "-" + nPad(this.releaseDate.getMonth() + 1) + "-" + nPad(this.releaseDate.getDate())) + .up() + .ele("season") + .text(season) + .up() + .ele("episode") + .text(episode) + .up() + .end({ pretty: true }); + await fs.writeFile(this.nfoPath, nfo, "utf8"); + await fs.utimes(this.nfoPath, new Date(), this.releaseDate); + } + + public async downloadArtwork() { + if (!this.artworkUrl) return; + // If the file already exists + if (await this.artworkFileExtension()) return; + + // Make sure the folder for the video exists + await fs.mkdir(this.folderPath, { recursive: true }); + + // Fetch the thumbnail and get its content type + const response = await fApi.got(this.artworkUrl, { responseType: "buffer" }); + const contentType = response.headers["content-type"]; + + // Map the content type to a file extension + const fileExtension = contentType ? `.${extension(contentType)}` : Attachment.Extensions.Thumbnail; + + // Update the artworkPath with the correct file extension + const artworkPathWithExtension = `${this.artworkPath}${fileExtension}`; + + // Save the thumbnail with the correct file extension + await fs.writeFile(artworkPathWithExtension, response.body); + await fs.utimes(artworkPathWithExtension, new Date(), this.releaseDate); + } + + // The number of available slots for making delivery requests, + // limiting the rate of requests to avoid exceeding the API rate limit. + private static DeliveryTimeout = 65000; + private static DeliverySemaphore = new Semaphore(2); + private async getDelivery() { + // Ensure that we only call the delivery endpoint twice a minute at most + await Video.DeliverySemaphore.obtain(); + + // Send download request video, assume the first video attached is the actual video as most will not have more than one video + const deliveryInfo = await fApi.cdn.delivery("download", this.attachmentId); + + // Release the semaphore after DeliveryTimeout + setTimeout(() => Video.DeliverySemaphore.release(), Video.DeliveryTimeout); + + return deliveryInfo?.groups?.[0]; + } + + protected async getVideoStream(quality: string): Promise> { + if ((await this.getState()) === VideoState.Muxed) throw new Error(`Attempting to download "${this.videoTitle}" video already downloaded!`); + + // Make sure the folder for the video exists + await fs.mkdir(this.folderPath, { recursive: true }); + + const delivery = await this.getDelivery(); + if (delivery?.origins === undefined) throw new Error("Video has no origins to download from!"); + + // Round robin edges with download enabled + const downloadOrigin = Selector.next(delivery.origins); + + // Sort qualities from highest to smallest + const availableVariants = delivery.variants.filter((variant) => variant.url !== "").sort((a, b) => (b.order || 0) - (a.order || 0)); + + if (availableVariants.length === 0) throw new Error("No variants available for download!"); + + // Set the quality to use based on whats given in the settings.json or the highest available + const downloadVariant = availableVariants.find((variant) => variant.label.includes(quality)) ?? availableVariants[0]; + + const downloadRequest = fApi.got.stream(`${downloadOrigin.url}${downloadVariant.url}`); + + // Set the videos expectedSize once we know how big it should be for download validation. + downloadRequest.once("downloadProgress", async (progress) => ((await this.attachmentInfo()).partialBytes = progress.total)); + + return downloadRequest; + } + + public async muxffmpegMetadata(): Promise { + if ((await this.getState()) !== VideoState.Partial) throw new Error(`Cannot mux ffmpeg metadata! Video not downloaded.`); + + let artworkEmbed: string[] = []; + const artworkExtension = await this.artworkFileExtension(); + if (settings.extras.downloadArtwork && this.artworkUrl !== null && artworkExtension) { + artworkEmbed = ["-i", `${this.artworkPath}${artworkExtension}`, "-map", "2", "-disposition:0", "attached_pic"]; + } + + await fs.unlink(this.muxedPath).catch(() => null); + + const description = htmlToText(this.description); + const metadata = { + title: this.videoTitle, + AUTHOR: this.channelTitle, + YEAR: this.releaseDate.getFullYear().toString(), + date: `${this.releaseDate.getFullYear().toString()}${nPad(this.releaseDate.getMonth() + 1)}${nPad(this.releaseDate.getDate())}`, + description: description, + synopsis: description, + }; + const metadataFilePath = `${this.muxedPath}.metadata.txt`; + const metadataContent = Object.entries(metadata) + .map(([key, value]) => `${key}=${value.replaceAll(/\n/g, "\\\n")}`) + .join("\n"); + await fs.writeFile(metadataFilePath, `;FFMETADATA\n${metadataContent}`); + + await new Promise((resolve, reject) => + execFile( + "./db/ffmpeg", + [ + "-i", + this.partialPath, + "-i", + metadataFilePath, // Include the metadata file as an input + ...artworkEmbed, + "-map", + "0", + "-map_metadata", + "1", + "-c", + "copy", + this.muxedPath, + ], + (error, stdout, stderr) => { + if (error !== null) { + error.message ??= ""; + error.message += stderr; + reject(error); + } else resolve(stdout); + }, + ), + ); + await fs.unlink(metadataFilePath).catch(() => null); + + await fs.unlink(this.partialPath); + // Set the files update time to when the video was released + await fs.utimes(this.muxedPath, new Date(), this.releaseDate); + + (await this.attachmentInfo()).muxedBytes = await Video.pathBytes(this.muxedPath); + } + + public async postProcessingCommand(): Promise { + const result = await exec(this.formatFilePath(settings.postProcessingCommand)); + if (result.stderr !== "") throw new Error(result.stderr); + } } diff --git a/src/lib/VideoBase.ts b/src/lib/VideoBase.ts deleted file mode 100644 index 569c7b7..0000000 --- a/src/lib/VideoBase.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { exec as execCallback, execFile } from "child_process"; -import { promisify } from "util"; -import fs from "fs/promises"; -import { settings, fApi } from "./helpers/index.js"; - -import { extension } from "mime-types"; - -const exec = promisify(execCallback); - -import { htmlToText } from "html-to-text"; -import builder from "xmlbuilder"; - -import { nPad } from "@inrixia/helpers/math"; - -import { Semaphore } from "./helpers/Semaphore.js"; -import { Attachment } from "./Attachment.js"; -import { fileExists } from "./helpers/fileExists.js"; -import { Selector } from "./helpers/Selector.js"; - -enum VideoState { - Missing, - Partial, - Muxed, -} -export type VideoInfo = { - description: string; - artworkUrl?: string; - attachmentId: string; - channelTitle: string; - videoTitle: string; - releaseDate: Date; -}; -export class VideoBase extends Attachment { - private readonly description: string; - private readonly artworkUrl?: string; - - public static State = VideoState; - - protected constructor(videoInfo: VideoInfo) { - super(videoInfo); - - this.description = videoInfo.description; - this.artworkUrl = videoInfo.artworkUrl; - } - - private static async pathBytes(path: string) { - const { size } = await fs.stat(path).catch(() => ({ size: -1 })); - return size; - } - - public async getState() { - const attrStore = await this.attachmentInfo(); - - const muxedBytes = await VideoBase.pathBytes(this.muxedPath); - // If considerAllNonPartialDownloaded is true, return true if the file exists. Otherwise check if the file is the correct size - if (settings.extras.considerAllNonPartialDownloaded && muxedBytes !== -1) attrStore.muxedBytes = muxedBytes; - if (attrStore.muxedBytes === muxedBytes) return VideoState.Muxed; - - const partialBytes = await VideoBase.pathBytes(this.partialPath); - if (attrStore.partialBytes === partialBytes) return VideoState.Partial; - - return VideoState.Missing; - } - - public async saveNfo() { - if (await fileExists(this.nfoPath)) return; - - // Make sure the folder for the video exists - await fs.mkdir(this.folderPath, { recursive: true }); - - let season = ""; - let episode = ""; - const match = /S(\d+)E(\d+)/i.exec(this.nfoPath); - if (match !== null) { - season = match[1]; - episode = match[2]; - } - const nfo = builder - .create("episodedetails") - .ele("title") - .text(this.videoTitle) - .up() - .ele("showtitle") - .text(this.channelTitle) - .up() - .ele("description") - .text(htmlToText(this.description)) - .up() - .ele("plot") // Kodi/Plex NFO format uses `plot` as the episode description - .text(htmlToText(this.description)) - .up() - .ele("aired") // format: yyyy-mm-dd required for Kodi/Plex - .text(this.releaseDate.getFullYear().toString() + "-" + nPad(this.releaseDate.getMonth() + 1) + "-" + nPad(this.releaseDate.getDate())) - .up() - .ele("season") - .text(season) - .up() - .ele("episode") - .text(episode) - .up() - .end({ pretty: true }); - await fs.writeFile(this.nfoPath, nfo, "utf8"); - await fs.utimes(this.nfoPath, new Date(), this.releaseDate); - } - - public async downloadArtwork() { - if (!this.artworkUrl) return; - // If the file already exists - if (await this.artworkFileExtension()) return; - - // Make sure the folder for the video exists - await fs.mkdir(this.folderPath, { recursive: true }); - - // Fetch the thumbnail and get its content type - const response = await fApi.got(this.artworkUrl, { responseType: "buffer" }); - const contentType = response.headers["content-type"]; - - // Map the content type to a file extension - const fileExtension = contentType ? `.${extension(contentType)}` : Attachment.Extensions.Thumbnail; - - // Update the artworkPath with the correct file extension - const artworkPathWithExtension = `${this.artworkPath}${fileExtension}`; - - // Save the thumbnail with the correct file extension - await fs.writeFile(artworkPathWithExtension, response.body); - await fs.utimes(artworkPathWithExtension, new Date(), this.releaseDate); - } - - // The number of available slots for making delivery requests, - // limiting the rate of requests to avoid exceeding the API rate limit. - private static DeliveryTimeout = 65000; - private static DeliverySemaphore = new Semaphore(2); - private async getDelivery() { - // Ensure that we only call the delivery endpoint twice a minute at most - await VideoBase.DeliverySemaphore.obtain(); - - // Send download request video, assume the first video attached is the actual video as most will not have more than one video - const deliveryInfo = await fApi.cdn.delivery("download", this.attachmentId); - - // Release the semaphore after DeliveryTimeout - setTimeout(() => VideoBase.DeliverySemaphore.release(), VideoBase.DeliveryTimeout); - - return deliveryInfo?.groups?.[0]; - } - - protected async getVideoStream(quality: string): Promise> { - if ((await this.getState()) === VideoState.Muxed) throw new Error(`Attempting to download "${this.videoTitle}" video already downloaded!`); - - // Make sure the folder for the video exists - await fs.mkdir(this.folderPath, { recursive: true }); - - const delivery = await this.getDelivery(); - if (delivery?.origins === undefined) throw new Error("Video has no origins to download from!"); - - // Round robin edges with download enabled - const downloadOrigin = Selector.next(delivery.origins); - - // Sort qualities from highest to smallest - const availableVariants = delivery.variants.filter((variant) => variant.url !== "").sort((a, b) => (b.order || 0) - (a.order || 0)); - - if (availableVariants.length === 0) throw new Error("No variants available for download!"); - - // Set the quality to use based on whats given in the settings.json or the highest available - const downloadVariant = availableVariants.find((variant) => variant.label.includes(quality)) ?? availableVariants[0]; - - const downloadRequest = fApi.got.stream(`${downloadOrigin.url}${downloadVariant.url}`); - - // Set the videos expectedSize once we know how big it should be for download validation. - downloadRequest.once("downloadProgress", async (progress) => ((await this.attachmentInfo()).partialBytes = progress.total)); - - return downloadRequest; - } - - public async muxffmpegMetadata(): Promise { - if ((await this.getState()) !== VideoState.Partial) throw new Error(`Cannot mux ffmpeg metadata! Video not downloaded.`); - - let artworkEmbed: string[] = []; - const artworkExtension = await this.artworkFileExtension(); - if (settings.extras.downloadArtwork && this.artworkUrl !== null && artworkExtension) { - artworkEmbed = ["-i", `${this.artworkPath}${artworkExtension}`, "-map", "2", "-disposition:0", "attached_pic"]; - } - - await fs.unlink(this.muxedPath).catch(() => null); - - const description = htmlToText(this.description); - const metadata = { - title: this.videoTitle, - AUTHOR: this.channelTitle, - YEAR: this.releaseDate.getFullYear().toString(), - date: `${this.releaseDate.getFullYear().toString()}${nPad(this.releaseDate.getMonth() + 1)}${nPad(this.releaseDate.getDate())}`, - description: description, - synopsis: description, - }; - const metadataFilePath = `${this.muxedPath}.metadata.txt`; - const metadataContent = Object.entries(metadata) - .map(([key, value]) => `${key}=${value.replaceAll(/\n/g, "\\\n")}`) - .join("\n"); - await fs.writeFile(metadataFilePath, `;FFMETADATA\n${metadataContent}`); - - await new Promise((resolve, reject) => - execFile( - "./db/ffmpeg", - [ - "-i", - this.partialPath, - "-i", - metadataFilePath, // Include the metadata file as an input - ...artworkEmbed, - "-map", - "0", - "-map_metadata", - "1", - "-c", - "copy", - this.muxedPath, - ], - (error, stdout, stderr) => { - if (error !== null) { - error.message ??= ""; - error.message += stderr; - reject(error); - } else resolve(stdout); - }, - ), - ); - await fs.unlink(metadataFilePath).catch(() => null); - - await fs.unlink(this.partialPath); - // Set the files update time to when the video was released - await fs.utimes(this.muxedPath, new Date(), this.releaseDate); - - (await this.attachmentInfo()).muxedBytes = await VideoBase.pathBytes(this.muxedPath); - } - - public async postProcessingCommand(): Promise { - const result = await exec(this.formatFilePath(settings.postProcessingCommand)); - if (result.stderr !== "") throw new Error(result.stderr); - } -} diff --git a/src/lib/prompts/settings.ts b/src/lib/prompts/settings.ts index f766932..1e43b31 100644 --- a/src/lib/prompts/settings.ts +++ b/src/lib/prompts/settings.ts @@ -1,6 +1,6 @@ import prompts from "prompts"; import type { Extras, Resolution } from "../types.js"; -import { VideoBase } from "../VideoBase.js"; +import { Video } from "../Video.js"; /** * Prompts user for the video resolution they want to download in. @@ -25,7 +25,7 @@ export const videoResolution = async (initial: Resolution, resolutions: Array} options File formatting options available * @returns {Promise} File formatting to use */ -export const fileFormatting = async (initial: string, options: typeof VideoBase.FilePathOptions): Promise => +export const fileFormatting = async (initial: string, options: typeof Video.FilePathOptions): Promise => ( await prompts({ type: "text", diff --git a/src/quickStart.ts b/src/quickStart.ts index 22a8315..3796daa 100644 --- a/src/quickStart.ts +++ b/src/quickStart.ts @@ -5,7 +5,7 @@ import { MyPlexAccount } from "@ctrl/plex"; import * as prompts from "./lib/prompts/index.js"; import type { Extras } from "./lib/types.js"; -import { VideoBase } from "./lib/VideoBase.js"; +import { Video } from "./lib/Video.js"; export const promptPlexSections = async (): Promise => { const plexApi = await new MyPlexAccount(undefined, undefined, undefined, settings.plex.token).connect(); @@ -55,7 +55,7 @@ export const quickStart = async (): Promise => { console.log("\n== \u001b[38;5;208mGeneral\u001b[0m ==\n"); settings.floatplane.videosToSearch = await prompts.floatplane.videosToSearch(settings.floatplane.videosToSearch); settings.floatplane.videoResolution = await prompts.settings.videoResolution(settings.floatplane.videoResolution, defaultResolutions); - settings.filePathFormatting = await prompts.settings.fileFormatting(settings.filePathFormatting, VideoBase.FilePathOptions); + settings.filePathFormatting = await prompts.settings.fileFormatting(settings.filePathFormatting, Video.FilePathOptions); const extras = await prompts.settings.extras(settings.extras); if (extras !== undefined) {