WIP Prometheus download metrics

package.json

@@ -22,7 +22,7 @@
 		"default-import": "^1.1.5",
 		"dotenv": "^16.4.5",
 		"ffbinaries": "^1.1.6",
-		"floatplane": "^4.5.1",
+		"floatplane": "^4.5.2",
 		"html-to-text": "^9.0.5",
 		"json5": "^2.2.3",
 		"mime-types": "^2.1.35",
@@ -34,7 +34,8 @@
 		"semver": "^7.6.0",
 		"stream-throttle": "^0.1.3",
 		"tough-cookie": "^4.1.3",
-		"tough-cookie-file-store": "^2.0.3"
+		"tough-cookie-file-store": "^2.0.3",
+		"ws": "^8.16.0"
 	},
 	"devDependencies": {
 		"@types/html-to-text": "^9.0.4",
@@ -44,6 +45,7 @@
 		"@types/semver": "^7.5.8",
 		"@types/stream-throttle": "^0.1.4",
 		"@types/tough-cookie-file-store": "^2.0.4",
+		"@types/ws": "^8.5.10",
 		"@typescript-eslint/eslint-plugin": "^7.4.0",
 		"esbuild": "^0.20.2",
 		"eslint": "^8.57.0",

src/Downloader.ts

@@ -1,3 +1,4 @@
+import { Counter, Gauge } from "prom-client";
 import { MultiProgressBars, UpdateOptions } from "multi-progress-bars";
 import { Video } from "./lib/Video.js";
 
@@ -32,8 +33,39 @@ const summaryStats: { [key: string]: { totalMB: number; downloadedMB: number; do
 let AvalibleDeliverySlots = DownloadThreads;
 const DownloadQueue: (() => void)[] = [];
 
+const videoLabels = ["title", "channel", "creator"];
+const promVideosQueued = new Gauge({
+	name: "queued",
+	help: "Videos waiting to download",
+	labelNames: videoLabels,
+});
+const promVideoErrors = new Counter({
+	name: "errors",
+	help: "Video errors",
+	labelNames: ["message", ...videoLabels],
+});
+const promVideosDownloaded = new Counter({
+	name: "downloaded",
+	help: "Videos downloaded",
+	labelNames: videoLabels,
+});
+const promDownloadedBytes = new Counter({
+	name: "downloaded_bytes",
+	help: "Video downloaded bytes",
+	labelNames: videoLabels,
+});
+const promVideosDownloadedTotal = new Counter({
+	name: "downloaded_total",
+	help: "Videos downloaded",
+});
+const promDownloadedBytesTotal = new Counter({
+	name: "downloaded_bytes_total",
+	help: "Video downloaded bytes",
+});
+
 const getDownloadSempahore = async () => {
 	totalVideos++;
+	promVideosQueued.inc();
 	// If there is an available request slot, proceed immediately
 	if (AvalibleDeliverySlots > 0) return AvalibleDeliverySlots--;
 
@@ -43,6 +75,7 @@ const getDownloadSempahore = async () => {
 
 const releaseDownloadSemaphore = () => {
 	AvalibleDeliverySlots++;
+	promVideosQueued.dec();
 
 	// If there are queued requests, resolve the first one in the queue
 	DownloadQueue.shift()?.();
@@ -90,6 +123,11 @@ export const queueVideo = async (video: Video) => {
 };
 
 const processVideo = async (fTitle: string, video: Video, retries = 0) => {
+	const promLabels = {
+		title: video.post.title,
+		creator: video.post.creator.title,
+		channel: typeof video.post.channel !== "string" ? video.post.channel.title : undefined,
+	};
 	try {
 		mpb?.addTask(fTitle, {
 			type: "percentage",
@@ -112,10 +150,18 @@ const processVideo = async (fTitle: string, video: Video, retries = 0) => {
 
 				const downloadRequest = await video.download(settings.floatplane.videoResolution);
 
+				let _promDownloadedBytesLast = 0;
+				const _promDownloadedBytes = promDownloadedBytes.labels(promLabels);
+
 				downloadRequest.on("downloadProgress", (downloadProgress: DownloadProgress) => {
 					startTime ??= Date.now();
 					const timeElapsed = (Date.now() - startTime) / 1000;
 
+					const bytes = downloadProgress.transferred - _promDownloadedBytesLast;
+					_promDownloadedBytes.inc(bytes);
+					promDownloadedBytesTotal.inc(bytes);
+					_promDownloadedBytesLast = downloadProgress.transferred;
+
 					const totalMB = downloadProgress.total / 1024000;
 					const downloadedMB = downloadProgress.transferred / 1024000;
 					const downloadSpeed = downloadProgress.transferred / timeElapsed;
@@ -167,6 +213,8 @@ const processVideo = async (fTitle: string, video: Video, retries = 0) => {
 			// eslint-disable-next-line no-fallthrough
 			case Video.State.Muxed: {
 				completedVideos++;
+				promVideosDownloaded.labels(promLabels).inc();
+				promVideosDownloadedTotal.inc();
 				updateSummaryBar();
 				mpb?.done(fTitle);
 				setTimeout(() => mpb?.removeTask(fTitle), 10000 + Math.floor(Math.random() * 6000));
@@ -176,6 +224,7 @@ const processVideo = async (fTitle: string, video: Video, retries = 0) => {
 		let info;
 		if (!(error instanceof Error)) info = new Error(`Something weird happened, whatever was thrown was not a error! ${error}`);
 		else info = error;
+		promVideoErrors.labels({ message: info.message.includes("ffmpeg") ? "ffmpeg" : info.message, ...promLabels }).inc();
 		// Handle errors when downloading nicely
 		if (retries < MaxRetries) {
 			log(fTitle, { message: `\u001b[31m\u001b[1mERR\u001b[0m: ${info.message} - Retrying in ${retries}s [${retries}/${MaxRetries}]` });

src/float.ts

@@ -1,4 +1,4 @@
-import { initProm } from "./prometheus.js";
+import { initProm } from "./lib/prometheus/index.js";
 import { quickStart, validatePlexSettings } from "./quickStart.js";
 import { settings, fetchFFMPEG, fApi, args, DownloaderVersion } from "./lib/helpers.js";
 import { defaultSettings } from "./lib/defaults.js";

src/lib/Subscription.ts

@@ -90,20 +90,20 @@ export default class Subscription {
 		for (const attachmentId of blogPost.videoAttachments.sort((a, b) => blogPost.attachmentOrder.indexOf(a) - blogPost.attachmentOrder.indexOf(b))) {
 			// Make sure we have a unique object for each attachment
 			const post = { ...blogPost };
+			let videoTitle = post.title;
 			let video: VideoContent | undefined = undefined;
 			if (blogPost.videoAttachments.length > 1) {
 				dateOffset++;
 				video = await Subscription.AttachmentsCache.get(attachmentId);
-				post.title = removeRepeatedSentences(post.title, video.title);
+				videoTitle = removeRepeatedSentences(videoTitle, video.title);
 			}
 
-		 for (const channel of this.channels) {
+			for (const channel of this.channels) {
 				if (channel.isChannel === undefined) continue;
 				const isChannel =
 					Subscription.isChannelCache[channel.isChannel] ??
 					(Subscription.isChannelCache[channel.isChannel] = new Function(`${Subscription.isChannelHelper};return ${channel.isChannel};`)() as isChannel);
 
-
 				if (!isChannel(blogPost, video)) continue;
 				if (channel.skip) break;
 				if (channel.daysToKeepVideos !== undefined && new Date(post.releaseDate).getTime() < Subscription.getIgnoreBeforeTimestamp(channel)) return;
@@ -125,15 +125,15 @@ export default class Subscription {
 						/ : /i,
 					];
 					for (const regIdCheck of replacers) {
-						post.title = post.title.replace(regIdCheck, "");
+						videoTitle = videoTitle.replace(regIdCheck, "");
 					}
 
-					post.title = post.title.replaceAll("  ", " ");
-					if (post.title.startsWith(": ")) post.title = post.title.replace(": ", "");
+					videoTitle = videoTitle.replaceAll("  ", " ");
+					if (videoTitle.startsWith(": ")) videoTitle = videoTitle.replace(": ", "");
 
-					post.title = post.title.trim();
+					videoTitle = videoTitle.trim();
 				}
-				yield new Video(post, attachmentId, channel.title, dateOffset * 1000);
+				yield new Video(post, attachmentId, channel.title, videoTitle, dateOffset * 1000);
 				break;
 			}
 		}

src/lib/Video.ts

@@ -52,9 +52,10 @@ export class Video {
 	private releaseDate: Date;
 	private thumbnail: BlogPost["thumbnail"];
 
-	private attachmentId: string;
+	public readonly attachmentId: string;
+	public readonly post: BlogPost;
 
-	public title: BlogPost["title"];
+	public title: string;
 	public channelTitle: string;
 
 	public static State = VideoState;
@@ -74,11 +75,13 @@ export class Video {
 	private partialPath: string;
 	private muxedPath: string;
 
-	constructor(post: BlogPost, attachmentId: string, channelTitle: string, dateOffset: number) {
+	constructor(post: BlogPost, attachmentId: string, channelTitle: string, videoTitle: string, dateOffset: number) {
 		this.channelTitle = channelTitle;
-		this.title = post.title;
+		this.title = videoTitle;
 		this.attachmentId = attachmentId;
 
+		this.post = post;
+
 		this.description = post.text;
 		this.releaseDate = new Date(new Date(post.releaseDate).getTime() + dateOffset);
 		this.thumbnail = post.thumbnail;

src/lib/prometheus/fApi.ts

@@ -0,0 +1,29 @@
+import { Histogram } from "prom-client";
+import { fApi } from "../helpers.js";
+
+const httpRequestDurationmS = new Histogram({
+	name: "request_duration_ms",
+	help: "Duration of HTTP requests in ms",
+	labelNames: ["method", "hostname", "pathname", "status"],
+	buckets: [1, 10, 50, 100, 250, 500],
+});
+type WithStartTime<T> = T & { _startTime: number };
+fApi.extend({
+	hooks: {
+		beforeRequest: [
+			(options) => {
+				(<WithStartTime<typeof options>>options)._startTime = Date.now();
+			},
+		],
+		afterResponse: [
+			(res) => {
+				const url = res.requestUrl;
+				const options = <WithStartTime<typeof res.request.options>>res.request.options;
+				const thumbsIndex = url.pathname.indexOf("thumbnails");
+				const pathname = thumbsIndex !== -1 ? url.pathname.substring(0, thumbsIndex + thumbsIndex + 10) : url.pathname;
+				httpRequestDurationmS.observe({ method: options.method, hostname: url.hostname, pathname, status: res.statusCode }, Date.now() - options._startTime);
+				return res;
+			},
+		],
+	},
+});