From 81654ceec9a685974ff662d2ef853c1cc1c7b2ed Mon Sep 17 00:00:00 2001
From: William Candillon <wcandillon@gmail.com>
Date: Thu, 13 Jun 2024 09:47:13 +0200
Subject: [PATCH] =?UTF-8?q?fix(=F0=9F=93=B9):=20Video=20support=20on=20Web?=
 =?UTF-8?q?=20(#2477)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/docs/video.md                            |  4 +-
 package/src/external/reanimated/useVideo.ts   | 41 ++------
 .../external/reanimated/useVideoLoading.ts    | 28 ++++++
 .../reanimated/useVideoLoading.web.ts         | 17 ++++
 .../src/renderer/__tests__/e2e/Video.spec.tsx |  5 +-
 .../types/NativeBuffer/NativeBufferFactory.ts | 12 ++-
 package/src/skia/types/Skia.ts                |  2 +-
 .../src/skia/web/CanvasKitWebGLBufferImpl.ts  | 22 +++++
 package/src/skia/web/JsiSkImageFactory.ts     | 19 +++-
 package/src/skia/web/JsiSkia.ts               |  5 +-
 package/src/skia/web/JsiVideo.ts              | 96 +++++++++++++++++++
 11 files changed, 208 insertions(+), 43 deletions(-)
 create mode 100644 package/src/external/reanimated/useVideoLoading.ts
 create mode 100644 package/src/external/reanimated/useVideoLoading.web.ts
 create mode 100644 package/src/skia/web/CanvasKitWebGLBufferImpl.ts
 create mode 100644 package/src/skia/web/JsiVideo.ts

diff --git a/docs/docs/video.md b/docs/docs/video.md
index 670ed29118..213efe4efd 100644
--- a/docs/docs/video.md
+++ b/docs/docs/video.md
@@ -5,7 +5,9 @@ sidebar_label: Video
 slug: /video
 ---
 
-React Native Skia provides a way to load video frames as images, enabling rich multimedia experiences within your applications. A video frame can be used anywhere a Skia image is accepted: `Image`, `ImageShader`, and `Atlas`.
+React Native Skia provides a way to load video frames as images, enabling rich multimedia experiences within your applications.
+A video frame can be used anywhere a Skia image is accepted: `Image`, `ImageShader`, and `Atlas`.
+Videos are also supported on Web.
 
 ### Requirements
 
diff --git a/package/src/external/reanimated/useVideo.ts b/package/src/external/reanimated/useVideo.ts
index d7dbd4d961..9ec1f29084 100644
--- a/package/src/external/reanimated/useVideo.ts
+++ b/package/src/external/reanimated/useVideo.ts
@@ -1,17 +1,11 @@
-import {
-  type SharedValue,
-  type FrameInfo,
-  createWorkletRuntime,
-  runOnJS,
-  runOnRuntime,
-} from "react-native-reanimated";
-import { useEffect, useMemo, useState } from "react";
+import type { SharedValue, FrameInfo } from "react-native-reanimated";
+import { useEffect, useMemo } from "react";
 
-import { Skia } from "../../skia/Skia";
 import type { SkImage, Video } from "../../skia/types";
 import { Platform } from "../../Platform";
 
 import Rea from "./ReanimatedProxy";
+import { useVideoLoading } from "./useVideoLoading";
 
 type Animated<T> = SharedValue<T> | T;
 
@@ -28,7 +22,6 @@ const copyFrameOnAndroid = (currentFrame: SharedValue<SkImage | null>) => {
   if (Platform.OS === "android") {
     const tex = currentFrame.value;
     if (tex) {
-      console.log("Copying frame on Android");
       currentFrame.value = tex.makeNonTextureImage();
       tex.dispose();
     }
@@ -70,25 +63,6 @@ const disposeVideo = (video: Video | null) => {
   video?.dispose();
 };
 
-const runtime = createWorkletRuntime("video-metadata-runtime");
-
-type VideoSource = string | null;
-
-const useVideoLoading = (source: VideoSource) => {
-  const [video, setVideo] = useState<Video | null>(null);
-  const cb = (src: string) => {
-    "worklet";
-    const vid = Skia.Video(src);
-    runOnJS(setVideo)(vid);
-  };
-  useEffect(() => {
-    if (source) {
-      runOnRuntime(runtime, cb)(source);
-    }
-  }, [source]);
-  return video;
-};
-
 export const useVideo = (
   source: string | null,
   userOptions?: Partial<PlaybackOptions>
@@ -102,7 +76,10 @@ export const useVideo = (
   const currentTime = Rea.useSharedValue(0);
   const lastTimestamp = Rea.useSharedValue(-1);
   const duration = useMemo(() => video?.duration() ?? 0, [video]);
-  const framerate = useMemo(() => video?.framerate() ?? 0, [video]);
+  const framerate = useMemo(
+    () => (Platform.OS === "web" ? -1 : video?.framerate() ?? 0),
+    [video]
+  );
   const size = useMemo(() => video?.size() ?? { width: 0, height: 0 }, [video]);
   const rotation = useMemo(() => video?.rotation() ?? 0, [video]);
   const frameDuration = 1000 / framerate;
@@ -155,7 +132,9 @@ export const useVideo = (
       currentTime.value = seek.value;
       lastTimestamp.value = currentTimestamp;
     }
-    if (delta >= currentFrameDuration && !isOver) {
+    // On Web the framerate is uknown.
+    // This could be optimized by using requestVideoFrameCallback (Chrome only)
+    if ((delta >= currentFrameDuration && !isOver) || Platform.OS === "web") {
       setFrame(video, currentFrame);
       currentTime.value += delta;
       lastTimestamp.value = currentTimestamp;
diff --git a/package/src/external/reanimated/useVideoLoading.ts b/package/src/external/reanimated/useVideoLoading.ts
new file mode 100644
index 0000000000..7049956ac1
--- /dev/null
+++ b/package/src/external/reanimated/useVideoLoading.ts
@@ -0,0 +1,28 @@
+import { useEffect, useState } from "react";
+import {
+  createWorkletRuntime,
+  runOnJS,
+  runOnRuntime,
+} from "react-native-reanimated";
+
+import type { Video } from "../../skia/types";
+import { Skia } from "../../skia";
+
+const runtime = createWorkletRuntime("video-metadata-runtime");
+
+type VideoSource = string | null;
+
+export const useVideoLoading = (source: VideoSource) => {
+  const [video, setVideo] = useState<Video | null>(null);
+  const cb = (src: string) => {
+    "worklet";
+    const vid = Skia.Video(src) as Video;
+    runOnJS(setVideo)(vid);
+  };
+  useEffect(() => {
+    if (source) {
+      runOnRuntime(runtime, cb)(source);
+    }
+  }, [source]);
+  return video;
+};
diff --git a/package/src/external/reanimated/useVideoLoading.web.ts b/package/src/external/reanimated/useVideoLoading.web.ts
new file mode 100644
index 0000000000..43b8263400
--- /dev/null
+++ b/package/src/external/reanimated/useVideoLoading.web.ts
@@ -0,0 +1,17 @@
+import { useEffect, useState } from "react";
+
+import type { Video } from "../../skia/types";
+import { Skia } from "../../skia";
+
+type VideoSource = string | null;
+
+export const useVideoLoading = (source: VideoSource) => {
+  const [video, setVideo] = useState<Video | null>(null);
+  useEffect(() => {
+    if (source) {
+      const vid = Skia.Video(source) as Promise<Video>;
+      vid.then((v) => setVideo(v));
+    }
+  }, [source]);
+  return video;
+};
diff --git a/package/src/renderer/__tests__/e2e/Video.spec.tsx b/package/src/renderer/__tests__/e2e/Video.spec.tsx
index f65764ba6c..2094e2ce4e 100644
--- a/package/src/renderer/__tests__/e2e/Video.spec.tsx
+++ b/package/src/renderer/__tests__/e2e/Video.spec.tsx
@@ -1,10 +1,11 @@
 import { itRunsE2eOnly } from "../../../__tests__/setup";
+import type { Video } from "../../../skia/types";
 import { surface } from "../setup";
 
 describe("Videos", () => {
   itRunsE2eOnly("get video duration and framerate", async () => {
     const result = await surface.eval((Skia, ctx) => {
-      const video = Skia.Video(ctx.localAssets[0]);
+      const video = Skia.Video(ctx.localAssets[0]) as Video;
       return {
         duration: video.duration(),
         framerate: video.framerate(),
@@ -57,7 +58,7 @@ describe("Videos", () => {
   // });
   itRunsE2eOnly("seek non existing frame returns null", async () => {
     const result = await surface.eval((Skia, ctx) => {
-      const video = Skia.Video(ctx.localAssets[0]);
+      const video = Skia.Video(ctx.localAssets[0]) as Video;
       video.seek(100000);
       const image = video.nextImage();
       return image === null;
diff --git a/package/src/skia/types/NativeBuffer/NativeBufferFactory.ts b/package/src/skia/types/NativeBuffer/NativeBufferFactory.ts
index acd9e83448..51160cedd6 100644
--- a/package/src/skia/types/NativeBuffer/NativeBufferFactory.ts
+++ b/package/src/skia/types/NativeBuffer/NativeBufferFactory.ts
@@ -1,7 +1,14 @@
 import type { SkImage } from "../Image";
 
+export abstract class CanvasKitWebGLBuffer {}
+
 export type NativeBuffer<
-  T extends bigint | ArrayBuffer | CanvasImageSource | unknown = unknown
+  T extends
+    | bigint
+    | ArrayBuffer
+    | CanvasImageSource
+    | CanvasKitWebGLBuffer
+    | unknown = unknown
 > = T;
 
 export type NativeBufferAddr = NativeBuffer<bigint>;
@@ -20,7 +27,8 @@ export const isNativeBufferWeb = (
   buffer instanceof OffscreenCanvas ||
   buffer instanceof VideoFrame ||
   buffer instanceof HTMLImageElement ||
-  buffer instanceof SVGImageElement;
+  buffer instanceof SVGImageElement ||
+  buffer instanceof CanvasKitWebGLBuffer;
 
 export const isNativeBufferNode = (
   buffer: NativeBuffer
diff --git a/package/src/skia/types/Skia.ts b/package/src/skia/types/Skia.ts
index f0904c36e6..ca0c2edcf2 100644
--- a/package/src/skia/types/Skia.ts
+++ b/package/src/skia/types/Skia.ts
@@ -96,6 +96,6 @@ export interface Skia {
   TextBlob: TextBlobFactory;
   Surface: SurfaceFactory;
   ParagraphBuilder: ParagraphBuilderFactory;
-  Video: (url: string) => Video;
+  Video: (url: string) => Promise<Video> | Video;
   NativeBuffer: NativeBufferFactory;
 }
diff --git a/package/src/skia/web/CanvasKitWebGLBufferImpl.ts b/package/src/skia/web/CanvasKitWebGLBufferImpl.ts
new file mode 100644
index 0000000000..02ebd7c5e6
--- /dev/null
+++ b/package/src/skia/web/CanvasKitWebGLBufferImpl.ts
@@ -0,0 +1,22 @@
+import type { Surface, TextureSource, Image } from "canvaskit-wasm";
+
+import { CanvasKitWebGLBuffer } from "../types";
+
+export class CanvasKitWebGLBufferImpl extends CanvasKitWebGLBuffer {
+  public image: Image | null = null;
+
+  constructor(public surface: Surface, private source: TextureSource) {
+    super();
+  }
+
+  toImage() {
+    if (this.image === null) {
+      this.image = this.surface.makeImageFromTextureSource(this.source);
+    }
+    if (this.image === null) {
+      throw new Error("Failed to create image from texture source");
+    }
+    this.surface.updateTextureFromSource(this.image, this.source);
+    return this.image;
+  }
+}
diff --git a/package/src/skia/web/JsiSkImageFactory.ts b/package/src/skia/web/JsiSkImageFactory.ts
index fd3a4f94c8..09b6521542 100644
--- a/package/src/skia/web/JsiSkImageFactory.ts
+++ b/package/src/skia/web/JsiSkImageFactory.ts
@@ -1,6 +1,6 @@
 import type { CanvasKit, Image } from "canvaskit-wasm";
 
-import { isNativeBufferWeb } from "../types";
+import { CanvasKitWebGLBuffer, isNativeBufferWeb } from "../types";
 import type {
   SkData,
   ImageInfo,
@@ -13,6 +13,7 @@ import { Host, getEnum } from "./Host";
 import { JsiSkImage } from "./JsiSkImage";
 import { JsiSkData } from "./JsiSkData";
 import type { JsiSkSurface } from "./JsiSkSurface";
+import type { CanvasKitWebGLBufferImpl } from "./CanvasKitWebGLBufferImpl";
 
 export class JsiSkImageFactory extends Host implements ImageFactory {
   constructor(CanvasKit: CanvasKit) {
@@ -35,8 +36,20 @@ export class JsiSkImageFactory extends Host implements ImageFactory {
       throw new Error("Invalid NativeBuffer");
     }
     if (!surface) {
-      // TODO: this is way to slow
-      const img = this.CanvasKit.MakeImageFromCanvasImageSource(buffer);
+      let img: Image;
+      if (
+        buffer instanceof HTMLImageElement ||
+        buffer instanceof HTMLVideoElement ||
+        buffer instanceof ImageBitmap
+      ) {
+        img = this.CanvasKit.MakeLazyImageFromTextureSource(buffer);
+      } else if (buffer instanceof CanvasKitWebGLBuffer) {
+        img = (
+          buffer as CanvasKitWebGLBuffer as CanvasKitWebGLBufferImpl
+        ).toImage();
+      } else {
+        img = this.CanvasKit.MakeImageFromCanvasImageSource(buffer);
+      }
       return new JsiSkImage(this.CanvasKit, img);
     } else if (!image) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
diff --git a/package/src/skia/web/JsiSkia.ts b/package/src/skia/web/JsiSkia.ts
index c6908c745f..165ffc01b4 100644
--- a/package/src/skia/web/JsiSkia.ts
+++ b/package/src/skia/web/JsiSkia.ts
@@ -42,6 +42,7 @@ import { JsiSkFontMgrFactory } from "./JsiSkFontMgrFactory";
 import { JsiSkAnimatedImageFactory } from "./JsiSkAnimatedImageFactory";
 import { JsiSkParagraphBuilderFactory } from "./JsiSkParagraphBuilderFactory";
 import { JsiSkNativeBufferFactory } from "./JsiSkNativeBufferFactory";
+import { createVideo } from "./JsiVideo";
 
 export const JsiSkApi = (CanvasKit: CanvasKit): Skia => ({
   Point: (x: number, y: number) =>
@@ -127,7 +128,5 @@ export const JsiSkApi = (CanvasKit: CanvasKit): Skia => ({
   FontMgr: new JsiSkFontMgrFactory(CanvasKit),
   ParagraphBuilder: new JsiSkParagraphBuilderFactory(CanvasKit),
   NativeBuffer: new JsiSkNativeBufferFactory(CanvasKit),
-  Video: (_localUri: string) => {
-    throw new Error("Not implemented on React Native Web");
-  },
+  Video: createVideo.bind(null, CanvasKit),
 });
diff --git a/package/src/skia/web/JsiVideo.ts b/package/src/skia/web/JsiVideo.ts
new file mode 100644
index 0000000000..abbb556a4f
--- /dev/null
+++ b/package/src/skia/web/JsiVideo.ts
@@ -0,0 +1,96 @@
+import type { CanvasKit, Surface } from "canvaskit-wasm";
+
+import type { CanvasKitWebGLBuffer, Video, ImageFactory } from "../types";
+
+import { CanvasKitWebGLBufferImpl } from "./CanvasKitWebGLBufferImpl";
+import { JsiSkImageFactory } from "./JsiSkImageFactory";
+
+export const createVideo = async (
+  CanvasKit: CanvasKit,
+  url: string
+): Promise<Video> => {
+  const video = document.createElement("video");
+  return new Promise((resolve, reject) => {
+    video.src = url;
+    video.style.display = "none";
+    video.crossOrigin = "anonymous";
+    video.volume = 0;
+    video.addEventListener("loadedmetadata", () => {
+      document.body.appendChild(video);
+      resolve(new JsiVideo(new JsiSkImageFactory(CanvasKit), video));
+    });
+    video.addEventListener("error", () => {
+      reject(new Error(`Failed to load video from URL: ${url}`));
+    });
+  });
+};
+
+export class JsiVideo implements Video {
+  __typename__ = "Video" as const;
+
+  private webglBuffer: CanvasKitWebGLBuffer | null = null;
+
+  constructor(
+    private ImageFactory: ImageFactory,
+    private videoElement: HTMLVideoElement
+  ) {
+    document.body.appendChild(this.videoElement);
+  }
+
+  duration() {
+    return this.videoElement.duration * 1000;
+  }
+
+  framerate(): number {
+    throw new Error("Video.frame is not available on React Native Web");
+  }
+
+  setSurface(surface: Surface) {
+    // If we have the surface, we can use the WebGL buffer which is slightly faster
+    // This is because WebGL cannot be shared across contextes.
+    // This can be removed with WebGPU
+    this.webglBuffer = new CanvasKitWebGLBufferImpl(surface, this.videoElement);
+  }
+
+  nextImage() {
+    return this.ImageFactory.MakeImageFromNativeBuffer(
+      this.webglBuffer ? this.webglBuffer : this.videoElement
+    );
+  }
+
+  seek(time: number) {
+    if (isNaN(time)) {
+      throw new Error(`Invalid time: ${time}`);
+    }
+    this.videoElement.currentTime = time / 1000;
+  }
+
+  rotation() {
+    return 0 as const;
+  }
+
+  size() {
+    return {
+      width: this.videoElement.videoWidth,
+      height: this.videoElement.videoHeight,
+    };
+  }
+
+  pause() {
+    this.videoElement.pause();
+  }
+
+  play() {
+    this.videoElement.play();
+  }
+
+  setVolume(volume: number) {
+    this.videoElement.volume = volume;
+  }
+
+  dispose() {
+    if (this.videoElement.parentNode) {
+      this.videoElement.parentNode.removeChild(this.videoElement);
+    }
+  }
+}