feat: Make asset loading asynchronous and introduce hooks (#44)

Co-authored-by: Marc Rousavy <me@mrousavy.com>

package/android/src/main/cpp/AndroidFilamentProxy.cpp

@@ -3,6 +3,7 @@
 //
 
 #include "AndroidFilamentProxy.h"
+#include "WithJNIScope.h"
 
 namespace margelo {
 
@@ -17,12 +18,12 @@ AndroidFilamentProxy::~AndroidFilamentProxy() {
   jni::ThreadScope::WithClassLoader([&] { _proxy.reset(); });
 }
 
-std::shared_ptr<FilamentBuffer> AndroidFilamentProxy::getAssetByteBuffer(std::string path) {
-  return _proxy->cthis()->getAssetByteBuffer(path);
+std::shared_ptr<FilamentBuffer> AndroidFilamentProxy::loadAsset(std::string path) {
+  return jni::WithJNIScope<std::shared_ptr<FilamentBuffer>>([=] { return _proxy->cthis()->loadAsset(path); });
 }
 
 std::shared_ptr<FilamentView> AndroidFilamentProxy::findFilamentView(int id) {
-  return _proxy->cthis()->findFilamentView(id);
+  return jni::WithJNIScope<std::shared_ptr<FilamentView>>([=] { return _proxy->cthis()->findFilamentView(id); });
 }
 
 std::shared_ptr<Choreographer> AndroidFilamentProxy::createChoreographer() {

package/android/src/main/cpp/AndroidFilamentProxy.h

@@ -22,7 +22,7 @@ class AndroidFilamentProxy : public FilamentProxy {
   ~AndroidFilamentProxy();
 
 private:
-  std::shared_ptr<FilamentBuffer> getAssetByteBuffer(std::string path) override;
+  std::shared_ptr<FilamentBuffer> loadAsset(std::string path) override;
   std::shared_ptr<FilamentView> findFilamentView(int id) override;
   std::shared_ptr<Choreographer> createChoreographer() override;
 

package/android/src/main/cpp/WithJNIScope.h

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 12.03.24.
+//
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <functional>
+
+namespace facebook {
+namespace jni {
+
+  template <typename T> T WithJNIScope(std::function<T()>&& lambda) {
+    T result;
+    jni::ThreadScope::WithClassLoader([&result, lambda = std::move(lambda)]() { result = lambda(); });
+    return std::move(result);
+  }
+
+} // namespace jni
+} // namespace facebook

package/android/src/main/cpp/java-bindings/JFilamentProxy.cpp

@@ -22,8 +22,8 @@ JFilamentProxy::~JFilamentProxy() {
   // TODO(hanno): Cleanup?
 }
 
-std::shared_ptr<FilamentBuffer> JFilamentProxy::getAssetByteBuffer(const std::string& path) {
-  static const auto method = javaClassLocal()->getMethod<jni::alias_ref<jni::JByteBuffer>(jni::alias_ref<jstring>)>("getAssetByteBuffer");
+std::shared_ptr<FilamentBuffer> JFilamentProxy::loadAsset(const std::string& path) {
+  static const auto method = javaClassLocal()->getMethod<jni::alias_ref<jni::JByteBuffer>(jni::alias_ref<jstring>)>("loadAsset");
   jni::local_ref<jni::JByteBuffer> buffer = method(_javaPart, jni::make_jstring(path));
   auto managedBuffer = std::make_shared<AndroidManagedBuffer>(buffer);
   return std::make_shared<FilamentBuffer>(managedBuffer);

package/android/src/main/cpp/java-bindings/JFilamentProxy.h

@@ -24,7 +24,7 @@ class JFilamentProxy : public jni::HybridClass<JFilamentProxy> {
   ~JFilamentProxy();
   static void registerNatives();
 
-  std::shared_ptr<FilamentBuffer> getAssetByteBuffer(const std::string& path);
+  std::shared_ptr<FilamentBuffer> loadAsset(const std::string& path);
   std::shared_ptr<FilamentView> findFilamentView(int id);
   std::shared_ptr<Choreographer> createChoreographer();
 

package/android/src/main/java/com/margelo/filament/FilamentProxy.java

@@ -67,7 +67,7 @@ private static byte[] readAllBytes(InputStream inputStream) throws IOException {
     /** @noinspection unused*/
     @DoNotStrip
     @Keep
-    ByteBuffer getAssetByteBuffer(String assetName) throws IOException {
+    ByteBuffer loadAsset(String assetName) throws IOException {
         try (InputStream inputStream = reactContext.getAssets().open(assetName)) {
             byte[] bytes = readAllBytes(inputStream);
             ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.length);

package/cpp/FilamentProxy.cpp

@@ -17,12 +17,24 @@ namespace margelo {
 using namespace facebook;
 
 void FilamentProxy::loadHybridMethods() {
-  registerHybridMethod("getAssetByteBuffer", &FilamentProxy::getAssetByteBuffer, this);
+  registerHybridMethod("loadAsset", &FilamentProxy::loadAssetAsync, this);
   registerHybridMethod("findFilamentView", &FilamentProxy::findFilamentView, this);
   registerHybridMethod("createTestObject", &FilamentProxy::createTestObject, this);
   registerHybridMethod("createEngine", &FilamentProxy::createEngine, this);
 }
 
+std::future<std::shared_ptr<FilamentBuffer>> FilamentProxy::loadAssetAsync(std::string path) {
+  auto weakThis = std::weak_ptr<FilamentProxy>(shared<FilamentProxy>());
+  return std::async(std::launch::async, [=]() {
+    auto sharedThis = weakThis.lock();
+    if (sharedThis != nullptr) {
+      return this->loadAsset(path);
+    } else {
+      throw std::runtime_error("Failed to load asset, FilamentProxy has already been destroyed!");
+    }
+  });
+}
+
 std::shared_ptr<TestHybridObject> FilamentProxy::createTestObject() {
   return std::make_shared<TestHybridObject>();
 }

package/cpp/FilamentProxy.h

@@ -7,6 +7,7 @@
 #include <jsi/jsi.h>
 
 #include <core/EngineWrapper.h>
+#include <future>
 #include <string>
 #include <vector>
 
@@ -27,7 +28,8 @@ class FilamentProxy : public HybridObject {
   explicit FilamentProxy() : HybridObject("FilamentProxy") {}
 
 private:
-  virtual std::shared_ptr<FilamentBuffer> getAssetByteBuffer(std::string path) = 0;
+  std::future<std::shared_ptr<FilamentBuffer>> loadAssetAsync(std::string path);
+  virtual std::shared_ptr<FilamentBuffer> loadAsset(std::string path) = 0;
   virtual std::shared_ptr<FilamentView> findFilamentView(int id) = 0;
   virtual std::shared_ptr<Choreographer> createChoreographer() = 0;
 

package/example/src/App.tsx

@@ -1,8 +1,8 @@
 import * as React from 'react'
-import { useCallback, useEffect, useMemo, useRef } from 'react'
+import { useEffect, useRef } from 'react'
 
 import { Platform, StyleSheet } from 'react-native'
-import { FilamentProxy, Filament, useEngine, Float3, useRenderCallback, RenderCallback } from 'react-native-filament'
+import { Filament, useEngine, Float3, useRenderCallback, useAsset, useModel } from 'react-native-filament'
 
 const penguModelPath = Platform.select({
   android: 'custom/pengu.glb',
@@ -25,46 +25,40 @@ const far = 1000
 export default function App() {
   const engine = useEngine()
 
-  const [_pengu, penguAnimator] = useMemo(() => {
-    const modelBuffer = FilamentProxy.getAssetByteBuffer(penguModelPath)
-    const asset = engine.loadAsset(modelBuffer)
-    const animator = asset.getAnimator()
-    asset.releaseSourceData()
+  const pengu = useModel({ engine: engine, path: penguModelPath })
+  const light = useAsset({ path: indirectLightPath })
 
-    return [asset, animator]
-  }, [engine])
-
-  const prevAspectRatio = useRef(0)
-  const renderCallback: RenderCallback = useCallback(
-    (_timestamp, _startTime, passedSeconds) => {
-      const view = engine.getView()
-      const aspectRatio = view.aspectRatio
-      if (prevAspectRatio.current !== aspectRatio) {
-        prevAspectRatio.current = aspectRatio
-        // Setup camera lens:
-        const camera = engine.getCamera()
-        camera.setLensProjection(focalLengthInMillimeters, aspectRatio, near, far)
-      }
+  useEffect(() => {
+    if (light == null) return
+    // create a default light
+    engine.setIndirectLight(light)
 
-      penguAnimator.applyAnimation(0, passedSeconds)
-      penguAnimator.updateBoneMatrices()
+    // Create a directional light for supporting shadows
+    const directionalLight = engine.createLightEntity('directional', 6500, 10000, 0, -1, 0, true)
+    engine.getScene().addEntity(directionalLight)
+    return () => {
+      // TODO: Remove directionalLight from scene
+    }
+  }, [engine, light])
 
-      engine.getCamera().lookAt(cameraPosition, cameraTarget, cameraUp)
-    },
-    [engine, penguAnimator]
-  )
-  useRenderCallback(engine, renderCallback)
+  const prevAspectRatio = useRef(0)
+  useRenderCallback(engine, (_timestamp, _startTime, passedSeconds) => {
+    const view = engine.getView()
+    const aspectRatio = view.aspectRatio
+    if (prevAspectRatio.current !== aspectRatio) {
+      prevAspectRatio.current = aspectRatio
+      // Setup camera lens:
+      const camera = engine.getCamera()
+      camera.setLensProjection(focalLengthInMillimeters, aspectRatio, near, far)
+    }
 
-  // Setup the 3D scene:
-  useEffect(() => {
-    // Create a default light:
-    const indirectLightBuffer = FilamentProxy.getAssetByteBuffer(indirectLightPath)
-    engine.setIndirectLight(indirectLightBuffer)
+    if (pengu.state === 'loaded') {
+      pengu.animator.applyAnimation(0, passedSeconds)
+      pengu.animator.updateBoneMatrices()
+    }
 
-    // Create a directional light for supporting shadows
-    const light = engine.createLightEntity('directional', 6500, 10000, 0, -1, 0, true)
-    engine.getScene().addEntity(light)
-  }, [engine])
+    engine.getCamera().lookAt(cameraPosition, cameraTarget, cameraUp)
+  })
 
   return <Filament style={styles.filamentView} engine={engine} />
 }

package/ios/src/AppleFilamentProxy.h

@@ -22,7 +22,7 @@ class AppleFilamentProxy : public FilamentProxy {
   ~AppleFilamentProxy();
 
 public:
-  std::shared_ptr<FilamentBuffer> getAssetByteBuffer(std::string path) override;
+  std::shared_ptr<FilamentBuffer> loadAsset(std::string path) override;
   std::shared_ptr<FilamentView> findFilamentView(int modelId) override;
   std::shared_ptr<Choreographer> createChoreographer() override;
 

package/ios/src/AppleFilamentProxy.mm

@@ -26,7 +26,7 @@
   // TODO(hanno): cleanup here?
 }
 
-std::shared_ptr<FilamentBuffer> AppleFilamentProxy::getAssetByteBuffer(std::string path) {
+std::shared_ptr<FilamentBuffer> AppleFilamentProxy::loadAsset(std::string path) {
   NSString* filePath = [NSString stringWithUTF8String:path.c_str()];
 
   // Split the path at the last dot to separate name and extension

package/src/hooks/useAsset.ts

@@ -0,0 +1,25 @@
+import { useEffect, useState } from 'react'
+import { Asset } from '../native/FilamentBuffer'
+import { FilamentProxy } from '../native/FilamentProxy'
+
+export interface AssetProps {
+  /**
+   * A web URL (http:// or https://), local file (file://) or resource ID of the bundled asset.
+   */
+  path: string
+}
+
+/**
+ * Asynchronously load an asset from the given web URL, local file path, or resource ID.
+ */
+export function useAsset({ path }: AssetProps): Asset | undefined {
+  const [asset, setAsset] = useState<Asset>()
+
+  useEffect(() => {
+    FilamentProxy.loadAsset(path)
+      .then((a) => setAsset(a))
+      .catch((e) => console.error(`Failed to load asset ${path}!`, e))
+  }, [path])
+
+  return asset
+}

package/src/hooks/useModel.ts

@@ -0,0 +1,66 @@
+import { useEffect, useMemo } from 'react'
+import { AssetProps, useAsset } from './useAsset'
+import { Animator, Engine } from '../types'
+import { FilamentAsset } from '../types/FilamentAsset'
+
+interface ModelProps extends AssetProps {
+  /**
+   * The Filament engine this model should be loaded into.
+   */
+  engine: Engine
+  /**
+   * Whether source data of the model should be released after loading, or not.
+   * @default true
+   */
+  shouldReleaseSourceData?: boolean
+}
+
+/**
+ * The resulting filament model, or `'loading'` if not yet available.
+ */
+export type FilamentModel =
+  | {
+      state: 'loaded'
+      animator: Animator
+      asset: FilamentAsset
+    }
+  | {
+      state: 'loading'
+    }
+
+/**
+ * Use a Filament Model that gets asynchronously loaded into the given Engine.
+ * @example
+ * ```ts
+ * const engine = useEngine()
+ * const pengu = useModel({ engine: engine, path: PENGU_PATH })
+ * ```
+ */
+export function useModel({ path, engine, shouldReleaseSourceData }: ModelProps): FilamentModel {
+  const asset = useAsset({ path: path })
+
+  const engineAsset = useMemo(() => {
+    if (asset == null) return undefined
+    return engine.loadAsset(asset)
+  }, [asset, engine])
+
+  const animator = useMemo(() => engineAsset?.getAnimator(), [engineAsset])
+
+  useEffect(() => {
+    if (shouldReleaseSourceData) {
+      // releases CPU memory for bindings
+      engineAsset?.releaseSourceData()
+    }
+  }, [engineAsset, shouldReleaseSourceData])
+
+  if (asset == null || engineAsset == null || animator == null) {
+    return {
+      state: 'loading',
+    }
+  }
+  return {
+    state: 'loaded',
+    asset: engineAsset,
+    animator: animator,
+  }
+}

package/src/index.tsx

@@ -8,3 +8,5 @@ export * from './Filament'
 // hooks
 export * from './hooks/useEngine'
 export * from './hooks/useRenderCallback'
+export * from './hooks/useAsset'
+export * from './hooks/useModel'

package/src/native/FilamentBuffer.ts

@@ -1 +1,3 @@
 export interface FilamentBuffer {}
+
+export type Asset = FilamentBuffer

package/src/native/FilamentProxy.ts

@@ -18,12 +18,11 @@ interface TestHybridObject {
 }
 
 export interface TFilamentProxy {
-  // TODO: rename to loadModelBytes
   /**
-   * Loads the byte buffer for any given path.
+   * Asynchronously loads the the given asset into a ByteBuffer.
    * @param path A web URL (http:// or https://), local file (file://) or resource ID. (Only resource ID supported for now)
    */
-  getAssetByteBuffer(path: string): FilamentBuffer
+  loadAsset(path: string): Promise<FilamentBuffer>
   /**
    * @private
    */