feat: Add API for controlling the scene (#15) (#27)

>[!WARNING]
> Based on the following PR which should be merged first
> - #13

**Goal:**

To provide an API from JS that can be used to setup the scene and
control the transform of the elements.

(Most important) **Changes:**

- **Camera:**
- Renamed `lookAt` -> `lookAtCameraManipulator` Can be used to make
camera look at camera manipulator
- New `lookAt` function, which can be used to manually control the
position, target and up vector of the camera
- New `setLensProjection` & `setProjection` function to Control the lens
configuration from JS
- **⚠️ Note:** @mrousavy The user would also need to call this when the
surface size changes (as we need to pass the aspect ratio to this
function). Lets discuss in a quick meeting how to solve this best

- **Engine:**
- Renamed `createDefaultLight` -> `setIndirectLight`, only sets the
indirect light from an FilamentBuffer now (decouples the function from
setting a default directional light, which can be now configured by the
user)
- New `createLightEntity` Function to create a light with the
configuration you want
- Exposed `transformToUnitCube`. Pass an asset and it will transform the
asset to fit into a unit cube
- New `setEntityPosition`, `setEntityRotation` and `setEntityScale` to
set or update an entities transform
- `loadAsset` now returns a `FilamentAsset`. The asset can be used for
various operations in the scene.

---------

---------

Co-authored-by: Hanno J. Gödecke <hanno@margelo.io>
Co-authored-by: Hanno J. Gödecke <die.drei99@yahoo.de>

package/android/CMakeLists.txt

@@ -36,6 +36,7 @@ add_library(
         ../cpp/core/CameraWrapper.cpp
         ../cpp/core/ViewWrapper.cpp
         ../cpp/core/SwapChainWrapper.cpp
+        ../cpp/core/FilamentAssetWrapper.cpp
 
         # Filament Utils
         ../cpp/core/utils/EntityWrapper.cpp
@@ -44,7 +45,6 @@ add_library(
         # Java JNI
         src/main/cpp/AndroidFilamentProxy.cpp
         src/main/cpp/AndroidSurface.cpp
-        src/main/cpp/AndroidManagedBuffer.cpp
         src/main/cpp/Filament.cpp
         src/main/cpp/JNISharedPtr.cpp
         src/main/cpp/FilamentInstaller.cpp

package/android/src/main/cpp/AndroidManagedBuffer.h

@@ -18,7 +18,7 @@ class AndroidManagedBuffer : public ManagedBuffer {
     _buffer->rewind();
   }
 
-  ~AndroidManagedBuffer() {
+  ~AndroidManagedBuffer() override {
     jni::ThreadScope::WithClassLoader([&] { _buffer.reset(); });
   }
 

package/cpp/ManagedBuffer.h

@@ -11,7 +11,7 @@ namespace margelo {
 
 class ManagedBuffer {
 public:
-  virtual ~ManagedBuffer() = 0;
+  virtual ~ManagedBuffer() {}
   virtual const uint8_t* getData() const = 0;
   virtual size_t getSize() const = 0;
 };

package/cpp/core/CameraFovEnum.h

@@ -0,0 +1,35 @@
+//
+// Created by Marc Rousavy on 22.02.24.
+//
+
+#pragma once
+
+#include "jsi/EnumMapper.h"
+#include <filament/Camera.h>
+
+namespace margelo {
+using namespace filament;
+
+namespace EnumMapper {
+  static void convertJSUnionToEnum(const std::string& inUnion, Camera::Fov* outEnum) {
+    if (inUnion == "horizontal")
+      *outEnum = Camera::Fov::HORIZONTAL;
+    else if (inUnion == "vertical")
+      *outEnum = Camera::Fov::VERTICAL;
+    else
+      throw invalidUnion(inUnion);
+  }
+  static void convertEnumToJSUnion(Camera::Fov inEnum, std::string* outUnion) {
+    switch (inEnum) {
+      case filament::Camera::Fov::HORIZONTAL:
+        *outUnion = "horizontal";
+        break;
+      case filament::Camera::Fov::VERTICAL:
+        *outUnion = "vertical";
+        break;
+      default:
+        throw invalidEnum(inEnum);
+    }
+  }
+} // namespace EnumMapper
+} // namespace margelo

package/cpp/core/CameraWrapper.cpp

@@ -1,10 +1,14 @@
 #include "CameraWrapper.h"
+#include "CameraFovEnum.h"
 
 void margelo::CameraWrapper::loadHybridMethods() {
+  registerHybridMethod("lookAtCameraManipulator", &CameraWrapper::lookAtCameraManipulator, this);
   registerHybridMethod("lookAt", &CameraWrapper::lookAt, this);
+  registerHybridMethod("setLensProjection", &CameraWrapper::setLensProjection, this);
+  registerHybridMethod("setProjection", &CameraWrapper::setProjection, this);
 }
 
-void margelo::CameraWrapper::lookAt(std::shared_ptr<ManipulatorWrapper> cameraManipulator) {
+void margelo::CameraWrapper::lookAtCameraManipulator(std::shared_ptr<ManipulatorWrapper> cameraManipulator) {
   if (!cameraManipulator) {
     throw std::invalid_argument("CameraManipulator is null");
   }
@@ -13,3 +17,23 @@ void margelo::CameraWrapper::lookAt(std::shared_ptr<ManipulatorWrapper> cameraMa
   cameraManipulator->getManipulator()->getLookAt(&eye, &center, &up);
   _camera->lookAt(eye, center, up);
 }
+
+void margelo::CameraWrapper::lookAt(std::vector<double> eye, std::vector<double> center, std::vector<double> up) {
+  math::float3 eyeVec = {static_cast<float>(eye[0]), static_cast<float>(eye[1]), static_cast<float>(eye[2])};
+  math::float3 centerVec = {static_cast<float>(center[0]), static_cast<float>(center[1]), static_cast<float>(center[2])};
+  math::float3 upVec = {static_cast<float>(up[0]), static_cast<float>(up[1]), static_cast<float>(up[2])};
+  _camera->lookAt(eyeVec, centerVec, upVec);
+}
+
+void margelo::CameraWrapper::setLensProjection(double fov, double aspect, double near, double far) {
+  _camera->setLensProjection(static_cast<float>(fov), static_cast<float>(aspect), static_cast<float>(near), static_cast<float>(far));
+}
+
+void margelo::CameraWrapper::setProjection(double fovInDegrees, double aspect, double near, double far,
+                                           std::string directionStr = "vertical") {
+  Camera::Fov direction;
+  EnumMapper::convertJSUnionToEnum(directionStr, &direction);
+
+  _camera->setProjection(static_cast<float>(fovInDegrees), static_cast<float>(aspect), static_cast<float>(near), static_cast<float>(far),
+                         direction);
+}

package/cpp/core/CameraWrapper.h

@@ -22,7 +22,11 @@ class CameraWrapper : public HybridObject {
   std::shared_ptr<Camera> _camera;
 
 private:
+  void lookAt(std::vector<double> eye, std::vector<double> center, std::vector<double> up);
+  void setLensProjection(double fov, double aspect, double near, double far);
+  // TODO(Hanno): Add directionStr , Camera::Fov directionStr = Camera::Fov::VERTICAL
+  void setProjection(double fovInDegrees, double aspect, double near, double far, std::string directionStr);
   // Convenience methods
-  void lookAt(std::shared_ptr<ManipulatorWrapper> cameraManipulator);
+  void lookAtCameraManipulator(std::shared_ptr<ManipulatorWrapper> cameraManipulator);
 };
 } // namespace margelo

package/cpp/core/EngineWrapper.cpp

@@ -4,7 +4,10 @@
 
 #include "EngineWrapper.h"
 
+#include "LightEnum.h"
 #include "References.h"
+#include "utils/Converter.h"
+
 #include <filament/Color.h>
 #include <filament/Engine.h>
 #include <filament/Fence.h>
@@ -42,17 +45,22 @@ EngineWrapper::EngineWrapper(std::shared_ptr<Choreographer> choreographer) {
         delete provider;
       });
 
+  gltfio::AssetConfiguration assetConfig {.engine = _engine.get(), .materials = _materialProvider.get()};
   gltfio::AssetLoader* assetLoaderPtr =
-      gltfio::AssetLoader::create(filament::gltfio::AssetConfiguration{.engine = _engine.get(), .materials = _materialProvider.get()});
+      gltfio::AssetLoader::create(assetConfig);
   _assetLoader = References<gltfio::AssetLoader>::adoptEngineRef(
       _engine, assetLoaderPtr, [](const std::shared_ptr<Engine>& engine, gltfio::AssetLoader* assetLoader) {
         auto* ncm = assetLoader->getNames();
         delete ncm;
         gltfio::AssetLoader::destroy(&assetLoader);
       });
 
-  gltfio::ResourceLoader* resourceLoaderPtr =
-      new filament::gltfio::ResourceLoader({.engine = _engine.get(), .normalizeSkinningWeights = true});
+
+  filament::gltfio::ResourceConfiguration resourceConfig {
+    .engine = _engine.get(),
+    .normalizeSkinningWeights = true
+  };
+  auto* resourceLoaderPtr = new filament::gltfio::ResourceLoader(resourceConfig);
   // Add texture providers to the resource loader
   auto stbProvider = filament::gltfio::createStbProvider(_engine.get());
   auto ktx2Provider = filament::gltfio::createKtx2Provider(_engine.get());
@@ -82,7 +90,7 @@ EngineWrapper::EngineWrapper(std::shared_ptr<Choreographer> choreographer) {
 void EngineWrapper::loadHybridMethods() {
   registerHybridMethod("setSurfaceProvider", &EngineWrapper::setSurfaceProvider, this);
   registerHybridMethod("setRenderCallback", &EngineWrapper::setRenderCallback, this);
-  registerHybridMethod("createDefaultLight", &EngineWrapper::createDefaultLight, this);
+  registerHybridMethod("setIndirectLight", &EngineWrapper::setIndirectLight, this);
 
   registerHybridMethod("loadAsset", &EngineWrapper::loadAsset, this);
 
@@ -92,6 +100,11 @@ void EngineWrapper::loadHybridMethods() {
   registerHybridMethod("getView", &EngineWrapper::getView, this);
   registerHybridMethod("getCamera", &EngineWrapper::getCamera, this);
   registerHybridMethod("getCameraManipulator", &EngineWrapper::getCameraManipulator, this);
+  registerHybridMethod("createLightEntity", &EngineWrapper::createLightEntity, this);
+  registerHybridMethod("transformToUnitCube", &EngineWrapper::transformToUnitCube, this);
+  registerHybridMethod("setEntityPosition", &EngineWrapper::setEntityPosition, this);
+  registerHybridMethod("setEntityRotation", &EngineWrapper::setEntityRotation, this);
+  registerHybridMethod("setEntityScale", &EngineWrapper::setEntityScale, this);
 }
 
 void EngineWrapper::setSurfaceProvider(std::shared_ptr<SurfaceProvider> surfaceProvider) {
@@ -136,7 +149,8 @@ void EngineWrapper::surfaceSizeChanged(int width, int height) {
     _view->setViewport(0, 0, width, height);
   }
 
-  updateCameraProjection();
+  // TODO: when the surface resizes we need to update the camera projection, but that one is owned by JS now.
+  //  updateCameraProjection();
 }
 
 void EngineWrapper::destroySurface() {
@@ -227,8 +241,9 @@ std::shared_ptr<CameraWrapper> EngineWrapper::createCamera() {
   return std::make_shared<CameraWrapper>(camera);
 }
 
-void EngineWrapper::loadAsset(std::shared_ptr<FilamentBuffer> modelBuffer) {
-  filament::gltfio::FilamentAsset* asset = _assetLoader->createAsset(modelBuffer->getData(), modelBuffer->getSize());
+std::shared_ptr<FilamentAssetWrapper> EngineWrapper::loadAsset(std::shared_ptr<FilamentBuffer> modelBuffer) {
+  std::shared_ptr<ManagedBuffer> buffer = modelBuffer->getBuffer();
+  gltfio::FilamentAsset* asset = _assetLoader->createAsset(buffer->getData(), buffer->getSize());
   if (asset == nullptr) {
     throw std::runtime_error("Failed to load asset");
   }
@@ -246,23 +261,27 @@ void EngineWrapper::loadAsset(std::shared_ptr<FilamentBuffer> modelBuffer) {
   _animator = asset->getInstance()->getAnimator();
   asset->releaseSourceData();
 
-  transformToUnitCube(asset);
+  auto assetLoader = _assetLoader;
+  auto sharedPtr = References<gltfio::FilamentAsset>::adoptRef(asset, [assetLoader](gltfio::FilamentAsset* asset) {
+    assetLoader->destroyAsset(asset);
+  });
+  return std::make_shared<FilamentAssetWrapper>(sharedPtr);
 }
 
 // Default light is a directional light for shadows + a default IBL
-void EngineWrapper::createDefaultLight(std::shared_ptr<FilamentBuffer> iblBuffer) {
+void EngineWrapper::setIndirectLight(std::shared_ptr<FilamentBuffer> iblBuffer) {
   if (!_scene) {
     throw std::runtime_error("Scene not initialized");
   }
   if (!iblBuffer) {
     throw std::runtime_error("IBL buffer is null");
   }
-  if (iblBuffer->getSize() == 0) {
+  auto buffer = iblBuffer->getBuffer();
+  if (buffer->getSize() == 0) {
     throw std::runtime_error("IBL buffer is empty");
   }
 
-  auto* iblBundle = new image::Ktx1Bundle(iblBuffer->getData(), iblBuffer->getSize());
-
+  auto* iblBundle = new image::Ktx1Bundle(buffer->getData(), buffer->getSize());
   Texture* cubemap = ktxreader::Ktx1Reader::createTexture(
       _engine.get(), *iblBundle, false,
       [](void* userdata) {
@@ -278,17 +297,23 @@ void EngineWrapper::createDefaultLight(std::shared_ptr<FilamentBuffer> iblBuffer
       IndirectLight::Builder().reflections(cubemap).irradiance(3, harmonics).intensity(30000.0f).build(*_engine);
 
   _scene->getScene()->setIndirectLight(_indirectLight);
+}
 
-  // Add directional light for supporting shadows
+std::shared_ptr<EntityWrapper> EngineWrapper::createLightEntity(std::string lightTypeStr, double colorFahrenheit, double intensity,
+                                                                double directionX, double directionY, double directionZ, bool castShadows) {
   auto lightEntity = _engine->getEntityManager().create();
-  LightManager::Builder(LightManager::Type::DIRECTIONAL)
-      .color(Color::cct(6500.0f))
-      .intensity(10000)
-      .direction({0, -1, 0})
-      .castShadows(true)
-      .build(*_engine, lightEntity);
 
-  _scene->getScene()->addEntity(lightEntity);
+  // TODO(Marc): Fix enum converter
+  LightManager::Type lightType;
+  EnumMapper::convertJSUnionToEnum(lightTypeStr, &lightType);
+
+  LightManager::Builder(lightType)
+      .color(Color::cct(static_cast<float>(colorFahrenheit)))
+      .intensity(static_cast<float>(intensity))
+      .direction({directionX, directionY, directionZ})
+      .castShadows(castShadows)
+      .build(*_engine, lightEntity);
+  return std::make_shared<EntityWrapper>(lightEntity);
 }
 
 std::shared_ptr<ManipulatorWrapper> EngineWrapper::createCameraManipulator(int width, int height) {
@@ -305,31 +330,9 @@ std::shared_ptr<ManipulatorWrapper> EngineWrapper::createCameraManipulator(int w
 /**
  * Sets up a root transform on the current model to make it fit into a unit cube.
  */
-void EngineWrapper::transformToUnitCube(filament::gltfio::FilamentAsset* asset) {
+void EngineWrapper::transformToUnitCube(std::shared_ptr<FilamentAssetWrapper> asset) {
   TransformManager& tm = _engine->getTransformManager();
-  Aabb aabb = asset->getBoundingBox();
-  math::details::TVec3<float> center = aabb.center();
-  math::details::TVec3<float> halfExtent = aabb.extent();
-  float maxExtent = max(halfExtent) * 2.0f;
-  float scaleFactor = 2.0f / maxExtent;
-  math::mat4f transform = math::mat4f::scaling(scaleFactor) * math::mat4f::translation(-center);
-  EntityInstance<TransformManager> transformInstance = tm.getInstance(asset->getRoot());
-  tm.setTransform(transformInstance, transform);
-}
-
-void EngineWrapper::updateCameraProjection() {
-  if (!_view) {
-    throw std::runtime_error("View not initialized");
-  }
-  if (!_camera) {
-    throw std::runtime_error("Camera not initialized");
-  }
-
-  const double aspect = (double)_view->getView()->getViewport().width / _view->getView()->getViewport().height;
-  double focalLength = 28.0;
-  double near = 0.05;  // 5cm
-  double far = 1000.0; // 1km
-  _camera->getCamera()->setLensProjection(focalLength, aspect, near, far);
+  asset->transformToUnitCube(tm);
 }
 
 void EngineWrapper::synchronizePendingFrames() {
@@ -344,4 +347,42 @@ void EngineWrapper::synchronizePendingFrames() {
   _engine->destroy(fence);
 }
 
+/**
+ * Internal method that will help updating the transform of an entity.
+ * @param transform The transform matrix to apply
+ * @param entity The entity to apply the transform to
+ * @param multiplyCurrent If true, the current transform will be multiplied with the new transform, otherwise it will be replaced
+ */
+void EngineWrapper::updateTransform(math::mat4 transform, std::shared_ptr<EntityWrapper> entity, bool multiplyCurrent) {
+  if (!entity) {
+    throw std::invalid_argument("Entity is null");
+  }
+
+  TransformManager& tm = _engine->getTransformManager();
+  EntityInstance<TransformManager> entityInstance = tm.getInstance(entity->getEntity());
+  auto currentTransform = tm.getTransform(entityInstance);
+  auto newTransform = multiplyCurrent ? (currentTransform * transform) : transform;
+  tm.setTransform(entityInstance, newTransform);
+}
+
+// TODO(Marc): Ideally i want to do this in the entity wrapper, but i dont have access to the transform manager there
+void EngineWrapper::setEntityPosition(std::shared_ptr<EntityWrapper> entity, std::vector<double> positionVec, bool multiplyCurrent) {
+  math::float3 position = Converter::VecToFloat3(positionVec);
+  auto translationMatrix = math::mat4::translation(position);
+  updateTransform(translationMatrix, entity, multiplyCurrent);
+}
+
+void EngineWrapper::setEntityRotation(std::shared_ptr<EntityWrapper> entity, double angleRadians, std::vector<double> axisVec,
+                                      bool multiplyCurrent) {
+  math::float3 axis = Converter::VecToFloat3(axisVec);
+  auto rotationMatrix = math::mat4::rotation(angleRadians, axis);
+  updateTransform(rotationMatrix, entity, multiplyCurrent);
+}
+
+void EngineWrapper::setEntityScale(std::shared_ptr<EntityWrapper> entity, std::vector<double> scaleVec, bool multiplyCurrent) {
+  math::float3 scale = Converter::VecToFloat3(scaleVec);
+  auto scaleMatrix = math::mat4::scaling(scale);
+  updateTransform(scaleMatrix, entity, multiplyCurrent);
+}
+
 } // namespace margelo