Add an option to load H264 decoder with dlopen()

CMakeLists.txt

@@ -25,6 +25,7 @@ cmake_dependent_option(TG_OWT_USE_X11 "Use X11 for desktop capture." ON "UNIX; N
 cmake_dependent_option(TG_OWT_USE_PIPEWIRE "Use pipewire for desktop capture." ON "UNIX; NOT APPLE" OFF)
 cmake_dependent_option(TG_OWT_DLOPEN_PIPEWIRE "dlopen pipewire for desktop capture." ${not_packaged_build} TG_OWT_USE_PIPEWIRE OFF)
 option(TG_OWT_BUILD_AUDIO_BACKENDS "Build webrtc audio backends." OFF)
+option(TG_OWT_DLOPEN_H264 "dlopen H264 for video coding." OFF)
 
 if (BUILD_SHARED_LIBS)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -168,7 +169,7 @@ link_openssl(tg_owt)
 link_ffmpeg(tg_owt)
 link_opus(tg_owt)
 link_libabsl(tg_owt)
-link_libopenh264(tg_owt)
+link_libopenh264(tg_owt ${TG_OWT_DLOPEN_H264})
 link_libvpx(tg_owt)
 link_crc32c(tg_owt)
 link_dl(tg_owt)

cmake/external.cmake

@@ -129,7 +129,7 @@ endfunction()
 
 # libopenh264
 set(TG_OWT_OPENH264_INCLUDE_PATH "" CACHE STRING "Include path for openh264.")
-function(link_libopenh264 target_name)
+function(link_libopenh264 target_name with_dlopen)
     if (TG_OWT_PACKAGED_BUILD)
         find_package(PkgConfig REQUIRED)
         pkg_check_modules(LIBOPENH264 openh264)
@@ -148,6 +148,10 @@ function(link_libopenh264 target_name)
             ${TG_OWT_OPENH264_INCLUDE_PATH}
         )
     endif()
+    if (with_dlopen)
+        target_compile_definitions(${target_name} PRIVATE -DWEBRTC_USE_H264_DLOPEN)
+        target_sources(${target_name} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/src/modules/video_coding/codecs/h264/h264_dlopen.cc)
+    endif()
 endfunction()
 
 # libvpx

src/modules/video_coding/codecs/h264/h264_dlopen.cc

@@ -0,0 +1,128 @@
+/*
+ * OpenH264 dlopen code
+ *
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <dlfcn.h>
+#include <cstddef>
+
+#include "h264_dlopen.h"
+
+/*
+ * The symbol binding makes sure we do not run into strict aliasing issues which
+ * can lead into segfaults.
+ */
+typedef int (*__oh264_WelsCreateSVCEncoder)(ISVCEncoder **);
+typedef void (*__oh264_WelsDestroySVCEncoder)(ISVCEncoder *);
+typedef int (*__oh264_WelsGetDecoderCapability)(SDecoderCapability *);
+typedef long (*__oh264_WelsCreateDecoder)(ISVCDecoder **);
+typedef void (*__oh264_WelsDestroyDecoder)(ISVCDecoder *);
+typedef OpenH264Version (*__oh264_WelsGetCodecVersion)(void);
+typedef void (*__oh264_WelsGetCodecVersionEx)(OpenH264Version *);
+
+#define OH264_SYMBOL_ENTRY(i)                                                  \
+  union {                                                                      \
+    __oh264_##i f;                                                             \
+    void *obj;                                                                 \
+  } _oh264_##i
+
+struct oh264_symbols {
+  OH264_SYMBOL_ENTRY(WelsCreateSVCEncoder);
+  OH264_SYMBOL_ENTRY(WelsDestroySVCEncoder);
+  OH264_SYMBOL_ENTRY(WelsGetDecoderCapability);
+  OH264_SYMBOL_ENTRY(WelsCreateDecoder);
+  OH264_SYMBOL_ENTRY(WelsDestroyDecoder);
+  OH264_SYMBOL_ENTRY(WelsGetCodecVersion);
+  OH264_SYMBOL_ENTRY(WelsGetCodecVersionEx);
+};
+
+/* Symbols are bound by loadLibOpenH264() */
+static struct oh264_symbols openh264_symbols;
+
+int oh264_WelsCreateSVCEncoder(ISVCEncoder **ppEncoder) {
+  return openh264_symbols._oh264_WelsCreateSVCEncoder.f(ppEncoder);
+}
+
+void oh264_WelsDestroySVCEncoder(ISVCEncoder *pEncoder) {
+  return openh264_symbols._oh264_WelsDestroySVCEncoder.f(pEncoder);
+}
+
+int oh264_WelsGetDecoderCapability(SDecoderCapability *pDecCapability) {
+  return openh264_symbols._oh264_WelsGetDecoderCapability.f(pDecCapability);
+}
+
+long oh264_WelsCreateDecoder(ISVCDecoder **ppDecoder) {
+  return openh264_symbols._oh264_WelsCreateDecoder.f(ppDecoder);
+}
+
+void oh264_WelsDestroyDecoder(ISVCDecoder *pDecoder) {
+  return openh264_symbols._oh264_WelsDestroyDecoder.f(pDecoder);
+}
+
+OpenH264Version oh264_WelsGetCodecVersion(void) {
+  return openh264_symbols._oh264_WelsGetCodecVersion.f();
+}
+
+void oh264_WelsGetCodecVersionEx(OpenH264Version *pVersion) {
+  openh264_symbols._oh264_WelsGetCodecVersionEx.f(pVersion);
+}
+
+static void *_oh264_bind_symbol(void *handle,
+                                const char *sym_name) {
+    void *sym = NULL;
+
+    sym = dlsym(handle, sym_name);
+    if (sym == NULL) {
+        const char *err = dlerror();
+        return NULL;
+    }
+
+    return sym;
+}
+
+#define oh264_bind_symbol(handle, sym_name)                           \
+  if (openh264_symbols._oh264_##sym_name.obj == NULL) {                      \
+    openh264_symbols._oh264_##sym_name.obj = _oh264_bind_symbol(handle, #sym_name); \
+    if (openh264_symbols._oh264_##sym_name.obj == NULL) {                    \
+      return 1;                                                              \
+    }                                                                        \
+  }
+
+int loadLibOpenH264(void) {
+  static bool initialized = false;
+  void *libopenh264 = NULL;
+  const char *err = NULL;
+
+  if (initialized) {
+      return 0;
+  }
+
+#define OPENH264_LIB "libopenh264.so.7"
+  libopenh264 = dlopen(OPENH264_LIB, RTLD_LAZY);
+  err = dlerror();
+  if (err != NULL) {
+    if (libopenh264 != NULL) {
+      dlclose(libopenh264);
+    }
+    return 1;
+  }
+
+  oh264_bind_symbol(libopenh264, WelsCreateSVCEncoder);
+  oh264_bind_symbol(libopenh264, WelsDestroySVCEncoder);
+  oh264_bind_symbol(libopenh264, WelsGetDecoderCapability);
+  oh264_bind_symbol(libopenh264, WelsCreateDecoder);
+  oh264_bind_symbol(libopenh264, WelsDestroyDecoder);
+  oh264_bind_symbol(libopenh264, WelsGetCodecVersion);
+  oh264_bind_symbol(libopenh264, WelsGetCodecVersionEx);
+
+  initialized = true;
+
+  return 0;
+}

src/modules/video_coding/codecs/h264/h264_dlopen.h

@@ -0,0 +1,46 @@
+/*
+ * OpenH264 dlopen code
+ *
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef HAVE_LIBOPENH264_DLOPEN_H
+#define HAVE_LIBOPENH264_DLOPEN_H
+
+#ifdef WEBRTC_USE_H264_DLOPEN
+
+#include <wels/codec_api.h>
+#include <wels/codec_ver.h>
+
+int oh264_WelsCreateSVCEncoder(ISVCEncoder **ppEncoder);
+#define WelsCreateSVCEncoder oh264_WelsCreateSVCEncoder
+
+void oh264_WelsDestroySVCEncoder(ISVCEncoder *pEncoder);
+#define WelsDestroySVCEncoder oh264_WelsDestroySVCEncoder
+
+int oh264_WelsGetDecoderCapability(SDecoderCapability *pDecCapability);
+#define WelsGetDecoderCapability oh264_WelsGetDecoderCapability
+
+long oh264_WelsCreateDecoder(ISVCDecoder **ppDecoder);
+#define WelsCreateDecoder oh264_WelsCreateDecoder
+
+void oh264_WelsDestroyDecoder(ISVCDecoder *pDecoder);
+#define WelsDestroyDecoder oh264_WelsDestroyDecoder
+
+OpenH264Version oh264_WelsGetCodecVersion(void);
+#define WelsGetCodecVersion oh264_WelsGetCodecVersion
+
+void oh264_WelsGetCodecVersionEx(OpenH264Version *pVersion);
+#define WelsGetCodecVersionEx oh264_WelsGetCodecVersionEx
+
+int loadLibOpenH264(void);
+
+#endif /* WEBRTC_USE_H264_DLOPEN */
+
+#endif /* HAVE_LIBOPENH264_DLOPEN_H */

src/modules/video_coding/codecs/h264/h264_encoder_impl.cc

@@ -218,6 +218,12 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
     return release_ret;
   }
 
+  #ifdef WEBRTC_USE_H264_DLOPEN
+  if (loadLibOpenH264()) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  #endif
+
   int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
   bool doing_simulcast = (number_of_streams > 1);
 

src/modules/video_coding/codecs/h264/h264_encoder_impl.h

@@ -30,7 +30,12 @@
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/svc/scalable_video_controller.h"
 #include "modules/video_coding/utility/quality_scaler.h"
+
+#ifdef WEBRTC_USE_H264_DLOPEN
+#include "h264_dlopen.h"
+#else
 #include <wels/codec_app_def.h>
+#endif
 
 class ISVCEncoder;