Deferred compilation on Android

.github/workflows/integration-tests.yml

@@ -238,18 +238,8 @@ jobs:
           key: android-build-dir-${{ hashFiles('src/**', 'react-native/android/**', 'vendor/**') }}
           restore-keys: |
             android-build-dir-
-      - uses: actions/cache@v2
-        name: Restore Android librealm.so from cache
-        id: cache-android-so
-        with:
-          path: react-native/android/**/librealm.so
-          key: android-so-${{ hashFiles('src/**', 'react-native/android/**', 'vendor/**') }}
-
       - run: npm ci
         if: ${{ steps.cache-node-modules.outputs.cache-hit != 'true' }}
-      # Build the Android shared object file (if cache missed)
-      - run: ANDROID_NDK=$ANDROID_NDK_HOME node scripts/build-android.js --arch=x86 --build-type=${{ matrix.type }}
-        if: ${{ steps.cache-android-so.outputs.cache-hit != 'true' }}
       # Bootstrap lerna sub-packages (builds the packages, the Realm JS native module and pod install)
       - run: npx lerna bootstrap --scope '{realm-integration-tests,realm-react-native-tests}' --include-dependencies
       # Run the tests

CMakeLists.txt

@@ -39,6 +39,21 @@ set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 
+set(PACKAGE_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+
+# Load dependency info from dependencies.list into REALM_FOO_VERSION variables.
+set(DEPENDENCIES_FILE "dependencies.list" CACHE STRING "path to dependencies list")
+file(STRINGS ${DEPENDENCIES_FILE} DEPENDENCIES)
+foreach(DEPENDENCY IN LISTS DEPENDENCIES)
+    string(REGEX MATCHALL "([^=]+)" COMPONENT_AND_VERSION ${DEPENDENCY})
+    list(GET COMPONENT_AND_VERSION 0 COMPONENT)
+    list(GET COMPONENT_AND_VERSION 1 VERSION)
+    if(COMPONENT MATCHES "_VERSION$")
+        message(DEBUG "Setting ${COMPONENT}=${VERSION}")
+        set(${COMPONENT} ${VERSION})
+    endif()
+endforeach()
+
 if(DEFINED CMAKE_JS_VERSION)
     include(NodeJSTargets)
 endif()
@@ -53,6 +68,57 @@ if(REALM_JS_BUILD_CORE_FROM_SOURCE)
     endif()
 
     add_subdirectory(vendor/realm-core EXCLUDE_FROM_ALL)
+else()
+    if(NOT ANDROID)
+        message(FATAL_ERROR "Building with Realm Core prebuilt binaries is only supported on Android")
+    endif()
+
+    message(STATUS "Using Realm Core ${REALM_CORE_VERSION} prebuilt binaries")
+
+    set(REALM_CORE_PLATFORM Android-${ANDROID_ABI})
+    # TODO: Revising the "devel" part of the following line
+    set(REALM_CORE_FILENAME realm-${CMAKE_BUILD_TYPE}-v${REALM_CORE_VERSION}-${REALM_CORE_PLATFORM}-devel)
+    set(REALM_CORE_PATH ${CMAKE_BINARY_DIR}/${REALM_CORE_FILENAME})
+    set(REALM_CORE_CMAKE_CONFIG ${REALM_CORE_PATH}/lib/cmake/Realm/RealmConfig.cmake)
+    if(EXISTS ${REALM_CORE_CMAKE_CONFIG})
+        message(STATUS "Skipping download of Realm Core (it's already there)")
+    else()
+        set(REALM_CORE_ARCHIVE_PATH ${CMAKE_BINARY_DIR}/${REALM_CORE_FILENAME}.tar.gz)
+        message(STATUS "Downloading Realm Core prebuilt binaries")
+        file(DOWNLOAD "https://static.realm.io/downloads/core/${REALM_CORE_FILENAME}.tar.gz" "${REALM_CORE_ARCHIVE_PATH}" STATUS DOWNLOAD_STATUS)
+        # Separate the returned status code, and error message.
+        list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
+        list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
+        # Check if download was successful.
+        if(${STATUS_CODE} EQUAL 0)
+            message(STATUS "Download of Realm Core completed successfully")
+        else()
+            message(FATAL_ERROR "Error occurred during download: ${ERROR_MESSAGE} (${STATUS_CODE})")
+        endif()
+
+        # TODO: Await a core release with fixed binaries and uri.hpp file and remove this
+        if(${ANDROID_ABI} MATCHES "^x86")
+            set(REALM_CORE_ARCHIVE_PATH "/Users/kraen.hansen/Downloads/realm-Debug-v11.4.1-48-g1787cfadf-Android-${ANDROID_ABI}-devel.tar.gz")
+        else()
+            message(FATAL_ERROR "Unsupported ANDROID_ABI (got ${ANDROID_ABI})")
+        endif()
+
+        message(STATUS "Uncompressing Realm Core")
+        file(MAKE_DIRECTORY ${REALM_CORE_PATH})
+        execute_process(COMMAND "${CMAKE_COMMAND}"
+            -E tar xfz ${REALM_CORE_ARCHIVE_PATH}
+            WORKING_DIRECTORY ${REALM_CORE_PATH}
+        )
+        # TODO: Solve this
+        execute_process(COMMAND "${CMAKE_COMMAND}"
+            -E copy ${PACKAGE_ROOT_DIR}/vendor/realm-core/src/realm/util/uri.hpp ${REALM_CORE_PATH}/include/realm/util/uri.hpp
+        )
+    endif()
+
+    message(STATUS "Incuding Realm Core CMAKE configuration")
+    include(${REALM_CORE_CMAKE_CONFIG})
+
+    # set_property(TARGET realm PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${REALM_CORE_PATH}/include)
 endif()
 
 add_subdirectory(src)

Jenkinsfile

@@ -92,7 +92,6 @@ stage('build') {
     parallelExecutors["Windows ia32 NAPI ${nodeTestVersion}"] = buildWindows(nodeTestVersion, 'ia32')
     parallelExecutors["Windows x64 NAPI ${nodeTestVersion}"] = buildWindows(nodeTestVersion, 'x64')
 
-    parallelExecutors["Android RN"] = buildAndroid()
     parallelExecutors["iOS RN"] = buildiOS()
 
     parallel parallelExecutors
@@ -291,25 +290,6 @@ def inAndroidContainer(workerFunction) {
   }
 }
 
-def buildAndroid() {
-  return {
-    myNode('docker') {
-      unstash 'source'
-      def image = buildDockerEnv('ci/realm-js:android-build', '-f Dockerfile.android')
-      image.inside('-e HOME=/tmp') {
-        // Using --ignore-scripts to skip building for node
-        sh "./scripts/nvm-wrapper.sh ${nodeTestVersion} npm ci --ignore-scripts"
-        sh "./scripts/nvm-wrapper.sh ${nodeTestVersion} node scripts/build-android.js"
-      }
-      dir('react-native/android/src/main') {
-        // Uncomment this when testing build changes if you want to be able to download pre-built artifacts from Jenkins.
-        // archiveArtifacts('jniLibs/**')
-        stash includes: 'jniLibs/**', name: 'android-jnilibs'
-      }
-    }
-  }
-}
-
 def publish(dependencies, tag) {
   myNode('docker') {
 

contrib/building.md

@@ -171,12 +171,11 @@ Note: If you have cloned the repo previously make sure you remove your `node_mod
 
 ### Building for iOS
 
-* Run `./scripts/build-ios.sh` from the `realm-js` root directory
+Run `./scripts/build-ios.sh` from the `realm-js` root directory
 
 ### Building for Android
 
-* Run `node scripts/build-android.js` from the `realm-js` root directory
-  -  The compiled version of the Android module is output to `<project-root>/android`
+Building for Android is deferred, such that compilation happens on the users machine when building their app.
 
 ### Building for Node.js
 

contrib/how-to-release.md

@@ -19,7 +19,6 @@ The procedure is:
 - Tag the commit: `git tag vX.Y.Z`
 - Push the changes: `git push origin --tag master` (if you are releasing from another branch, then use that instead of `master`)
 - Our CI system will build and push binaries for node.js. You can follow the progress at https://ci.realm.io. Once the "Publish" stage is completed, the binaries are uploaded.
-- Build Android binaries: `node ./scripts/build-android.js`
 - Build iOS binaries: `./scripts/build-ios.sh`
 - Publish the package: `npm publish`
 - Manually create a new release on Github

integration-tests/README.md

@@ -16,11 +16,7 @@ Building the native module (xcframework) for the iOS simulator in debug mode:
 ./scripts/build-ios.sh -c Debug simulator
 ```
 
-Building the native module (.so) for the Android simulator in debug mode:
-
-```bash
-node scripts/build-android.js --arch=x86 --build-type=Debug
-```
+Building the native module (.so) for the Android simulator will happen once the integration test app is being built.
 
 ## Installing the integration tests and environments
 

react-native/android/build.gradle

@@ -47,6 +47,31 @@ android {
     defaultConfig {
         minSdkVersion rootProject.hasProperty("minSdkVersion") ? rootProject.minSdkVersion : 16
         targetSdkVersion rootProject.hasProperty("targetSdkVersion") ? rootProject.targetSdkVersion : 28
+        externalNativeBuild {
+            cmake {
+                arguments '-DANDROID_STL=c++_shared', '-DREALM_JS_BUILD_CORE_FROM_SOURCE=off'
+                targets 'realm-js-android'
+                cppFlags ''
+                abiFilters 'x86', 'x86_64'
+            }
+        }
+    }
+    buildTypes {
+        debug {
+            jniDebuggable true
+        }
+    }
+    externalNativeBuild {
+        cmake {
+            path file('../../CMakeLists.txt')
+            version '3.18.1'
+        }
+    }
+
+    packagingOptions {
+        excludes = [
+            "**/libc++_shared.so"
+        ]
     }
 
     tasks.withType(JavaCompile) {