[AOT] provide libc and libm stubs so NDK is not required (#7475)

Context: 346a933

Commit 346a933 made it possible to build and link AOT shared libraries
without having to have an Android NDK available.  However, it seems
that the produced shared libraries do not depend on the `libc.so` and
`libm.so ` standard libraries, leading to runtime errors similar to:

	D monodroid-assembly: monodroid_dlopen: hash match found, DSO name is 'libaot-Mono.Android.dll.so'
	I monodroid-assembly: Trying to load shared library '/data/app/~~4kGTzxWG6HKO7nzyg6ryBg==/com.companyname.hellomaui.old-h8Jlutcvf8Dzfpuuq1ouUA==/lib/arm64/libaot-Mono.Android.dll.so'
	I monodroid-assembly: Failed to load shared library '/data/app/~~4kGTzxWG6HKO7nzyg6ryBg==/com.companyname.hellomaui.old-h8Jlutcvf8Dzfpuuq1ouUA==/lib/arm64/libaot-Mono.Android.dll.so'. dlopen failed: cannot locate symbol "memset" referenced by "/data/app/~~4kGTzxWG6HKO7nzyg6ryBg==/com.companyname.hellomaui.old-h8Jlutcvf8Dzfpuuq1ouUA==/lib/arm64/libaot-Mono.Android.dll.so"...

This is caused by us not passing the `-lc` and `-lm` flags to the
linker (or full paths to the `libc.so` and `libm.so` libraries).
However, without the NDK available we do not have either of these
standard libraries to link against.

Fortunately, ELF shared libraries by default will resolve symbols
from anywhere in the process address space, so there is no need to
provide a `libc.so` which contains *all* public symbols.  We can
instead build and use a "dummy"/"stub" version of `libc.so` and
`libm.so`, existing only so that the linker will add the appropriate
`NEEDED` entries and resolve symbols from those libraries.

Update `src/monodroid` to also produce "stub" `libc.so` and `libm.so`
native libraries, and update the `<Aot/>` task to add the equivalent
of `-L path/to/libstubs -lc -lm` to the AOT command-line.

This ensures that the resulting `.so` files now require `libc.so`:

	$ llvm-readelf -a Mono.Android.dll.so | grep NEEDED
	  0x0000000000000001 (NEEDED)    Shared library: [libc.so]
	  0x0000000000000001 (NEEDED)    Shared library: [libm.so]

build-tools/installers/create-installers.targets

@@ -318,6 +318,14 @@
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)K4os.Compression.LZ4.dll" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ELFSharp.dll" />
     <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ManifestOverlays\Timing.xml" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libc.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libm.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libc.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libm.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libc.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libm.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libc.so" />
+    <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libm.so" />
   </ItemGroup>
   <ItemGroup>
     <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" />

src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs

@@ -169,9 +169,6 @@ IEnumerable<Config> GetAotConfigs (NdkTools ndk)
 					if (!string.IsNullOrEmpty (LdName)) {
 						aotOptions.Add ($"ld-name={LdName}");
 					}
-					if (!string.IsNullOrEmpty (LdFlags)) {
-						aotOptions.Add ($"ld-flags={LdFlags}");
-					}
 
 					// We don't check whether any mode option was added via `AotAdditionalArguments`, the `AndroidAotMode` property should always win here.
 					// Modes not supported by us directly can be set by setting `AndroidAotMode` to "normal" and adding the desired mode name to the
@@ -186,6 +183,10 @@ IEnumerable<Config> GetAotConfigs (NdkTools ndk)
 							break;
 					}
 
+					if (!string.IsNullOrEmpty (LdFlags)) {
+						aotOptions.Add ($"ld-flags={LdFlags}");
+					}
+
 					// we need to quote the entire --aot arguments here to make sure it is parsed
 					// on windows as one argument. Otherwise it will be split up into multiple
 					// values, which wont work.

src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs

@@ -255,6 +255,7 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP
 		{
 			var toolchainPath = toolPrefix.Substring (0, toolPrefix.LastIndexOf (Path.DirectorySeparatorChar));
 			var ldFlags = new StringBuilder ();
+			var libs = new List<string> ();
 			if (UseAndroidNdk && EnableLLVM) {
 				string androidLibPath = string.Empty;
 				try {
@@ -273,7 +274,6 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP
 				} else
 					toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath);
 
-				var libs = new List<string> ();
 				if (ndk.UsesClang) {
 					if (!String.IsNullOrEmpty (toolchainLibDir)) {
 						libs.Add ($"-L{toolchainLibDir.TrimEnd ('\\')}");
@@ -292,24 +292,36 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP
 				}
 				libs.Add (Path.Combine (androidLibPath, "libc.so"));
 				libs.Add (Path.Combine (androidLibPath, "libm.so"));
+			} else if (!UseAndroidNdk && EnableLLVM) {
+				// We need to link against libc and libm, but since NDK is not in use, the linker won't be able to find the actual Android libraries.
+				// Therefore, we will use their stubs to satisfy the linker. At runtime they will, of course, use the actual Android libraries.
+				string relPath = Path.Combine ("..", "..");
+				if (!OS.IsWindows) {
+					// the `binutils` directory is one level down (${OS}/binutils) than the Windows one
+					relPath = Path.Combine (relPath, "..");
+				}
+				string libstubsPath = Path.GetFullPath (Path.Combine (AndroidBinUtilsDirectory, relPath, "libstubs", ArchToRid (arch)));
+				libs.Add (Path.Combine (libstubsPath, "libc.so"));
+				libs.Add (Path.Combine (libstubsPath, "libm.so"));
+			}
 
+			if (libs.Count > 0) {
 				ldFlags.Append ($"\\\"{string.Join ("\\\";\\\"", libs)}\\\"");
-			} else {
+			}
+
+			//
+			// This flag is needed for Mono AOT to work correctly with the LLVM 14 `lld` linker due to the following change:
+			//
+			//   The AArch64 port now supports adrp+ldr and adrp+add optimizations. --no-relax can suppress the optimization.
+			//
+			// Without the flag, `lld` will modify AOT-generated code in a way that the Mono runtime doesn't support. Until
+			// the runtime issue is fixed, we need to pass this flag then.
+			//
+			if (!UseAndroidNdk) {
 				if (ldFlags.Length > 0) {
 					ldFlags.Append (' ');
 				}
-
-				//
-				// This flag is needed for Mono AOT to work correctly with the LLVM 14 `lld` linker due to the following change:
-				//
-				//   The AArch64 port now supports adrp+ldr and adrp+add optimizations. --no-relax can suppress the optimization.
-				//
-				// Without the flag, `lld` will modify AOT-generated code in a way that the Mono runtime doesn't support. Until
-				// the runtime issue is fixed, we need to pass this flag then.
-				//
-				if (!UseAndroidNdk) {
-					ldFlags.Append ("--no-relax");
-				}
+				ldFlags.Append ("--no-relax");
 			}
 
 			if (StripLibraries) {
@@ -320,6 +332,26 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP
 			}
 
 			return ldFlags.ToString ();
+
+			string ArchToRid (AndroidTargetArch arch)
+			{
+				switch (arch) {
+					case AndroidTargetArch.Arm64:
+						return "android-arm64";
+
+					case AndroidTargetArch.Arm:
+						return "android-arm";
+
+					case AndroidTargetArch.X86:
+						return "android-x86";
+
+					case AndroidTargetArch.X86_64:
+						return "android-x64";
+
+					default:
+						throw new InvalidOperationException ($"Internal error: unsupported ABI '{arch}'");
+				}
+			}
 		}
 
 		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null)

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs

@@ -82,6 +82,42 @@ public void CheckMonoComponentsMask (bool enableProfiler, bool useInterpreter, b
 			}
 		}
 
+		[Test]
+		public void CheckWhetherLibcAndLibmAreReferencedInAOTLibraries ()
+		{
+			var proj = new XamarinAndroidApplicationProject {
+				IsRelease = true,
+				EmbedAssembliesIntoApk = true,
+				AotAssemblies = true,
+			};
+			proj.SetProperty ("EnableLLVM", "True");
+
+			var abis = new [] { "arm64-v8a", "x86_64" };
+			proj.SetAndroidSupportedAbis (abis);
+
+			var libPaths = new List<string> ();
+			if (Builder.UseDotNet) {
+				libPaths.Add (Path.Combine ("android-arm64", "aot", "Mono.Android.dll.so"));
+				libPaths.Add (Path.Combine ("android-x64", "aot", "Mono.Android.dll.so"));
+			} else {
+				libPaths.Add (Path.Combine ("aot", "arm64-v8a", "libaot-Mono.Android.dll.so"));
+				libPaths.Add (Path.Combine ("aot", "x86_64", "libaot-Mono.Android.dll.so"));
+			}
+
+			using (var b = CreateApkBuilder ()) {
+				Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
+				string objPath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath);
+
+				foreach (string libPath in libPaths) {
+					string lib = Path.Combine (objPath, libPath);
+
+					Assert.IsTrue (File.Exists (lib), $"Library {lib} should exist on disk");
+					Assert.IsTrue (ELFHelper.ReferencesLibrary (lib, "libc.so"), $"Library {lib} should reference libc.so");
+					Assert.IsTrue (ELFHelper.ReferencesLibrary (lib, "libm.so"), $"Library {lib} should reference libm.so");
+				}
+			}
+		}
+
 		static object [] CheckAssemblyCountsSource = new object [] {
 			new object[] {
 				/*isRelease*/ false,

src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs

@@ -28,7 +28,47 @@ public static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path)
 				log.LogWarningFromException (ex, showStackTrace: true);
 				return false;
 			}
+		}
+
+		public static bool ReferencesLibrary (string libraryPath, string referencedLibraryName)
+		{
+			if (String.IsNullOrEmpty (libraryPath) || !File.Exists (libraryPath)) {
+				return false;
+			}
+
+			IELF elf = ELFReader.Load (libraryPath);
+			var dynstr = GetSection (elf, ".dynstr") as IStringTable;
+			if (dynstr == null) {
+				return false;
+			}
+
+			foreach (IDynamicSection section in elf.GetSections<IDynamicSection> ()) {
+				foreach (IDynamicEntry entry in section.Entries) {
+					if (IsLibraryReference (dynstr, entry, referencedLibraryName)) {
+						return true;
+					}
+				}
+			}
+
+			return false;
+		}
+
+		static bool IsLibraryReference (IStringTable stringTable, IDynamicEntry dynEntry, string referencedLibraryName)
+		{
+			if (dynEntry.Tag != DynamicTag.Needed) {
+				return false;
+			}
+
+			ulong index;
+			if (dynEntry is DynamicEntry<ulong> entry64) {
+				index = entry64.Value;
+			} else if (dynEntry is DynamicEntry<uint> entry32) {
+				index = (ulong)entry32.Value;
+			} else {
+				return false;
+			}
 
+			return String.Compare (referencedLibraryName, stringTable[(long)index], StringComparison.Ordinal) == 0;
 		}
 
 		static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path, IELF elf)

src/monodroid/CMakeLists.txt

@@ -427,6 +427,7 @@ endif()
 if(ANDROID)
   if(ENABLE_NET)
     set(XA_LIBRARY_OUTPUT_DIRECTORY "${XA_LIB_TOP_DIR}/lib/${ANDROID_RID}")
+    set(XA_LIBRARY_STUBS_OUTPUT_DIRECTORY "${XA_LIB_TOP_DIR}/libstubs/${ANDROID_RID}")
     link_directories("${NET_RUNTIME_DIR}/native")
   else()
     set(XA_LIBRARY_OUTPUT_DIRECTORY "${XA_LIB_TOP_DIR}/lib/${ANDROID_ABI}")
@@ -570,6 +571,10 @@ set(XAMARIN_DEBUG_APP_HELPER_SOURCES
   ${SOURCES_DIR}/shared-constants.cc
   )
 
+set(XAMARIN_STUB_LIB_SOURCES
+  libstub/stub.cc
+)
+
 # Build
 configure_file(jni/host-config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/host-config.h)
 
@@ -716,3 +721,57 @@ target_link_libraries(
   ${XAMARIN_MONO_ANDROID_LIB}
   ${LINK_LIBS} xamarin-app
   )
+
+if(ANDROID AND ENABLE_NET)
+  add_library(
+    c
+    SHARED ${XAMARIN_STUB_LIB_SOURCES}
+  )
+
+  target_compile_definitions(
+    c
+    PRIVATE STUB_LIB_NAME=libc
+  )
+
+  target_compile_options(
+    c
+    PRIVATE -nostdlib
+  )
+
+  target_link_options(
+    c
+    PRIVATE -nostdlib
+  )
+
+  set_target_properties(
+    c
+    PROPERTIES
+    LIBRARY_OUTPUT_DIRECTORY "${XA_LIBRARY_STUBS_OUTPUT_DIRECTORY}"
+  )
+
+  add_library(
+    m
+    SHARED ${XAMARIN_STUB_LIB_SOURCES}
+  )
+
+  target_compile_definitions(
+    m
+    PRIVATE STUB_LIB_NAME=libm
+  )
+
+  target_compile_options(
+    m
+    PRIVATE -nostdlib
+  )
+
+  target_link_options(
+    m
+    PRIVATE -nostdlib
+  )
+
+  set_target_properties(
+    m
+    PROPERTIES
+    LIBRARY_OUTPUT_DIRECTORY "${XA_LIBRARY_STUBS_OUTPUT_DIRECTORY}"
+  )
+endif()

src/monodroid/libstub/stub.cc

@@ -0,0 +1,8 @@
+#if !defined (STUB_LIB_NAME)
+#error STUB_LIB_NAME must be defined on command line
+#endif
+
+void STUB_LIB_NAME ()
+{
+	// no-op
+}

tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs

@@ -694,8 +694,30 @@ public void RunWithLLVMEnabled ()
 			Assert.IsTrue (builder.Install (proj), "Install should have succeeded.");
 			RunProjectAndAssert (proj, builder);
 
-			Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity",
-				Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log")));
+			var activityNamespace   = proj.PackageName;
+			var activityName        = "MainActivity";
+			var logcatFilePath      = Path.Combine (Root, builder.ProjectDirectory, "startup-logcat.log");
+			var failedToLoad        = new List<string> ();
+			bool appLaunched        = MonitorAdbLogcat ((line) => {
+				if (SeenFailedToLoad (line))
+					failedToLoad.Add (line);
+				return SeenActivityDisplayed (line);
+			}, logcatFilePath, timeout: 120);
+
+			Assert.IsTrue (appLaunched, "LLVM app did not launch");
+			Assert.AreEqual (0, failedToLoad.Count, $"LLVM .so files not loaded:\n{string.Join ("\n", failedToLoad)}");
+
+			bool SeenActivityDisplayed (string line)
+			{
+				var idx1 = line.IndexOf ("ActivityManager: Displayed", StringComparison.OrdinalIgnoreCase);
+				var idx2 = idx1 > 0 ? 0 : line.IndexOf ("ActivityTaskManager: Displayed", StringComparison.OrdinalIgnoreCase);
+				return (idx1 > 0 || idx2 > 0) && line.Contains (activityNamespace) && line.Contains (activityName);
+			}
+
+			bool SeenFailedToLoad (string line)
+			{
+				return line.Contains ("Failed to load shared library");
+			}
 		}
 
 		[Test]