Singlefile compression of native files

src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs

@@ -17,5 +17,6 @@ public enum BundleOptions
         BundleOtherFiles = 2,
         BundleSymbolFiles = 4,
         BundleAllContent = BundleNativeBinaries | BundleOtherFiles,
+        EnableCompression = 8,
     };
 }

src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs

@@ -9,6 +9,7 @@
 using System.IO;
 using System.Reflection.PortableExecutable;
 using System.Runtime.InteropServices;
+using System.IO.Compression;
 
 namespace Microsoft.NET.HostModel.Bundle
 {
@@ -18,6 +19,9 @@ namespace Microsoft.NET.HostModel.Bundle
     /// </summary>
     public class Bundler
     {
+        public const uint BundlerMajorVersion = 6;
+        public const uint BundlerMinorVersion = 0;
+
         private readonly string HostName;
         private readonly string OutputDir;
         private readonly string DepsJson;
@@ -49,17 +53,67 @@ public Bundler(string hostName,
             RuntimeConfigJson = appAssemblyName + ".runtimeconfig.json";
             RuntimeConfigDevJson = appAssemblyName + ".runtimeconfig.dev.json";
 
-            BundleManifest = new Manifest(Target.BundleVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent));
+            BundleManifest = new Manifest(Target.BundleMajorVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent));
             Options = Target.DefaultOptions | options;
         }
 
+        private bool ShouldCompress(FileType type)
+        {
+            // compression is not supported before bundle vesion 6
+            if (Target.BundleMajorVersion < 6)
+            {
+                return false;
+            }
+
+            if (!Options.HasFlag(BundleOptions.EnableCompression))
+            {
+                return false;
+            }
+
+            switch (type)
+            {
+                case FileType.Symbols:
+                case FileType.NativeBinary:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
+
         /// <summary>
         /// Embed 'file' into 'bundle'
         /// </summary>
-        /// <returns>Returns the offset of the start 'file' within 'bundle'</returns>
-
-        private long AddToBundle(Stream bundle, Stream file, FileType type)
+        /// <returns>
+        /// startOffset: offset of the start 'file' within 'bundle'
+        /// compressedSize: size of the compressed data, if entry was compressed, otherwise 0
+        /// </returns>
+        private (long startOffset, long compressedSize) AddToBundle(Stream bundle, Stream file, FileType type)
         {
+            long startOffset = bundle.Position;
+            if (ShouldCompress(type))
+            {
+                long fileLength = file.Length;
+                file.Position = 0;
+
+                // We use DeflateStream here.
+                // It uses GZip algorithm, but with a trivial header that does not contain file info.
+                using (DeflateStream compressionStream = new DeflateStream(bundle, CompressionLevel.Optimal, leaveOpen: true))
+                {
+                    file.CopyTo(compressionStream);
+                }
+
+                long compressedSize = bundle.Position - startOffset;
+                if (compressedSize < fileLength * 0.75)
+                {
+                    return (startOffset, compressedSize);
+                }
+
+                // compression rate was not good enough
+                // roll back the bundle offset and let the uncompressed code path take care of the entry.
+                bundle.Seek(startOffset, SeekOrigin.Begin);
+            }
+
             if (type == FileType.Assembly)
             {
                 long misalignment = (bundle.Position % Target.AssemblyAlignment);
@@ -72,10 +126,10 @@ private long AddToBundle(Stream bundle, Stream file, FileType type)
             }
 
             file.Position = 0;
-            long startOffset = bundle.Position;
+            startOffset = bundle.Position;
             file.CopyTo(bundle);
 
-            return startOffset;
+            return (startOffset, 0);
         }
 
         private bool IsHost(string fileRelativePath)
@@ -186,8 +240,8 @@ private FileType InferType(FileSpec fileSpec)
         /// </exceptions>
         public string GenerateBundle(IReadOnlyList<FileSpec> fileSpecs)
         {
-            Tracer.Log($"Bundler version: {Manifest.CurrentVersion}");
-            Tracer.Log($"Bundler Header: {BundleManifest.DesiredVersion}");
+            Tracer.Log($"Bundler Version: {BundlerMajorVersion}.{BundlerMinorVersion}");
+            Tracer.Log($"Bundle  Version: {BundleManifest.BundleVersion}");
             Tracer.Log($"Target Runtime: {Target}");
             Tracer.Log($"Bundler Options: {Options}");
 
@@ -254,8 +308,8 @@ public string GenerateBundle(IReadOnlyList<FileSpec> fileSpecs)
                     using (FileStream file = File.OpenRead(fileSpec.SourcePath))
                     {
                         FileType targetType = Target.TargetSpecificFileType(type);
-                        long startOffset = AddToBundle(bundle, file, targetType);
-                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length);
+                        (long startOffset, long compressedSize) = AddToBundle(bundle, file, targetType);
+                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length, compressedSize, Target.BundleMajorVersion);
                         Tracer.Log($"Embed: {entry}");
                     }
                 }

src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs

@@ -15,28 +15,40 @@ namespace Microsoft.NET.HostModel.Bundle
     /// * Name       ("NameLength" Bytes)
     /// * Offset     (Int64)
     /// * Size       (Int64)
+    /// === present only in bundle version 3+
+    /// * CompressedSize   (Int64)  0 indicates No Compression
     /// </summary>
     public class FileEntry
     {
+        public readonly uint BundleMajorVersion;
+
         public readonly long Offset;
         public readonly long Size;
+        public readonly long CompressedSize;
         public readonly FileType Type;
         public readonly string RelativePath; // Path of an embedded file, relative to the Bundle source-directory.
 
         public const char DirectorySeparatorChar = '/';
 
-        public FileEntry(FileType fileType, string relativePath, long offset, long size)
+        public FileEntry(FileType fileType, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
         {
+            BundleMajorVersion = bundleMajorVersion;
             Type = fileType;
             RelativePath = relativePath.Replace('\\', DirectorySeparatorChar);
             Offset = offset;
             Size = size;
+            CompressedSize = compressedSize;
         }
 
         public void Write(BinaryWriter writer)
         {
             writer.Write(Offset);
             writer.Write(Size);
+            // compression is used only in version 6.0+
+            if (BundleMajorVersion >= 6)
+            {
+                writer.Write(CompressedSize);
+            }
             writer.Write((byte)Type);
             writer.Write(RelativePath);
         }

src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs

@@ -66,32 +66,26 @@ private enum HeaderFlags : ulong
         // with path-names so that the AppHost can use it in
         // extraction path.
         public readonly string BundleID;
-
-        public const uint CurrentMajorVersion = 2;
-        public readonly uint DesiredMajorVersion;
+        public readonly uint BundleMajorVersion;
         // The Minor version is currently unused, and is always zero
-        public const uint MinorVersion = 0;
-
-        public static string CurrentVersion => $"{CurrentMajorVersion}.{MinorVersion}";
-        public string DesiredVersion => $"{DesiredMajorVersion}.{MinorVersion}";
-
+        public const uint BundleMinorVersion = 0;
         private FileEntry DepsJsonEntry;
         private FileEntry RuntimeConfigJsonEntry;
         private HeaderFlags Flags;
-
         public List<FileEntry> Files;
+        public string BundleVersion => $"{BundleMajorVersion}.{BundleMinorVersion}";
 
-        public Manifest(uint desiredVersion, bool netcoreapp3CompatMode = false)
+        public Manifest(uint bundleMajorVersion, bool netcoreapp3CompatMode = false)
         {
-            DesiredMajorVersion = desiredVersion;
+            BundleMajorVersion = bundleMajorVersion;
             Files = new List<FileEntry>();
             BundleID = Path.GetRandomFileName();
-            Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode: HeaderFlags.None;
+            Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode : HeaderFlags.None;
         }
 
-        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size)
+        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
         {
-            FileEntry entry = new FileEntry(type, relativePath, offset, size);
+            FileEntry entry = new FileEntry(type, relativePath, offset, size, compressedSize, bundleMajorVersion);
             Files.Add(entry);
 
             switch (entry.Type)
@@ -118,12 +112,12 @@ public long Write(BinaryWriter writer)
             long startOffset = writer.BaseStream.Position;
 
             // Write the bundle header
-            writer.Write(DesiredMajorVersion);
-            writer.Write(MinorVersion);
+            writer.Write(BundleMajorVersion);
+            writer.Write(BundleMinorVersion);
             writer.Write(Files.Count);
             writer.Write(BundleID);
 
-            if (DesiredMajorVersion == 2)
+            if (BundleMajorVersion >= 2)
             {
                 writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Offset : 0);
                 writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Size : 0);

src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs

@@ -25,26 +25,31 @@ public class TargetInfo
         public readonly OSPlatform OS;
         public readonly Architecture Arch;
         public readonly Version FrameworkVersion;
-        public readonly uint BundleVersion;
+        public readonly uint BundleMajorVersion;
         public readonly BundleOptions DefaultOptions;
         public readonly int AssemblyAlignment;
 
         public TargetInfo(OSPlatform? os, Architecture? arch, Version targetFrameworkVersion)
         {
             OS = os ?? HostOS;
             Arch = arch ?? RuntimeInformation.OSArchitecture;
-            FrameworkVersion = targetFrameworkVersion ?? net50;
+            FrameworkVersion = targetFrameworkVersion ?? net60;
 
             Debug.Assert(IsLinux || IsOSX || IsWindows);
 
-            if (FrameworkVersion.CompareTo(net50) >= 0)
+            if (FrameworkVersion.CompareTo(net60) >= 0)
             {
-                BundleVersion = 2u;
+                BundleMajorVersion = 6u;
+                DefaultOptions = BundleOptions.None;
+            }
+            else if (FrameworkVersion.CompareTo(net50) >= 0)
+            {
+                BundleMajorVersion = 2u;
                 DefaultOptions = BundleOptions.None;
             }
             else if (FrameworkVersion.Major == 3 && (FrameworkVersion.Minor == 0 || FrameworkVersion.Minor == 1))
             {
-                BundleVersion = 1u;
+                BundleMajorVersion = 1u;
                 DefaultOptions = BundleOptions.BundleAllContent;
             }
             else
@@ -94,7 +99,7 @@ public override string ToString()
 
         // The .net core 3 apphost doesn't care about semantics of FileType -- all files are extracted at startup.
         // However, the apphost checks that the FileType value is within expected bounds, so set it to the first enumeration.
-        public FileType TargetSpecificFileType(FileType fileType) => (BundleVersion == 1) ? FileType.Unknown : fileType;
+        public FileType TargetSpecificFileType(FileType fileType) => (BundleMajorVersion == 1) ? FileType.Unknown : fileType;
 
         // In .net core 3.x, bundle processing happens within the AppHost.
         // Therefore HostFxr and HostPolicy can be bundled within the single-file app.
@@ -105,6 +110,7 @@ public override string ToString()
         public bool ShouldExclude(string relativePath) =>
             (FrameworkVersion.Major != 3) && (relativePath.Equals(HostFxr) || relativePath.Equals(HostPolicy));
 
+        private readonly Version net60 = new Version(6, 0);
         private readonly Version net50 = new Version(5, 0);
         private string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib";
         private string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib";

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs

@@ -32,7 +32,8 @@ private void Bundle_Extraction_To_Specific_Path_Succeeds()
 
             // Publish the bundle
             UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options: BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options);
 
             // Verify expected files in the bundle directory
             var bundleDir = BundleHelper.GetBundleDir(fixture);
@@ -81,6 +82,7 @@ private void Bundle_Extraction_To_Relative_Path_Succeeds(string relativePath, Bu
 
             var fixture = sharedTestState.TestFixture.Copy();
             UseSingleFileSelfContainedHost(fixture);
+            bundleOptions |= BundleOptions.EnableCompression;
             var bundler = BundleHelper.BundleApp(fixture, out var singleFile, bundleOptions);
 
             // Run the bundled app (extract files to <path>)
@@ -111,7 +113,8 @@ private void Bundle_extraction_is_reused()
 
             // Publish the bundle
             UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options);
 
             // Create a directory for extraction.
             var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture);
@@ -161,7 +164,8 @@ private void Bundle_extraction_can_recover_missing_files()
 
             // Publish the bundle
             UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options);
 
             // Create a directory for extraction.
             var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture);

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs

@@ -47,6 +47,7 @@ public static string BundleSelfContainedApp(
             Version targetFrameworkVersion = null)
         {
             UseSingleFileSelfContainedHost(testFixture);
+            options |= BundleOptions.EnableCompression;
             return BundleHelper.BundleApp(testFixture, options, targetFrameworkVersion);
         }
 

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs

@@ -59,6 +59,8 @@ public void Bundled_Framework_dependent_App_Run_Succeeds(BundleOptions options)
         public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options)
         {
             var fixture = sharedTestState.TestSelfContainedFixture.Copy();
+            UseSingleFileSelfContainedHost(fixture);
+            options |= BundleOptions.EnableCompression;
             var singleFile = BundleHelper.BundleApp(fixture, options);
 
             // Run the bundled app (extract files)
@@ -75,6 +77,8 @@ public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options)
         public void Bundled_With_Empty_File_Succeeds(BundleOptions options)
         {
             var fixture = sharedTestState.TestAppWithEmptyFileFixture.Copy();
+            UseSingleFileSelfContainedHost(fixture);
+            options |= BundleOptions.EnableCompression;
             var singleFile = BundleHelper.BundleApp(fixture, options);
 
             // Run the app

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs

@@ -27,7 +27,8 @@ public void Bundle_Is_Extracted()
         {
             var fixture = sharedTestState.TestFixture.Copy();
             UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleAllContent);
+            BundleOptions options = BundleOptions.EnableCompression | BundleOptions.BundleAllContent;
+            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options);
             var extractionBaseDir = BundleHelper.GetExtractionRootDir(fixture);
 
             Command.Create(singleFile, "executing_assembly_location trusted_platform_assemblies assembly_location System.Console")

src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs

@@ -91,6 +91,7 @@ public void GetCommandLineArgs_0_Non_Bundled_App()
         public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir()
         {
             var fixture = sharedTestState.TestFixture.Copy();
+            UseSingleFileSelfContainedHost(fixture);
             Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile);
             string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name;
             string bundleDir = BundleHelper.GetBundleDir(fixture).FullName;
@@ -110,6 +111,7 @@ public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir()
         public void AppContext_Native_Search_Dirs_Contains_Bundle_And_Extraction_Dirs()
         {
             var fixture = sharedTestState.TestFixture.Copy();
+            UseSingleFileSelfContainedHost(fixture);
             Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
             string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name;
             string bundleDir = BundleHelper.GetBundleDir(fixture).FullName;

src/installer/tests/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs

@@ -60,7 +60,7 @@ public static string[] GetBundledFiles(TestProjectFixture fixture)
 
         public static string[] GetExtractedFiles(TestProjectFixture fixture, BundleOptions bundleOptions)
         {
-            switch (bundleOptions)
+            switch (bundleOptions & ~BundleOptions.EnableCompression)
             {
                 case BundleOptions.None:
                 case BundleOptions.BundleOtherFiles:

src/native/corehost/apphost/static/CMakeLists.txt

@@ -19,6 +19,10 @@ set(SKIP_VERSIONING 1)
 include_directories(..)
 include_directories(../../json)
 
+if(NOT CLR_CMAKE_TARGET_WIN32)
+    include_directories(../../../../libraries/Native/Unix/Common)
+endif()
+
 set(SOURCES
     ../bundle_marker.cpp
     ./hostfxr_resolver.cpp