Support History/Build configs in an immutable way (#358)

Microsoft.NET.Build.Containers.IntegrationTests/Microsoft.NET.Build.Containers.IntegrationTests.csproj

@@ -27,6 +27,9 @@
 
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.NET.Build.Containers\Microsoft.NET.Build.Containers.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetFramework)' != 'net472'">
     <ProjectReference Include="..\Microsoft.NET.Build.Containers.UnitTests\Microsoft.NET.Build.Containers.UnitTests.csproj" />
   </ItemGroup>
 

Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs

@@ -2,10 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Text.Json.Nodes;
-using Microsoft.NET.Build.Containers;
 using Xunit;
 
-namespace Test.Microsoft.NET.Build.Containers;
+namespace Microsoft.NET.Build.Containers.UnitTests;
 
 public class ImageBuilderTests
 {

Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs

@@ -0,0 +1,58 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Text.Json.Nodes;
+using Xunit;
+
+namespace Microsoft.NET.Build.Containers.UnitTests;
+
+public class ImageConfigTests
+{
+    private const string SampleImageConfig = """
+                {
+                    "architecture": "amd64",
+                    "config": {
+                      "User": "app",
+                      "Env": [
+                        "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+                        "ASPNETCORE_URLS=http://+:80",
+                        "DOTNET_RUNNING_IN_CONTAINER=true",
+                        "DOTNET_VERSION=7.0.2",
+                        "ASPNET_VERSION=7.0.2"
+                      ],
+                      "Cmd": ["bash"],
+                      "Volumes": {
+                        "/var/log/": {}
+                      },
+                      "WorkingDir": "/app",
+                      "Entrypoint": null,
+                      "Labels": null,
+                      "StopSignal": "SIGTERM"
+                    },
+                    "created": "2023-02-04T08:14:52.000901321Z",
+                    "os": "linux",
+                    "rootfs": {
+                      "type": "layers",
+                      "diff_ids": [
+                        "sha256:bd2fe8b74db65d82ea10db97368d35b92998d4ea0e7e7dc819481fe4a68f64cf",
+                        "sha256:94100d1041b650c6f7d7848c550cd98c25d0bdc193d30692e5ea5474d7b3b085",
+                        "sha256:53c2a75a33c8f971b4b5036d34764373e134f91ee01d8053b4c3573c42e1cf5d",
+                        "sha256:49a61320e585180286535a2545be5722b09e40ad44c7c190b20ec96c9e42e4a3",
+                        "sha256:8a379cce2ac272aa71aa029a7bbba85c852ba81711d9f90afaefd3bf5036dc48"
+                      ]
+                    }
+                }
+                """;
+
+    [InlineData("User")]
+    [InlineData("Volumes")]
+    [InlineData("StopSignal")]
+    [Theory]
+    public void PassesThroughPropertyEvenThoughPropertyIsntExplicitlyHandled(string property)
+    {
+        ImageConfig c = new(SampleImageConfig);
+        JsonNode after = JsonNode.Parse(c.BuildConfig())!;
+        JsonNode? prop = after["config"]?[property];
+        Assert.NotNull(prop);
+    }
+}

Microsoft.NET.Build.Containers.UnitTests/Microsoft.NET.Build.Containers.UnitTests.csproj

@@ -1,11 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net7.0;net472</TargetFrameworks>
+    <TargetFrameworks>net7.0</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <LangVersion>preview</LangVersion>
     <Nullable>enable</Nullable>
-
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
@@ -23,12 +22,4 @@
     <ProjectReference Include="..\Microsoft.NET.Build.Containers\Microsoft.NET.Build.Containers.csproj" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(TargetFramework)' == 'net472'">
-    <Compile Remove="DockerDaemonAvailableUtils.cs" />
-    <Compile Remove="DockerDaemonTests.cs" />
-    <Compile Remove="RegistryTests.cs" />
-    <Compile Remove="DescriptorTests.cs" />
-    <Compile Remove="ImageBuilderTests.cs" />
-  </ItemGroup>
-
 </Project>

Microsoft.NET.Build.Containers.UnitTests/Resources/ResourceTests.cs

@@ -4,7 +4,7 @@
 using Microsoft.NET.Build.Containers.Resources;
 using Xunit;
 
-namespace Test.Microsoft.NET.Build.Containers.UnitTests.Resources
+namespace Microsoft.NET.Build.Containers.UnitTests.Resources
 {
     public class ResourceTests
     {

Microsoft.NET.Build.Containers/Constants.cs

@@ -0,0 +1,11 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Reflection;
+
+namespace Microsoft.NET.Build.Containers;
+
+public static class Constants {
+    public static readonly string Version = typeof(Registry).Assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? "";
+
+}

Microsoft.NET.Build.Containers/ImageConfig.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text.Json;
 using System.Text.Json.Nodes;
 
 namespace Microsoft.NET.Build.Containers;
@@ -12,12 +13,23 @@ internal sealed class ImageConfig
 {
     private readonly JsonObject _config;
     private readonly Dictionary<string, string> _labels;
-    private readonly List<Layer> _newLayers = new();
     private readonly HashSet<Port> _exposedPorts;
     private readonly Dictionary<string, string> _environmentVariables;
     private string? _newWorkingDirectory;
     private (string[] ExecutableArgs, string[]? Args)? _newEntryPoint;
 
+    /// <summary>
+    /// Models the file system of the image. Typically has a key 'type' with value 'layers' and a key 'diff_ids' with a list of layer digests.
+    /// </summary>
+    private readonly List<string> _rootFsLayers;
+    private readonly string _architecture;
+    private readonly string _os;
+    private readonly List<HistoryEntry> _history;
+
+    internal ImageConfig(string imageConfigJson) : this(JsonNode.Parse(imageConfigJson)!)
+    {
+    }
+
     internal ImageConfig(JsonNode config)
     {
         _config = config as JsonObject ?? throw new ArgumentException($"{nameof(config)} should be a JSON object.", nameof(config));
@@ -29,53 +41,121 @@ internal ImageConfig(JsonNode config)
         _labels = GetLabels();
         _exposedPorts = GetExposedPorts();
         _environmentVariables = GetEnvironmentVariables();
+        _rootFsLayers = GetRootFileSystemLayers();
+        _architecture = GetArchitecture();
+        _os = GetOs();
+        _history = GetHistory();
     }
 
+    private List<HistoryEntry> GetHistory() => _config["history"]?.AsArray().Select(node => node.Deserialize<HistoryEntry>()!).ToList() ?? new List<HistoryEntry>();
+    private string GetOs() => _config["os"]?.ToString() ?? throw new ArgumentException("Base image configuration should contain an 'os' property.");
+    private string GetArchitecture() => _config["architecture"]?.ToString() ?? throw new ArgumentException("Base image configuration should contain an 'architecture' property.");
+
     /// <summary>
     /// Builds in additional configuration and returns updated image configuration in JSON format as string.
     /// </summary>
     internal string BuildConfig()
     {
-        if (_config["config"] is not JsonObject config)
+        var newConfig = new JsonObject();
+
+        if (_exposedPorts.Any())
         {
-            throw new InvalidOperationException("Base image configuration should contain a 'config' node.");
+            newConfig["ExposedPorts"] = CreatePortMap();
+        }
+        if (_labels.Any())
+        {
+            newConfig["Labels"] = CreateLabelMap();
+        }
+        if (_environmentVariables.Any())
+        {
+            newConfig["Env"] = CreateEnvironmentVariablesMapping();
         }
-
-        //update creation date
-        _config["created"] = DateTime.UtcNow;
-
-        config["ExposedPorts"] = CreatePortMap();
-        config["Env"] = CreateEnvironmentVariablesMapping();
-        config["Labels"] = CreateLabelMap();
 
         if (_newWorkingDirectory is not null)
         {
-            config["WorkingDir"] = _newWorkingDirectory;
+            newConfig["WorkingDir"] = _newWorkingDirectory;
         }
 
         if (_newEntryPoint.HasValue)
         {
-            config["Entrypoint"] = ToJsonArray(_newEntryPoint.Value.ExecutableArgs);
+            newConfig["Entrypoint"] = ToJsonArray(_newEntryPoint.Value.ExecutableArgs);
 
             if (_newEntryPoint.Value.Args is null)
             {
-                config.Remove("Cmd");
+                newConfig.Remove("Cmd");
             }
             else
             {
-                config["Cmd"] = ToJsonArray(_newEntryPoint.Value.Args);
+                newConfig["Cmd"] = ToJsonArray(_newEntryPoint.Value.Args);
             }
         }
 
-        foreach (Layer l in _newLayers)
+        // These fields aren't (yet) supported by the task layer, but we should
+        // preserve them if they're already set in the base image.
+        foreach (string propertyName in new [] { "User", "Volumes", "StopSignal" })
+        {
+            if (_config["config"]?[propertyName] is JsonNode propertyValue)
+            {
+                // we can't just copy the property value because JsonValues have Parents
+                // and they cannot be re-parented. So we need to Clone them, but there's
+                // not an API for cloning, so the recommendation is to stringify and parse.
+                newConfig[propertyName] = JsonNode.Parse(propertyValue.ToJsonString());
+            }
+        }
+
+        // add a history entry for ourselves so folks can map generated layers to the Dockerfile commands
+        _history.Add(new HistoryEntry(created: DateTime.UtcNow, author: ".NET SDK", created_by: $".NET SDK Container Tooling, version {Constants.Version}"));
+
+        var configContainer = new JsonObject()
+        {
+            ["config"] = newConfig,
+            //update creation date
+            ["created"] = RFC3339Format(DateTime.UtcNow),
+            ["rootfs"] = new JsonObject()
+            {
+                ["type"] = "layers",
+                ["diff_ids"] = ToJsonArray(_rootFsLayers)
+            },
+            ["architecture"] = _architecture,
+            ["os"] = _os,
+            ["history"] = new JsonArray(_history.Select(CreateHistory).ToArray()),
+        };
+
+        return configContainer.ToJsonString();
+
+        static JsonArray ToJsonArray(IEnumerable<string> items) => new(items.Where(s => !string.IsNullOrEmpty(s)).Select(s => JsonValue.Create(s)).ToArray());
+    }
+
+    private JsonObject CreateHistory(HistoryEntry h)
+    {
+        var history = new JsonObject();
+
+        if (h.author is not null)
+        {
+            history["author"] = h.author;
+        }
+        if (h.comment is not null)
         {
-            _config["rootfs"]!["diff_ids"]!.AsArray().Add(l.Descriptor.UncompressedDigest);
+            history["comment"] = h.comment;
+        }
+        if (h.created is {} date)
+        {
+            history["created"] = RFC3339Format(date);
+        }
+        if (h.created_by is not null)
+        {
+            history["created_by"] = h.created_by;
+        }
+        if (h.empty_layer is not null)
+        {
+            history["empty_layer"] = h.empty_layer;
         }
 
-        return _config.ToJsonString();
-        static JsonArray ToJsonArray(string[] items) => new(items.Where(s => !string.IsNullOrEmpty(s)).Select(s => JsonValue.Create(s)).ToArray());
+        return history;
     }
 
+    static string RFC3339Format(DateTimeOffset dateTime) => dateTime.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", System.Globalization.CultureInfo.InvariantCulture);
+
     internal void ExposePort(int number, PortType type)
     {
         _exposedPorts.Add(new(number, type));
@@ -103,7 +183,7 @@ internal void SetEntryPoint(string[] executableArgs, string[]? args = null)
 
     internal void AddLayer(Layer l)
     {
-        _newLayers.Add(l);
+        _rootFsLayers.Add(l.Descriptor.UncompressedDigest!);
     }
 
     private HashSet<Port> GetExposedPorts()
@@ -137,7 +217,7 @@ private Dictionary<string, string> GetLabels()
                 {
                     labels[propertyName] = propertyValue.ToString();
                 }
-            }    
+            }
         }
         return labels;
     }
@@ -195,4 +275,25 @@ private JsonObject CreateLabelMap()
         }
         return container;
     }
+
+    private List<string> GetRootFileSystemLayers()
+    {
+        if (_config["rootfs"] is { } rootfs)
+        {
+            if (rootfs["type"]?.GetValue<string>() == "layers" && rootfs["diff_ids"] is JsonArray layers)
+            {
+                return layers.Select(l => l!.GetValue<string>()).ToList();
+            }
+            else
+            {
+                return new();
+            }
+        }
+        else
+        {
+            throw new InvalidOperationException("Base image configuration should contain a 'rootfs' node.");
+        }
+    }
+
+    private record HistoryEntry(DateTimeOffset? created = null, string? created_by = null, bool? empty_layer = null, string? comment = null, string? author = null);
 }

Microsoft.NET.Build.Containers/Layer.cs

@@ -3,6 +3,7 @@
 
 using System.Diagnostics;
 using System.Formats.Tar;
+using System.Globalization;
 using System.IO.Compression;
 using System.Security.Cryptography;
 using System.Text;
@@ -114,9 +115,9 @@ private static void EnsureDirectoryEntries(TarWriter tar, HashSet<string> direct
         var pathBuilder = new StringBuilder();
         for (var i = 0; i < filePathSegments.Count - 1; i++)
         {
-            pathBuilder.Append($"{filePathSegments[i]}/");
+            pathBuilder.Append(CultureInfo.InvariantCulture, $"{filePathSegments[i]}/");
 
-            var fullPath = pathBuilder.ToString(); 
+            var fullPath = pathBuilder.ToString();
             if (!directoryEntries.Contains(fullPath))
             {
                 tar.WriteEntry(new PaxTarEntry(TarEntryType.Directory, fullPath));

Microsoft.NET.Build.Containers/PublicAPI/net7.0/PublicAPI.Unshipped.txt

@@ -1,5 +1,7 @@
 const Microsoft.NET.Build.Containers.KnownDaemonTypes.Docker = "Docker" -> string!
 Microsoft.NET.Build.Containers.BaseImageNotFoundException
+Microsoft.NET.Build.Containers.Constants
+static readonly Microsoft.NET.Build.Containers.Constants.Version -> string!
 Microsoft.NET.Build.Containers.ContainerBuilder
 Microsoft.NET.Build.Containers.ContainerHelpers
 Microsoft.NET.Build.Containers.ContainerHelpers.ParsePortError
@@ -117,6 +119,7 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerizeDirectory.set ->
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.get -> string!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.CreateNewImage() -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Dispose() -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Entrypoint.get -> Microsoft.Build.Framework.ITaskItem![]!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Entrypoint.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.EntrypointArgs.get -> Microsoft.Build.Framework.ITaskItem![]!

Microsoft.NET.Build.Containers/Resources/Resource.cs

@@ -46,7 +46,7 @@ public static string FormatString(string name, params object?[] args)
 
             return resource is null ?
                 $"<{name}>" :
-                string.Format(resource, args);
+                string.Format(CultureInfo.CurrentCulture, resource, args);
         }
     }
 }