From e65d04254c68697d1ee59e370c4de47c422021e2 Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Fri, 2 Jun 2023 15:24:47 -0500 Subject: [PATCH] Add alpine source build CI leg (#15765) --- eng/pipelines/templates/stages/vmr-build.yml | 29 +++++++++++ .../DotNetHelper.cs | 2 +- ...osoft.DotNet.SourceBuild.SmokeTests.csproj | 1 + .../OmniSharpTests.cs | 2 +- .../SdkContentTests.cs | 2 +- .../SourceBuiltArtifactsTests.cs | 15 +++--- .../Utilities.cs | 49 +++++++++++++++++++ 7 files changed, 89 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index f7837e84ebad..d90588a29134 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -13,6 +13,7 @@ parameters: # The following parameters aren't expected to be passed in rather they are used for encapsulation # ----------------------------------------------------------------------------------------------- + alpine317Container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17 centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 centOSStream9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 debian11Arm64Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-arm64v8 @@ -65,6 +66,9 @@ stages: value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} jobs: + + # PR and CI legs ------------------------------------ + - template: ../jobs/vmr-build.yml parameters: buildName: CentOSStream8_Online_MsftSdk @@ -84,6 +88,27 @@ stages: withPreviousSDK: false # 🚫 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + + # CI - Stage 1 x64 legs ------------------------------------ + + - template: ../jobs/vmr-build.yml + parameters: + buildName: Alpine317_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.alpine317Container }} + buildFromArchive: false # βœ… + enablePoison: false # 🚫 + excludeOmniSharpTests: true # βœ… + overrideDistroDisablingSha1: false # 🚫 + runOnline: false # 🚫 + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + - template: ../jobs/vmr-build.yml parameters: buildName: CentOSStream8_Online_PreviousSourceBuiltSdk @@ -192,6 +217,8 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 + # CI - Stage 1 arm64 Legs ------------------------------------ + - template: ../jobs/vmr-build.yml parameters: buildName: Debian11_Offline_MsftSdk @@ -208,6 +235,8 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 + # CI - Stage 2 x64 Legs ------------------------------------ + - template: ../jobs/vmr-build.yml parameters: buildName: CentOSStream8_Online_CurrentSourceBuiltSdk diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs index 80ff2e29ea60..bce9b5424502 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs @@ -36,7 +36,7 @@ public DotNetHelper(ITestOutputHelper outputHelper) } Directory.CreateDirectory(Config.DotNetDirectory); - ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"xzf {Config.SdkTarballPath} -C {Config.DotNetDirectory}", outputHelper); + Utilities.ExtractTarball(Config.SdkTarballPath, Config.DotNetDirectory); } IsMonoRuntime = DetermineIsMonoRuntime(Config.DotNetDirectory); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj index d13dabf93052..c491778a80b0 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj @@ -9,6 +9,7 @@ + diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs index 56f14043b9a1..fdcb48ad5e01 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs @@ -65,7 +65,7 @@ private async Task InitializeOmniSharp() await client.DownloadFileAsync(omniSharpTarballUrl, omniSharpTarballFile, OutputHelper); Directory.CreateDirectory(OmniSharpDirectory); - ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"xzf {omniSharpTarballFile} -C {OmniSharpDirectory}", OutputHelper); + Utilities.ExtractTarball(omniSharpTarballFile, OmniSharpDirectory); } } } diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs index 4d02ed3ae687..88c0e473d7e4 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs @@ -43,7 +43,7 @@ private void WriteTarballFileList(string? tarballPath, string outputFileName, bo throw new InvalidOperationException($"Tarball path '{tarballPath}' does not exist."); } - string fileListing = ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"tf {tarballPath}", OutputHelper); + string fileListing = Utilities.GetTarballContentNames(tarballPath).Aggregate((a, b) => $"{a}{Environment.NewLine}{b}"); fileListing = BaselineHelper.RemoveRids(fileListing, isPortable); fileListing = BaselineHelper.RemoveVersions(fileListing); IEnumerable files = fileListing.Split(Environment.NewLine).OrderBy(path => path); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs index 49c03008f7b8..be7bd9395ace 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs @@ -1,4 +1,8 @@ -ο»Ώusing System; +ο»Ώ// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -22,7 +26,7 @@ public void VerifyVersionFile() try { // Extract the .version file - ExtractFileFromTarball(Config.SourceBuiltArtifactsPath, ".version", outputDir); + Utilities.ExtractTarball(Config.SourceBuiltArtifactsPath, outputDir, ".version"); string[] versionLines = File.ReadAllLines(Path.Combine(outputDir, ".version")); Assert.Equal(2, versionLines.Length); @@ -48,7 +52,7 @@ public void VerifyVersionFile() string sdkVersion = versionLines[1]; // Find the expected SDK version by getting it from the SDK tarball - ExtractFileFromTarball(Config.SdkTarballPath ?? string.Empty, "./sdk/*/.version", outputDir); + Utilities.ExtractTarball(Config.SdkTarballPath ?? string.Empty, outputDir, "./sdk/*/.version"); DirectoryInfo sdkDir = new DirectoryInfo(Path.Combine(outputDir, "sdk")); string sdkVersionPath = sdkDir.GetFiles(".version", SearchOption.AllDirectories).Single().FullName; string[] sdkVersionLines = File.ReadAllLines(Path.Combine(outputDir, sdkVersionPath)); @@ -61,9 +65,4 @@ public void VerifyVersionFile() Directory.Delete(outputDir, recursive: true); } } - - private void ExtractFileFromTarball(string tarballPath, string filePath, string outputDir) - { - ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"--wildcards -xzf {tarballPath} -C {outputDir} {filePath}", OutputHelper); - } } diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs index d706d3972417..86fc70b4bba7 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs @@ -2,7 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.Extensions.FileSystemGlobbing; using System; +using System.Collections.Generic; +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; using System.Threading; using System.Threading.Tasks; using Xunit.Abstractions; @@ -11,6 +16,50 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; public static class Utilities { + public static void ExtractTarball(string tarballPath, string outputDir) + { + using FileStream fileStream = File.OpenRead(tarballPath); + using GZipStream decompressorStream = new(fileStream, CompressionMode.Decompress); + TarFile.ExtractToDirectory(decompressorStream, outputDir, true); + } + + public static void ExtractTarball(string tarballPath, string outputDir, string targetFilePath) + { + Matcher matcher = new(); + matcher.AddInclude(targetFilePath); + + using FileStream fileStream = File.OpenRead(tarballPath); + using GZipStream decompressorStream = new(fileStream, CompressionMode.Decompress); + using TarReader reader = new(decompressorStream); + + TarEntry entry; + while ((entry = reader.GetNextEntry()) is not null) + { + if (matcher.Match(entry.Name).HasMatches) + { + string outputPath = Path.Join(outputDir, entry.Name); + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); + + using FileStream outputFileStream = File.Create(outputPath); + entry.DataStream.CopyTo(outputFileStream); + break; + } + } + } + + public static IEnumerable GetTarballContentNames(string tarballPath) + { + using FileStream fileStream = File.OpenRead(tarballPath); + using GZipStream decompressorStream = new(fileStream, CompressionMode.Decompress); + using TarReader reader = new(decompressorStream); + + TarEntry entry; + while ((entry = reader.GetNextEntry()) is not null) + { + yield return entry.Name; + } + } + public static async Task RetryAsync(Func executor, ITestOutputHelper outputHelper) { await Utilities.RetryAsync(