[main] Add CI for building with previously source-built SDK

eng/pipelines/templates/jobs/vmr-build.yml

@@ -80,12 +80,8 @@ jobs:
   - ${{ else }}:
     - name: BotAccount-dotnet-bot-repo-PAT
       value: N/A
-  - ${{ if or(not(eq(parameters.reuseBuildArtifactsFrom, '')), eq(parameters.withPreviousSDK, 'true')) }}:
-    - name: additionalBuildArgs
-      value: ' --with-sdk /vmr/.dotnet'
-  - ${{ else }}:
-    - name: additionalBuildArgs
-      value: ''
+  - name: additionalBuildArgs
+    value: ''
 
   # Location of the VMR sources
   # We either build the repo directly, or we extract them outside (which is what partners do)
@@ -155,23 +151,17 @@ jobs:
     - script: |
         set -euo pipefail
 
-        sourceBuiltSDKUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/"
         packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props"
         notFoundMessage="No source-built SDK found to download..."
 
         echo "Looking for source-built SDK to download..."
-        archiveVersionLine=`grep -m 1 "<PrivateSourceBuiltSDKVersion>" "$packageVersionsPath" || :`
-        versionPattern="<PrivateSourceBuiltSDKVersion>(.*)</PrivateSourceBuiltSDKVersion>"
+        archiveUrlLine=`grep -m 1 "<PrivateSourceBuiltSdkUrl>" "$packageVersionsPath" || :`
+        urlPattern="<PrivateSourceBuiltSdkUrl>(.*)</PrivateSourceBuiltSdkUrl>"
 
-        if [[ $archiveVersionLine =~ $versionPattern ]]; then
-          filename="dotnet-sdk-${BASH_REMATCH[1]}.tar.gz"
-          archiveUrl="${sourceBuiltSDKUrl}${filename}"
-          downloadDir=$(mktemp -d)
+        if [[ $archiveUrlLine =~ $urlPattern ]]; then
+          archiveUrl="${BASH_REMATCH[1]}"
+          downloadDir="$(sourcesPath)/prereqs/packages/archive/"
           (cd $downloadDir && curl --retry 5 -O $archiveUrl)
-
-          mkdir ${{ variables.sourcesPath }}/.dotnet
-          tar -xzf $downloadDir/$filename -C ${{ variables.sourcesPath }}/.dotnet
-          rm -rf $downloadDir
         else
           echo "$notFoundMessage"
           exit 1
@@ -183,13 +173,19 @@ jobs:
 
       if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then
         docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh
-      elif [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
-        docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh --no-bootstrap
       else
         mkdir $(sourcesPath)/.dotnet
         previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz"
         eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet"
         eval rm -f "$previousSdkPath"
+
+        if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
+          # Source-built artifacts are from CentOS 8 Stream. We want to download them without
+          # downloading portable versions from the internet.
+          docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh --no-bootstrap
+        fi
+
+        echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /tarball/.dotnet"
       fi
     displayName: Prep the Build
 

eng/pipelines/templates/stages/vmr-build.yml

@@ -67,7 +67,7 @@ stages:
   jobs:
   - template: ../jobs/vmr-build.yml
     parameters:
-      buildName: CentOSStream8_Online
+      buildName: CentOSStream8_Online_MsftSdk
       isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
       vmrBranch: ${{ variables.VmrBranch }}
       architecture: x64
@@ -86,7 +86,7 @@ stages:
   # Temporarily move this to PR build just to test the N-1 CI
   - template: ../jobs/vmr-build.yml
     parameters:
-      buildName: CentOSStream8_WithPreviousSDK
+      buildName: CentOSStream8_Online_PreviousSourceBuiltSdk
       isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
       vmrBranch: ${{ variables.VmrBranch }}
       architecture: x64
@@ -105,7 +105,7 @@ stages:
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: CentOSStream8_Offline
+        buildName: CentOSStream8_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -123,7 +123,7 @@ stages:
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: CentOSStream9_Offline
+        buildName: CentOSStream9_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -141,7 +141,7 @@ stages:
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Fedora36_Offline
+        buildName: Fedora36_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -159,7 +159,7 @@ stages:
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Ubuntu2004_Offline
+        buildName: Ubuntu2004_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -177,7 +177,7 @@ stages:
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Debian11_Offline
+        buildName: Debian11_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: arm64
@@ -193,7 +193,7 @@ stages:
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Fedora36_Offline_Using_Previous
+        buildName: Fedora36_Offline_CurrentSourceBuiltSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64

src/SourceBuild/content/eng/Versions.props

@@ -12,20 +12,14 @@
 
   <PropertyGroup>
     <!--
-      Building .NET from source depends on one or two tar.gz files depending on the branch's current
+      Building .NET from source depends on several archives, depending on the branch's current
       source-buildability status.
 
-      PrivateSourceBuiltArtifactsPackageVersion is a tar.gz of .NET build outputs from a previous
-      build needed to build the current version of .NET. This is always defined, because .NET needs
-      to be bootstrappable at any point in time.
-
-      PrivateSourceBuiltPrebuiltsPackageVersion is a tar.gz of assets downloaded from the internet
-      that are needed to build the current version of .NET. Early in the lifecycle of a .NET major
-      or minor release, prebuilts may be needed. When the release is mature, prebuilts are not
-      necessary, and this property is removed from the file.
+      These URLs can't be composed from their base URL and version as we read them from the
+      prep.sh and pipeline scripts, outside of MSBuild.
     -->
-    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-8.0.100-5.centos.8-x64</PrivateSourceBuiltArtifactsPackageVersion>
-    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-8.0.100-8.centos.8-x64</PrivateSourceBuiltPrebuiltsPackageVersion>
-    <PrivateSourceBuiltSDKVersion>8.0.100-alpha.1.23080.1-centos.8-x64</PrivateSourceBuiltSDKVersion>
+    <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/dotnet/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.0.1.0-8.0.100-5.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsBaseUrl>
+    <PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/dotnet/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-8.0.100-8.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
+    <PrivateSourceBuiltSdkUrl>https://dotnetcli.azureedge.net/dotnet/source-built-artifacts/sdks/dotnet-sdk-8.0.100-preview.1.23115.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl>
   </PropertyGroup>
 </Project>

src/SourceBuild/content/prep.sh

@@ -79,10 +79,10 @@ function DownloadArchive {
     notFoundMessage="No source-built $archiveType found to download..."
 
     echo "  Looking for source-built $archiveType to download..."
-    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}PackageVersion>" "$packageVersionsPath" || :`
-    versionPattern="<PrivateSourceBuilt${archiveType}PackageVersion>(.*)</PrivateSourceBuilt${archiveType}PackageVersion>"
+    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :`
+    versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
     if [[ $archiveVersionLine =~ $versionPattern ]]; then
-        archiveUrl="${sourceBuiltArtifactsTarballUrl}${baseFileName}.${BASH_REMATCH[1]}.tar.gz"
+        archiveUrl="${BASH_REMATCH[1]}"
         echo "  Downloading source-built $archiveType from $archiveUrl..."
         (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl)
     elif [ "$isRequired" == "true" ]; then