ArPow stage 1: local source-build infrastructure (#53294)

* Initial arcade-powered source-build infra * Add patches, fixup from 5.0 * Rename patches to match 6.0-p1 work * Add source-build specific build script * Incorporate build-source-build.sh, into eng/ * Run inner build script, through coreclr * Initial source-build args based on live build (not props file) * Cleanup: add RID comments, rm empty/absolute args * Fix subsets * Disable sourcelink in env, not args * Fix CI NZEC * Revert "Use same code to DetectCiphersuiteConfiguration for portable and non-portable builds" This reverts commit 464010d. * Fix portability * Fix AllJits build * Fix missing crossgen2 for non-portable RID * Create supplemental intermediate nupkgs * Tweak category names * Use centralized supplemental nupkg infra * Add additional 6.0 patches * Patch updates after merging in main * SourceBuild.props cleanup * Fix issue with incorrect patch merge * Patch updates * Edit clr.tools patch * patch updates * Revert patch integration * Patch updates * Edits per code review feedback * ILAsm workaround * patch updates * Move logic to set ILAsmToolPath for source-build * Update eng/SourceBuild.props Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com> * Remove libraries specific patches * Patch updates necessary with latest main merge * Add back libraries-packages.proj harvesting patch * Refactor intermediate package split to be chunkier * Integrate patch 0017 * Subsets update per code review * Remove obsolete patch * Removed patches that were integrated into main * Remove two additional patches * Remove remaining patches Co-authored-by: Davis Goodin <dagood@microsoft.com> Co-authored-by: dseefeld <dseefeld@microsoft.com> Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>
dotnet · Jun 22, 2021 · 849033a · 849033a
1 parent 8a20ae0
commit 849033a
Show file tree

Hide file tree

Showing 9 changed files with 105 additions and 27 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -83,6 +83,7 @@
     <WasmBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WasmBuildTasksDir)', 'WasmBuildTasks.dll'))</WasmBuildTasksAssemblyPath>
     <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
     <RuntimeConfigParserTasksAssemblyPath>$([MSBuild]::NormalizePath('$(RuntimeConfigParserDir)', 'RuntimeConfigParser.dll'))</RuntimeConfigParserTasksAssemblyPath>
+    <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath>
   </PropertyGroup>
 
   <PropertyGroup Label="CalculateConfiguration">

diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props
@@ -0,0 +1,73 @@
+<Project>
+
+  <PropertyGroup>
+    <GitHubRepositoryName>runtime</GitHubRepositoryName>
+  </PropertyGroup>
+
+  <!-- Set up the dotnet/runtime source-build command. -->
+  <PropertyGroup>
+    <BaseInnerSourceBuildCommand>./build.sh</BaseInnerSourceBuildCommand>
+
+    <SourceBuildPortable>true</SourceBuildPortable>
+    <SourceBuildPortable Condition="'$(SourceBuildNonPortable)' == 'true'">false</SourceBuildPortable>
+
+    <!-- If TargetRid not specified, detect RID based on portability. -->
+    <TargetRid Condition="'$(TargetRid)' == '' and '$(SourceBuildNonPortable)' == 'true'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier)</TargetRid>
+    <TargetRid Condition="'$(TargetRid)' == ''">$(__DistroRid)</TargetRid>
+
+    <!-- Split e.g. 'fedora.33-x64' into 'fedora.33' and 'x64'. -->
+    <_targetRidPlatformIndex>$(TargetRid.LastIndexOfAny("-"))</_targetRidPlatformIndex>
+    <TargetRidWithoutPlatform>$(TargetRid.Substring(0, $(_targetRidPlatformIndex)))</TargetRidWithoutPlatform>
+    <TargetRidPlatform>$(TargetRid.Substring($(_targetRidPlatformIndex)).TrimStart('-'))</TargetRidPlatform>
+
+    <LogVerbosity Condition="'$(LogVerbosity)' == ''">minimal</LogVerbosity>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- Work around issue where local clone may cause failure using non-origin remote fallback: https://github.com/dotnet/sourcelink/issues/629 -->
+    <InnerBuildEnv Include="EnableSourceControlManagerQueries=false" />
+    <InnerBuildEnv Include="EnableSourceLink=false" />
+    <InnerBuildEnv Include="DisableSourceLink=true" />
+    <InnerBuildEnv Include="DeterministicSourcePaths=false" />
+  </ItemGroup>
+
+  <Target Name="GetRuntimeSourceBuildCommandConfiguration"
+          BeforeTargets="GetSourceBuildCommandConfiguration">
+    <PropertyGroup>
+      <InnerBuildArgs>$(InnerBuildArgs) --arch $(TargetRidPlatform)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --configuration $(Configuration)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --ci</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --allconfigurations</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --verbosity $(LogVerbosity)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --nodereuse false</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --warnAsError false</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:ContinuousIntegrationBuild=true</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:PackageRid=$(TargetRid)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:NoPgoOptimize=true</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:KeepNativeSymbols=true</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:RuntimeOS=$(TargetRidWithoutPlatform)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:PortableBuild=$(SourceBuildPortable)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:BuildDebPackage=false</InnerBuildArgs>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="CategorizeRuntimeSupplementalArtifacts"
+          BeforeTargets="GetCategorizedIntermediateNupkgContents">
+    <ItemGroup>
+      <!--
+        Runtime artifacts are too large to fit into a single package (Azure DevOps feeds 500 mb constraint).
+        Split large components into separate packages.
+      -->
+      <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\dotnet-runtime-*.tar.gz" Category="runtime" />
+
+      <IntermediateNupkgArtifactFile
+        Include="
+          $(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\dotnet-crossgen2-*.tar.gz;
+          $(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\Microsoft.NETCore.App.Crossgen2.*.nupkg"
+        Category="Crossgen2Pack" />
+    </ItemGroup>
+  </Target>
+
+</Project>
diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml
@@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>
diff --git a/eng/Subsets.props b/eng/Subsets.props
@@ -25,8 +25,8 @@
   <PropertyGroup>
     <DefaultSubsets>clr+mono+libs+host+packs</DefaultSubsets>
     <DefaultSubsets Condition="'$(TargetsMobile)' == 'true'">mono+libs+packs</DefaultSubsets>
-    <!-- On Windows mono is supported for x86/x64 only -->
-    <DefaultSubsets Condition="'$(TargetOS)' == 'windows' and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'x64'">clr+libs+host+packs</DefaultSubsets>
+    <!-- mono is not supported in source build. On Windows mono is supported for x86/x64 only. -->
+    <DefaultSubsets Condition="'$(DotNetBuildFromSource)' == 'true' or ('$(TargetOS)' == 'windows' and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'x64')">clr+libs+host+packs</DefaultSubsets>
   </PropertyGroup>
 
   <!-- Init _subset here in to allow RuntimeFlavor to be set as early as possible -->
@@ -49,18 +49,21 @@
     <DefaultMonoSubsets Condition="'$(TargetOS)' == 'Browser'">$(DefaultMonoSubsets)mono.wasmruntime+</DefaultMonoSubsets>
     <DefaultMonoSubsets Condition="'$(MonoCrossAOTTargetOS)' != ''">$(DefaultMonoSubsets)mono.aotcross+</DefaultMonoSubsets>
     <DefaultMonoSubsets>$(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages</DefaultMonoSubsets>
-     
+
     <DefaultLibrariesSubsets Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or
                                         '$(BuildTargetFramework)' == '' or
                                         '$(BuildAllConfigurations)' == 'true'">libs.native+</DefaultLibrariesSubsets>
-    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src+libs.pretest+libs.packages</DefaultLibrariesSubsets>
+    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src+libs.packages</DefaultLibrariesSubsets>
+    <DefaultLibrariesSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultLibrariesSubsets)+libs.pretest</DefaultLibrariesSubsets>
 
-    <DefaultHostSubsets>host.native+host.pkg+host.tools+host.tests</DefaultHostSubsets>
+    <DefaultHostSubsets>host.native+host.tools</DefaultHostSubsets>
+    <DefaultHostSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultHostSubsets)+host.pkg+host.tests</DefaultHostSubsets>
     <DefaultHostSubsets Condition="'$(RuntimeFlavor)' == 'Mono'"></DefaultHostSubsets>
     <DefaultHostSubsets Condition="'$(RuntimeFlavor)' == 'Mono' and '$(TargetsMobile)' != 'true'">host.native</DefaultHostSubsets>
 
     <DefaultPacksSubsets>packs.product</DefaultPacksSubsets>
     <DefaultPacksSubsets Condition="'$(BuildMonoAOTCrossCompilerOnly)' != 'true'">$(DefaultPacksSubsets)+packs.tests</DefaultPacksSubsets>
+    <DefaultPacksSubsets Condition="'$(DotNetBuildFromSource)' == 'true'">$(DefaultPacksSubsets)+packs.installers</DefaultPacksSubsets>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -204,12 +207,12 @@
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\runincontext\runincontext.csproj;
                              $(CoreClrProjectRoot)tools\r2rdump\R2RDump.csproj;
                              $(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo.csproj;
-                             $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj;
-                             $(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
+                             $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/>
+    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
 
     <ProjectToBuild Condition="'$(TargetArchitecture)' != 'x64' and '$(BuildArchitecture)' == 'x64'" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_crossarch.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler.TypeSystem.ReadyToRun.Tests\ILCompiler.TypeSystem.ReadyToRun.Tests.csproj"
-      Test="true" Category="clr" Condition="'$(__DistroRid)' != 'linux-musl-x64'"/>
+      Test="true" Category="clr" Condition="'$(__DistroRid)' != 'linux-musl-x64' and '$(DotNetBuildFromSource)' != 'true'"/>
   </ItemGroup>
 
   <ItemGroup Condition="$(_subset.Contains('+clr.nativecorelib+'))">
@@ -221,7 +224,7 @@
 
   <ItemGroup Condition="$(_subset.Contains('+clr.packages+')) and '$(PgoInstrument)' != 'true'">
     <ProjectToBuild Include="$(CoreClrProjectRoot).nuget\coreclr-packages.proj" Pack="true" Category="clr" />
-    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo-pack.proj" Pack="true" BuildInParallel="false" Category="clr" />
+    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo-pack.proj" Pack="true" BuildInParallel="false" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/>
   </ItemGroup>
 
   <!-- Mono sets -->

diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -9,6 +9,7 @@
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21311.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>4a2b475948d498b89fedef7cf890883f49bc1ea3</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21311.3">
       <Uri>https://github.com/dotnet/arcade</Uri>

diff --git a/eng/packaging.props b/eng/packaging.props
@@ -16,10 +16,6 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'">
-    <!-- Don't build referenced projects -->
-    <BuildPackageLibraryReferences>false</BuildPackageLibraryReferences>
-    <!-- Omit any files that were not built -->
-    <AllowPartialPackages>true</AllowPartialPackages>
     <!-- Don't permit harvesting since this requires pre-builts -->
     <HarvestStablePackage>false</HarvestStablePackage>
     <!-- Validation will fail in case we were relying on harvested assets or assets not built to satisfy stated support -->

diff --git a/eng/restore/harvestPackages.targets b/eng/restore/harvestPackages.targets
@@ -1,4 +1,14 @@
 <Project InitialTargets="AddPackageDownload">
+
+  <PropertyGroup>
+    <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly>
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp3.1\</PackagingTaskAssembly>
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly>
+    <!-- TODO: Remove when https://github.com/dotnet/arcade/issues/7413 is fixed. -->
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core' and '$(DotNetBuildFromSource)' == 'true'">$(PackagingTaskAssembly)net5.0\</PackagingTaskAssembly>
+    <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly>
+  </PropertyGroup>
+
   <UsingTask TaskName="GetLastStablePackage" AssemblyFile="$(PackagingTaskAssembly)"/>
   <Target Name="AddPackageDownload">
     <ItemGroup>

diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props
@@ -28,7 +28,6 @@
   <PropertyGroup>
     <!-- Build all .NET Framework configurations when net48 is passed in. This is for convenience. -->
     <AdditionalBuildTargetFrameworks Condition="'$(BuildTargetFramework)' == 'net48'">net461;net462;net47;net471;net472</AdditionalBuildTargetFrameworks>
-    <AdditionalBuildTargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true' and '$(BuildAllProjects)' == 'true'">$(AdditionalBuildTargetFrameworks);netstandard2.0</AdditionalBuildTargetFrameworks>
   </PropertyGroup>
 
   <!-- Define test projects and companions -->

diff --git a/src/libraries/libraries-packages.proj b/src/libraries/libraries-packages.proj
@@ -1,20 +1,11 @@
 <Project Sdk="Microsoft.Build.Traversal" DefaultTargets="Pack">
 
-  <PropertyGroup>
-    <AdditionalBuildTargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true'">$(AdditionalBuildTargetFrameworks);package-$(Configuration)</AdditionalBuildTargetFrameworks>
-  </PropertyGroup>
-
-  <PropertyGroup>
-    <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly>
-    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp3.1\</PackagingTaskAssembly>
-    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly>
-    <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly>
-  </PropertyGroup>
-
   <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Packaging" Version="$(MicrosoftDotNetBuildTasksPackagingVersion)" />
+
     <ProjectReference Include="$(PkgDir)*\*.proj" Exclude="$(PkgDir)test\*" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)*\src\*.*proj"
-                      Condition="('$(BuildAllConfigurations)' == 'true' or '$(DotNetBuildFromSource)' == 'true')"
+                      Condition="'$(BuildAllConfigurations)' == 'true'"
                       Exclude="$(MSBuildThisFileDirectory)*\src\**\*.shproj"/>
   </ItemGroup>
 
@@ -31,7 +22,6 @@
     ones that might do this. After we ship a stable set of packages this target should be ran and the
     changes to the package index should be commited to the repo.
   -->
-  <UsingTask TaskName="UpdatePackageIndex" AssemblyFile="$(PackagingTaskAssembly)"/>
   <Target Name="UpdatePackageIndexWithStableVersions"
           BeforeTargets="Build;Pack"
           Condition="'$(DotNetFinalVersionKind)' == 'release'">