Skip to content

Commit

Permalink
Various fixes for merged test runners on Mono (#67665)
Browse files Browse the repository at this point in the history
  • Loading branch information
jkoritzinsky committed Apr 7, 2022
1 parent c61cb71 commit c160754
Show file tree
Hide file tree
Showing 21 changed files with 260 additions and 60 deletions.
2 changes: 2 additions & 0 deletions Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
<WorkloadBuildTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WorkloadBuildTasksDir>
<MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoAOTCompilerDir>
<MonoTargetsTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoTargetsTasksDir>
<TestExclusionListTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</TestExclusionListTasksDir>
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
<Crossgen2SdkOverridePropsPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Crossgen2Tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'Microsoft.NET.CrossGen.props'))</Crossgen2SdkOverridePropsPath>
Expand All @@ -104,6 +105,7 @@
<WorkloadBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll'))</WorkloadBuildTasksAssemblyPath>
<MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
<MonoTargetsTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll'))</MonoTargetsTasksAssemblyPath>
<TestExclusionListTasksAssemblyPath>$([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll'))</TestExclusionListTasksAssemblyPath>
<ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath>
</PropertyGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ parameters:
runtimeVariant: ''
variables: {}
pool: ''
runtimeFlavorDisplayName: 'Mono'
dependsOn: []
#arcade-specific parameters
condition: always()
Expand All @@ -38,8 +37,8 @@ steps:
osGroup: ${{ parameters.osGroup }}
osSubgroup: ${{ parameters.osSubgroup}}
coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr
runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
runtimeFlavor: ${{ parameters.runtimeFlavor }}
runtimeVariant: ${{ parameters.runtimeVariant }}

${{ if eq(variables['System.TeamProject'], 'public') }}:
Expand Down
5 changes: 2 additions & 3 deletions eng/pipelines/common/templates/runtimes/build-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ parameters:
variables: {}
pool: ''
runtimeFlavor: 'coreclr'
runtimeFlavorDisplayName: 'CoreCLR'
runtimeVariant: ''
dependsOn: []
dependOnEvaluatePaths: false
Expand Down Expand Up @@ -52,10 +51,10 @@ jobs:
# Compute job name from template parameters
${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavor }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavor }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'

# Since the condition is being altered, merge the default with the additional conditions.
# See https://docs.microsoft.com/azure/devops/pipelines/process/conditions
Expand Down
7 changes: 3 additions & 4 deletions eng/pipelines/common/templates/runtimes/run-test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ parameters:
variables: {}
pool: ''
runtimeFlavor: 'coreclr'
runtimeFlavorDisplayName: 'CoreCLR'
shouldContinueOnError: false
dependsOn: []
dependOnEvaluatePaths: false
Expand Down Expand Up @@ -81,10 +80,10 @@ jobs:
# Compute job name from template parameters
${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
name: 'run_test_p0_${{ parameters.runtimeFlavor }}${{ parameters.runtimeVariant }}_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'

variables:

Expand Down Expand Up @@ -335,7 +334,7 @@ jobs:
archType: ${{ parameters.archType }}
osGroup: ${{ parameters.osGroup }}
osSubgroup: ${{ parameters.osSubgroup}}
runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
runtimeFlavor: ${{ parameters.runtimeFlavor }}
shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
runtimeVariant: ${{ parameters.runtimeVariant }}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ parameters:
tieringTest: ''
longRunningGcTests: ''
gcSimulatorTests: ''
runtimeFlavorDisplayName: 'CoreCLR'
runtimeFlavor: 'CoreCLR'
runtimeVariant: ''
shouldContinueOnError: false

Expand Down Expand Up @@ -57,7 +57,7 @@ steps:
_PALTestsDir: ${{ parameters.runPALTestsDir }}
_TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
_TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
RuntimeFlavor: ${{ parameters.runtimeFlavor }}
_RuntimeVariant: ${{ parameters.runtimeVariant }}
${{ if eq(parameters.publishTestResults, 'true') }}:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ parameters:
runtimeVariant: ''
variables: {}
pool: ''
runtimeFlavor: 'mono'
runtimeFlavorDisplayName: 'Mono'
dependsOn: []
#arcade-specific parameters
condition: always()
Expand All @@ -32,7 +30,7 @@ parameters:


steps:
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=monointerpreter /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper)
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=${{ parameters.runtimeVariant }} /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper)
displayName: Build Tests

# Send tests to Helix
Expand All @@ -44,8 +42,9 @@ steps:
osGroup: ${{ parameters.osGroup }}
osSubgroup: ${{ parameters.osSubgroup}}
coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr
runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
runtimeFlavor: ${{ parameters.runtimeFlavor }}
runtimeVariant: ${{ parameters.runtimeVariant }}

${{ if eq(variables['System.TeamProject'], 'public') }}:
creator: $(Build.DefinitionName)
Expand Down
1 change: 1 addition & 0 deletions eng/pipelines/common/templates/wasm-runtime-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
testGroup: innerloop
isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }}
nameSuffix: AllSubsets_Mono_RuntimeTests
runtimeVariant: monointerpreter
buildArgs: -s mono+libs -c $(_BuildConfig)
timeoutInMinutes: 180
condition: >-
Expand Down
5 changes: 0 additions & 5 deletions eng/pipelines/common/xplat-setup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,6 @@ parameters:
jobs:
- template: ${{ coalesce(parameters.helixQueuesTemplate, parameters.jobTemplate) }}
parameters:
${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}:
runtimeFlavorDisplayName: 'CoreCLR'
${{ if eq(parameters.jobParameters.runtimeFlavor, 'mono') }}:
runtimeFlavorDisplayName: 'Mono'

shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }}

# keep in sync with /eng/pipelines/common/variables.yml
Expand Down
4 changes: 1 addition & 3 deletions eng/pipelines/coreclr/templates/helix-queues-setup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ parameters:
container: ''
pool: ''
platform: ''
runtimeFlavorDisplayName: ''
shouldContinueOnError: false
dependOnEvaluatePaths: false
jobParameters: {}
Expand All @@ -23,8 +22,7 @@ jobs:
pool: ${{ parameters.pool }}
platform: ${{ parameters.platform }}
shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}}
runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
helixQueues:

# iOS/tvOS simulator x64/x86
Expand Down
2 changes: 0 additions & 2 deletions eng/pipelines/libraries/helix-queues-setup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ parameters:
container: ''
pool: ''
platform: ''
runtimeFlavorDisplayName: ''
shouldContinueOnError: false
dependOnEvaluatePaths: false
jobParameters: {}
Expand All @@ -24,7 +23,6 @@ jobs:
platform: ${{ parameters.platform }}
shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}}
runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
helixQueues:

# Linux arm
Expand Down
32 changes: 31 additions & 1 deletion src/tasks/AndroidAppBuilder/ApkBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ public ApkBuilder(TaskLoggingHelper logger)
// 4. Align APK

string alignedApk = Path.Combine(OutputDir, "bin", $"{ProjectName}.apk");
Utils.RunProcess(logger, zipalign, $"-v 4 {apkFile} {alignedApk}", workingDir: OutputDir);
AlignApk(apkFile, alignedApk, zipalign);
// we don't need the unaligned one any more
File.Delete(apkFile);

Expand All @@ -475,6 +475,11 @@ public ApkBuilder(TaskLoggingHelper logger)
return (alignedApk, packageId);
}

private void AlignApk(string unalignedApkPath, string apkOutPath, string zipalign)
{
Utils.RunProcess(logger, zipalign, $"-v 4 {unalignedApkPath} {apkOutPath}", workingDir: OutputDir);
}

private void SignApk(string apkPath, string apksigner)
{
string defaultKey = Path.Combine(OutputDir, "debug.keystore");
Expand All @@ -495,6 +500,31 @@ private void SignApk(string apkPath, string apksigner)
$"--ks-pass pass:android --key-pass pass:android {apkPath}", workingDir: OutputDir);
}

public void ZipAndSignApk(string apkPath)
{
if (string.IsNullOrEmpty(AndroidSdk))
AndroidSdk = Environment.GetEnvironmentVariable("ANDROID_SDK_ROOT");

if (string.IsNullOrEmpty(AndroidSdk) || !Directory.Exists(AndroidSdk))
throw new ArgumentException($"Android SDK='{AndroidSdk}' was not found or incorrect (can be set via ANDROID_SDK_ROOT envvar).");

if (string.IsNullOrEmpty(BuildToolsVersion))
BuildToolsVersion = GetLatestBuildTools(AndroidSdk);

if (string.IsNullOrEmpty(MinApiLevel))
MinApiLevel = DefaultMinApiLevel;

string buildToolsFolder = Path.Combine(AndroidSdk, "build-tools", BuildToolsVersion);
string zipalign = Path.Combine(buildToolsFolder, "zipalign");
string apksigner = Path.Combine(buildToolsFolder, "apksigner");

string alignedApkPath = $"{apkPath}.aligned";
AlignApk(apkPath, alignedApkPath, zipalign);
logger.LogMessage(MessageImportance.High, $"\nMoving '{alignedApkPath}' to '{apkPath}'.\n");
File.Move(alignedApkPath, apkPath, overwrite: true);
SignApk(apkPath, apksigner);
}

public void ReplaceFileInApk(string file)
{
if (string.IsNullOrEmpty(AndroidSdk))
Expand Down
56 changes: 56 additions & 0 deletions src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
namespace TestExclusionListTasks;

public class PatchExclusionListInApks : Task
{
[Required]
public ITaskItem[]? ApkPaths { get; set; }

[Required]
public ITaskItem[]? ExcludedTests { get; set; }

public string? AndroidSdk { get; set; }

public string? MinApiLevel { get; set; }

public string? BuildToolsVersion { get; set; }

public string? KeyStorePath { get; set; }

public override bool Execute()
{
var apkBuilder = new ApkBuilder(Log);
apkBuilder.AndroidSdk = AndroidSdk;
apkBuilder.MinApiLevel = MinApiLevel;
apkBuilder.BuildToolsVersion = BuildToolsVersion;
apkBuilder.KeyStorePath = KeyStorePath;

string testExclusionList = string.Join(
'\n',
(ExcludedTests ?? Enumerable.Empty<ITaskItem>()).Select(t => t.ItemSpec));
foreach (ITaskItem apk in ApkPaths ?? Enumerable.Empty<ITaskItem>())
{
string apkPath = apk.GetMetadata("FullPath")!;
apkBuilder.OutputDir = Path.GetDirectoryName(apkPath)!;
using (ZipArchive apkArchive = ZipFile.Open(apkPath, ZipArchiveMode.Update))
{
ZipArchiveEntry assetsZipEntry = apkArchive.GetEntry("assets/assets.zip")!;
using ZipArchive assetsArchive = new ZipArchive(assetsZipEntry.Open(), ZipArchiveMode.Update);
ZipArchiveEntry testExclusionListEntry = assetsArchive.GetEntry("TestExclusionList.txt")!;
using StreamWriter textExclusionListWriter = new StreamWriter(testExclusionListEntry.Open());
textExclusionListWriter.WriteLine(testExclusionList);
}
apkBuilder.ZipAndSignApk(apkPath);
}
return true;
}
}
19 changes: 19 additions & 0 deletions src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TargetFrameworkForNETCoreTasks)</TargetFramework>
<OutputType>Library</OutputType>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<Nullable>enable</Nullable>
<NoWarn>$(NoWarn),CA1050</NoWarn>
</PropertyGroup>

<ItemGroup>
<Compile Include="PatchExclusionListInApks.cs" />
<Compile Include="../AndroidAppBuilder/ApkBuilder.cs" />
<Compile Include="../Common/Utils.cs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
</ItemGroup>
</Project>
6 changes: 6 additions & 0 deletions src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ static OutOfProcessTest()
}
}

public static bool OutOfProcessTestsSupported =>
!OperatingSystem.IsIOS()
&& !OperatingSystem.IsTvOS()
&& !OperatingSystem.IsAndroid()
&& !OperatingSystem.IsBrowser();

public static void RunOutOfProcessTest(string basePath, string assemblyPath)
{
int ret = -100;
Expand Down
5 changes: 5 additions & 0 deletions src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ public static async Task<int> RunTests(
string? filter,
HashSet<string> testExclusionList)
{
// If an exclusion list is passed as a filter, treat it as though no filter is provided here.
if (filter?.StartsWith("--exclusion-list=") == true)
{
filter = null;
}
ApplicationEntryPoint? entryPoint = null;
if (OperatingSystem.IsAndroid())
{
Expand Down
7 changes: 6 additions & 1 deletion src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,12 @@ public OutOfProcessTest(string displayName, string relativeAssemblyPath)
Method = displayName;
DisplayNameForFiltering = displayName;
TestNameExpression = $"@\"{displayName}\"";
ExecutionStatement = $@"TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");";
ExecutionStatement = $@"
if (TestLibrary.OutOfProcessTest.OutOfProcessTestsSupported)
{{
TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");
}}
";
}

public string TestNameExpression { get; }
Expand Down
Loading

0 comments on commit c160754

Please sign in to comment.