From c16075477d048a42a8dac970ca3f637c48b56de6 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 6 Apr 2022 21:53:41 -0700 Subject: [PATCH] Various fixes for merged test runners on Mono (#67665) --- Directory.Build.props | 2 + .../android-runtime-and-send-to-helix.yml | 3 +- .../templates/runtimes/build-test-job.yml | 5 +- .../templates/runtimes/run-test-job.yml | 7 +- .../templates/runtimes/send-to-helix-step.yml | 4 +- .../wasm-runtime-and-send-to-helix.yml | 7 +- .../common/templates/wasm-runtime-tests.yml | 1 + eng/pipelines/common/xplat-setup.yml | 5 -- .../coreclr/templates/helix-queues-setup.yml | 4 +- .../libraries/helix-queues-setup.yml | 2 - src/tasks/AndroidAppBuilder/ApkBuilder.cs | 32 ++++++- .../PatchExclusionListInApks.cs | 56 ++++++++++++ .../TestExclusionListTasks.csproj | 19 ++++ .../CoreCLRTestLibrary/OutOfProcessTest.cs | 6 ++ .../XHarnessRunnerLibrary/RunnerEntryPoint.cs | 5 ++ .../Common/XUnitWrapperGenerator/ITestInfo.cs | 7 +- .../Common/XUnitWrapperLibrary/TestFilter.cs | 32 +++++++ src/tests/Common/helixpublishwitharcade.proj | 89 +++++++++++++------ src/tests/Common/mergedrunnermobile.targets | 18 ++++ src/tests/build.proj | 8 +- src/tests/issues.targets | 8 ++ 21 files changed, 260 insertions(+), 60 deletions(-) create mode 100644 src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs create mode 100644 src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj diff --git a/Directory.Build.props b/Directory.Build.props index 5104b073389d7..ea12ca71919bc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -93,6 +93,7 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Crossgen2Tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'Microsoft.NET.CrossGen.props')) @@ -104,6 +105,7 @@ $([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll')) $([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll')) $([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll')) + $([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll')) $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)')) diff --git a/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml index 2e883dc9a6c6e..0868ffb1ac174 100644 --- a/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml @@ -13,7 +13,6 @@ parameters: runtimeVariant: '' variables: {} pool: '' - runtimeFlavorDisplayName: 'Mono' dependsOn: [] #arcade-specific parameters condition: always() @@ -38,8 +37,8 @@ steps: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup}} coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} runtimeVariant: ${{ parameters.runtimeVariant }} ${{ if eq(variables['System.TeamProject'], 'public') }}: diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index b1bc6a78bb3fa..d32ca3b66137f 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -11,7 +11,6 @@ parameters: variables: {} pool: '' runtimeFlavor: 'coreclr' - runtimeFlavorDisplayName: 'CoreCLR' runtimeVariant: '' dependsOn: [] dependOnEvaluatePaths: false @@ -52,10 +51,10 @@ jobs: # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}' # Since the condition is being altered, merge the default with the additional conditions. # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 03288003d85f0..c553827ab39c1 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -21,7 +21,6 @@ parameters: variables: {} pool: '' runtimeFlavor: 'coreclr' - runtimeFlavorDisplayName: 'CoreCLR' shouldContinueOnError: false dependsOn: [] dependOnEvaluatePaths: false @@ -81,10 +80,10 @@ jobs: # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: 'run_test_p0_${{ parameters.runtimeFlavor }}${{ parameters.runtimeVariant }}_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' + displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' variables: @@ -335,7 +334,7 @@ jobs: archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup}} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} runtimeVariant: ${{ parameters.runtimeVariant }} diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index fe67ff8377224..99a3de5196ac1 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -24,7 +24,7 @@ parameters: tieringTest: '' longRunningGcTests: '' gcSimulatorTests: '' - runtimeFlavorDisplayName: 'CoreCLR' + runtimeFlavor: 'CoreCLR' runtimeVariant: '' shouldContinueOnError: false @@ -57,7 +57,7 @@ steps: _PALTestsDir: ${{ parameters.runPALTestsDir }} _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} + RuntimeFlavor: ${{ parameters.runtimeFlavor }} _RuntimeVariant: ${{ parameters.runtimeVariant }} ${{ if eq(parameters.publishTestResults, 'true') }}: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml index 6dd5f6b858b3c..f1b3bbe955a7c 100644 --- a/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml @@ -18,8 +18,6 @@ parameters: runtimeVariant: '' variables: {} pool: '' - runtimeFlavor: 'mono' - runtimeFlavorDisplayName: 'Mono' dependsOn: [] #arcade-specific parameters condition: always() @@ -32,7 +30,7 @@ parameters: steps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=monointerpreter /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=${{ parameters.runtimeVariant }} /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper) displayName: Build Tests # Send tests to Helix @@ -44,8 +42,9 @@ steps: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup}} coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} + runtimeVariant: ${{ parameters.runtimeVariant }} ${{ if eq(variables['System.TeamProject'], 'public') }}: creator: $(Build.DefinitionName) diff --git a/eng/pipelines/common/templates/wasm-runtime-tests.yml b/eng/pipelines/common/templates/wasm-runtime-tests.yml index aa7e4257764bd..e6bb45d98887c 100644 --- a/eng/pipelines/common/templates/wasm-runtime-tests.yml +++ b/eng/pipelines/common/templates/wasm-runtime-tests.yml @@ -28,6 +28,7 @@ jobs: testGroup: innerloop isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} nameSuffix: AllSubsets_Mono_RuntimeTests + runtimeVariant: monointerpreter buildArgs: -s mono+libs -c $(_BuildConfig) timeoutInMinutes: 180 condition: >- diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 6ea24574d3a76..e6ac905afb064 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -14,11 +14,6 @@ parameters: jobs: - template: ${{ coalesce(parameters.helixQueuesTemplate, parameters.jobTemplate) }} parameters: - ${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}: - runtimeFlavorDisplayName: 'CoreCLR' - ${{ if eq(parameters.jobParameters.runtimeFlavor, 'mono') }}: - runtimeFlavorDisplayName: 'Mono' - shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }} # keep in sync with /eng/pipelines/common/variables.yml diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 45ac52c4dcd92..43b78cee145dc 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -7,7 +7,6 @@ parameters: container: '' pool: '' platform: '' - runtimeFlavorDisplayName: '' shouldContinueOnError: false dependOnEvaluatePaths: false jobParameters: {} @@ -23,8 +22,7 @@ jobs: pool: ${{ parameters.pool }} platform: ${{ parameters.platform }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} + dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} helixQueues: # iOS/tvOS simulator x64/x86 diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index d67e4cec12332..ecca458ba5932 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -7,7 +7,6 @@ parameters: container: '' pool: '' platform: '' - runtimeFlavorDisplayName: '' shouldContinueOnError: false dependOnEvaluatePaths: false jobParameters: {} @@ -24,7 +23,6 @@ jobs: platform: ${{ parameters.platform }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}} - runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }} helixQueues: # Linux arm diff --git a/src/tasks/AndroidAppBuilder/ApkBuilder.cs b/src/tasks/AndroidAppBuilder/ApkBuilder.cs index b6acdc48592f1..2d54e5192f899 100644 --- a/src/tasks/AndroidAppBuilder/ApkBuilder.cs +++ b/src/tasks/AndroidAppBuilder/ApkBuilder.cs @@ -463,7 +463,7 @@ public ApkBuilder(TaskLoggingHelper logger) // 4. Align APK string alignedApk = Path.Combine(OutputDir, "bin", $"{ProjectName}.apk"); - Utils.RunProcess(logger, zipalign, $"-v 4 {apkFile} {alignedApk}", workingDir: OutputDir); + AlignApk(apkFile, alignedApk, zipalign); // we don't need the unaligned one any more File.Delete(apkFile); @@ -475,6 +475,11 @@ public ApkBuilder(TaskLoggingHelper logger) return (alignedApk, packageId); } + private void AlignApk(string unalignedApkPath, string apkOutPath, string zipalign) + { + Utils.RunProcess(logger, zipalign, $"-v 4 {unalignedApkPath} {apkOutPath}", workingDir: OutputDir); + } + private void SignApk(string apkPath, string apksigner) { string defaultKey = Path.Combine(OutputDir, "debug.keystore"); @@ -495,6 +500,31 @@ private void SignApk(string apkPath, string apksigner) $"--ks-pass pass:android --key-pass pass:android {apkPath}", workingDir: OutputDir); } + public void ZipAndSignApk(string apkPath) + { + if (string.IsNullOrEmpty(AndroidSdk)) + AndroidSdk = Environment.GetEnvironmentVariable("ANDROID_SDK_ROOT"); + + if (string.IsNullOrEmpty(AndroidSdk) || !Directory.Exists(AndroidSdk)) + throw new ArgumentException($"Android SDK='{AndroidSdk}' was not found or incorrect (can be set via ANDROID_SDK_ROOT envvar)."); + + if (string.IsNullOrEmpty(BuildToolsVersion)) + BuildToolsVersion = GetLatestBuildTools(AndroidSdk); + + if (string.IsNullOrEmpty(MinApiLevel)) + MinApiLevel = DefaultMinApiLevel; + + string buildToolsFolder = Path.Combine(AndroidSdk, "build-tools", BuildToolsVersion); + string zipalign = Path.Combine(buildToolsFolder, "zipalign"); + string apksigner = Path.Combine(buildToolsFolder, "apksigner"); + + string alignedApkPath = $"{apkPath}.aligned"; + AlignApk(apkPath, alignedApkPath, zipalign); + logger.LogMessage(MessageImportance.High, $"\nMoving '{alignedApkPath}' to '{apkPath}'.\n"); + File.Move(alignedApkPath, apkPath, overwrite: true); + SignApk(apkPath, apksigner); + } + public void ReplaceFileInApk(string file) { if (string.IsNullOrEmpty(AndroidSdk)) diff --git a/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs b/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs new file mode 100644 index 0000000000000..254bef87ef627 --- /dev/null +++ b/src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +namespace TestExclusionListTasks; + +public class PatchExclusionListInApks : Task +{ + [Required] + public ITaskItem[]? ApkPaths { get; set; } + + [Required] + public ITaskItem[]? ExcludedTests { get; set; } + + public string? AndroidSdk { get; set; } + + public string? MinApiLevel { get; set; } + + public string? BuildToolsVersion { get; set; } + + public string? KeyStorePath { get; set; } + + public override bool Execute() + { + var apkBuilder = new ApkBuilder(Log); + apkBuilder.AndroidSdk = AndroidSdk; + apkBuilder.MinApiLevel = MinApiLevel; + apkBuilder.BuildToolsVersion = BuildToolsVersion; + apkBuilder.KeyStorePath = KeyStorePath; + + string testExclusionList = string.Join( + '\n', + (ExcludedTests ?? Enumerable.Empty()).Select(t => t.ItemSpec)); + foreach (ITaskItem apk in ApkPaths ?? Enumerable.Empty()) + { + string apkPath = apk.GetMetadata("FullPath")!; + apkBuilder.OutputDir = Path.GetDirectoryName(apkPath)!; + using (ZipArchive apkArchive = ZipFile.Open(apkPath, ZipArchiveMode.Update)) + { + ZipArchiveEntry assetsZipEntry = apkArchive.GetEntry("assets/assets.zip")!; + using ZipArchive assetsArchive = new ZipArchive(assetsZipEntry.Open(), ZipArchiveMode.Update); + ZipArchiveEntry testExclusionListEntry = assetsArchive.GetEntry("TestExclusionList.txt")!; + using StreamWriter textExclusionListWriter = new StreamWriter(testExclusionListEntry.Open()); + textExclusionListWriter.WriteLine(testExclusionList); + } + apkBuilder.ZipAndSignApk(apkPath); + } + return true; + } +} diff --git a/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj b/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj new file mode 100644 index 0000000000000..d17dc95c41af6 --- /dev/null +++ b/src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj @@ -0,0 +1,19 @@ + + + $(TargetFrameworkForNETCoreTasks) + Library + true + false + enable + $(NoWarn),CA1050 + + + + + + + + + + + diff --git a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs index 89fafa77da433..976f542a6b21e 100644 --- a/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs +++ b/src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs @@ -42,6 +42,12 @@ static OutOfProcessTest() } } + public static bool OutOfProcessTestsSupported => + !OperatingSystem.IsIOS() + && !OperatingSystem.IsTvOS() + && !OperatingSystem.IsAndroid() + && !OperatingSystem.IsBrowser(); + public static void RunOutOfProcessTest(string basePath, string assemblyPath) { int ret = -100; diff --git a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs index b1b1f4e44e72e..daf22773002ef 100644 --- a/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs +++ b/src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs @@ -16,6 +16,11 @@ public static async Task RunTests( string? filter, HashSet testExclusionList) { + // If an exclusion list is passed as a filter, treat it as though no filter is provided here. + if (filter?.StartsWith("--exclusion-list=") == true) + { + filter = null; + } ApplicationEntryPoint? entryPoint = null; if (OperatingSystem.IsAndroid()) { diff --git a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs index 6559a6db44f5d..d0db16db340b5 100644 --- a/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs +++ b/src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs @@ -249,7 +249,12 @@ public OutOfProcessTest(string displayName, string relativeAssemblyPath) Method = displayName; DisplayNameForFiltering = displayName; TestNameExpression = $"@\"{displayName}\""; - ExecutionStatement = $@"TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");"; + ExecutionStatement = $@" +if (TestLibrary.OutOfProcessTest.OutOfProcessTestsSupported) +{{ +TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}""); +}} +"; } public string TestNameExpression { get; } diff --git a/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs b/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs index a2cf5f0c70418..6c4c4f6b57b1b 100644 --- a/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs +++ b/src/tests/Common/XUnitWrapperLibrary/TestFilter.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; namespace XUnitWrapperLibrary; @@ -47,6 +48,11 @@ public bool IsMatch(string fullyQualifiedName, string displayName, string[] trai } return stringToSearch == Filter; } + + public override string ToString() + { + return $"{Kind}{(Substring ? "~" : "=")}{Filter}"; + } } public sealed class AndClause : ISearchClause @@ -61,6 +67,11 @@ public AndClause(ISearchClause left, ISearchClause right) } public bool IsMatch(string fullyQualifiedName, string displayName, string[] traits) => _left.IsMatch(fullyQualifiedName, displayName, traits) && _right.IsMatch(fullyQualifiedName, displayName, traits); + + public override string ToString() + { + return $"({_left}) && ({_right})"; + } } public sealed class OrClause : ISearchClause @@ -75,6 +86,11 @@ public OrClause(ISearchClause left, ISearchClause right) } public bool IsMatch(string fullyQualifiedName, string displayName, string[] traits) => _left.IsMatch(fullyQualifiedName, displayName, traits) || _right.IsMatch(fullyQualifiedName, displayName, traits); + + public override string ToString() + { + return $"({_left}) || ({_right})"; + } } public sealed class NotClause : ISearchClause @@ -87,6 +103,11 @@ public NotClause(ISearchClause inner) } public bool IsMatch(string fullyQualifiedName, string displayName, string[] traits) => !_inner.IsMatch(fullyQualifiedName, displayName, traits); + + public override string ToString() + { + return $"!({_inner})"; + } } private readonly ISearchClause? _filter; @@ -133,6 +154,17 @@ public bool ShouldRunTest(string fullyQualifiedName, string displayName, string[ public static HashSet LoadTestExclusionList() { HashSet output = new (); + + // Try reading the exclusion list as a base64-encoded semicolon-delimited string as a commmand-line arg. + string[] arguments = Environment.GetCommandLineArgs(); + string? testExclusionListArg = arguments.FirstOrDefault(arg => arg.StartsWith("--exclusion-list=")); + if (testExclusionListArg is not null) + { + string testExclusionListPathFromCommandLine = testExclusionListArg.Substring("--exclusion-list=".Length); + output.UnionWith(File.ReadAllLines(testExclusionListPathFromCommandLine)); + } + + // Try reading the exclusion list as a line-delimited file. string? testExclusionListPath = Environment.GetEnvironmentVariable("TestExclusionListPath"); if (!string.IsNullOrEmpty(testExclusionListPath)) { diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 4bf0fbd060468..ccb695afd9c0b 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -69,7 +69,7 @@ - + @@ -324,14 +324,36 @@ + + + + + + + + - + + + + + + <_ExcludeListProject Include="$(RepoRoot)src/tests/issues.targets"> + XunitTestBinBase=$(TestBinDir);$(_PropertiesToPass) + + + + + + + + DependsOnTargets="DiscoverMergedTestWrappers;GetTestExclusionList"> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) <_MergedWrapperName>%(_MergedWrapperMarker.FileName) @@ -343,12 +365,15 @@ + + DependsOnTargets="DiscoverMergedTestWrappers;GetTestExclusionList"> <_MergedWrapperRunScript Include="@(_MergedWrapperMarker->'%(RootDir)%(Directory)AppBundle/RunTests.$(TestScriptExtension)')" /> @@ -367,16 +392,29 @@ MakeRelative function calls Escape function internally that replaces all the Unicode characters with %. --> $([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)AppBundle', %(FullPath))) + + <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> + <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> + + + + + + + - + + + + DependsOnTargets="DiscoverMergedTestWrappers;GetTestExclusionList"> <_MergedWrapperDirectory>%(_MergedWrapperMarker.RootDir)%(Directory) <_MergedWrapperName>%(_MergedWrapperMarker.FileName) @@ -400,10 +438,16 @@ MakeRelative function calls Escape function internally that replaces all the Unicode characters with %. --> $([System.IO.Path]::GetRelativePath('$(_MergedWrapperDirectory)AppBundle/$(_MergedWrapperName)/$(Configuration)-$(XCodeSdk)', %(FullPath))) + + <_TestExclusionListPlaceholder Include="@(_MergedPayloadFiles)" Condition="$([System.String]::new('%(FileName)').EndsWith('TestExclusionList'))" /> + <_MergedPayloadFiles Remove="@(_TestExclusionListPlaceholder)" /> - + + + @@ -416,7 +460,10 @@ - + @@ -428,7 +475,10 @@ - + @@ -443,7 +493,7 @@ false false false - $(RuntimeFlavorDisplayName) + $(RuntimeFlavor) R2R R2R-CG2 $(TestRunNamePrefix)$(TargetOS) $(TargetArchitecture) $(Configuration) @ @@ -542,16 +592,7 @@ - - - $(MergedPayloadsRootDirectory)%(FileName)%(Extension)\HelixCommand.txt - $([System.IO.File]::ReadAllText('%(PayloadHelixCommandFile)')) - $([System.IO.Path]::GetDirectoryName('%(MergedTestHelixCommand)').Replace('/', '-').Replace('\', '-')) - $(MergedPayloadsRootDirectory)%(PayloadGroup).zip - - - - + $(MergedPayloadsRootDirectory)%(FileName)%(Extension)\HelixCommand.txt $([System.IO.File]::ReadAllText('%(PayloadHelixCommandFile)')) @@ -560,12 +601,6 @@ - - - chmod +x %(MergedPayloads.MergedTestHelixCommand)%0A%(MergedPayloads.MergedTestHelixCommand) - - - %(FileName)%(Extension) @@ -595,6 +630,7 @@ + --arg=env:TestExclusionListPath=TestExclusionList.txt net.dot.%(PayloadGroup) net.dot.MonoRunner $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) @@ -609,6 +645,7 @@ + --set-env=TestExclusionListPath=TestExclusionList.txt ios-simulator-64 ios-simulator-64 $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) diff --git a/src/tests/Common/mergedrunnermobile.targets b/src/tests/Common/mergedrunnermobile.targets index f595bb35745b1..7371cdc702c5a 100644 --- a/src/tests/Common/mergedrunnermobile.targets +++ b/src/tests/Common/mergedrunnermobile.targets @@ -21,6 +21,24 @@ + + + + <_PlaceholderExclusionList Include="$(IntermediateOutputPath)/PlaceholderTestExclusionList.txt" /> + + + + + diff --git a/src/tests/build.proj b/src/tests/build.proj index c292d42fe4cf1..78700b40b7c7f 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -597,17 +597,11 @@ Properties="@(CreateLibProperty)" /> - + $(CORE_ROOT)\TestExclusionList.txt - - - - mobile and wasm don't support tests with native libraries. wasm also needs static linking + + + + + +