From 4415472b81adecf3a12b24f8c808da3f33de966b Mon Sep 17 00:00:00 2001 From: Mike Voorhees Date: Mon, 7 Aug 2023 03:42:42 -0400 Subject: [PATCH] Improve test failure messages (#89967) --- .../ExpectNonZeroExitCodeAttribute.cs | 14 +++++++++++ .../CommandLine/InvalidArguments.cs | 1 + .../FeatureSubstitutionsInvalid.cs | 1 + .../LinkAttributes/TypedArgumentsErrors.cs | 1 + .../Logging/CommonLogs.cs | 1 + .../Warnings/CanDisableWarnAsError.cs | 1 + .../Warnings/CanSingleWarnWithWarnAsError.cs | 1 + .../Warnings/CanWarnAsError.cs | 1 + .../Warnings/InvalidWarningVersion.cs | 1 + .../TestCasesRunner/LinkedTestCaseResult.cs | 4 ++- .../TestCasesRunner/ResultChecker.cs | 25 +++++++++++++++++++ .../TestCasesRunner/TestRunner.cs | 4 +-- 12 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Assertions/ExpectNonZeroExitCodeAttribute.cs diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Assertions/ExpectNonZeroExitCodeAttribute.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Assertions/ExpectNonZeroExitCodeAttribute.cs new file mode 100644 index 0000000000000..bd95bc61daf96 --- /dev/null +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Assertions/ExpectNonZeroExitCodeAttribute.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Mono.Linker.Tests.Cases.Expectations.Assertions; + +[AttributeUsage (AttributeTargets.Class)] +public class ExpectNonZeroExitCodeAttribute : BaseExpectedLinkedBehaviorAttribute +{ + public ExpectNonZeroExitCodeAttribute (int value) + { + } +} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/CommandLine/InvalidArguments.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/CommandLine/InvalidArguments.cs index 22f21400a2d92..55dc4d416a5cc 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/CommandLine/InvalidArguments.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/CommandLine/InvalidArguments.cs @@ -3,6 +3,7 @@ namespace Mono.Linker.Tests.Cases.CommandLine { + [ExpectNonZeroExitCode (1)] [SetupLinkerArgument ("--verbose", "--invalidArgument")] [LogContains ("Unrecognized command-line option")] [NoLinkedOutput] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/FeatureSettings/FeatureSubstitutionsInvalid.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/FeatureSettings/FeatureSubstitutionsInvalid.cs index a7723e812bf36..2c7eef3331fba 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/FeatureSettings/FeatureSubstitutionsInvalid.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/FeatureSettings/FeatureSubstitutionsInvalid.cs @@ -3,6 +3,7 @@ namespace Mono.Linker.Tests.Cases.FeatureSettings { + [ExpectNonZeroExitCode (1)] [SetupLinkerSubstitutionFile ("FeatureSubstitutionsInvalid.xml")] [SetupLinkerArgument ("--feature", "NoValueFeature", "true")] [LogContains ("FeatureSubstitutionsInvalid.xml'. Feature 'NoValueFeature' does not specify a 'featurevalue' attribute")] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/LinkAttributes/TypedArgumentsErrors.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/LinkAttributes/TypedArgumentsErrors.cs index 44513d83e07ba..f2a99cbbc8572 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/LinkAttributes/TypedArgumentsErrors.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/LinkAttributes/TypedArgumentsErrors.cs @@ -4,6 +4,7 @@ namespace Mono.Linker.Tests.Cases.LinkAttributes { + [ExpectNonZeroExitCode (1)] [SetupLinkAttributesFile ("TypedArgumentsErrors.xml")] [IgnoreLinkAttributes (false)] [NoLinkedOutput] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Logging/CommonLogs.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Logging/CommonLogs.cs index a65685c90e663..7fa0c3edcf0f9 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Logging/CommonLogs.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Logging/CommonLogs.cs @@ -3,6 +3,7 @@ namespace Mono.Linker.Tests.Cases.Logging { + [ExpectNonZeroExitCode (1)] [SetupCompileBefore ("LogStep.dll", new[] { "Dependencies/LogStep.cs" }, new[] { "illink.dll", "Mono.Cecil.dll" })] [SetupLinkerArgument ("--custom-step", "Log.LogStep,LogStep.dll")] [SetupLinkerArgument ("--verbose")] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanDisableWarnAsError.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanDisableWarnAsError.cs index 988a036d4e945..3cda9e17d9770 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanDisableWarnAsError.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanDisableWarnAsError.cs @@ -3,6 +3,7 @@ namespace Mono.Linker.Tests.Cases.Warnings { + [ExpectNonZeroExitCode (1)] [IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)] [SkipKeptItemsValidation] [SkipRemainingErrorsValidation] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanSingleWarnWithWarnAsError.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanSingleWarnWithWarnAsError.cs index df9e6534f1f53..4a38bd1f4c88b 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanSingleWarnWithWarnAsError.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanSingleWarnWithWarnAsError.cs @@ -5,6 +5,7 @@ namespace Mono.Linker.Tests.Cases.Warnings { + [ExpectNonZeroExitCode (1)] [IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)] [SkipKeptItemsValidation] [SetupCompileBefore ("library.dll", new[] { typeof (TriggerWarnings_Lib) })] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanWarnAsError.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanWarnAsError.cs index b0e931ab4fabe..6499028609557 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanWarnAsError.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/CanWarnAsError.cs @@ -3,6 +3,7 @@ namespace Mono.Linker.Tests.Cases.Warnings { + [ExpectNonZeroExitCode (1)] [IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)] [SkipKeptItemsValidation] [SetupLinkerSubstitutionFile ("CanWarnAsErrorSubstitutions.xml")] diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/InvalidWarningVersion.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/InvalidWarningVersion.cs index 825d3f639328e..5998b5264621b 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/InvalidWarningVersion.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Warnings/InvalidWarningVersion.cs @@ -3,6 +3,7 @@ namespace Mono.Linker.Tests.Cases.Warnings { + [ExpectNonZeroExitCode (1)] [IgnoreTestCase ("Ignore in NativeAOT, see https://github.com/dotnet/runtime/issues/82447", IgnoredBy = Tool.NativeAot)] [SetupLinkerArgument ("--verbose")] [SetupLinkerArgument ("--warn", "invalid")] diff --git a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/LinkedTestCaseResult.cs b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/LinkedTestCaseResult.cs index b1da7c0970ded..430225aa68851 100644 --- a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/LinkedTestCaseResult.cs +++ b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/LinkedTestCaseResult.cs @@ -17,8 +17,9 @@ public class LinkedTestCaseResult public readonly ManagedCompilationResult CompilationResult; public readonly LinkerTestLogger Logger; public readonly LinkerCustomizations Customizations; + public readonly int ExitCode; - public LinkedTestCaseResult (TestCase testCase, NPath inputAssemblyPath, NPath outputAssemblyPath, NPath expectationsAssemblyPath, TestCaseSandbox sandbox, TestCaseMetadataProvider metadataProvider, ManagedCompilationResult compilationResult, LinkerTestLogger logger, LinkerCustomizations customizations) + public LinkedTestCaseResult (TestCase testCase, NPath inputAssemblyPath, NPath outputAssemblyPath, NPath expectationsAssemblyPath, TestCaseSandbox sandbox, TestCaseMetadataProvider metadataProvider, ManagedCompilationResult compilationResult, LinkerTestLogger logger, LinkerCustomizations customizations, int exitCode) { TestCase = testCase; InputAssemblyPath = inputAssemblyPath; @@ -29,6 +30,7 @@ public LinkedTestCaseResult (TestCase testCase, NPath inputAssemblyPath, NPath o CompilationResult = compilationResult; Logger = logger; Customizations = customizations; + ExitCode = exitCode; } } } \ No newline at end of file diff --git a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/ResultChecker.cs b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/ResultChecker.cs index 7b37d2ab5a686..641036229a215 100644 --- a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/ResultChecker.cs +++ b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/ResultChecker.cs @@ -7,6 +7,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using Mono.Cecil; using Mono.Cecil.Cil; @@ -110,8 +111,12 @@ public virtual void Check (LinkedTestCaseResult linkResult) try { var original = ResolveOriginalsAssembly (linkResult.ExpectationsAssemblyPath.FileNameWithoutExtension); + + VerifyExitCode (linkResult, original); + if (!HasAttribute (original, nameof (NoLinkedOutputAttribute))) { Assert.IsTrue (linkResult.OutputAssemblyPath.FileExists (), $"The linked output assembly was not found. Expected at {linkResult.OutputAssemblyPath}"); + var linked = ResolveLinkedAssembly (linkResult.OutputAssemblyPath.FileNameWithoutExtension); if (ShouldValidateIL (original)) { @@ -239,6 +244,26 @@ void PerformOutputSymbolChecks (AssemblyDefinition original, NPath outputDirecto } } + void VerifyExitCode (LinkedTestCaseResult linkResult, AssemblyDefinition original) + { + if (TryGetCustomAttribute (original, nameof(ExpectNonZeroExitCodeAttribute), out var attr)) { + var expectedExitCode = (int) attr.ConstructorArguments[0].Value; + Assert.AreEqual (expectedExitCode, linkResult.ExitCode, $"Expected exit code {expectedExitCode} but got {linkResult.ExitCode}. Output was:\n{FormatLinkerOutput()}"); + } else { + if (linkResult.ExitCode != 0) { + Assert.Fail($"Linker exited with an unexpected non-zero exit code of {linkResult.ExitCode} and output:\n{FormatLinkerOutput()}"); + } + } + + string FormatLinkerOutput () + { + var sb = new StringBuilder (); + foreach (var message in linkResult.Logger.GetLoggedMessages ()) + sb.AppendLine (message.ToString ()); + return sb.ToString (); + } + } + void VerifyKeptSymbols (CustomAttribute symbolsAttribute) { var assemblyName = (string) symbolsAttribute.ConstructorArguments[0].Value; diff --git a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/TestRunner.cs b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/TestRunner.cs index 7d50e5ee2a115..52a91c049e6e6 100644 --- a/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/TestRunner.cs +++ b/src/tools/illink/test/Mono.Linker.Tests/TestCasesRunner/TestRunner.cs @@ -109,9 +109,9 @@ private LinkedTestCaseResult Link (TestCase testCase, TestCaseSandbox sandbox, M AddLinkOptions (sandbox, compilationResult, builder, metadataProvider); LinkerTestLogger logger = new LinkerTestLogger (); - linker.Link (builder.ToArgs (), linkerCustomizations, logger); + var exitCode = linker.Link (builder.ToArgs (), linkerCustomizations, logger); - return new LinkedTestCaseResult (testCase, compilationResult.InputAssemblyPath, sandbox.OutputDirectory.Combine (compilationResult.InputAssemblyPath.FileName), compilationResult.ExpectationsAssemblyPath, sandbox, metadataProvider, compilationResult, logger, linkerCustomizations); + return new LinkedTestCaseResult (testCase, compilationResult.InputAssemblyPath, sandbox.OutputDirectory.Combine (compilationResult.InputAssemblyPath.FileName), compilationResult.ExpectationsAssemblyPath, sandbox, metadataProvider, compilationResult, logger, linkerCustomizations, exitCode); } protected virtual void AddLinkOptions (TestCaseSandbox sandbox, ManagedCompilationResult compilationResult, LinkerArgumentBuilder builder, TestCaseMetadataProvider metadataProvider)