From 5898930d87e4acca8fbcea5f38f08c967a506f79 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 9 Dec 2021 13:12:08 -0800 Subject: [PATCH] Refactor target-specific ApplicationEntryPoint subclasses to have non-xunit-specific base classes (#791) --- .../AndroidApplicationEntryPointBase.cs | 35 +++++ .../ApplicationEntryPoint.cs | 55 +++++--- .../ApplicationOptions.cs | 2 +- .../LogWriter.cs | 11 +- .../WasmApplicationEntryPointBase.cs | 30 ++++ .../iOSApplicationEntryPointBase.cs | 39 ++++++ .../AndroidApplicationEntryPoint.cs | 56 +------- .../ThreadlessXunitTestRunner.cs | 130 +++++++++++------- .../WasmApplicationEntryPoint.cs | 54 ++++---- .../XUnitTestRunner.cs | 65 +-------- .../XunitTestRunnerBase.cs | 87 ++++++++++++ .../iOSApplicationEntryPoint.cs | 46 +------ 12 files changed, 352 insertions(+), 258 deletions(-) create mode 100644 src/Microsoft.DotNet.XHarness.TestRunners.Common/AndroidApplicationEntryPointBase.cs create mode 100644 src/Microsoft.DotNet.XHarness.TestRunners.Common/WasmApplicationEntryPointBase.cs create mode 100644 src/Microsoft.DotNet.XHarness.TestRunners.Common/iOSApplicationEntryPointBase.cs create mode 100644 src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XunitTestRunnerBase.cs diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Common/AndroidApplicationEntryPointBase.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Common/AndroidApplicationEntryPointBase.cs new file mode 100644 index 0000000000000..2214c80e503f0 --- /dev/null +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Common/AndroidApplicationEntryPointBase.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Common; + +/// +/// Implementors should provide a text writter than will be used to +/// write the logging of the tests that are executed. +/// +public abstract class AndroidApplicationEntryPointBase : ApplicationEntryPoint +{ + public abstract TextWriter? Logger { get; } + + /// + /// Implementors should provide a full path in which the final + /// results of the test run will be written. This property must not + /// return null. + /// + public abstract string TestsResultsFinalPath { get; } + + public override async Task RunAsync() + { + var options = ApplicationOptions.Current; + using TextWriter? resultsFileMaybe = options.EnableXml ? File.CreateText(TestsResultsFinalPath) : null; + await InternalRunAsync(options, Logger, resultsFileMaybe); + } +} diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationEntryPoint.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationEntryPoint.cs index 041015b8381c5..e8e727740039f 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationEntryPoint.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationEntryPoint.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; - +#nullable enable namespace Microsoft.DotNet.XHarness.TestRunners.Common; /// @@ -39,24 +39,24 @@ public abstract class ApplicationEntryPoint /// /// Event raised when the test run has started. /// - public event EventHandler TestsStarted; + public event EventHandler? TestsStarted; /// /// Event raised when the test run has completed. /// - public event EventHandler TestsCompleted; + public event EventHandler? TestsCompleted; // fwd the events from the runner so that clients can connect to them /// /// Event raised when a test has started. /// - public event EventHandler TestStarted; + public event EventHandler? TestStarted; /// /// Event raised when a test has completed or has been skipped. /// - public event EventHandler<(string TestName, TestResult TestResult)> TestCompleted; + public event EventHandler<(string TestName, TestResult TestResult)>? TestCompleted; protected abstract int? MaxParallelThreads { get; } @@ -64,7 +64,7 @@ public abstract class ApplicationEntryPoint /// Must be implemented and return a class that returns the information /// of a device. It can return null. /// - protected abstract IDevice Device { get; } + protected abstract IDevice? Device { get; } /// /// Returns the IEnumerable with the asseblies that contain the tests @@ -97,7 +97,7 @@ public abstract class ApplicationEntryPoint /// * the 'KLASS:' prefix can be used to ignore all the tests in a class. /// * the 'Platform32:' prefix can be used to ignore a test but only in a 32b arch device. /// - protected virtual string IgnoreFilesDirectory => null; + protected virtual string? IgnoreFilesDirectory => null; /// /// Returns the path to a file that contains the list of traits to ignore in the following format: @@ -105,7 +105,7 @@ public abstract class ApplicationEntryPoint /// /// The default implementation will return null and therefore no traits will be ignored. /// - protected virtual string IgnoredTraitsFilePath => null; + protected virtual string? IgnoredTraitsFilePath => null; /// /// States if the skipped tests should be logged. Helpful to determine why some tests are executed and others @@ -129,9 +129,9 @@ public abstract class ApplicationEntryPoint /// public MinimumLogLevel MinimumLogLevel { get; set; } = MinimumLogLevel.Info; - private void OnTestStarted(object sender, string testName) => TestStarted?.Invoke(sender, testName); + private void OnTestStarted(object? sender, string testName) => TestStarted?.Invoke(sender, testName); - private void OnTestCompleted(object sender, (string TestName, TestResult Testresult) result) => TestCompleted?.Invoke(sender, result); + private void OnTestCompleted(object? sender, (string TestName, TestResult Testresult) result) => TestCompleted?.Invoke(sender, result); private async Task> GetIgnoredCategories() { @@ -173,14 +173,13 @@ internal static void ConfigureRunnerFilters(TestRunner runner, ApplicationOption } } - internal static string WriteResults(TestRunner runner, ApplicationOptions options, LogWriter logger, TextWriter writer) + private static void WriteResults(TestRunner runner, ApplicationOptions options, LogWriter logger, TextWriter writer) { if (options.EnableXml && writer == null) { throw new ArgumentNullException(nameof(writer)); } - string resultsFilePath = null; if (options.EnableXml) { runner.WriteResultsToFile(writer, options.XmlVersion); @@ -188,14 +187,12 @@ internal static string WriteResults(TestRunner runner, ApplicationOptions option } else { - resultsFilePath = runner.WriteResultsToFile(options.XmlVersion); + string resultsFilePath = runner.WriteResultsToFile(options.XmlVersion); logger.Info($"XML results can be found in '{resultsFilePath}'"); } - - return resultsFilePath; } - protected async Task InternalRunAsync(LogWriter logger) + private async Task InternalRunAsync(LogWriter logger) { logger.MinimumLogLevel = MinimumLogLevel; var runner = GetTestRunner(logger); @@ -220,4 +217,30 @@ protected async Task InternalRunAsync(LogWriter logger) TestsCompleted?.Invoke(this, result); return runner; } + + protected async Task InternalRunAsync(ApplicationOptions options, TextWriter? loggerWriter, TextWriter? resultsFile) + { + // we generate the logs in two different ways depending if the generate xml flag was + // provided. If it was, we will write the xml file to the provided writer if present, else + // we will write the normal console output using the LogWriter + var logger = (loggerWriter == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, loggerWriter); + logger.MinimumLogLevel = MinimumLogLevel.Info; + var runner = await InternalRunAsync(logger); + + WriteResults(runner, options, logger, resultsFile ?? Console.Out); + + logger.Info($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.FilteredTests} Skipped: {runner.SkippedTests}"); + + if (options.AppEndTag != null) + { + logger.Info(options.AppEndTag); + } + + if (options.TerminateAfterExecution) + { + TerminateWithSuccess(); + } + + return runner; + } } diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationOptions.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationOptions.cs index 240bf66baef23..4e9ca09611a2e 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationOptions.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Common/ApplicationOptions.cs @@ -16,7 +16,7 @@ internal enum XmlMode Wrapped = 1, } -internal class ApplicationOptions +public class ApplicationOptions { public static ApplicationOptions Current = new(); private readonly List _singleMethodFilters = new(); diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Common/LogWriter.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Common/LogWriter.cs index ce2e2ccbc7d11..54e2f4d88dd5c 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Common/LogWriter.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Common/LogWriter.cs @@ -3,28 +3,30 @@ // See the LICENSE file in the project root for more information. using System; +using System.Diagnostics; using System.IO; +#nullable enable namespace Microsoft.DotNet.XHarness.TestRunners.Common; public class LogWriter { private readonly TextWriter _writer; - private readonly IDevice _device; + private readonly IDevice? _device; public MinimumLogLevel MinimumLogLevel { get; set; } = MinimumLogLevel.Info; public LogWriter() : this(null, Console.Out) { } - public LogWriter(IDevice device) : this(device, Console.Out) { } + public LogWriter(IDevice? device) : this(device, Console.Out) { } public LogWriter(TextWriter w) : this(null, w) { } - public LogWriter(IDevice device, TextWriter writer) + public LogWriter(IDevice? device, TextWriter writer) { _writer = writer ?? Console.Out; _device = device; - if (_device != null) // we just write the header if we do have the device info + if (_device is not null) // we just write the header if we do have the device info { InitLogging(); } @@ -35,6 +37,7 @@ public LogWriter(IDevice device, TextWriter writer) public void InitLogging() { + Debug.Assert(_device is not null); // print some useful info _writer.WriteLine("[Runner executing:\t{0}]", "Run everything"); _writer.WriteLine("[{0}:\t{1} v{2}]", _device.Model, _device.SystemName, _device.SystemVersion); diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Common/WasmApplicationEntryPointBase.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Common/WasmApplicationEntryPointBase.cs new file mode 100644 index 0000000000000..c249ae9a59c35 --- /dev/null +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Common/WasmApplicationEntryPointBase.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Common; + +public abstract class WasmApplicationEntryPointBase : ApplicationEntryPoint +{ + protected override int? MaxParallelThreads => 1; + + protected override IDevice? Device => null; + + public override async Task RunAsync() + { + var options = ApplicationOptions.Current; + var runner = await InternalRunAsync(options, null, Console.Out); + + LastRunHadFailedTests = runner.FailedTests != 0; + } + + public bool LastRunHadFailedTests { get; set; } + + protected override void TerminateWithSuccess() => Environment.Exit(0); +} diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Common/iOSApplicationEntryPointBase.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Common/iOSApplicationEntryPointBase.cs new file mode 100644 index 0000000000000..b06b88b455ee9 --- /dev/null +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Common/iOSApplicationEntryPointBase.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Common; + +public abstract class iOSApplicationEntryPointBase : ApplicationEntryPoint +{ + public override async Task RunAsync() + { + var options = ApplicationOptions.Current; + TcpTextWriter? writer; + + try + { + writer = options.UseTunnel + ? TcpTextWriter.InitializeWithTunnelConnection(options.HostPort) + : TcpTextWriter.InitializeWithDirectConnection(options.HostName, options.HostPort); + } + catch (Exception ex) + { + Console.WriteLine("Failed to initialize TCP writer. Continuing on console." + Environment.NewLine + ex); + writer = null; // null means we will fall back to console output + } + + using (writer) + { + var logger = (writer == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, writer); + logger.MinimumLogLevel = MinimumLogLevel.Info; + + await InternalRunAsync(options, writer, writer); + } + } +} diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/AndroidApplicationEntryPoint.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/AndroidApplicationEntryPoint.cs index 57949b6128af5..33fa3120bec15 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/AndroidApplicationEntryPoint.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/AndroidApplicationEntryPoint.cs @@ -11,21 +11,8 @@ namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; -public abstract class AndroidApplicationEntryPoint : ApplicationEntryPoint +public abstract class AndroidApplicationEntryPoint : AndroidApplicationEntryPointBase { - /// - /// Implementors should provide a text writter than will be used to - /// write the logging of the tests that are executed. - /// - public abstract TextWriter? Logger { get; } - - /// - /// Implementors should provide a full path in which the final - /// results of the test run will be written. This property must not - /// return null. - /// - public abstract string TestsResultsFinalPath { get; } - protected override bool IsXunit => true; protected override TestRunner GetTestRunner(LogWriter logWriter) @@ -34,45 +21,4 @@ protected override TestRunner GetTestRunner(LogWriter logWriter) ConfigureRunnerFilters(runner, ApplicationOptions.Current); return runner; } - - public override async Task RunAsync() - { - var options = ApplicationOptions.Current; - // we generate the logs in two different ways depending if the generate xml flag was - // provided. If it was, we will write the xml file to the tcp writer if present, else - // we will write the normal console output using the LogWriter - var logger = (Logger == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, Logger); - logger.MinimumLogLevel = MinimumLogLevel.Info; - - var runner = await InternalRunAsync(logger); - if (options.EnableXml) - { - if (TestsResultsFinalPath == null) - { - throw new InvalidOperationException("Tests results final path cannot be null."); - } - - using (var stream = File.Create(TestsResultsFinalPath)) - using (var writer = new StreamWriter(stream)) - { - WriteResults(runner, options, logger, writer); - } - } - else - { - WriteResults(runner, options, logger, Console.Out); - } - - logger.Info($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.FilteredTests}"); - - if (options.AppEndTag != null) - { - logger.Info(options.AppEndTag); - } - - if (options.TerminateAfterExecution) - { - TerminateWithSuccess(); - } - } } diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs index 65cd1120a9f25..de2d27bdfc639 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; @@ -13,36 +14,49 @@ using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; - +using Microsoft.DotNet.XHarness.Common; +using Microsoft.DotNet.XHarness.TestRunners.Common; using Xunit; using Xunit.Abstractions; namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; -internal class ThreadlessXunitTestRunner +internal class ThreadlessXunitTestRunner : XunitTestRunnerBase { - public static async Task Run(string assemblyFileName, bool printXml, XunitFilters filters, bool oneLineResults = false) + public ThreadlessXunitTestRunner(LogWriter logger, bool oneLineResults = false) : base(logger) { - try + _oneLineResults = oneLineResults; + } + + protected override string ResultsFileName { get => string.Empty; set => throw new InvalidOperationException("This runner outputs its results to stdout."); } + + private readonly XElement _assembliesElement = new XElement("assemblies"); + private readonly bool _oneLineResults; + + public override async Task Run(IEnumerable testAssemblies) + { + var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; + var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); + var discoverySink = new TestDiscoverySink(); + var diagnosticSink = new ConsoleDiagnosticMessageSink(); + var testOptions = TestFrameworkOptions.ForExecution(configuration); + var testSink = new TestMessageSink(); + + var totalSummary = new ExecutionSummary(); + foreach (var testAsmInfo in testAssemblies) { - var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; - var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); - var discoverySink = new TestDiscoverySink(); - var diagnosticSink = new ConsoleDiagnosticMessageSink(); - var testOptions = TestFrameworkOptions.ForExecution(configuration); - var testSink = new TestMessageSink(); + string assemblyFileName = testAsmInfo.FullPath; var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})"); - var assembly = Assembly.LoadFrom(assemblyFileName); - var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); + var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(testAsmInfo.Assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); - var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); + var testCasesToRun = discoverySink.TestCases.Where(t => !_filters.IsExcluded(t)).ToList(); Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)"); var summaryTaskSource = new TaskCompletionSource(); @@ -50,53 +64,77 @@ public static async Task Run(string assemblyFileName, bool printXml, XunitF var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); + if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) { testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); }; } - testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; - testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; - testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; + testSink.Execution.TestPassedEvent += args => + { + Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); + PassedTests++; + }; + testSink.Execution.TestSkippedEvent += args => + { + Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); + SkippedTests++; + }; + testSink.Execution.TestFailedEvent += args => + { + Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); + FailedTests++; + }; + testSink.Execution.TestFinishedEvent += args => ExecutedTests++; testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); - var summary = await summaryTaskSource.Task; - Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); + totalSummary = Combine(totalSummary, await summaryTaskSource.Task); + _assembliesElement.Add(resultsXmlAssembly); + } + TotalTests = totalSummary.Total; + Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {totalSummary.Total}, Errors: 0, Failed: {totalSummary.Failed}, Skipped: {totalSummary.Skipped}, Time: {TimeSpan.FromSeconds((double)totalSummary.Time).TotalSeconds}s{Environment.NewLine}"); + } - if (printXml) - { - if (oneLineResults) - { - var resultsXml = new XElement("assemblies"); - resultsXml.Add(resultsXmlAssembly); - using var ms = new MemoryStream(); - resultsXml.Save(ms); - var bytes = ms.ToArray(); - var base64 = Convert.ToBase64String(bytes, Base64FormattingOptions.None); - Console.WriteLine($"STARTRESULTXML {bytes.Length} {base64} ENDRESULTXML"); - Console.WriteLine($"Finished writing {bytes.Length} bytes of RESULTXML"); - } - else - { - Console.WriteLine($"STARTRESULTXML"); - var resultsXml = new XElement("assemblies"); - resultsXml.Add(resultsXmlAssembly); - resultsXml.Save(Console.Out); - Console.WriteLine(); - Console.WriteLine($"ENDRESULTXML"); - } - } + private ExecutionSummary Combine(ExecutionSummary aggregateSummary, ExecutionSummary assemblySummary) + { + return new ExecutionSummary + { + Total = aggregateSummary.Total + assemblySummary.Total, + Failed = aggregateSummary.Failed + assemblySummary.Failed, + Skipped = aggregateSummary.Skipped + assemblySummary.Skipped, + Errors = aggregateSummary.Errors + assemblySummary.Errors, + Time = aggregateSummary.Time + assemblySummary.Time + }; + } + + public override string WriteResultsToFile(XmlResultJargon xmlResultJargon) + { + Debug.Assert(xmlResultJargon == XmlResultJargon.xUnit); + WriteResultsToFile(Console.Out, xmlResultJargon); + return ""; + } + + public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon) + { + if (_oneLineResults) + { - var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; - return failed ? 1 : 0; + using var ms = new MemoryStream(); + _assembliesElement.Save(ms); + var bytes = ms.ToArray(); + var base64 = Convert.ToBase64String(bytes, Base64FormattingOptions.None); + Console.WriteLine($"STARTRESULTXML {bytes.Length} {base64} ENDRESULTXML"); + Console.WriteLine($"Finished writing {bytes.Length} bytes of RESULTXML"); } - catch (Exception ex) + else { - Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}"); - return 2; + writer.WriteLine($"STARTRESULTXML"); + _assembliesElement.Save(writer); + writer.WriteLine(); + writer.WriteLine($"ENDRESULTXML"); } } } diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs index 358140a75912f..cd6987a34c707 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs @@ -6,12 +6,14 @@ using System; using System.Collections.Generic; +using System.Reflection; using System.Threading.Tasks; +using Microsoft.DotNet.XHarness.TestRunners.Common; using Xunit; namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; -public abstract class WasmApplicationEntryPoint +public abstract class WasmApplicationEntryPoint : WasmApplicationEntryPointBase { protected virtual string TestAssembly { get; set; } = ""; protected virtual IEnumerable ExcludedTraits { get; set; } = Array.Empty(); @@ -20,39 +22,37 @@ public abstract class WasmApplicationEntryPoint protected virtual IEnumerable IncludedMethods { get; set; } = Array.Empty(); protected virtual IEnumerable IncludedNamespaces { get; set; } = Array.Empty(); - public async Task Run() - { - var filters = new XunitFilters(); - - foreach (var trait in ExcludedTraits) ParseEqualSeparatedArgument(filters.ExcludedTraits, trait); - foreach (var trait in IncludedTraits) ParseEqualSeparatedArgument(filters.IncludedTraits, trait); - foreach (var ns in IncludedNamespaces) filters.IncludedNamespaces.Add(ns); - foreach (var cl in IncludedClasses) filters.IncludedClasses.Add(cl); - foreach (var me in IncludedMethods) filters.IncludedMethods.Add(me); - - var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters, true); + protected override bool IsXunit => true; - return result; - } - - private static void ParseEqualSeparatedArgument(Dictionary> targetDictionary, string argument) + protected override TestRunner GetTestRunner(LogWriter logWriter) { - var parts = argument.Split('='); - if (parts.Length != 2 || string.IsNullOrEmpty(parts[0]) || string.IsNullOrEmpty(parts[1])) + var runner = new ThreadlessXunitTestRunner(logWriter, true); + ConfigureRunnerFilters(runner, ApplicationOptions.Current); + + runner.SkipCategories(ExcludedTraits); + runner.SkipCategories(IncludedTraits, isExcluded: false); + foreach (var cls in IncludedClasses) { - throw new ArgumentException($"Invalid argument value '{argument}'.", nameof(argument)); + runner.SkipClass(cls, false); } - - var name = parts[0]; - var value = parts[1]; - List excludedTraits; - if (targetDictionary.TryGetValue(name, out excludedTraits!)) + foreach (var method in IncludedMethods) { - excludedTraits.Add(value); + runner.SkipMethod(method, false); } - else + foreach (var ns in IncludedNamespaces) { - targetDictionary[name] = new List { value }; + runner.SkipNamespace(ns, isExcluded: false); } + return runner; + } + + protected override IEnumerable GetTestAssemblies() + => new[] { new TestAssemblyInfo(Assembly.LoadFrom(TestAssembly), TestAssembly) }; + + public async Task Run() + { + await RunAsync(); + + return LastRunHadFailedTests ? 1 : 0; } } diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XUnitTestRunner.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XUnitTestRunner.cs index 2672b6c56d5be..03f31fdc42ae5 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XUnitTestRunner.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XUnitTestRunner.cs @@ -27,14 +27,13 @@ internal class XsltIdGenerator public int GenerateHash() => _seed++; } -internal class XUnitTestRunner : TestRunner +internal class XUnitTestRunner : XunitTestRunnerBase { private readonly TestMessageSink _messageSink; public int? MaxParallelThreads { get; set; } private XElement _assembliesElement; - private XUnitFiltersCollection _filters = new(); public AppDomainSupport AppDomainSupport { get; set; } = AppDomainSupport.Denied; protected override string ResultsFileName { get; set; } = "TestResults.xUnit.xml"; @@ -1126,66 +1125,4 @@ private async Task Run(Assembly assembly, string assemblyPath) } } } - - public override void SkipTests(IEnumerable tests) - { - if (tests.Any()) - { - // create a single filter per test - foreach (var t in tests) - { - if (t.StartsWith("KLASS:", StringComparison.Ordinal)) - { - var klass = t.Replace("KLASS:", ""); - _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); - } - else if (t.StartsWith("KLASS32:", StringComparison.Ordinal) && IntPtr.Size == 4) - { - var klass = t.Replace("KLASS32:", ""); - _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); - } - else if (t.StartsWith("KLASS64:", StringComparison.Ordinal) && IntPtr.Size == 8) - { - var klass = t.Replace("KLASS32:", ""); - _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); - } - else if (t.StartsWith("Platform32:", StringComparison.Ordinal) && IntPtr.Size == 4) - { - var filter = t.Replace("Platform32:", ""); - _filters.Add(XUnitFilter.CreateSingleFilter(filter, true)); - } - else - { - _filters.Add(XUnitFilter.CreateSingleFilter(t, true)); - } - } - } - } - - public override void SkipCategories(IEnumerable categories) - { - if (categories == null) - { - throw new ArgumentNullException(nameof(categories)); - } - - foreach (var c in categories) - { - var traitInfo = c.Split('='); - if (traitInfo.Length == 2) - { - _filters.Add(XUnitFilter.CreateTraitFilter(traitInfo[0], traitInfo[1], true)); - } - else - { - _filters.Add(XUnitFilter.CreateTraitFilter(c, null, true)); - } - } - } - - public override void SkipMethod(string method, bool isExcluded) - => _filters.Add(XUnitFilter.CreateSingleFilter(singleTestName: method, exclude: isExcluded)); - - public override void SkipClass(string className, bool isExcluded) - => _filters.Add(XUnitFilter.CreateClassFilter(className: className, exclude: isExcluded)); } diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XunitTestRunnerBase.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XunitTestRunnerBase.cs new file mode 100644 index 0000000000000..95e7831ec1d20 --- /dev/null +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/XunitTestRunnerBase.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.DotNet.XHarness.TestRunners.Common; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; +public abstract class XunitTestRunnerBase : TestRunner +{ + private protected XUnitFiltersCollection _filters = new(); + + protected XunitTestRunnerBase(LogWriter logger) : base(logger) + { + } + + public override void SkipTests(IEnumerable tests) + { + if (tests.Any()) + { + // create a single filter per test + foreach (var t in tests) + { + if (t.StartsWith("KLASS:", StringComparison.Ordinal)) + { + var klass = t.Replace("KLASS:", ""); + _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); + } + else if (t.StartsWith("KLASS32:", StringComparison.Ordinal) && IntPtr.Size == 4) + { + var klass = t.Replace("KLASS32:", ""); + _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); + } + else if (t.StartsWith("KLASS64:", StringComparison.Ordinal) && IntPtr.Size == 8) + { + var klass = t.Replace("KLASS32:", ""); + _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); + } + else if (t.StartsWith("Platform32:", StringComparison.Ordinal) && IntPtr.Size == 4) + { + var filter = t.Replace("Platform32:", ""); + _filters.Add(XUnitFilter.CreateSingleFilter(filter, true)); + } + else + { + _filters.Add(XUnitFilter.CreateSingleFilter(t, true)); + } + } + } + } + + public override void SkipCategories(IEnumerable categories) => SkipCategories(categories, isExcluded: true); + + public virtual void SkipCategories(IEnumerable categories, bool isExcluded) + { + if (categories == null) + { + throw new ArgumentNullException(nameof(categories)); + } + + foreach (var c in categories) + { + var traitInfo = c.Split('='); + if (traitInfo.Length == 2) + { + _filters.Add(XUnitFilter.CreateTraitFilter(traitInfo[0], traitInfo[1], isExcluded)); + } + else + { + _filters.Add(XUnitFilter.CreateTraitFilter(c, null, isExcluded)); + } + } + } + + public override void SkipMethod(string method, bool isExcluded) + => _filters.Add(XUnitFilter.CreateSingleFilter(singleTestName: method, exclude: isExcluded)); + + public override void SkipClass(string className, bool isExcluded) + => _filters.Add(XUnitFilter.CreateClassFilter(className: className, exclude: isExcluded)); + + public virtual void SkipNamespace(string namespaceName, bool isExcluded) + => _filters.Add(XUnitFilter.CreateNamespaceFilter(namespaceName, exclude: isExcluded)); +} diff --git a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/iOSApplicationEntryPoint.cs b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/iOSApplicationEntryPoint.cs index 95f52b37a6414..9d1c57120882a 100644 --- a/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/iOSApplicationEntryPoint.cs +++ b/src/Microsoft.DotNet.XHarness.TestRunners.Xunit/iOSApplicationEntryPoint.cs @@ -9,7 +9,7 @@ #nullable enable namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; -public abstract class iOSApplicationEntryPoint : ApplicationEntryPoint +public abstract class iOSApplicationEntryPoint : iOSApplicationEntryPointBase { protected override TestRunner GetTestRunner(LogWriter logWriter) { @@ -19,48 +19,4 @@ protected override TestRunner GetTestRunner(LogWriter logWriter) } protected override bool IsXunit => true; - - public override async Task RunAsync() - { - var options = ApplicationOptions.Current; - TcpTextWriter? writer; - - try - { - writer = options.UseTunnel - ? TcpTextWriter.InitializeWithTunnelConnection(options.HostPort) - : TcpTextWriter.InitializeWithDirectConnection(options.HostName, options.HostPort); - } - catch (Exception ex) - { - Console.WriteLine("Failed to initialize TCP writer. Continuing on console." + Environment.NewLine + ex); - writer = null; // null means we will fall back to console output - } - - // we generate the logs in two different ways depending if the generate xml flag was - // provided. If it was, we will write the xml file to the tcp writer if present, else - // we will write the normal console output using the LogWriter - using (writer) - { - var logger = (writer == null || options.EnableXml) ? new LogWriter(Device) : new LogWriter(Device, writer); - logger.MinimumLogLevel = MinimumLogLevel.Info; - - // if we have ignore files, ignore those tests - var runner = await InternalRunAsync(logger); - - WriteResults(runner, options, logger, writer ?? Console.Out); - - logger.Info($"Tests run: {runner.TotalTests} Passed: {runner.PassedTests} Inconclusive: {runner.InconclusiveTests} Failed: {runner.FailedTests} Ignored: {runner.FilteredTests + runner.SkippedTests}"); - - if (options.AppEndTag != null) - { - logger.Info(options.AppEndTag); - } - - if (options.TerminateAfterExecution) - { - TerminateWithSuccess(); - } - } - } }