Improve exit codes returned by XHarness (#747)

src/Microsoft.DotNet.XHarness.Apple/ErrorKnowledgeBase.cs

@@ -6,68 +6,76 @@
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
+using Microsoft.DotNet.XHarness.Common.CLI;
 using Microsoft.DotNet.XHarness.Common.Logging;
 using Microsoft.DotNet.XHarness.iOS.Shared;
 
 namespace Microsoft.DotNet.XHarness.Apple
 {
     public class ErrorKnowledgeBase : IErrorKnowledgeBase
     {
-        private static readonly Dictionary<string, (string HumanMessage, string? IssueLink)> s_testErrorMaps = new()
+        private static readonly Dictionary<string, KnownIssue> s_testErrorMaps = new()
         {
-            ["Failed to communicate with the device"] = // Known issue but not a failure.
-                ("Failed to communicate with the device. Please ensure the cable is properly connected, and try rebooting the device", null),
+            ["Failed to communicate with the device"] =
+                new("Failed to communicate with the device. Please ensure the cable is properly connected, and try rebooting the device",
+                    suggestedExitCode: (int)ExitCode.DEVICE_FAILURE),
 
             ["MT1031"] =
-                ("Cannot launch the application because the device is locked. Please unlock the device and try again", null),
+                new("Cannot launch the application because the device is locked. Please unlock the device and try again",
+                    suggestedExitCode: (int)ExitCode.DEVICE_FAILURE),
 
             ["the device is locked"] =
-                ("Cannot launch the application because the device is locked. Please unlock the device and try again", null),
+                new("Cannot launch the application because the device is locked. Please unlock the device and try again",
+                    suggestedExitCode: (int)ExitCode.DEVICE_FAILURE),
 
             ["while Setup Assistant is running"] =
-                ("Cannot launch the application because the device's update hasn't been finished. The setup assistant is still running. Please finish the device OS update on the device", null),
+                new("Cannot launch the application because the device's update hasn't been finished. The setup assistant is still running. Please finish the device OS update on the device",
+                    suggestedExitCode: (int)ExitCode.DEVICE_FAILURE),
 
             ["LSOpenURLsWithRole() failed with error -10825"] =
-                ("This application requires a newer version of MacOS", null),
+                new("This application requires a newer version of MacOS",
+                    suggestedExitCode: (int)ExitCode.GENERAL_FAILURE),
+
+            ["HE0018: Could not launch the simulator application"] =
+                new("Failed to launch the Simulator application. Please reboot the computer and try again",
+                    suggestedExitCode: (int)ExitCode.SIMULATOR_FAILURE),
         };
 
-        private static readonly Dictionary<string, (string HumanMessage, string? IssueLink)> s_buildErrorMaps = new();
+        private static readonly Dictionary<string, KnownIssue> s_buildErrorMaps = new();
 
-        private static readonly Dictionary<string, (string HumanMessage, string? IssueLink)> s_installErrorMaps = new()
+        private static readonly Dictionary<string, KnownIssue> s_installErrorMaps = new()
         {
             ["IncorrectArchitecture"] =
-                ("IncorrectArchitecture: Failed to find matching device arch for the application", null), // known failure, but not an issue
+                new("IncorrectArchitecture: Failed to find matching device arch for the application"), // known failure, but not an issue
 
             ["0xe8008015"] =
-                ("No valid provisioning profile found", null),
+                new("No valid provisioning profile found", suggestedExitCode: (int)ExitCode.APP_NOT_SIGNED),
 
             ["valid provisioning profile for this executable was not found"] =
-                ("No valid provisioning profile found", null),
+                new("No valid provisioning profile found", suggestedExitCode: (int)ExitCode.APP_NOT_SIGNED),
 
             ["0xe800801c"] =
-                ("App is not signed", null),
+                new("App is not signed", suggestedExitCode: (int)ExitCode.APP_NOT_SIGNED),
 
             ["No code signature found"] =
-                ("App is not signed", null),
+                new("App is not signed", suggestedExitCode: (int)ExitCode.APP_NOT_SIGNED),
+
+            ["HE0018: Could not launch the simulator application"] =
+                new("Failed to launch the Simulator application",
+                    suggestedExitCode: (int)ExitCode.SIMULATOR_FAILURE),
         };
 
-        public bool IsKnownBuildIssue(IFileBackedLog buildLog,
-                                      [NotNullWhen(true)]
-                                      out (string HumanMessage, string? IssueLink)? knownFailureMessage)
+        public bool IsKnownBuildIssue(IFileBackedLog buildLog, [NotNullWhen(true)] out KnownIssue? knownFailureMessage)
             => TryFindErrors(buildLog, s_buildErrorMaps, out knownFailureMessage);
 
-        public bool IsKnownTestIssue(IFileBackedLog runLog,
-                                    [NotNullWhen(true)]
-                                    out (string HumanMessage, string? IssueLink)? knownFailureMessage)
+        public bool IsKnownTestIssue(IFileBackedLog runLog, [NotNullWhen(true)] out KnownIssue? knownFailureMessage)
             => TryFindErrors(runLog, s_testErrorMaps, out knownFailureMessage);
 
-        public bool IsKnownInstallIssue(IFileBackedLog installLog,
-                                        [NotNullWhen(true)]
-                                        out (string HumanMessage, string? IssueLink)? knownFailureMessage)
+        public bool IsKnownInstallIssue(IFileBackedLog installLog, [NotNullWhen(true)] out KnownIssue? knownFailureMessage)
             => TryFindErrors(installLog, s_installErrorMaps, out knownFailureMessage);
 
-        private static bool TryFindErrors(IFileBackedLog log, Dictionary<string, (string HumanMessage, string? IssueLink)> errorMap,
-            [NotNullWhen(true)] out (string HumanMessage, string? IssueLink)? failureMessage)
+        private static bool TryFindErrors(IFileBackedLog log, Dictionary<string, KnownIssue> errorMap,
+            [NotNullWhen(true)] out KnownIssue? failureMessage)
         {
             failureMessage = null;
             if (log == null)

src/Microsoft.DotNet.XHarness.Apple/Orchestration/BaseOrchestrator.cs

@@ -108,13 +108,19 @@ protected async Task<ExitCode> OrchestrateRun(
                 catch (Exception e)
                 {
                     var message = new StringBuilder().AppendLine("Application run failed:");
+                    exitCode = ExitCode.APP_LAUNCH_FAILURE;
 
-                    if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failureMessage))
+                    if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure))
                     {
-                        message.Append(failureMessage.Value.HumanMessage);
-                        if (failureMessage.Value.IssueLink != null)
+                        message.Append(failure.HumanMessage);
+                        if (failure.IssueLink != null)
                         {
-                            message.AppendLine($" Find more information at {failureMessage.Value.IssueLink}");
+                            message.AppendLine($" Find more information at {failure.IssueLink}");
+                        }
+
+                        if (failure.SuggestedExitCode.HasValue)
+                        {
+                            exitCode = (ExitCode)failure.SuggestedExitCode.Value;
                         }
                     }
                     else
@@ -124,7 +130,7 @@ protected async Task<ExitCode> OrchestrateRun(
 
                     _logger.LogError(message.ToString());
 
-                    return ExitCode.APP_LAUNCH_FAILURE;
+                    return exitCode;
                 }
             }
 
@@ -195,14 +201,21 @@ protected async Task<ExitCode> OrchestrateRun(
             }
             catch (Exception e)
             {
+                exitCode = ExitCode.APP_LAUNCH_FAILURE;
+
                 var message = new StringBuilder().AppendLine("Application run failed:");
 
-                if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failureMessage))
+                if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure))
                 {
-                    message.Append(failureMessage.Value.HumanMessage);
-                    if (failureMessage.Value.IssueLink != null)
+                    message.Append(failure.HumanMessage);
+                    if (failure.IssueLink != null)
                     {
-                        message.AppendLine($" Find more information at {failureMessage.Value.IssueLink}");
+                        message.AppendLine($" Find more information at {failure.IssueLink}");
+                    }
+
+                    if (failure.SuggestedExitCode.HasValue)
+                    {
+                        exitCode = (ExitCode)failure.SuggestedExitCode.Value;
                     }
                 }
                 else
@@ -211,8 +224,6 @@ protected async Task<ExitCode> OrchestrateRun(
                 }
 
                 _logger.LogError(message.ToString());
-
-                exitCode = ExitCode.APP_LAUNCH_FAILURE;
             }
             finally
             {
@@ -272,19 +283,26 @@ protected virtual async Task<ExitCode> InstallApp(
                     return ExitCode.PACKAGE_INSTALLATION_TIMEOUT;
                 }
 
+                var exitCode = ExitCode.PACKAGE_INSTALLATION_FAILURE;
+
                 // use the knowledge base class to decide if the error is known, if it is, let the user know
                 // the failure reason
-                if (_errorKnowledgeBase.IsKnownInstallIssue(_mainLog, out var errorMessage))
+                if (_errorKnowledgeBase.IsKnownInstallIssue(_mainLog, out var failure))
                 {
                     var error = new StringBuilder()
                         .AppendLine("Failed to install the application")
-                        .AppendLine(errorMessage.Value.HumanMessage);
+                        .AppendLine(failure.HumanMessage);
 
-                    if (errorMessage.Value.IssueLink != null)
+                    if (failure.IssueLink != null)
                     {
                         error
                             .AppendLine()
-                            .AppendLine($" Find more information at {errorMessage.Value.IssueLink}");
+                            .AppendLine($" Find more information at {failure.IssueLink}");
+                    }
+
+                    if (failure.SuggestedExitCode.HasValue)
+                    {
+                        exitCode = (ExitCode)failure.SuggestedExitCode.Value;
                     }
 
                     _logger.LogError(error.ToString());
@@ -294,7 +312,7 @@ protected virtual async Task<ExitCode> InstallApp(
                     _logger.LogError($"Failed to install the application");
                 }
 
-                return ExitCode.PACKAGE_INSTALLATION_FAILURE;
+                return exitCode;
             }
 
             _logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on '{device.Name}'");

src/Microsoft.DotNet.XHarness.Apple/Orchestration/InstallOrchestrator.cs

@@ -66,7 +66,17 @@ public async Task<ExitCode> OrchestrateInstall(
             CancellationToken cancellationToken)
         {
             _consoleLogger.LogInformation($"Getting app bundle information from '{appPackagePath}'");
-            var appBundleInfo = await _appBundleInformationParser.ParseFromAppBundle(appPackagePath, target.Platform, _mainLog, cancellationToken);
+
+            AppBundleInformation appBundleInfo;
+            try
+            {
+                appBundleInfo = await _appBundleInformationParser.ParseFromAppBundle(appPackagePath, target.Platform, _mainLog, cancellationToken);
+            }
+            catch (Exception e)
+            {
+                _consoleLogger.LogError(e.Message);
+                return ExitCode.FAILED_TO_GET_BUNDLE_INFO;
+            }                
 
             Func<AppBundleInformation, Task<ExitCode>> executeMacCatalystApp = (appBundleInfo)
                 => throw new InvalidOperationException("install command not available on maccatalyst");

src/Microsoft.DotNet.XHarness.Apple/Orchestration/RunOrchestrator.cs

@@ -256,16 +256,24 @@ private ExitCode ParseResult(
             if (expectedExitCode != exitCode)
             {
                 _logger.LogError($"Application has finished with exit code {exitCode} but {expectedExitCode} was expected");
+                var cliExitCode = ExitCode.GENERAL_FAILURE;
 
                 foreach (var log in _logs)
                 {
-                    if (_errorKnowledgeBase.IsKnownTestIssue(log, out var failureMessage))
+                    if (_errorKnowledgeBase.IsKnownTestIssue(log, out var failure))
                     {
-                        _logger.LogError(failureMessage.Value.HumanMessage);
+                        _logger.LogError(failure.HumanMessage);
+
+                        if (failure.SuggestedExitCode.HasValue)
+                        {
+                            cliExitCode = (ExitCode)failure.SuggestedExitCode.Value;
+                        }
+
+                        break;
                     }
                 }
 
-                return ExitCode.GENERAL_FAILURE;
+                return cliExitCode;
             }
 
             _logger.LogInformation("Application has finished with exit code: " + exitCode +

src/Microsoft.DotNet.XHarness.Apple/Orchestration/TestOrchestrator.cs

@@ -274,14 +274,14 @@ private ExitCode ParseResult(TestExecutingResult testResult, string resultMessag
             string newLine = Environment.NewLine;
             const string checkLogsMessage = "Check logs for more information";
 
-            void LogProblem(string message)
+            ExitCode LogProblem(string message, ExitCode defaultExitCode)
             {
                 foreach (var log in _logs)
                 {
                     if (_errorKnowledgeBase.IsKnownTestIssue(log, out var issue))
                     {
-                        _logger.LogError(message + newLine + issue.Value.HumanMessage);
-                        return;
+                        _logger.LogError(message + newLine + issue.HumanMessage);
+                        return issue.SuggestedExitCode.HasValue ? (ExitCode)issue.SuggestedExitCode.Value : defaultExitCode;
                     }
                 }
 
@@ -293,6 +293,8 @@ void LogProblem(string message)
                 {
                     _logger.LogError(message + newLine + checkLogsMessage);
                 }
+
+                return defaultExitCode;
             }
 
             switch (testResult)
@@ -308,12 +310,10 @@ void LogProblem(string message)
                     return ExitCode.TESTS_FAILED;
 
                 case TestExecutingResult.LaunchFailure:
-                    LogProblem("Failed to launch the application");
-                    return ExitCode.APP_LAUNCH_FAILURE;
+                    return LogProblem("Failed to launch the application", ExitCode.APP_LAUNCH_FAILURE);
 
                 case TestExecutingResult.Crashed:
-                    LogProblem("Application test run crashed");
-                    return ExitCode.APP_CRASH;
+                    return LogProblem("Application test run crashed", ExitCode.APP_LAUNCH_FAILURE);
 
                 case TestExecutingResult.TimedOut:
                     _logger.LogWarning($"Application test run timed out");

src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/AppleRunCommand.cs

@@ -38,11 +38,21 @@ protected override async Task<ExitCode> InvokeInternal(ServiceProvider servicePr
 
             var mainLog = serviceProvider.GetRequiredService<IFileBackedLog>();
             var appBundleInformationParser = serviceProvider.GetRequiredService<IAppBundleInformationParser>();
-            var appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(
-                Arguments.AppBundlePath.Value ?? throw new ArgumentException("App bundle path not provided"),
-                Arguments.Target.Value.Platform,
-                mainLog,
-                cancellationToken);
+
+            AppBundleInformation appBundleInfo;
+            try
+            {
+                appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(
+                    Arguments.AppBundlePath.Value ?? throw new ArgumentException("App bundle path not provided"),
+                    Arguments.Target.Value.Platform,
+                    mainLog,
+                    cancellationToken);
+            }
+            catch (Exception e)
+            {
+                logger.LogError(e.Message);
+                return ExitCode.FAILED_TO_GET_BUNDLE_INFO;
+            }
 
             var orchestrator = serviceProvider.GetRequiredService<IRunOrchestrator>();
             return await orchestrator.OrchestrateRun(

src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/AppleTestCommand.cs

@@ -38,7 +38,21 @@ protected override async Task<ExitCode> InvokeInternal(ServiceProvider servicePr
 
             var mainLog = serviceProvider.GetRequiredService<IFileBackedLog>();
             var appBundleInformationParser = serviceProvider.GetRequiredService<IAppBundleInformationParser>();
-            var appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(Arguments.AppBundlePath.Value ?? throw new ArgumentException("App bundle path not set"), Arguments.Target.Value.Platform, mainLog, cancellationToken);
+
+            AppBundleInformation appBundleInfo;
+            try
+            {
+                appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(
+                    Arguments.AppBundlePath.Value ?? throw new ArgumentException("App bundle path not provided"),
+                    Arguments.Target.Value.Platform,
+                    mainLog,
+                    cancellationToken);
+            }
+            catch (Exception e)
+            {
+                logger.LogError(e.Message);
+                return ExitCode.FAILED_TO_GET_BUNDLE_INFO;
+            }
 
             var orchestrator = serviceProvider.GetRequiredService<ITestOrchestrator>();