diff --git a/bp/src/BPExitStatus.h b/bp/src/BPExitStatus.h index 47573461..1c63814f 100644 --- a/bp/src/BPExitStatus.h +++ b/bp/src/BPExitStatus.h @@ -10,18 +10,18 @@ #import typedef NS_ENUM(NSInteger, BPExitStatus) { - BPExitStatusTestsAllPassed = 0, - BPExitStatusTestsFailed = 1, - BPExitStatusSimulatorCreationFailed = 2, - BPExitStatusSimulatorCrashed = 3, - BPExitStatusInstallAppFailed = 4, - BPExitStatusLaunchAppFailed = 5, - BPExitStatusTestTimeout = 6, - BPExitStatusAppCrashed = 7, - BPExitStatusInterrupted = 8, - BPExitStatusSimulatorDeleted = 9, - BPExitStatusUninstallAppFailed = 10, - BPExitStatusSimulatorReuseFailed = 11, + BPExitStatusTestsAllPassed = 0, + BPExitStatusTestsFailed = 1 << 0, + BPExitStatusSimulatorCreationFailed = 1 << 1, + BPExitStatusInstallAppFailed = 1 << 2, + BPExitStatusInterrupted = 1 << 3, + BPExitStatusSimulatorCrashed = 1 << 4, + BPExitStatusLaunchAppFailed = 1 << 5, + BPExitStatusTestTimeout = 1 << 6, + BPExitStatusAppCrashed = 1 << 7, + BPExitStatusSimulatorDeleted = 1 << 8, + BPExitStatusUninstallAppFailed = 1 << 9, + BPExitStatusSimulatorReuseFailed = 1 << 10 }; @protocol BPExitStatusProtocol diff --git a/bp/src/BPExitStatus.m b/bp/src/BPExitStatus.m index 1519254f..ec977bef 100644 --- a/bp/src/BPExitStatus.m +++ b/bp/src/BPExitStatus.m @@ -13,8 +13,7 @@ @implementation BPExitStatusHelper -// Exit status to string -+ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { ++ (NSString *)simpleExitStatus:(BPExitStatus)exitStatus { switch (exitStatus) { case BPExitStatusTestsAllPassed: return @"BPExitStatusTestsAllPassed"; @@ -22,6 +21,10 @@ + (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { return @"BPExitStatusTestsFailed"; case BPExitStatusSimulatorCreationFailed: return @"BPExitStatusSimulatorCreationFailed"; + case BPExitStatusInstallAppFailed: + return @"BPExitStatusInstallAppFailed"; + case BPExitStatusInterrupted: + return @"BPExitStatusInterrupted"; case BPExitStatusSimulatorCrashed: return @"BPExitStatusSimulatorCrashed"; case BPExitStatusLaunchAppFailed: @@ -30,17 +33,30 @@ + (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { return @"BPExitStatusTestTimeout"; case BPExitStatusAppCrashed: return @"BPExitStatusAppCrashed"; - case BPExitStatusInstallAppFailed: - return @"BPExitStatusInstallAppFailed"; - case BPExitStatusInterrupted: - return @"BPExitStatusInterrupted"; case BPExitStatusSimulatorDeleted: return @"BPExitStatusSimulatorDeleted"; + case BPExitStatusUninstallAppFailed: + return @"BPExitStatusUninstallAppFailed"; case BPExitStatusSimulatorReuseFailed: return @"BPExitStatusSimulatorReuseFailed"; default: - return @"UNKNOWN_BPEXITSTATUS"; + return [NSString stringWithFormat:@"UNKNOWN_BPEXITSTATUS - %ld", (long)exitStatus]; + } +} + +// Exit status to string ++ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus { + if (exitStatus == BPExitStatusTestsAllPassed) + return @"BPExitStatusTestsAllPassed"; + + NSString *exitStatusString = @""; + while (exitStatus > 0) { + BPExitStatus prevExitStatus = exitStatus; + exitStatus = exitStatus & (exitStatus - 1); + exitStatusString = [exitStatusString stringByAppendingFormat:@"%@ ", [self simpleExitStatus:(prevExitStatus - exitStatus)]]; } + + return [exitStatusString stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]]; } @end diff --git a/bp/src/BPUtils.m b/bp/src/BPUtils.m index c6e36e56..10fd8c47 100644 --- a/bp/src/BPUtils.m +++ b/bp/src/BPUtils.m @@ -379,7 +379,6 @@ + (NSDictionary *)loadSimpleJsonFile:(NSString *)filePath NSDictionary *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config andXCTestFiles:xcTestFiles]; for(NSString *filePath in testsToRunByFilePath) { - NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]); NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath]; double __block testBundleExecutionTime = 0.0; [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) { @@ -419,7 +418,6 @@ + (double)getTotalTimeWithConfig:(BPConfiguration *)config NSDictionary *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config andXCTestFiles:xcTestFiles]; for(NSString *filePath in testsToRunByFilePath) { - NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]); NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath]; double __block testBundleExecutionTime = 0.0; [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) { diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m index ac88364f..d31f8bc0 100644 --- a/bp/src/Bluepill.m +++ b/bp/src/Bluepill.m @@ -120,11 +120,13 @@ - (void)retry { // There were test failures. If our failure tolerance is 0, then we're good with that. if (self.failureTolerance == 0) { // If there is no more retries, set the final exitCode to current context's exitCode - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + self.finalExitStatus |= self.context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; return; } + // Resetting the failed bit since the test is being retried + self.context.finalExitStatus &= ~self.context.exitStatus; [self.context.parser cleanup]; // Otherwise, reduce our failure tolerance count and retry self.failureTolerance -= 1; @@ -155,7 +157,7 @@ - (void)retry { - (void)recover { // If error retry reach to the max, then return if (self.retries == [self.config.errorRetriesCount integerValue]) { - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + self.finalExitStatus |= self.context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; @@ -183,7 +185,7 @@ - (void)recover { // Proceed to next test case - (void)proceed { if (self.retries == [self.config.errorRetriesCount integerValue]) { - self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus; + self.finalExitStatus |= self.context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."]; @@ -195,6 +197,7 @@ - (void)proceed { [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries]; self.context.attemptNumber = self.retries + 1; // set the attempt number self.context.exitStatus = BPExitStatusTestsAllPassed; // reset exitStatus + [BPUtils printInfo:INFO withString:@"Proceeding to next test"]; NEXT([self beginWithContext:self.context]); } @@ -578,36 +581,17 @@ - (void)deleteSimulatorOnlyTaskWithContext:(BPExecutionContext *)context { } } -- (BOOL)hasRemainingTestsInContext:(BPExecutionContext *)context { - // Make sure we're not doing unnecessary work on the next run. - NSMutableSet *testsRemaining = [[NSMutableSet alloc] initWithArray:context.config.allTestCases]; - NSSet *testsToSkip = [[NSSet alloc] initWithArray:context.config.testCasesToSkip]; - [testsRemaining minusSet:testsToSkip]; - return ([testsRemaining count] > 0); -} - /** Scenarios: - 1. crash/time out and proceed passes -> Crash/Timeout - 2. crash/time out and retry passes -> AllPass + 1. crash and proceed passes -> Crash + 2. time out and retry passes -> AllPass 3. failure and retry passes -> AllPass 4. happy all pass -> AllPassed 5. failure and still fails -> TestFailed */ - (void)finishWithContext:(BPExecutionContext *)context { - - // Because BPExitStatusTestsAllPassed is 0, we must check it explicitly against - // the run rather than the aggregate bitmask built with finalExitStatus - - if (![self hasRemainingTestsInContext:context] && (context.attemptNumber <= [context.config.errorRetriesCount integerValue])) { - [BPUtils printInfo:INFO withString:@"No more tests to run."]; - [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; - // TODO: Temporarily disabling the fix from PR#338 while the issue is being investigated - // self.finalExitStatus = context.exitStatus; - self.finalExitStatus = context.finalExitStatus | context.exitStatus; - self.exitLoop = YES; - return; - } + context.finalExitStatus |= context.exitStatus; + [BPUtils printInfo:INFO withString:@"Attempt's Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:context.exitStatus]]; switch (context.exitStatus) { // BP exit handler @@ -615,23 +599,15 @@ - (void)finishWithContext:(BPExecutionContext *)context { self.exitLoop = YES; return; - // MARK: Test suite completed - // If there is no test crash/time out, we retry from scratch case BPExitStatusTestsFailed: NEXT([self retry]); return; case BPExitStatusTestsAllPassed: - // Check previous result - if (context.finalExitStatus != BPExitStatusTestsAllPassed) { - // If there is a test crashed/timed out before, retry from scratch - NEXT([self retry]); - } else { - // If it is a real all pass, exit - self.exitLoop = YES; - return; - } + // Time to exit + self.finalExitStatus |= BPExitStatusTestsAllPassed; + self.exitLoop = YES; return; // Recover from scratch if there is tooling failure. @@ -645,21 +621,23 @@ - (void)finishWithContext:(BPExecutionContext *)context { // If it is test hanging or crashing, we set final exit code of current context and proceed. case BPExitStatusTestTimeout: - context.finalExitStatus = BPExitStatusTestTimeout; NEXT([self proceed]); return; + case BPExitStatusAppCrashed: - context.finalExitStatus = BPExitStatusAppCrashed; + // Remember the app crash and report whether a retry passes or not + self.finalExitStatus |= BPExitStatusAppCrashed; NEXT([self proceed]); return; + case BPExitStatusSimulatorDeleted: case BPExitStatusSimulatorReuseFailed: - self.finalExitStatus = context.exitStatus; + self.finalExitStatus |= context.finalExitStatus; [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]]; self.exitLoop = YES; return; } - + [BPUtils printInfo:ERROR withString:@"%s:%d YOU SHOULDN'T BE HERE. exitStatus = %@, finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:context.exitStatus], [BPExitStatusHelper stringFromExitStatus:context.finalExitStatus]]; } // MARK: Helpers diff --git a/bp/tests/BPUtilsTests.m b/bp/tests/BPUtilsTests.m index 5a0b2165..ca3032db 100644 --- a/bp/tests/BPUtilsTests.m +++ b/bp/tests/BPUtilsTests.m @@ -8,6 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import +#import "BPExitStatus.h" #import "BPUtils.h" #import "BPXCTestFile.h" #import "BPTestHelper.h" @@ -110,4 +111,41 @@ - (void) testTrailingParanthesesInTestNames { XCTAssert([testCasesWithParantheses count] == 0); } +- (void) testExitStatus { + BPExitStatus exitCode; + + exitCode = 0; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsAllPassed"]); + exitCode = 1; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed"]); + exitCode = 2; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed"]); + exitCode = 4; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInstallAppFailed"]); + exitCode = 8; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInterrupted"]); + exitCode = 16; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCrashed"]); + exitCode = 32; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusLaunchAppFailed"]); + exitCode = 64; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout"]); + exitCode = 128; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusAppCrashed"]); + exitCode = 256; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorDeleted"]); + exitCode = 512; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusUninstallAppFailed"]); + exitCode = 1024; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorReuseFailed"]); + exitCode = 3; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed BPExitStatusSimulatorCreationFailed"]); + exitCode = 192; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout BPExitStatusAppCrashed"]); + exitCode = 2048; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"UNKNOWN_BPEXITSTATUS - 2048"]); + exitCode = 2050; + XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed UNKNOWN_BPEXITSTATUS - 2048"]); +} + @end diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillTests.m index ceec2400..40056340 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillTests.m @@ -214,7 +214,6 @@ - (void)testReportWithAppCrashingAndRetryOnlyFailedTestsSet { NSString *tempDir = NSTemporaryDirectory(); NSError *error; NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppCrashingTestsSetTempDir", tempDir] withError:&error]; - // NSLog(@"output directory is %@", outputDir); self.config.outputDirectory = outputDir; self.config.errorRetriesCount = @1; self.config.failureTolerance = @1; @@ -337,7 +336,111 @@ - (void)testReportFailureOnTimeoutCrashAndPass { self.config.outputDirectory = outputDir; BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; - XCTAssertTrue(exitCode == BPExitStatusAppCrashed); + XCTAssertTrue(exitCode == (BPExitStatusTestTimeout | BPExitStatusAppCrashed)); +} + +/** + Execution plan: CRASH, TIMEOUT, PASS + */ +- (void)testReportFailureOnCrashTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"CRASH TIMEOUT PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == (BPExitStatusAppCrashed | BPExitStatusTestTimeout)); +} + +/** + Execution plan: FAIL, TIMEOUT, PASS + */ +- (void)testReportSuccessOnFailTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @1; + self.config.testing_ExecutionPlan = @"FAIL TIMEOUT PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestsAllPassed); +} + +/** + Execution plan: FAIL, TIMEOUT, PASS + */ +- (void)testReportFailureOnFailTimeoutAndPass { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @0; + self.config.testing_ExecutionPlan = @"FAIL TIMEOUT PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestsFailed); +} + +/** + Execution plan: TIMEOUT, PASS + */ +- (void)testReportSuccessOnTimeoutAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.testing_ExecutionPlan = @"TIMEOUT PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + self.config.failureTolerance = @1; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestsAllPassed); +} + +/** + Execution plan: FAIL, PASS + */ +- (void)testReportSuccessOnTestFailedAndPassOnRetry { + self.config.stuckTimeout = @6; + self.config.failureTolerance = @1; + self.config.testing_ExecutionPlan = @"FAIL PASS"; + self.config.errorRetriesCount = @4; + self.config.onlyRetryFailed = TRUE; + NSString *testBundlePath = [BPTestHelper sampleAppHangingTestsBundlePath]; + self.config.testBundlePath = testBundlePath; + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/AppHangingTestsSetTempDir", tempDir] withError:&error]; + NSLog(@"output directory is %@", outputDir); + self.config.outputDirectory = outputDir; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + XCTAssertTrue(exitCode == BPExitStatusTestsAllPassed); } - (void)testReportWithFailingTestsSetAndDiagnostics {