Add a few more tests and fixed the exit status issues

Changing the Exit Status to powers of 2 to make them mergable. Easy consolidation/aggregation of exit code. In case of failure reporting, report all exit status that happened over multiple attempts, if any. Prevent extra attempt, which does nothing, by eliminating the use of `hasRemainingTestsInContext` which is not accurate.
MobileNativeFoundation · Apr 8, 2020 · c3783a8 · c3783a8
1 parent f6229f0
commit c3783a8
Show file tree

Hide file tree

Showing 9 changed files with 238 additions and 99 deletions.
diff --git a/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m b/BPSampleApp/BPSampleAppHangingTests/BPSampleAppHangingTests.m
@@ -6,18 +6,14 @@
 //  Unless required by applicable law or agreed to in writing, software
 //  distributed under the License is distributed on an "AS IS" BASIS,
 //  WITHOUT WARRANTIES OF ANY KIND, either express or implied.  See the License for the specific language governing permissions and limitations under the License.
-
 #import <XCTest/XCTest.h>
-
 @interface BPSampleAppHangingTests : XCTestCase
-
 @end
-
 @implementation BPSampleAppHangingTests
-
-long attemptFromSimulatorVersionInfo(NSString *simulatorVersionInfo) {
+-(long)attemptFromSimulatorVersionInfo:(NSString *)simulatorVersionInfo {
     // simulatorVersionInfo is something like
     // CoreSimulator 587.35 - Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861) - Runtime: iOS 12.2 (16E226) - DeviceType: iPhone 7
+    NSLog(@"Dissecting version info %@ to extra attempt number.", simulatorVersionInfo);
     NSArray<NSString *> *parts = [simulatorVersionInfo componentsSeparatedByString:@" - "];
     NSString *deviceString = parts[1];
     // Device: BP93497-2-2 (7AB3D528-5473-401A-B23E-2E2E86C73861)
@@ -28,35 +24,25 @@ long attemptFromSimulatorVersionInfo(NSString *simulatorVersionInfo) {
     NSString *attempt = parts[1];
     return [attempt longLongValue];
 }
-
-- (void)setUp {
-    [super setUp];
-    // Put setup code here. This method is called before the invocation of each test method in the class.
-
-}
-
-- (void)tearDown {
-    // Put teardown code here. This method is called after the invocation of each test method in the class.
-    [super tearDown];
-}
-
-- (void)testSimpleTest {
-    XCTAssert(YES);
-}
-
-- (void)testBasedOnExecutionPlan {
+-(void)extractPlanAndExecuteActions:(int)index {
     NSDictionary *env = [[NSProcessInfo processInfo] environment];
     NSString *simulatorVersionInfo = [env objectForKey:@"SIMULATOR_VERSION_INFO"];
-    long attempt = attemptFromSimulatorVersionInfo(simulatorVersionInfo);
+    long attempt = [self attemptFromSimulatorVersionInfo:simulatorVersionInfo];
     NSString *executionPlan = [env objectForKey:@"_BP_TEST_EXECUTION_PLAN"];
     if (!executionPlan) {
         NSLog(@"No execution plan found in attempt#%ld. Failing the test.", attempt);
         XCTAssert(NO);
         return;
     }
     NSLog(@"Received execution plan %@ on attempt#%ld for this test.", executionPlan, attempt);
-
-    NSArray *array = [executionPlan componentsSeparatedByCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
+    NSArray *setsOfPlans = [executionPlan componentsSeparatedByString:@";"];
+    if (index >= [setsOfPlans count]) {
+        NSLog(@"Not enough plans for test#%d in execution plan: '%@'.", index, executionPlan);
+        XCTAssert(YES);
+        return;
+    }
+    NSString *currentPlan = setsOfPlans[index];
+    NSArray *array = [currentPlan componentsSeparatedByCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
     if (attempt > [array count]) {
         NSLog(@"Passing on attempt#%ld, by default, as there is no action defined in the execution plan", (long)attempt);
         XCTAssert(YES);
@@ -87,4 +73,28 @@ - (void)testBasedOnExecutionPlan {
     XCTAssert(NO);
     return;
 }
+- (void)setUp {
+    [super setUp];
+    // Put setup code here. This method is called before the invocation of each test method in the class.
+}
+- (void)tearDown {
+    // Put teardown code here. This method is called after the invocation of each test method in the class.
+    [super tearDown];
+}
+- (void)testASimpleTest {
+    XCTAssert(YES);
+}
+- (void)testBasedOnExecutionPlan {
+    [self extractPlanAndExecuteActions:0];
+}
+- (void)testCaseFinal {
+    XCTAssert(YES);
+}
+- (void)testDoubleBasedOnExecutionPlan {
+    [self extractPlanAndExecuteActions:1];
+}
+- (void)testEndFinal {
+    XCTAssert(YES);
+}
+
 @end
diff --git a/bp/src/BPExitStatus.h b/bp/src/BPExitStatus.h
@@ -10,18 +10,18 @@
 #import <Foundation/Foundation.h>
 
 typedef NS_ENUM(NSInteger, BPExitStatus) {
-    BPExitStatusTestsAllPassed = 0,
-    BPExitStatusTestsFailed = 1,
-    BPExitStatusSimulatorCreationFailed = 2,
-    BPExitStatusSimulatorCrashed = 3,
-    BPExitStatusInstallAppFailed = 4,
-    BPExitStatusLaunchAppFailed = 5,
-    BPExitStatusTestTimeout = 6,
-    BPExitStatusAppCrashed = 7,
-    BPExitStatusInterrupted = 8,
-    BPExitStatusSimulatorDeleted = 9,
-    BPExitStatusUninstallAppFailed = 10,
-    BPExitStatusSimulatorReuseFailed = 11,
+    BPExitStatusTestsAllPassed          = 0,
+    BPExitStatusTestsFailed             = 1 << 0,
+    BPExitStatusSimulatorCreationFailed = 1 << 1,
+    BPExitStatusInstallAppFailed        = 1 << 2,
+    BPExitStatusInterrupted             = 1 << 3,
+    BPExitStatusSimulatorCrashed        = 1 << 4,
+    BPExitStatusLaunchAppFailed         = 1 << 5,
+    BPExitStatusTestTimeout             = 1 << 6,
+    BPExitStatusAppCrashed              = 1 << 7,
+    BPExitStatusSimulatorDeleted        = 1 << 8,
+    BPExitStatusUninstallAppFailed      = 1 << 9,
+    BPExitStatusSimulatorReuseFailed    = 1 << 10
 };
 
 @protocol BPExitStatusProtocol <NSObject>

diff --git a/bp/src/BPExitStatus.m b/bp/src/BPExitStatus.m
@@ -13,15 +13,18 @@
 
 @implementation BPExitStatusHelper
 
-// Exit status to string
-+ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus {
++ (NSString *)simpleExitStatus:(BPExitStatus)exitStatus {
     switch (exitStatus) {
         case BPExitStatusTestsAllPassed:
             return @"BPExitStatusTestsAllPassed";
         case BPExitStatusTestsFailed:
             return @"BPExitStatusTestsFailed";
         case BPExitStatusSimulatorCreationFailed:
             return @"BPExitStatusSimulatorCreationFailed";
+        case BPExitStatusInstallAppFailed:
+            return @"BPExitStatusInstallAppFailed";
+        case BPExitStatusInterrupted:
+            return @"BPExitStatusInterrupted";
         case BPExitStatusSimulatorCrashed:
             return @"BPExitStatusSimulatorCrashed";
         case BPExitStatusLaunchAppFailed:
@@ -30,17 +33,30 @@ + (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus {
             return @"BPExitStatusTestTimeout";
         case BPExitStatusAppCrashed:
             return @"BPExitStatusAppCrashed";
-        case BPExitStatusInstallAppFailed:
-            return @"BPExitStatusInstallAppFailed";
-        case BPExitStatusInterrupted:
-            return @"BPExitStatusInterrupted";
         case BPExitStatusSimulatorDeleted:
             return @"BPExitStatusSimulatorDeleted";
+        case BPExitStatusUninstallAppFailed:
+            return @"BPExitStatusUninstallAppFailed";
         case BPExitStatusSimulatorReuseFailed:
             return @"BPExitStatusSimulatorReuseFailed";
         default:
-            return @"UNKNOWN_BPEXITSTATUS";
+            return [NSString stringWithFormat:@"UNKNOWN_BPEXITSTATUS - %ld", (long)exitStatus];
+    }
+}
+
+// Exit status to string
++ (NSString *)stringFromExitStatus:(BPExitStatus)exitStatus {
+    if (exitStatus == BPExitStatusTestsAllPassed)
+        return @"BPExitStatusTestsAllPassed";
+
+    NSString *exitStatusString = @"";
+    while (exitStatus > 0) {
+        BPExitStatus prevExitStatus = exitStatus;
+        exitStatus = exitStatus & (exitStatus - 1);
+        exitStatusString = [exitStatusString stringByAppendingFormat:@"%@ ", [self simpleExitStatus:(prevExitStatus - exitStatus)]];
     }
+
+    return [exitStatusString stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
 }
 
 @end
diff --git a/bp/src/BPUtils.m b/bp/src/BPUtils.m
@@ -379,7 +379,6 @@ + (NSDictionary *)loadSimpleJsonFile:(NSString *)filePath
     NSDictionary<NSString *, NSSet *> *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config
                                                                                           andXCTestFiles:xcTestFiles];
     for(NSString *filePath in testsToRunByFilePath) {
-        NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]);
         NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath];
         double __block testBundleExecutionTime = 0.0;
         [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) {
@@ -419,7 +418,6 @@ + (double)getTotalTimeWithConfig:(BPConfiguration *)config
     NSDictionary<NSString *, NSSet *> *testsToRunByFilePath = [BPUtils getTestsToRunByFilePathWithConfig:config
                                                                                           andXCTestFiles:xcTestFiles];
     for(NSString *filePath in testsToRunByFilePath) {
-        NSLog(@"filePath=%@ Tests to run in this filePath=%@", filePath, [testsToRunByFilePath objectForKey:filePath]);
         NSSet *bundleTestsToRun = [testsToRunByFilePath objectForKey:filePath];
         double __block testBundleExecutionTime = 0.0;
         [bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) {

diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m
@@ -120,11 +120,13 @@ - (void)retry {
     // There were test failures. If our failure tolerance is 0, then we're good with that.
     if (self.failureTolerance == 0) {
         // If there is no more retries, set the final exitCode to current context's exitCode
-        self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus;
+        self.finalExitStatus |= self.context.finalExitStatus;
         [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]];
         self.exitLoop = YES;
         return;
     }
+    // Resetting the failed bit since the test is being retried
+    self.context.finalExitStatus &= ~self.context.exitStatus;
     [self.context.parser cleanup];
     // Otherwise, reduce our failure tolerance count and retry
     self.failureTolerance -= 1;
@@ -155,7 +157,7 @@ - (void)retry {
 - (void)recover {
   // If error retry reach to the max, then return
   if (self.retries == [self.config.errorRetriesCount integerValue]) {
-      self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus;
+      self.finalExitStatus |= self.context.finalExitStatus;
       [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]];
       self.exitLoop = YES;
       [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."];
@@ -183,7 +185,7 @@ - (void)recover {
 // Proceed to next test case
 - (void)proceed {
     if (self.retries == [self.config.errorRetriesCount integerValue]) {
-        self.finalExitStatus = self.context.exitStatus | self.context.finalExitStatus;
+        self.finalExitStatus |= self.context.finalExitStatus;
         [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]];
         self.exitLoop = YES;
         [BPUtils printInfo:ERROR withString:@"Too many retries have occurred. Giving up."];
@@ -195,6 +197,7 @@ - (void)proceed {
     [BPUtils printInfo:INFO withString:@"Retry count: %lu", self.retries];
     self.context.attemptNumber = self.retries + 1; // set the attempt number
     self.context.exitStatus = BPExitStatusTestsAllPassed; // reset exitStatus
+
     [BPUtils printInfo:INFO withString:@"Proceeding to next test"];
     NEXT([self beginWithContext:self.context]);
 }
@@ -578,60 +581,33 @@ - (void)deleteSimulatorOnlyTaskWithContext:(BPExecutionContext *)context {
     }
 }
 
-- (BOOL)hasRemainingTestsInContext:(BPExecutionContext *)context {
-    // Make sure we're not doing unnecessary work on the next run.
-    NSMutableSet *testsRemaining = [[NSMutableSet alloc] initWithArray:context.config.allTestCases];
-    NSSet *testsToSkip = [[NSSet alloc] initWithArray:context.config.testCasesToSkip];
-    [testsRemaining minusSet:testsToSkip];
-    return ([testsRemaining count] > 0);
-}
-
 /**
  Scenarios:
- 1. crash/time out and proceed passes -> Crash/Timeout
- 2. crash/time out and retry passes -> AllPass
+ 1. crash and proceed passes -> Crash
+ 2. time out and retry passes -> AllPass
  3. failure and retry passes -> AllPass
  4. happy all pass -> AllPassed
  5. failure and still fails -> TestFailed
  */
 - (void)finishWithContext:(BPExecutionContext *)context {
-
-    // Because BPExitStatusTestsAllPassed is 0, we must check it explicitly against
-    // the run rather than the aggregate bitmask built with finalExitStatus
-
-    if (![self hasRemainingTestsInContext:context] && (context.attemptNumber <= [context.config.errorRetriesCount integerValue])) {
-        [BPUtils printInfo:INFO withString:@"No more tests to run."];
-        [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]];
-        // TODO: Temporarily disabling the fix from PR#338 while the issue is being investigated
-        // self.finalExitStatus = context.exitStatus;
-        self.finalExitStatus = context.finalExitStatus | context.exitStatus;
-        self.exitLoop = YES;
-        return;
-    }
+    context.finalExitStatus |= context.exitStatus;
+    [BPUtils printInfo:INFO withString:@"Attempt's Exit Status: %@", [BPExitStatusHelper stringFromExitStatus:context.exitStatus]];
 
     switch (context.exitStatus) {
         // BP exit handler
         case BPExitStatusInterrupted:
             self.exitLoop = YES;
             return;
 
-        // MARK: Test suite completed
-
         // If there is no test crash/time out, we retry from scratch
         case BPExitStatusTestsFailed:
             NEXT([self retry]);
             return;
 
         case BPExitStatusTestsAllPassed:
-            // Check previous result
-            if (context.finalExitStatus != BPExitStatusTestsAllPassed) {
-                // If there is a test crashed/timed out before, retry from scratch
-                NEXT([self retry]);
-            } else {
-                // If it is a real all pass, exit
-                self.exitLoop = YES;
-                return;
-            }
+            // Time to exit
+            self.finalExitStatus |= BPExitStatusTestsAllPassed;
+            self.exitLoop = YES;
             return;
 
         // Recover from scratch if there is tooling failure.
@@ -645,21 +621,23 @@ - (void)finishWithContext:(BPExecutionContext *)context {
 
         // If it is test hanging or crashing, we set final exit code of current context and proceed.
         case BPExitStatusTestTimeout:
-            context.finalExitStatus = BPExitStatusTestTimeout;
             NEXT([self proceed]);
             return;
+
         case BPExitStatusAppCrashed:
-            context.finalExitStatus = BPExitStatusAppCrashed;
+            // Remember the app crash and report whether a retry passes or not
+            self.finalExitStatus |= BPExitStatusAppCrashed;
             NEXT([self proceed]);
             return;
+
         case BPExitStatusSimulatorDeleted:
         case BPExitStatusSimulatorReuseFailed:
-            self.finalExitStatus = context.exitStatus;
+            self.finalExitStatus |= context.finalExitStatus;
             [BPUtils printInfo:INFO withString:@"%s:%d finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:self.finalExitStatus]];
             self.exitLoop = YES;
             return;
     }
-
+    [BPUtils printInfo:ERROR withString:@"%s:%d YOU SHOULDN'T BE HERE. exitStatus = %@, finalExitStatus = %@", __FILE__, __LINE__, [BPExitStatusHelper stringFromExitStatus:context.exitStatus], [BPExitStatusHelper stringFromExitStatus:context.finalExitStatus]];
 }
 
 // MARK: Helpers

diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m
@@ -234,11 +234,6 @@ - (void)onOutputReceived:(NSString *)output {
 }
 
 - (void)stopTestsWithErrorMessage:(NSString *)message forTestName:(NSString *)testName inClass:(NSString *)testClass {
-
-    // Timeout or crash on a test means we should skip it when we rerun the tests, unless we've enabled re-running failed tests
-    if (!self.config.onlyRetryFailed) {
-        [self updateExecutedTestCaseList:testName inClass:testClass];
-    }
     if (self.appState == Running && !self.config.testing_NoAppWillRun) {
         [BPUtils printInfo:ERROR withString:@"Will kill the process with appPID: %d", self.appPID];
         NSAssert(self.appPID > 0, @"Failed to find a valid PID");

diff --git a/bp/tests/BPUtilsTests.m b/bp/tests/BPUtilsTests.m
@@ -8,6 +8,7 @@
 //  WITHOUT WARRANTIES OF ANY KIND, either express or implied.  See the License for the specific language governing permissions and limitations under the License.
 
 #import <XCTest/XCTest.h>
+#import "BPExitStatus.h"
 #import "BPUtils.h"
 #import "BPXCTestFile.h"
 #import "BPTestHelper.h"
@@ -110,4 +111,41 @@ - (void) testTrailingParanthesesInTestNames {
     XCTAssert([testCasesWithParantheses count] == 0);
 }
 
+- (void) testExitStatus {
+    BPExitStatus exitCode;
+
+    exitCode = 0;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsAllPassed"]);
+    exitCode = 1;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed"]);
+    exitCode = 2;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed"]);
+    exitCode = 4;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInstallAppFailed"]);
+    exitCode = 8;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusInterrupted"]);
+    exitCode = 16;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCrashed"]);
+    exitCode = 32;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusLaunchAppFailed"]);
+    exitCode = 64;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout"]);
+    exitCode = 128;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusAppCrashed"]);
+    exitCode = 256;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorDeleted"]);
+    exitCode = 512;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusUninstallAppFailed"]);
+    exitCode = 1024;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorReuseFailed"]);
+    exitCode = 3;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestsFailed BPExitStatusSimulatorCreationFailed"]);
+    exitCode = 192;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusTestTimeout BPExitStatusAppCrashed"]);
+    exitCode = 2048;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"UNKNOWN_BPEXITSTATUS - 2048"]);
+    exitCode = 2050;
+    XCTAssert([[BPExitStatusHelper stringFromExitStatus: exitCode] isEqualToString:@"BPExitStatusSimulatorCreationFailed UNKNOWN_BPEXITSTATUS - 2048"]);
+}
+
 @end