Skip to content

Commit

Permalink
Ordering test bundles by test execution times read from an input json…
Browse files Browse the repository at this point in the history 
… file

Introducing an configuration to provide a json file with test case level execution times and subsequently use the execution times to order the test bundles from longest to shortest, for better concurrency.
Also added new tests for the new ordering logic and fixed some existing ones.

Ref: MobileNativeFoundation#336
  • Loading branch information
@ravimandala
ravimandala committed Aug 29, 2019
1 parent 1b9829b commit e6d7798
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 40 deletions.
17 changes: 15 additions & 2 deletions Bluepill-runner/Bluepill-runner/BPPacker.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ @implementation BPPacker
// TODO: Assign a sensible default if the estimate is not given
if ([testTimes objectForKey:test]) {
testBundleExecutionTime += [[testTimes objectForKey:test] doubleValue];
} else {
[BPUtils printInfo:INFO withString:@"Estimate not available for %@", test];
}
}];
estimatedTimesByTestFilePath[xctFile.testBundlePath] = [NSNumber numberWithDouble:testBundleExecutionTime];
Expand All @@ -148,12 +150,23 @@ @implementation BPPacker
[bundle setEstimatedExecutionTime:estimatedTimesByTestFilePath[xctFile.testBundlePath]];
[bundles addObject:bundle];
}
// Sort bundles by execution times
// Sort bundles by execution times from longest to shortest
NSMutableArray *sortedBundles = [NSMutableArray arrayWithArray:[bundles sortedArrayUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
NSNumber *estimatedTime1 = [(BPXCTestFile *)obj1 estimatedExecutionTime];
NSNumber *estimatedTime2 = [(BPXCTestFile *)obj2 estimatedExecutionTime];
return [estimatedTime2 doubleValue] - [estimatedTime1 doubleValue];
if ([estimatedTime1 doubleValue] < [estimatedTime2 doubleValue]) {
return NSOrderedDescending;
} else if([estimatedTime1 doubleValue] > [estimatedTime2 doubleValue]) {
return NSOrderedAscending;
} else {
return NSOrderedSame;
}
}]];
NSNumberFormatter *fmt = [[NSNumberFormatter alloc] init];
[fmt setPositiveFormat:@".02"];
for (BPXCTestFile *bundle in sortedBundles) {
[BPUtils printInfo:INFO withString:@"Estimated time is %@ for %@", [fmt stringFromNumber:bundle.estimatedExecutionTime], bundle.testBundlePath];
}
return sortedBundles;
}

Expand Down
81 changes: 45 additions & 36 deletions Bluepill-runner/BluepillRunnerTests/BPPackerTests.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,11 @@ - (void)testSortByTimeEstimates {
self.config.numSims = @4;
NSArray<BPXCTestFile *> *bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];
XCTAssert([bundles count] == [app.testBundles count]);
XCTAssert([[[bundles objectAtIndex:0] name] isEqualToString:@"BPSampleAppTests"]);
XCTAssert([[[bundles objectAtIndex:1] name] isEqualToString:@"BPSampleAppHangingTests"]);
XCTAssert([[[bundles objectAtIndex:2] name] isEqualToString:@"BPSampleAppCrashingTests"]);
XCTAssert([[[bundles objectAtIndex:3] name] isEqualToString:@"BPAppNegativeTests"]);
XCTAssert([[[bundles objectAtIndex:4] name] isEqualToString:@"BPSampleAppFatalErrorTests"]);
for (int i=0; i < bundles.count - 1; i++) {
double estimate1 = [[[bundles objectAtIndex:i] estimatedExecutionTime] doubleValue];
double estimate2 = [[[bundles objectAtIndex:(i+1)] estimatedExecutionTime] doubleValue];
XCTAssert(estimate1 >= estimate2);
}
}

- (void)testPacking {
Expand Down Expand Up @@ -180,14 +180,21 @@ - (void)testPacking {
// 4 unbreakable bundles (too few tests) and the big one broken into 4 bundles
XCTAssertEqual(bundles.count, 8);
// All we want to test is that we have full coverage
XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[3].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[4].skipTestIdentifiers count], 152);
XCTAssertEqual([bundles[5].skipTestIdentifiers count], 152);
XCTAssertEqual([bundles[6].skipTestIdentifiers count], 152);
XCTAssertEqual([bundles[7].skipTestIdentifiers count], 165);
long numSims = [self.config.numSims integerValue];
long testsPerBundle = [allTests count] / numSims;
long skipTestsPerBundle = 0;
long skipTestsInFinalBundle = 0;
for (int i = 0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
if (i < 4) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
} else if (i < bundles.count-1) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
} else { /* last bundle */
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
}
}

self.config.numSims = @1;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];
Expand All @@ -197,38 +204,40 @@ - (void)testPacking {
self.config.numSims = @16;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];

XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[3].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[4].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[5].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[6].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[7].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[8].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[9].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[10].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[11].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[12].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[13].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[14].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[15].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[16].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[17].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[18].skipTestIdentifiers count], 194);
XCTAssertEqual([bundles[19].skipTestIdentifiers count], 195);
numSims = [self.config.numSims integerValue];
testsPerBundle = [allTests count] / numSims;
for (int i = 0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
if (i < 4) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
} else if (i < bundles.count-1) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
} else { /* last bundle */
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
}
}

NSMutableArray *toRun = [[NSMutableArray alloc] init];
for (long i = 1; i <= 20; i++) {
[toRun addObject:[NSString stringWithFormat:@"BPSampleAppTests/testCase%03ld", i]];
}

self.config.numSims = @4;
self.config.testCasesToRun = toRun;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];

XCTAssertEqual(bundles.count, 4);
for (BPXCTestFile *bundle in bundles) {
XCTAssertEqual(bundle.skipTestIdentifiers.count, 202);
numSims = [self.config.numSims integerValue];
XCTAssertEqual(bundles.count, numSims);
testsPerBundle = [self.config.testCasesToRun count] / numSims;
for (int i=0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([self.config.testCasesToRun count] - (testsPerBundle * (numSims - 1)));
if (i < bundles.count - 1) {
XCTAssertEqual(bundles[i].skipTestIdentifiers.count, skipTestsPerBundle);
} else {
XCTAssertEqual(bundles[i].skipTestIdentifiers.count, skipTestsInFinalBundle);
}
}
}

Expand Down
5 changes: 3 additions & 2 deletions Source/Shared/BPConfiguration.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) {
"Number of simulators to run in parallel. (bluepill only)"},
{'o', "output-dir", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "outputDirectory",
"Directory where to put output log files (bluepill only)."},
{'j', "test-time-estimates-json", BP_MASTER | BP_SLAVE, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testTimeEstimatesJsonFile",
{'j', "test-time-estimates-json", BP_MASTER, NO, NO, required_argument, NULL, BP_VALUE | BP_PATH, "testTimeEstimatesJsonFile",
"Directory where Bluepill looks for input files with test execution time estimates (bluepill only)."},
{'r', "runtime", BP_MASTER | BP_SLAVE, NO, NO, required_argument, BP_DEFAULT_RUNTIME, BP_VALUE, "runtime",
"What runtime to use."},
Expand Down Expand Up @@ -530,6 +530,8 @@ - (BOOL)processOptionsWithError:(NSError **)errPtr {
if (printConfig) {
[self printConfig];
exit(0);
} else {
[self printConfig];
}
return TRUE;
}
Expand Down Expand Up @@ -593,7 +595,6 @@ - (BOOL)parseXcSchemeFile:(NSString *)schemePath withError:(NSError *__autorelea
if ([[[node attributeForName:@"isEnabled"] stringValue] boolValue]) {
environmentVariables[key] = value;
}

}
}
self.commandLineArguments = commandLineArgs;
Expand Down

0 comments on commit e6d7798

Please sign in to comment.