Skip to content

Commit

Permalink
Ordering test bundles by test execution times read from an input json…
Browse files Browse the repository at this point in the history 
… file

Introducing an configuration to provide a json file with test case level execution times and subsequently use the execution times to order the test bundles from longest to shortest, for better concurrency.
Also added new tests for the new ordering logic and fixed some existing ones.

Ref: MobileNativeFoundation#336
  • Loading branch information
@ravimandala
ravimandala committed Aug 21, 2019
1 parent a79047f commit 0243729
Show file tree
Hide file tree
Showing 13 changed files with 466 additions and 38 deletions.
4 changes: 4 additions & 0 deletions BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
BAB24F4C1DB5D45E00867756 /* BPSampleAppTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BAB24F4B1DB5D45E00867756 /* BPSampleAppTests.m */; };
BAB24F5D1DB5D83C00867756 /* BPAppNegativeTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BAB24F5C1DB5D83C00867756 /* BPAppNegativeTests.m */; };
BAFCCA601E36DC2000E33C31 /* BPSampleAppUITests.m in Sources */ = {isa = PBXBuildFile; fileRef = BAFCCA5F1E36DC2000E33C31 /* BPSampleAppUITests.m */; };
E492360122EF61F300395D98 /* BPSampleAppMoarTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E492360022EF61F300395D98 /* BPSampleAppMoarTests.m */; };
/* End PBXBuildFile section */

/* Begin PBXContainerItemProxy section */
Expand Down Expand Up @@ -101,6 +102,7 @@
BAFCCA5D1E36DC2000E33C31 /* BPSampleAppUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BPSampleAppUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
BAFCCA5F1E36DC2000E33C31 /* BPSampleAppUITests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPSampleAppUITests.m; sourceTree = "<group>"; };
BAFCCA611E36DC2000E33C31 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
E492360022EF61F300395D98 /* BPSampleAppMoarTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPSampleAppMoarTests.m; sourceTree = "<group>"; };
/* End PBXFileReference section */

/* Begin PBXFrameworksBuildPhase section */
Expand Down Expand Up @@ -240,6 +242,7 @@
BAB24F4A1DB5D45E00867756 /* BPSampleAppTests */ = {
isa = PBXGroup;
children = (
E492360022EF61F300395D98 /* BPSampleAppMoarTests.m */,
BAB24F4B1DB5D45E00867756 /* BPSampleAppTests.m */,
BAB24F4D1DB5D45E00867756 /* Info.plist */,
32FEEBBC2012B93D005916EF /* BPSampleAppSwiftTests.swift */,
Expand Down Expand Up @@ -567,6 +570,7 @@
buildActionMask = 2147483647;
files = (
BAB24F4C1DB5D45E00867756 /* BPSampleAppTests.m in Sources */,
E492360122EF61F300395D98 /* BPSampleAppMoarTests.m in Sources */,
32FEEBBD2012B93D005916EF /* BPSampleAppSwiftTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
35 changes: 35 additions & 0 deletions BPSampleApp/BPSampleAppTests/BPSampleAppMoarTests.m
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
//
// BPSampleAppMoarTests.m
// BPSampleAppTests
//
// Created by Ravi K. Mandala on 7/29/19.
// Copyright © 2019 LinkedIn. All rights reserved.
//

#import <XCTest/XCTest.h>

@interface BPSampleAppMoarTests : XCTestCase

@end


@implementation BPSampleAppMoarTests

- (void)setUp {
[super setUp];
// Put setup code here. This method is called before the invocation of each test method in the class.

}

- (void)tearDown {
// Put teardown code here. This method is called after the invocation of each test method in the class.
[super tearDown];
}

- (void)testCase000 { XCTAssert(YES);}
- (void)testCase001 { XCTAssert(YES);}
- (void)testCase002 { XCTAssert(YES);}
- (void)testCase003 { XCTAssert(YES);}
- (void)testCase004 { XCTAssert(YES);}

@end
4 changes: 4 additions & 0 deletions bluepill/bluepill.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
C4D6861A2267ABEF007D4237 /* bplib.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C4D686192267ABEF007D4237 /* bplib.framework */; };
C4D6861B2267ABFC007D4237 /* bplib.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C4D686192267ABEF007D4237 /* bplib.framework */; };
C4FD8C581DB6E09B000ED28C /* BPPacker.m in Sources */ = {isa = PBXBuildFile; fileRef = C4FD8C571DB6E09B000ED28C /* BPPacker.m */; };
E49235FF22EA847700395D98 /* times.json in Resources */ = {isa = PBXBuildFile; fileRef = E49235FE22EA847700395D98 /* times.json */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
Expand Down Expand Up @@ -62,6 +63,7 @@
C4D686192267ABEF007D4237 /* bplib.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = bplib.framework; sourceTree = BUILT_PRODUCTS_DIR; };
C4FD8C561DB6E09B000ED28C /* BPPacker.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPPacker.h; sourceTree = "<group>"; };
C4FD8C571DB6E09B000ED28C /* BPPacker.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPPacker.m; sourceTree = "<group>"; };
E49235FE22EA847700395D98 /* times.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = times.json; sourceTree = "<group>"; };
/* End PBXFileReference section */

/* Begin PBXFrameworksBuildPhase section */
Expand Down Expand Up @@ -138,6 +140,7 @@
BA9C2DAD1DD674AD007CB967 /* Resource Files */ = {
isa = PBXGroup;
children = (
E49235FE22EA847700395D98 /* times.json */,
BA9C2DB01DD67B66007CB967 /* testScheme.xcscheme */,
BA9C2DAE1DD674AD007CB967 /* BPSampleAppTests.xctest */,
);
Expand Down Expand Up @@ -285,6 +288,7 @@
BA70318A1DE4ED2F008B3539 /* result3.xml in Resources */,
BA7031881DE4ED2F008B3539 /* result1.xml in Resources */,
BA9C2DAF1DD674AD007CB967 /* BPSampleAppTests.xctest in Resources */,
E49235FF22EA847700395D98 /* times.json in Resources */,
BA7031891DE4ED2F008B3539 /* result2.xml in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
88 changes: 86 additions & 2 deletions bluepill/src/BPPacker.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,19 @@
@implementation BPPacker

+ (NSMutableArray<BPXCTestFile *> *)packTests:(NSArray<BPXCTestFile *> *)xcTestFiles
configuration:(BPConfiguration *)config
andError:(NSError **)errPtr {
configuration:(BPConfiguration *)config
andError:(NSError **)errPtr {
if (!config.testTimeEstimatesJsonFile) {
return [self packTestsByCount:xcTestFiles configuration:config andError:errPtr];
} else {
return [self packTestsByTime:xcTestFiles configuration:config andError:errPtr];
}
}

+ (NSMutableArray<BPXCTestFile *> *)packTestsByCount:(NSArray<BPXCTestFile *> *)xcTestFiles
configuration:(BPConfiguration *)config
andError:(NSError **)errPtr {
[BPUtils printInfo:INFO withString:@"Packing test bundles based on test counts."];
NSArray *testCasesToRun = config.testCasesToRun;
NSArray *noSplit = config.noSplit;
NSUInteger numBundles = [config.numSims integerValue];
Expand Down Expand Up @@ -86,5 +96,79 @@ @implementation BPPacker
return bundles;
}

+ (NSMutableArray<BPXCTestFile *> *)packTestsByTime:(NSArray<BPXCTestFile *> *)xcTestFiles
configuration:(BPConfiguration *)config
andError:(NSError **)errPtr {
[BPUtils printInfo:INFO withString:@"Packing based on individual test execution times in file path: %@", config.testTimeEstimatesJsonFile];
if (xcTestFiles.count == 0) {
BP_SET_ERROR(errPtr, @"Found no XCTest files.\n"
"Perhaps you forgot to 'build-for-testing'? (Cmd + Shift + U) in Xcode.");
return NULL;
}

// load the config file
NSDictionary *testTimes = [self loadConfigFile:config.testTimeEstimatesJsonFile withError:errPtr];
if ((errPtr && *errPtr) || !testTimes) {
[BPUtils printInfo:ERROR withString:@"%@", [*errPtr localizedDescription]];
return NULL;
}

NSMutableDictionary *estimatedTimesByTestFilePath = [[NSMutableDictionary alloc] init];
NSArray *testCasesToRun = config.testCasesToRun;

double totalTime = 0.0;
for (BPXCTestFile *xctFile in xcTestFiles) {
NSMutableSet *bundleTestsToRun = [[NSMutableSet alloc] initWithArray:[xctFile allTestCases]];
if (testCasesToRun) {
[bundleTestsToRun intersectSet:[[NSSet alloc] initWithArray:testCasesToRun]];
}
if (config.testCasesToSkip && [config.testCasesToSkip count] > 0) {
[bundleTestsToRun minusSet:[[NSSet alloc] initWithArray:config.testCasesToSkip]];
}
if (bundleTestsToRun.count > 0) {
double __block testBundleExecutionTime = 0.0;
[bundleTestsToRun enumerateObjectsUsingBlock:^(id _Nonnull test, BOOL * _Nonnull stop) {
// TODO: Assign a sensible default if the estimate is not given
if ([testTimes objectForKey:test]) {
testBundleExecutionTime += [[testTimes objectForKey:test] doubleValue];
}
}];
estimatedTimesByTestFilePath[xctFile.testBundlePath] = [NSNumber numberWithDouble:testBundleExecutionTime];
totalTime += testBundleExecutionTime;
}
}
assert([estimatedTimesByTestFilePath count] == [xcTestFiles count]);

NSMutableArray<BPXCTestFile *> *bundles = [[NSMutableArray alloc] init];
for (BPXCTestFile *xctFile in xcTestFiles) {
BPXCTestFile *bundle = [xctFile copy];
bundle.skipTestIdentifiers = config.testCasesToSkip;
[bundle setEstimatedExecutionTime:estimatedTimesByTestFilePath[xctFile.testBundlePath]];
[bundles addObject:bundle];
}
// Sort bundles by execution times from longest to shortest
NSMutableArray *sortedBundles = [NSMutableArray arrayWithArray:[bundles sortedArrayUsingComparator:^NSComparisonResult(id _Nonnull obj1, id _Nonnull obj2) {
NSNumber *estimatedTime1 = [(BPXCTestFile *)obj1 estimatedExecutionTime];
NSNumber *estimatedTime2 = [(BPXCTestFile *)obj2 estimatedExecutionTime];
if ([estimatedTime1 doubleValue] < [estimatedTime2 doubleValue]) {
return NSOrderedDescending;
} else if([estimatedTime1 doubleValue] > [estimatedTime2 doubleValue]) {
return NSOrderedAscending;
} else {
return NSOrderedSame;
}
}]];
return sortedBundles;
}

+ (NSDictionary *)loadConfigFile:(NSString *)file withError:(NSError **)errPtr{
NSData *data = [NSData dataWithContentsOfFile:file
options:NSDataReadingMappedIfSafe
error:errPtr];
if (!data) return nil;

return [NSJSONSerialization JSONObjectWithData:data
options:kNilOptions
error:errPtr];
}
@end
4 changes: 2 additions & 2 deletions bluepill/src/BPRunner.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ - (NSTask *)newTaskWithBundle:(BPXCTestFile *)bundle
cfg.outputDirectory = [self.config.outputDirectory
stringByAppendingPathComponent:
[NSString stringWithFormat:@"BP-%lu", (unsigned long)number]];
cfg.testTimeEstimatesJsonFile = self.config.testTimeEstimatesJsonFile;
[cfg printConfig];
NSTask *task = [[NSTask alloc] init];
[task setLaunchPath:self.bpExecutable];
Expand Down Expand Up @@ -306,8 +307,7 @@ - (int)runWithBPXCTestFiles:(NSArray<BPXCTestFile *> *)xcTestFiles {
@synchronized (self) {
listString = [taskList componentsJoinedByString:@", "];
}
[BPUtils printInfo:INFO withString:@"%lu BP%s still running. [%@]",
launchedTasks, launchedTasks == 1 ? "" : "s", listString];
[BPUtils printInfo:INFO withString:@"%lu BP(s) still running. [%@]", launchedTasks, listString];
[BPUtils printInfo:INFO withString:@"Using %d of %d processes.", numprocs(), maxProcs];
if (numprocs() > maxProcs * BP_MAX_PROCESSES_PERCENT) {
[BPUtils printInfo:WARNING withString:@"!!!The number of processes is more than %f percent of maxProcs!!! it may fail with error: Unable to boot device due to insufficient system resources. Please check with system admin to restart this node and for proper mainantance routine", BP_MAX_PROCESSES_PERCENT*100];
Expand Down
117 changes: 85 additions & 32 deletions bluepill/tests/BPPackerTests.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,51 @@ - (void)testPackingProvidesBalancedBundles {
}
}

- (void)testSmartPackIfJsonFound {
self.config.testTimeEstimatesJsonFile = [BPTestHelper sampleTimesJsonPath];
self.config.testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase000"];
self.config.numSims = @8;
BPApp *app = [BPApp appWithConfig:self.config withError:nil];
XCTAssert(app != nil);
NSArray<BPXCTestFile *> *bundles;
NSError *error;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:&error];
XCTAssert(error == nil);
XCTAssert([app.testBundles count] == [bundles count]);
}

- (void)testSmartPackIfJsonMissing {
self.config.testTimeEstimatesJsonFile = @"invalid/times/file/path.json";
self.config.testBundlePath = [BPTestHelper sampleAppBalancingTestsBundlePath];
self.config.testCasesToSkip = @[@"BPSampleAppTests/testCase000"];
self.config.numSims = @8;
BPApp *app = [BPApp appWithConfig:self.config withError:nil];
XCTAssert(app != nil);
NSArray<BPXCTestFile *> *bundles;
NSError *error;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:&error];
XCTAssert(error != nil);
XCTAssert([bundles count] == 0);
}

- (void)testSortByTimeEstimates {
self.config.testTimeEstimatesJsonFile = [BPTestHelper sampleTimesJsonPath];
self.config.testBundlePath = nil;
BPApp *app = [BPApp appWithConfig:self.config withError:nil];
XCTAssert(app != nil);
XCTAssert([app.testBundles count] == 5);
// Make sure we don't split when we don't want to
self.config.numSims = @4;
NSArray<BPXCTestFile *> *bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];
XCTAssert([bundles count] == [app.testBundles count]);
for (int i=0; i < bundles.count - 1; i++) {
double estimate1 = [[[bundles objectAtIndex:i] estimatedExecutionTime] doubleValue];
double estimate2 = [[[bundles objectAtIndex:(i+1)] estimatedExecutionTime] doubleValue];
XCTAssert(estimate1 >= estimate2);
}
}

- (void)testPacking {
NSArray *want, *got;
NSArray *allTests;
Expand Down Expand Up @@ -133,14 +178,21 @@ - (void)testPacking {
// 4 unbreakable bundles (too few tests) and the big one broken into 4 bundles
XCTAssertEqual(bundles.count, 8);
// All we want to test is that we have full coverage
XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[3].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[4].skipTestIdentifiers count], 149);
XCTAssertEqual([bundles[5].skipTestIdentifiers count], 149);
XCTAssertEqual([bundles[6].skipTestIdentifiers count], 149);
XCTAssertEqual([bundles[7].skipTestIdentifiers count], 159);
long numSims = [self.config.numSims integerValue];
long testsPerBundle = [allTests count] / numSims;
long skipTestsPerBundle = 0;
long skipTestsInFinalBundle = 0;
for (int i = 0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
if (i < 4) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
} else if (i < bundles.count-1) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
} else { /* last bundle */
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
}
}

self.config.numSims = @1;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];
Expand All @@ -150,38 +202,40 @@ - (void)testPacking {
self.config.numSims = @16;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];

XCTAssertEqual([bundles[0].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[1].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[2].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[3].skipTestIdentifiers count], 0);
XCTAssertEqual([bundles[4].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[5].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[6].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[7].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[8].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[9].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[10].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[11].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[12].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[13].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[14].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[15].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[16].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[17].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[18].skipTestIdentifiers count], 189);
XCTAssertEqual([bundles[19].skipTestIdentifiers count], 195);
numSims = [self.config.numSims integerValue];
testsPerBundle = [allTests count] / numSims;
for (int i = 0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = testsPerBundle * (numSims - 1);
if (i < 4) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], 0);
} else if (i < bundles.count-1) {
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsPerBundle);
} else { /* last bundle */
XCTAssertEqual([bundles[i].skipTestIdentifiers count], skipTestsInFinalBundle);
}
}

NSMutableArray *toRun = [[NSMutableArray alloc] init];
for (long i = 1; i <= 20; i++) {
[toRun addObject:[NSString stringWithFormat:@"BPSampleAppTests/testCase%03ld", i]];
}

self.config.numSims = @4;
self.config.testCasesToRun = toRun;
bundles = [BPPacker packTests:app.testBundles configuration:self.config andError:nil];

XCTAssertEqual(bundles.count, 4);
for (BPXCTestFile *bundle in bundles) {
XCTAssertEqual(bundle.skipTestIdentifiers.count, 197);
numSims = [self.config.numSims integerValue];
XCTAssertEqual(bundles.count, numSims);
testsPerBundle = [self.config.testCasesToRun count] / numSims;
for (int i=0; i < bundles.count; ++i) {
skipTestsPerBundle = ([[bundles[i] allTestCases] count] - testsPerBundle);
skipTestsInFinalBundle = [[bundles[i] allTestCases] count] - ([self.config.testCasesToRun count] - (testsPerBundle * (numSims - 1)));
if (i < bundles.count - 1) {
XCTAssertEqual(bundles[i].skipTestIdentifiers.count, skipTestsPerBundle);
} else {
XCTAssertEqual(bundles[i].skipTestIdentifiers.count, skipTestsInFinalBundle);
}
}
}

Expand All @@ -196,7 +250,6 @@ - (void)testPackingWithTestsToSkip {
for (BPXCTestFile *bundle in bundles) {
XCTAssertTrue([bundle.skipTestIdentifiers containsObject:@"BPSampleAppTests/testCase000"], @"testCase000 should be in testToSkip for all bundles");
}

}

@end
Loading

0 comments on commit 0243729

Please sign in to comment.