Merge pull request #1537 from matrix-org/manu/working_integration_tes…

…t_plan Add integration tests to the CI
matrix-org · Aug 1, 2022 · 93670ea · 93670ea
2 parents 840771e + db9114b
commit 93670ea
Show file tree

Hide file tree

Showing 44 changed files with 297 additions and 106 deletions.
diff --git a/.github/workflows/ci-crypto-tests.yml b/.github/workflows/ci-crypto-tests.yml
@@ -36,7 +36,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
-
       # Cache for Xcode env
       - uses: actions/cache@v2
         with:
@@ -50,7 +49,6 @@ jobs:
           key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
           restore-keys: |
             ${{ runner.os }}-gems-
-
       - name: Start synapse server
         uses: michaelkaye/setup-matrix-synapse@v1.0.3
         with:
@@ -62,21 +60,27 @@ jobs:
         run: |
           bundle config path vendor/bundle
           bundle install --jobs 4 --retry 3
-
       # Main step
       - name: Crypto tests
         run: bundle exec fastlane test testplan:CryptoTests
 
       # Store artifacts
       - uses: actions/upload-artifact@v2
+        if: always()
         with:
           name: report.html
           path: build/test/report.html
       - uses: actions/upload-artifact@v2
+        if: always()
         with:
           name: report.junit
           path: build/test/report.junit
       - uses: actions/upload-artifact@v2
+        if: always()
         with:
           name: MatrixSDK-macOS.xcresult
           path: build/test/MatrixSDK-macOS.xcresult/
+
+      # Upload coverage
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
diff --git a/.github/workflows/ci-integration-tests.yml b/.github/workflows/ci-integration-tests.yml
@@ -0,0 +1,91 @@
+name: Integration Tests CI
+
+on:
+  # Triggers the workflow on any pull request and push to develop
+  push:
+    branches: [ develop ]
+  pull_request:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  integration-tests:
+    name: Integration Tests
+    runs-on: macos-11
+
+    concurrency:
+      # When running on develop, use the sha to allow all runs of this workflow to run concurrently.
+      # Otherwise only allow a single run of this workflow on each branch, automatically cancelling older runs.
+      group: ${{ github.ref == 'refs/heads/develop' && format('tests-integration-develop-{0}', github.sha) || format('tests-integration-{0}', github.ref) }}
+      cancel-in-progress: true
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # Common cache
+      # Note: GH actions do not support yaml anchor yet. We need to duplicate this for every job
+      - uses: actions/cache@v2
+        with:
+          path: Pods
+          key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-pods-
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+      # Cache for python env for Synapse
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      # Set up a Synapse server
+      - name: Start synapse server
+        uses: michaelkaye/setup-matrix-synapse@v1.0.3
+        with:
+          uploadLogs: true
+          httpPort: 8080
+          disableRateLimiting: true
+
+      # Common setup
+      # Note: GH actions do not support yaml anchor yet. We need to duplicate this for every job
+      - name: Bundle install
+        run: |
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+      # Main step
+      - name: Integration tests
+        run: bundle exec fastlane test testplan:AllWorkingTests
+
+      # Store artifacts
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: report.html
+          path: build/test/report.html
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: report.junit
+          path: build/test/report.junit
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: MatrixSDK-macOS.xcresult
+          path: build/test/MatrixSDK-macOS.xcresult/
+
+      # Upload coverage
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,8 @@ DerivedData
 *.ipa
 *.xcuserstate
 *.DS_Store
+.vscode/
+vendor/
 
 # CocoaPods
 #

diff --git a/MatrixSDK.xcodeproj/project.pbxproj b/MatrixSDK.xcodeproj/project.pbxproj
@@ -315,6 +315,8 @@
 		32618E7220ED2DF500E1D2EA /* MXFilterJSONModel.m in Sources */ = {isa = PBXBuildFile; fileRef = 32618E7020ED2DF500E1D2EA /* MXFilterJSONModel.m */; };
 		32618E7B20EFA45B00E1D2EA /* MXRoomMembers.h in Headers */ = {isa = PBXBuildFile; fileRef = 32618E7920EFA45B00E1D2EA /* MXRoomMembers.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		32618E7C20EFA45B00E1D2EA /* MXRoomMembers.m in Sources */ = {isa = PBXBuildFile; fileRef = 32618E7A20EFA45B00E1D2EA /* MXRoomMembers.m */; };
+		326277F1288BC21A009A0508 /* AllWorkingTests.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 326277F0288BC219009A0508 /* AllWorkingTests.xctestplan */; };
+		326277F2288BC21A009A0508 /* AllWorkingTests.xctestplan in Resources */ = {isa = PBXBuildFile; fileRef = 326277F0288BC219009A0508 /* AllWorkingTests.xctestplan */; };
 		32637ED41E5B00400011E20D /* MXDeviceList.h in Headers */ = {isa = PBXBuildFile; fileRef = 32637ED21E5B00400011E20D /* MXDeviceList.h */; };
 		32637ED51E5B00400011E20D /* MXDeviceList.m in Sources */ = {isa = PBXBuildFile; fileRef = 32637ED31E5B00400011E20D /* MXDeviceList.m */; };
 		3264DB911CEC528D00B99881 /* MXAccountData.h in Headers */ = {isa = PBXBuildFile; fileRef = 3264DB8F1CEC528D00B99881 /* MXAccountData.h */; settings = {ATTRIBUTES = (Public, ); }; };
@@ -2117,6 +2119,7 @@
 		32618E7020ED2DF500E1D2EA /* MXFilterJSONModel.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MXFilterJSONModel.m; sourceTree = "<group>"; };
 		32618E7920EFA45B00E1D2EA /* MXRoomMembers.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MXRoomMembers.h; sourceTree = "<group>"; };
 		32618E7A20EFA45B00E1D2EA /* MXRoomMembers.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MXRoomMembers.m; sourceTree = "<group>"; };
+		326277F0288BC219009A0508 /* AllWorkingTests.xctestplan */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = AllWorkingTests.xctestplan; sourceTree = "<group>"; };
 		32637ED21E5B00400011E20D /* MXDeviceList.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MXDeviceList.h; sourceTree = "<group>"; };
 		32637ED31E5B00400011E20D /* MXDeviceList.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MXDeviceList.m; sourceTree = "<group>"; };
 		3264DB8F1CEC528D00B99881 /* MXAccountData.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MXAccountData.h; sourceTree = "<group>"; };
@@ -3742,6 +3745,7 @@
 		3298ABD42637FA3100E40B06 /* TestPlans */ = {
 			isa = PBXGroup;
 			children = (
+				326277F0288BC219009A0508 /* AllWorkingTests.xctestplan */,
 				3298ABD52637FA3100E40B06 /* AllTests.xctestplan */,
 				3209682F26396385005D64ED /* AllTestsWithSanitizers.xctestplan */,
 				EDC2A0E528369E740039F3D6 /* CryptoTests.xctestplan */,
@@ -5982,6 +5986,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				3298ABD62637FA3100E40B06 /* AllTests.xctestplan in Resources */,
+				326277F1288BC21A009A0508 /* AllWorkingTests.xctestplan in Resources */,
 				3209683326396385005D64ED /* UnitTestsWithSanitizers.xctestplan in Resources */,
 				3298ABDD2637FB1900E40B06 /* UnitTests.xctestplan in Resources */,
 				3209683126396385005D64ED /* AllTestsWithSanitizers.xctestplan in Resources */,
@@ -6001,6 +6006,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				3298ABD72637FA3100E40B06 /* AllTests.xctestplan in Resources */,
+				326277F2288BC21A009A0508 /* AllWorkingTests.xctestplan in Resources */,
 				3209683426396385005D64ED /* UnitTestsWithSanitizers.xctestplan in Resources */,
 				3298ABDE2637FB1900E40B06 /* UnitTests.xctestplan in Resources */,
 				3209683226396385005D64ED /* AllTestsWithSanitizers.xctestplan in Resources */,

diff --git a/MatrixSDK.xcodeproj/xcshareddata/xcschemes/MatrixSDK-macOS.xcscheme b/MatrixSDK.xcodeproj/xcshareddata/xcschemes/MatrixSDK-macOS.xcscheme
@@ -67,6 +67,9 @@
          <TestPlanReference
             reference = "container:MatrixSDKTests/TestPlans/CryptoTests.xctestplan">
          </TestPlanReference>
+         <TestPlanReference
+            reference = "container:MatrixSDKTests/TestPlans/AllWorkingTests.xctestplan">
+         </TestPlanReference>
       </TestPlans>
       <Testables>
          <TestableReference

diff --git a/MatrixSDK/Data/Store/MXMemoryStore/MXMemoryStore.m b/MatrixSDK/Data/Store/MXMemoryStore/MXMemoryStore.m
@@ -195,6 +195,11 @@ - (NSAttributedString *)partialAttributedTextMessageOfRoom:(NSString *)roomId
     return roomStore.partialAttributedTextMessage;
 }
 
+- (void)stateOfRoom:(NSString *)roomId success:(void (^)(NSArray<MXEvent *> * _Nonnull))success failure:(void (^)(NSError * _Nonnull))failure
+{
+    success(@[]);
+}
+
 - (void)loadReceiptsForRoom:(NSString *)roomId completion:(void (^)(void))completion
 {
     [self getOrCreateRoomReceiptsStore:roomId];

diff --git a/MatrixSDKTests/DirectRoomTests.m b/MatrixSDKTests/DirectRoomTests.m
@@ -42,9 +42,9 @@ - (void)setUp
 
 - (void)tearDown
 {
+    matrixSDKTestsData = nil;
     [super tearDown];
 
-    matrixSDKTestsData = nil;
 }
 
 // Create the following scenario with 3 rooms

diff --git a/MatrixSDKTests/MXAccountDataTests.m b/MatrixSDKTests/MXAccountDataTests.m
@@ -42,9 +42,9 @@ - (void)setUp
 
 - (void)tearDown
 {
-    [super tearDown];
-
     matrixSDKTestsData = nil;
+
+    [super tearDown];
 }
 
 - (void)testIgnoreUser

diff --git a/MatrixSDKTests/MXAggregatedEditsTests.m b/MatrixSDKTests/MXAggregatedEditsTests.m
@@ -57,6 +57,8 @@ - (void)tearDown
 {
     matrixSDKTestsData = nil;
     matrixSDKTestsE2EData = nil;
+
+    [super tearDown];
 }
 
 // Create a room with an event with an edit it on it

diff --git a/MatrixSDKTests/MXAggregatedReactionTests.m b/MatrixSDKTests/MXAggregatedReactionTests.m
@@ -48,6 +48,8 @@ - (void)setUp
 - (void)tearDown
 {
     matrixSDKTestsData = nil;
+
+    [super tearDown];
 }
 
 // Create a room with an event with a reaction on it

diff --git a/MatrixSDKTests/MXAggregatedReferenceTests.m b/MatrixSDKTests/MXAggregatedReferenceTests.m
@@ -53,6 +53,8 @@ - (void)tearDown
 {
     matrixSDKTestsData = nil;
     matrixSDKTestsE2EData = nil;
+
+    [super tearDown];
 }
 
 

diff --git a/MatrixSDKTests/MXAsyncTaskQueueUnitTests.swift b/MatrixSDKTests/MXAsyncTaskQueueUnitTests.swift
@@ -18,14 +18,6 @@ import XCTest
 
 class MXAsyncTaskQueueUnitTests: XCTestCase {
 
-    override func setUpWithError() throws {
-        // Put setup code here. This method is called before the invocation of each test method in the class.
-    }
-
-    override func tearDownWithError() throws {
-        // Put teardown code here. This method is called after the invocation of each test method in the class.
-    }
-
     /// Check that tasks and async tasks are run
     func test() throws {
         let expectation = self.expectation(description: "test")

diff --git a/MatrixSDKTests/MXAuthenticationSessionUnitTests.swift b/MatrixSDKTests/MXAuthenticationSessionUnitTests.swift
@@ -20,14 +20,6 @@ import MatrixSDK
 
 class MXAuthenticationSessionUnitTests: XCTestCase {
 
-    override func setUpWithError() throws {
-        // Put setup code here. This method is called before the invocation of each test method in the class.
-    }
-
-    override func tearDownWithError() throws {
-        // Put teardown code here. This method is called after the invocation of each test method in the class.
-    }
-
     func testParsing() throws {
 
         let json: [String: Any] = [

diff --git a/MatrixSDKTests/MXBackgroundSyncServiceTests.swift b/MatrixSDKTests/MXBackgroundSyncServiceTests.swift
@@ -29,6 +29,7 @@ class MXBackgroundSyncServiceTests: XCTestCase {
     }
 
     override func setUp() {
+        super.setUp()
         testData = MatrixSDKTestsData()
         e2eTestData = MatrixSDKTestsE2EData(matrixSDKTestsData: testData)
     }
@@ -37,6 +38,7 @@ class MXBackgroundSyncServiceTests: XCTestCase {
         testData = nil
         e2eTestData = nil
         bgSyncService = nil
+        super.tearDown()
     }
 
 

diff --git a/MatrixSDKTests/MXBeaconAggregationsTests.swift b/MatrixSDKTests/MXBeaconAggregationsTests.swift
@@ -28,12 +28,14 @@ class MXBeaconAggregationsTests: XCTestCase {
     // MARK: - Setup
 
     override func setUp() {
+        super.setUp()
         testData = MatrixSDKTestsData()
         MXSDKOptions.sharedInstance().enableThreads = true
     }
 
     override func tearDown() {
         testData = nil
+        super.tearDown()
     }
 
     // MARK: - Tests

diff --git a/MatrixSDKTests/MXCoreDataRoomListDataManagerUnitTests.swift b/MatrixSDKTests/MXCoreDataRoomListDataManagerUnitTests.swift
@@ -34,6 +34,7 @@ class MXCoreDataRoomListDataManagerUnitTests: XCTestCase {
     }
 
     override class func setUp() {
+        super.setUp()
         MXSDKOptions.sharedInstance().roomListDataManagerClass = MXCoreDataRoomListDataManager.self
         MXRealmCryptoStore.deleteAllStores()
     }
@@ -42,6 +43,7 @@ class MXCoreDataRoomListDataManagerUnitTests: XCTestCase {
         fetcher?.stop()
         fetcher = nil
         delegate = nil
+        super.tearDown()
     }
 
     private var basicFetchOptions: MXRoomListDataFetchOptions {

diff --git a/MatrixSDKTests/MXCrossSigningTests.m b/MatrixSDKTests/MXCrossSigningTests.m
@@ -70,6 +70,8 @@ - (void)tearDown
     {
         [[NSNotificationCenter defaultCenter] removeObserver:observer];
     }
+
+    [super tearDown];
 }
 
 

diff --git a/MatrixSDKTests/MXCryptoRecoveryServiceTests.m b/MatrixSDKTests/MXCryptoRecoveryServiceTests.m
@@ -45,6 +45,8 @@ - (void)tearDown
 {
     matrixSDKTestsData = nil;
     matrixSDKTestsE2EData = nil;
+
+    [super tearDown];
 }
 
 

diff --git a/MatrixSDKTests/MXCryptoSecretShareTests.m b/MatrixSDKTests/MXCryptoSecretShareTests.m
@@ -48,6 +48,8 @@ - (void)tearDown
 {
     matrixSDKTestsData = nil;
     matrixSDKTestsE2EData = nil;
+
+    [super tearDown];
 }
 
 /**

diff --git a/MatrixSDKTests/MXCryptoSecretStorageTests.m b/MatrixSDKTests/MXCryptoSecretStorageTests.m
@@ -66,6 +66,8 @@ - (void)tearDown
 {
     matrixSDKTestsData = nil;
     matrixSDKTestsE2EData = nil;
+
+    [super tearDown];
 }
 
 

diff --git a/MatrixSDKTests/MXEventAnnotationUnitTests.swift b/MatrixSDKTests/MXEventAnnotationUnitTests.swift
@@ -40,12 +40,6 @@ class MXEventAnnotationUnitTests: XCTestCase {
         ]
         ]
 
-    override func setUp() {
-    }
-
-    override func tearDown() {
-    }
-
     func testModelFromJSON() {
         let event = MXEvent(fromJSON: eventJSON)
 

diff --git a/MatrixSDKTests/MXEventReferenceUnitTests.swift b/MatrixSDKTests/MXEventReferenceUnitTests.swift
@@ -39,12 +39,6 @@ class MXEventReferenceUnitTests: XCTestCase {
         ]
     ]
 
-    override func setUp() {
-    }
-
-    override func tearDown() {
-    }
-
     func testModelFromJSON() {
         let event = MXEvent(fromJSON: eventJSON)