AWS test run

.github/workflows/e2ePerformanceTests.yml

@@ -125,6 +125,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Setup Node
+        uses: Expensify/App/.github/actions/composite/setupNode@main
+
       - name: Make zip directory for everything to send to AWS Device Farm
         run: mkdir zip
 
@@ -137,7 +140,7 @@ jobs:
 
       # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
       - name: Rename baseline APK
-        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk"
+        run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-main.apk"
 
       - name: Download delta APK
         uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
@@ -147,7 +150,7 @@ jobs:
           path: zip
 
       - name: Rename delta APK
-        run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk"
+        run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-delta.apk"
 
       - name: Copy e2e code into zip folder
         run: cp -r tests/e2e zip
@@ -162,44 +165,72 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: us-west-2
 
-      - name: Schedule AWS Device Farm test run
+      - name: Schedule AWS Device Farm test run on main branch
         uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
+        id: schedule-awsdf-main
         with:
           name: App E2E Performance Regression Tests
           project_arn: ${{ secrets.AWS_PROJECT_ARN }}
           device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
-          app_file: zip/app-e2eRelease-baseline.apk
+          app_file: zip/app-e2eRelease-main.apk
           app_type: ANDROID_APP
           test_type: APPIUM_NODE
           test_package_file: App.zip
           test_package_type: APPIUM_NODE_TEST_PACKAGE
-          test_spec_file: tests/e2e/TestSpec.yml
+          test_spec_file: tests/e2e/TestSpecMain.yml
           test_spec_type: APPIUM_NODE_TEST_SPEC
           remote_src: false
           file_artifacts: Customer Artifacts.zip
+          log_artifacts: debug.log
           cleanup: true
 
-      - name: Unzip AWS Device Farm results
-        if: ${{ always() }}
-        run: unzip "Customer Artifacts.zip"
-
-      - name: Print AWS Device Farm run results
-        if: ${{ always() }}
-        run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"
-
-      - name: Print AWS Device Farm verbose run results
-        if: ${{ always() && runner.debug != null && fromJSON(runner.debug) }}
-        run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/debug.log"
-
-# TODO: Once tests are more reliable we should uncomment this
-#      - name: Check if test failed, if so post the results and add the DeployBlocker label
-#        run: |
-#          if grep -q '🔴' ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; then
-#            gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
-#            gh pr comment ${{ inputs.PR_NUMBER }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md
-#            gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
-#          else
-#            echo '✅ no performance regression detected'
-#          fi
-#        env:
-#          GITHUB_TOKEN: ${{ github.token }}
+      - name: Print logs if run failed
+        if: failure()
+        run: |
+          echo ${{ steps.schedule-awsdf-main.outputs.data }}
+          unzip "Customer Artifacts.zip" -d mainResults
+          cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log
+
+      - name: Unzip AWS Device Farm main results
+        run: unzip "Customer Artifacts.zip" -d mainResults
+
+      - name: Delete Customer Artifacts.zip
+        run: rm "Customer Artifacts.zip"
+
+      - name: Schedule AWS Device Farm test run on delta branch
+        uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
+        with:
+          name: App E2E Performance Regression Tests
+          project_arn: ${{ secrets.AWS_PROJECT_ARN }}
+          device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
+          app_file: zip/app-e2eRelease-delta.apk
+          app_type: ANDROID_APP
+          test_type: APPIUM_NODE
+          test_package_file: App.zip
+          test_package_type: APPIUM_NODE_TEST_PACKAGE
+          test_spec_file: tests/e2e/TestSpecDelta.yml
+          test_spec_type: APPIUM_NODE_TEST_SPEC
+          remote_src: false
+          file_artifacts: Customer Artifacts.zip
+          cleanup: true
+
+      - name: Unzip AWS Device Farm delta results
+        run: unzip "Customer Artifacts.zip" -d deltaResults
+
+      - name: Compare results
+        run: node tests/e2e/merge.js --mainPath ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/main.json --deltaPath ./deltaResults//Host_Machine_Files/\$WORKING_DIRECTORY/delta.json --outputPath ./output.md
+
+      - name: Print results
+        run: cat "./output.md"
+
+      - name: Check if test failed, if so post the results and add the DeployBlocker label
+        run: |
+          if grep -q '🔴' ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; then
+            gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
+            gh pr comment ${{ inputs.PR_NUMBER }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md
+            gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
+          else
+            echo '✅ no performance regression detected'
+          fi
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.tool-versions

@@ -0,0 +1 @@
+nodejs 16.15.1

metro.config.js

@@ -7,9 +7,10 @@ require('dotenv').config();
 const defaultConfig = getDefaultConfig(__dirname);
 
 const isUsingMockAPI = process.env.E2E_TESTING === 'true';
+
 if (isUsingMockAPI) {
     // eslint-disable-next-line no-console
-    console.warn('⚠️ Using mock API');
+    console.log('⚠️⚠️⚠️⚠️ Using mock API ⚠️⚠️⚠️⚠️');
 }
 
 /**
@@ -25,10 +26,14 @@ const config = {
         resolveRequest: (context, moduleName, platform) => {
             const resolution = context.resolveRequest(context, moduleName, platform);
             if (isUsingMockAPI && moduleName.includes('/API')) {
+                const originalPath = resolution.filePath;
+                const mockPath = originalPath.replace('src/libs/API.ts', 'src/libs/E2E/API.mock.js').replace('/src/libs/API.js/', 'src/libs/E2E/API.mock.js');
+                // eslint-disable-next-line no-console
+                console.log('⚠️⚠️⚠️⚠️ Replacing resolution path', originalPath, ' => ', mockPath);
+
                 return {
                     ...resolution,
-                    // TODO: Change API.mock.js extension once it is migrated to TypeScript
-                    filePath: resolution.filePath.replace(/src\/libs\/API.js/, 'src/libs/E2E/API.mock.js'),
+                    filePath: mockPath,
                 };
             }
             return resolution;

package.json

@@ -49,7 +49,9 @@
     "analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production",
     "symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map",
     "symbolicate:ios": "npx metro-symbolicate main.jsbundle.map",
-    "test:e2e": "node tests/e2e/testRunner.js --development",
+    "test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout",
+    "test:e2e:delta": "node tests/e2e/testRunner.js --development --branch main --label delta --skipCheckout",
+    "test:e2e:compare": "node tests/e2e/merge.js",
     "gh-actions-unused-styles": "./.github/scripts/findUnusedKeys.sh",
     "workflow-test": "./workflow_tests/scripts/runWorkflowTests.sh",
     "workflow-test:generate": "node workflow_tests/utils/preGenerateTest.js"

tests/e2e/TestSpecDelta.yml

@@ -0,0 +1,26 @@
+version: 0.1
+
+phases:
+  install:
+    commands:
+      # Install correct version of node
+      - export NVM_DIR=$HOME/.nvm
+      - . $NVM_DIR/nvm.sh
+      - nvm install 16.15.1
+      - nvm use 16.15.1
+
+      # Reverse ports using AWS magic
+      - PORT=4723
+      - IP_ADDRESS=$(ip -4 addr show eth0 | grep -Po "(?<=inet\s)\d+(\.\d+){3}")
+      - reverse_values="{\"ip_address\":\"$IP_ADDRESS\",\"local_port\":\"$PORT\",\"remote_port\":\"$PORT\"}"
+      - "curl -H \"Content-Type: application/json\" -X POST -d \"$reverse_values\" http://localhost:31007/reverse_forward_tcp"
+      - adb reverse tcp:$PORT tcp:$PORT
+
+  test:
+    commands:
+      - cd zip
+      - npm install underscore
+      - node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout --label delta --appPath app-e2eRelease-delta.apk
+
+artifacts:
+- $WORKING_DIRECTORY

tests/e2e/TestSpec.yml → tests/e2e/TestSpecMain.yml

@@ -20,7 +20,7 @@ phases:
     commands:
       - cd zip
       - npm install underscore
-      - node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout
+      - node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout --branch main --appPath app-e2eRelease-main.apk
 
 artifacts:
 - $WORKING_DIRECTORY

tests/e2e/compare/compare.js

@@ -1,7 +1,6 @@
 const fs = require('fs/promises');
 const fsSync = require('fs');
 const _ = require('underscore');
-const {OUTPUT_DIR} = require('../config');
 const {computeProbability, computeZ} = require('./math');
 const printToConsole = require('./output/console');
 const writeToMarkdown = require('./output/markdown');
@@ -119,7 +118,7 @@ function compareResults(compareEntries, baselineEntries) {
     };
 }
 
-module.exports = (baselineFile = `${OUTPUT_DIR}/baseline.json`, compareFile = `${OUTPUT_DIR}/compare.json`, outputFormat = 'all') => {
+module.exports = (baselineFile, compareFile, outputFile, outputFormat = 'all') => {
     const hasBaselineFile = fsSync.existsSync(baselineFile);
     if (!hasBaselineFile) {
         throw new Error(`Baseline results files "${baselineFile}" does not exists.`);
@@ -136,7 +135,7 @@ module.exports = (baselineFile = `${OUTPUT_DIR}/baseline.json`, compareFile = `$
                 printToConsole(outputData);
             }
             if (outputFormat === 'markdown' || outputFormat === 'all') {
-                return writeToMarkdown(`${OUTPUT_DIR}/output.md`, outputData);
+                return writeToMarkdown(outputFile, outputData);
             }
         });
     });

tests/e2e/config.js

@@ -23,24 +23,18 @@ const TEST_NAMES = {
 module.exports = {
     APP_PACKAGE: 'com.expensify.chat.adhoc',
 
-    APP_PATHS: {
-        baseline: './app-e2eRelease-baseline.apk',
-        compare: './app-e2eRelease-compare.apk',
-    },
+    APP_PATH: './app-e2eRelease-main.apk',
 
     ENTRY_FILE: 'src/libs/E2E/reactNativeLaunchingTest.js',
 
     // The port of the testing server that communicates with the app
     SERVER_PORT: 4723,
 
     // The amount of times a test should be executed for average performance metrics
-    RUNS: 90,
+    RUNS: 30,
 
     DEFAULT_BASELINE_BRANCH: 'main',
 
-    // The amount of runs that should happen without counting test results
-    WARM_UP_RUNS: 3,
-
     OUTPUT_DIR,
 
     // The file to write intermediate results to

tests/e2e/config.local.js

@@ -1,10 +1,5 @@
 module.exports = {
-    APP_PACKAGE: 'com.expensify.chat.dev',
-
-    WARM_UP_RUNS: 1,
+    APP_PACKAGE: 'com.expensify.chat.adhoc',
+    APP_PATH: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
     RUNS: 8,
-    APP_PATHS: {
-        baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
-        compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
-    },
 };

tests/e2e/manually_trigger_aws_df.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -ex
+
+PROJECT_ARN=arn:aws:devicefarm:us-west-2:015174472939:project:741d1dac-84e7-4c6f-84ed-b41a2209319d
+DEVICE_POOL_ARN=arn:aws:devicefarm:us-west-2:015174472939:devicepool:741d1dac-84e7-4c6f-84ed-b41a2209319d/efa4b3bb-85ef-45f2-a063-14393eb4d2c2
+UPLOAD_ARN=arn:aws:devicefarm:us-west-2:015174472939:upload:741d1dac-84e7-4c6f-84ed-b41a2209319d/e65b711b-9dba-45dd-aeb4-e7c0449bccc8
+TEST_SPEC_ARN=arn:aws:devicefarm:us-west-2:015174472939:upload:741d1dac-84e7-4c6f-84ed-b41a2209319d/8e694a23-1dbb-40ff-a0ef-a8077dbfa729
+TEST_PACKAGE_ARN=arn:aws:devicefarm:us-west-2:015174472939:upload:741d1dac-84e7-4c6f-84ed-b41a2209319d/6d61e606-5450-43f2-be94-0bee759569e5
+
+aws devicefarm schedule-run --project-arn "${PROJECT_ARN}" --app-arn "${UPLOAD_ARN}" --device-pool-arn "${DEVICE_POOL_ARN}" --name "Test run" --test type=APPIUM_NODE,testPackageArn="${TEST_PACKAGE_ARN}",testSpecArn="${TEST_SPEC_ARN}"
+# aws devicefarm list-uploads --arn "${PROJECT_ARN}"
+# 
+# aws devicefarm create-upload --name TestSpecMain.yml --type APPIUM_NODE_TEST_SPEC --project-arn "${PROJECT_ARN}"

tests/e2e/merge.js

@@ -0,0 +1,27 @@
+const compare = require('./compare/compare');
+const {OUTPUT_DIR} = require('./config');
+
+const args = process.argv.slice(2);
+
+let mainPath = `${OUTPUT_DIR}/main.json`;
+if (args.includes('--mainPath')) {
+    mainPath = args[args.indexOf('--mainPath') + 1];
+}
+
+let deltaPath = `${OUTPUT_DIR}/delta.json`;
+if (args.includes('--deltaPath')) {
+    deltaPath = args[args.indexOf('--deltaPath') + 1];
+}
+
+let outputPath = `${OUTPUT_DIR}/output.md`;
+if (args.includes('--outputPath')) {
+    outputPath = args[args.indexOf('--outputPath') + 1];
+}
+
+async function run() {
+    await compare(mainPath, deltaPath, outputPath, 'all');
+
+    process.exit(0);
+}
+
+run();