ci(tests): add windows tests to CI as non blocking (#3289)

.github/workflows/run-distributed-tests.yml

@@ -100,5 +100,55 @@ jobs:
           echo "LAMBDA_IMAGE_VERSION=${{ inputs.COMMIT_SHA }}" >> $GITHUB_ENV
       # runs the single test file from `package` workspace in the `file`, as defined in the matrix output
       - run: npm run test:aws:ci --workspace ${{fromJson(needs.generate-test-matrix.outputs.matrix).namesToFiles[matrix.testName].package }} -- --files ${{ fromJson(needs.generate-test-matrix.outputs.matrix).namesToFiles[matrix.testName].file }}
+        env:
+          FORCE_COLOR: 1
+
+  run-tests-windows:
+    needs: generate-test-matrix
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      ARTILLERY_CLOUD_ENDPOINT: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }}
+      ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }}
+      DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }}
+      DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }}
+      GITHUB_REPO: ${{ github.repository }}
+      GITHUB_ACTOR: ${{ github.actor }}
+      HAS_ARM64_BUILD: ${{ inputs.HAS_ARM64_BUILD }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.COMMIT_SHA || null }} # in a PR we make a collaborator check, otherwise this would override pull_request_target
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        env:
+          SHOW_STACK_TRACE: true
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.AWS_TEST_EXECUTION_ROLE_ARN_TEST5 }}
+          role-session-name: OIDCSession
+          mask-aws-account-id: true
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18.x
+      - run: .github/workflows/scripts/npm-command-retry.sh install
+      - run: npm run build
+      - name: Install Specific Artillery Version if needed
+        if: ${{ inputs.ARTILLERY_VERSION_OVERRIDE || false }}
+        run: mkdir __artillery__ && cd __artillery__ && npm init -y && ../.github/workflows/scripts/npm-command-retry.sh install artillery@${{ inputs.ARTILLERY_VERSION_OVERRIDE }}
+      - name: Set A9_PATH
+        if: ${{ inputs.ARTILLERY_VERSION_OVERRIDE || false }}
+        run: echo "A9_PATH=${{ github.workspace }}/__artillery__/node_modules/.bin/artillery" >> $GITHUB_ENV
+      - name: Set ECR Image Version if needed
+        if: ${{ inputs.COMMIT_SHA }}
+        run: |
+          echo "ECR_IMAGE_VERSION=${{ inputs.COMMIT_SHA }}" >> $GITHUB_ENV
+          echo "LAMBDA_IMAGE_VERSION=${{ inputs.COMMIT_SHA }}" >> $GITHUB_ENV
+      - run: npm run test:aws:windows --workspace artillery
         env:
           FORCE_COLOR: 1

.github/workflows/run-tests.yml

@@ -49,5 +49,42 @@ jobs:
         with:
           status: ${{ job.status }}
           fields: repo,message,commit,author,eventName,job,took,pullRequest
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  test-windows:
+    timeout-minutes: 60
+    runs-on: windows-latest
+    needs: generate-matrix-with-packages
+    continue-on-error: true
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js 20.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+      - run: npm install
+      - run: npm run build
+      - name: Run windows tests and capture exit code
+        continue-on-error: true
+        run: |
+          try {
+            npm run test:windows --workspace artillery
+            $global:ExitCode = 0
+          } catch {
+            $global:ExitCode = $LASTEXITCODE
+          }
+          echo "ExitCode=$global:ExitCode" >> $env:GITHUB_ENV
+        env:
+          FORCE_COLOR: 1
+      - name: Notify about failures
+        if: ${{ env.ExitCode }} != 0
+        uses: 8398a7/action-slack@v3.15.1
+        with:
+          status: failure
+          fields: repo,message,commit,author,eventName,job,took,pullRequest
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

packages/artillery/package.json

@@ -49,7 +49,7 @@
     "test:windows": "npm run test:unit && tap --timeout=420 test/cli/*.test.js",
     "test:aws": "tap --timeout=4200 test/cloud-e2e/**/*.test.js",
     "test:aws:ci": "tap --timeout=4200",
-    "test:aws:windows": "tap --timeout=420 test/cloud-e2e/fargate/*.test.js --grep \"Run simple-bom\"",
+    "test:aws:windows": "tap --timeout=420 test/cloud-e2e/**/*.test.js --grep \"@windows\"",
     "lint": "eslint --ext \".js,.ts,.tsx\" .",
     "lint-fix": "npm run lint -- --fix"
   },

packages/artillery/test/cloud-e2e/fargate/bom.test.js

@@ -26,7 +26,7 @@ beforeEach(async (t) => {
   reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
-test('Run simple-bom', async (t) => {
+test('Run simple-bom @windows', async (t) => {
   const scenarioPath = toCorrectPath(
     `${__dirname}/fixtures/simple-bom/simple-bom.yml`
   );

packages/artillery/test/cloud-e2e/fargate/processors.test.js

@@ -2,7 +2,11 @@ const { test, before, beforeEach } = require('tap');
 const { $ } = require('zx');
 const fs = require('fs');
 const path = require('path');
-const { generateTmpReportPath, getTestTags } = require('../../helpers');
+const {
+  generateTmpReportPath,
+  getTestTags,
+  toCorrectPath
+} = require('../../helpers');
 const {
   checkForNegativeValues,
   checkAggregateCounterSums
@@ -21,11 +25,15 @@ beforeEach(async (t) => {
   reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
-test('Run with typescript processor and external package', async (t) => {
-  const scenarioPath = `${__dirname}/fixtures/ts-external-pkg/with-external-foreign-pkg.yml`;
+test('Run with typescript processor and external package @windows', async (t) => {
+  const scenarioPath = toCorrectPath(
+    `${__dirname}/fixtures/ts-external-pkg/with-external-foreign-pkg.yml`
+  );
 
   const output =
-    await $`${A9_PATH} run-fargate ${scenarioPath} --output ${reportFilePath} --record --tags ${baseTags},typescript:true`;
+    await $`${A9_PATH} run-fargate ${scenarioPath} --output ${toCorrectPath(
+      reportFilePath
+    )} --record --tags ${baseTags},typescript:true`;
 
   t.equal(output.exitCode, 0, 'CLI Exit Code should be 0');
 
@@ -45,14 +53,16 @@ test('Run with typescript processor and external package', async (t) => {
   checkAggregateCounterSums(t, report);
 });
 
-test('Run a test with an ESM processor', async (t) => {
+test('Run a test with an ESM processor @windows', async (t) => {
   // The main thing we're checking here is that ESM + dependencies get bundled correctly by BOM
-  const scenarioPath = path.resolve(
-    `${__dirname}/../../scripts/scenario-async-esm-hooks/test.yml`
+  const scenarioPath = toCorrectPath(
+    path.resolve(`${__dirname}/../../scripts/scenario-async-esm-hooks/test.yml`)
   );
 
   const output =
-    await $`${A9_PATH} run-fargate ${scenarioPath} --output ${reportFilePath} --record --tags ${baseTags}`;
+    await $`${A9_PATH} run-fargate ${scenarioPath} --output ${toCorrectPath(
+      reportFilePath
+    )} --record --tags ${baseTags}`;
 
   t.equal(output.exitCode, 0, 'CLI exit code should be 0');
 

packages/artillery/test/cloud-e2e/lambda/lambda-bom.test.js

@@ -4,7 +4,8 @@ const { $ } = require('zx');
 const {
   getTestTags,
   generateTmpReportPath,
-  getImageArchitecture
+  getImageArchitecture,
+  toCorrectPath
 } = require('../../helpers');
 const {
   checkForNegativeValues,
@@ -26,8 +27,10 @@ tap.beforeEach(async (t) => {
   reportFilePath = generateTmpReportPath(t.name, 'json');
 });
 
-tap.test('Run simple-bom', async (t) => {
-  const scenarioPath = `${__dirname}/../fargate/fixtures/simple-bom/simple-bom.yml`;
+tap.test('Run simple-bom @windows', async (t) => {
+  const scenarioPath = toCorrectPath(
+    `${__dirname}/../fargate/fixtures/simple-bom/simple-bom.yml`
+  );
 
   const output =
     await $`${A9_PATH} run-lambda ${scenarioPath} --architecture ${ARCHITECTURE} -e test --tags ${tags} --output ${reportFilePath} --count 51 --record`;

packages/artillery/test/cloud-e2e/lambda/lambda-smoke.test.js

@@ -4,7 +4,8 @@ const { $ } = require('zx');
 const {
   getTestTags,
   generateTmpReportPath,
-  getImageArchitecture
+  getImageArchitecture,
+  toCorrectPath
 } = require('../../helpers');
 const {
   checkForNegativeValues,
@@ -57,12 +58,16 @@ tap.test('Run a test on AWS Lambda using containers', async (t) => {
 });
 
 tap.test(
-  'Run in Lambda container with typescript processor and external package',
+  'Run in Lambda container with typescript processor and external package @windows',
   async (t) => {
-    const scenarioPath = `${__dirname}/fixtures/ts-external-pkg/with-external-foreign-pkg.yml`;
+    const scenarioPath = toCorrectPath(
+      `${__dirname}/fixtures/ts-external-pkg/with-external-foreign-pkg.yml`
+    );
 
     const output =
-      await $`${A9_PATH} run-lambda ${scenarioPath} --architecture ${ARCHITECTURE} --record --output ${reportFilePath} --tags ${tags},typescript:true`;
+      await $`${A9_PATH} run-lambda ${scenarioPath} --architecture ${ARCHITECTURE} --record --output ${toCorrectPath(
+        reportFilePath
+      )} --tags ${tags},typescript:true`;
 
     t.equal(output.exitCode, 0, 'CLI Exit Code should be 0');
 

packages/artillery/test/helpers/index.js

@@ -43,8 +43,11 @@ function getTestTags(additionalTags) {
   const actorTag = `actor:${process.env.GITHUB_ACTOR || 'localhost'}`;
   const repoTag = `repo:${process.env.GITHUB_REPO || 'artilleryio/artillery'}`;
   const ciTag = `ci:${process.env.GITHUB_ACTIONS ? 'true' : 'false'}`;
+  const platformTag = `plat:${process.platform}`;
 
-  return `${repoTag},${actorTag},${ciTag},${additionalTags.join(',')}`;
+  return `${repoTag},${actorTag},${ciTag},${platformTag},${additionalTags.join(
+    ','
+  )}`;
 }
 
 const getImageArchitecture = () => {