test(cli): add assertions for not containing negative summaries (#3237)

artilleryio · Jul 1, 2024 · c3245a5 · c3245a5
1 parent 046418f
commit c3245a5
Show file tree

Hide file tree

Showing 16 changed files with 84 additions and 1 deletion.
diff --git a/packages/artillery/test/cli/async-hooks-esm.test.js b/packages/artillery/test/cli/async-hooks-esm.test.js
@@ -1,5 +1,6 @@
 const tap = require('tap');
 const { execute, generateTmpReportPath } = require('../helpers');
+const { checkForNegativeValues } = require('../helpers/expectations');
 const fs = require('fs');
 
 let reportFilePath;
@@ -45,4 +46,6 @@ tap.test('async hooks with ESM', async (t) => {
     10,
     'Should have no completed VUs'
   );
+
+  checkForNegativeValues(t, json);
 });
diff --git a/packages/artillery/test/cli/command-run.test.js b/packages/artillery/test/cli/command-run.test.js
@@ -6,6 +6,7 @@ const {
   returnTmpPath,
   generateTmpReportPath
 } = require('../helpers');
+const { checkForNegativeValues } = require('../helpers/expectations');
 const fs = require('fs');
 const path = require('path');
 const execa = require('execa');
@@ -509,6 +510,10 @@ tap.test('Script using a plugin', async (t) => {
     pluginCount,
     'Should have same number of requests in report as plugin'
   );
+  checkForNegativeValues(
+    t,
+    JSON.parse(fs.readFileSync(reportFilePath, 'utf8'))
+  );
 });
 
 tap.test(
@@ -723,5 +728,7 @@ tap.test("Script with 'parallel' behaves as expected", async (t) => {
     expectedRequests,
     `Should have made ${expectedRequests} successful requests`
   );
+
+  checkForNegativeValues(t, report);
   deleteFile(reportFilePath);
 });
diff --git a/packages/artillery/test/cli/run-typescript.test.js b/packages/artillery/test/cli/run-typescript.test.js
@@ -1,5 +1,6 @@
 const tap = require('tap');
 const { execute, generateTmpReportPath, deleteFile } = require('../helpers');
+const { checkForNegativeValues } = require('../helpers/expectations');
 const fs = require('fs');
 const path = require('path');
 
@@ -33,6 +34,8 @@ tap.test('Can run a Typescript processor', async (t) => {
     2,
     'Should have emitted 2 custom metrics from ts processor'
   );
+
+  checkForNegativeValues(t, json);
 });
 
 tap.test('Runs correctly when package is marked as external', async (t) => {
@@ -76,6 +79,7 @@ tap.test('Runs correctly when package is marked as external', async (t) => {
     2,
     'Should have emitted 2 custom metrics from ts processor'
   );
+  checkForNegativeValues(t, json);
 
   //assert that the bundle was created and marked as external
   const bundleLocation = path.join(

diff --git a/packages/artillery/test/cloud-e2e/fargate/bom.test.js b/packages/artillery/test/cloud-e2e/fargate/bom.test.js
@@ -2,6 +2,7 @@ const { test, before, beforeEach } = require('tap');
 const { $ } = require('zx');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -56,4 +57,6 @@ test('Run mixed-hierarchy', async (t) => {
     20,
     'Should have 20 "200 OK" responses'
   );
+
+  checkForNegativeValues(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/fargate/cw-adot.test.js b/packages/artillery/test/cloud-e2e/fargate/cw-adot.test.js
@@ -9,6 +9,7 @@ const {
   getTestTags
 } = require('../../helpers');
 const { getTestId, getXRayTraces } = require('./fixtures/adot/helpers.js');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 // NOTE: This test reports to Artillery Dashboard to dogfood and improve visibility
@@ -44,6 +45,7 @@ test('traces succesfully arrive to cloudwatch', async (t) => {
 
   const testId = getTestId(output.stdout);
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
+  checkForNegativeValues(t, report);
 
   let traceMap;
   try {

diff --git a/packages/artillery/test/cloud-e2e/fargate/dd-adot.test.js b/packages/artillery/test/cloud-e2e/fargate/dd-adot.test.js
@@ -8,7 +8,7 @@ const {
   deleteFile,
   getTestTags
 } = require('../../helpers');
-
+const { checkForNegativeValues } = require('../../helpers/expectations');
 const { getDatadogSpans, getTestId } = require('./fixtures/adot/helpers.js');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
@@ -52,6 +52,7 @@ test('traces succesfully arrive to datadog', async (t) => {
 
   const testId = getTestId(output.stdout);
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
+  checkForNegativeValues(t, report);
 
   let spanList;
   try {

diff --git a/packages/artillery/test/cloud-e2e/fargate/ensure-plugin.test.js b/packages/artillery/test/cloud-e2e/fargate/ensure-plugin.test.js
@@ -3,6 +3,7 @@ const { $ } = require('zx');
 const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -39,5 +40,6 @@ test('Run uses ensure', async (t) => {
       300,
       'Should have 300 "200 OK" responses'
     );
+    checkForNegativeValues(t, report);
   }
 });
diff --git a/packages/artillery/test/cloud-e2e/fargate/expect-plugin.test.js b/packages/artillery/test/cloud-e2e/fargate/expect-plugin.test.js
@@ -3,6 +3,7 @@ const { $ } = require('zx');
 const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -40,6 +41,7 @@ test('CLI should exit with non-zero exit code when there are failed expectations
       10,
       'Should have 10 "200 OK" responses'
     );
+    checkForNegativeValues(t, report);
   }
 });
 
@@ -67,5 +69,6 @@ test('Ensure (with new interface) should still run when workers exit from expect
       10,
       'Should have 10 "200 OK" responses'
     );
+    checkForNegativeValues(t, report);
   }
 });
diff --git a/packages/artillery/test/cloud-e2e/fargate/misc.test.js b/packages/artillery/test/cloud-e2e/fargate/misc.test.js
@@ -3,6 +3,7 @@ const { $ } = require('zx');
 const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -77,6 +78,7 @@ test('Kitchen Sink Test - multiple features together', async (t) => {
     40,
     'Should have 40 /pony "200 OK" responses'
   );
+  checkForNegativeValues(t, report);
 });
 
 test('Run lots-of-output', async (t) => {

diff --git a/packages/artillery/test/cloud-e2e/fargate/processors.test.js b/packages/artillery/test/cloud-e2e/fargate/processors.test.js
@@ -3,6 +3,7 @@ const { $ } = require('zx');
 const fs = require('fs');
 const path = require('path');
 const { generateTmpReportPath, getTestTags } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const A9_PATH = process.env.A9_PATH || 'artillery';
 
@@ -36,6 +37,7 @@ test('Run with typescript processor and external package', async (t) => {
     2,
     'Should have emitted 2 errors'
   );
+  checkForNegativeValues(t, report);
 });
 
 test('Run a test with an ESM processor', async (t) => {
@@ -61,4 +63,5 @@ test('Run a test with an ESM processor', async (t) => {
     10,
     'Should have emitted 10 custom metrics from ts processor'
   );
+  checkForNegativeValues(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-bom.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-bom.test.js
@@ -6,6 +6,7 @@ const {
   generateTmpReportPath,
   getImageArchitecture
 } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 
@@ -60,4 +61,5 @@ tap.test('Run mixed-hierarchy test in Lambda Container', async (t) => {
     20,
     'Should have 20 "200 OK" responses'
   );
+  checkForNegativeValues(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-dotenv.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-dotenv.test.js
@@ -6,6 +6,7 @@ const {
   generateTmpReportPath,
   getImageArchitecture
 } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 const A9_PATH = process.env.A9_PATH || 'artillery';
@@ -43,4 +44,5 @@ tap.test('Run dotenv test in Lambda Container', async (t) => {
     50,
     'Should have custom counter for env variable fruit'
   );
+  checkForNegativeValues(t, report);
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-ensure.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-ensure.test.js
@@ -7,6 +7,7 @@ const {
   getTestTags,
   getImageArchitecture
 } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
 const tags = getTestTags(['type:acceptance']);
@@ -45,5 +46,6 @@ tap.test('Lambda Container run uses ensure', async (t) => {
       300,
       'Should have 300 "200 OK" responses'
     );
+    checkForNegativeValues(t, report);
   }
 });
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-expect.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-expect.test.js
@@ -7,6 +7,7 @@ const {
   getTestTags,
   getImageArchitecture
 } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 let reportFilePath;
@@ -53,6 +54,7 @@ tap.test(
         10,
         'Should have 10 "200 OK" responses'
       );
+      checkForNegativeValues(t, report);
     }
   }
 );
diff --git a/packages/artillery/test/cloud-e2e/lambda/lambda-smoke.test.js b/packages/artillery/test/cloud-e2e/lambda/lambda-smoke.test.js
@@ -6,6 +6,7 @@ const {
   generateTmpReportPath,
   getImageArchitecture
 } = require('../../helpers');
+const { checkForNegativeValues } = require('../../helpers/expectations');
 
 const tags = getTestTags(['type:acceptance']);
 
@@ -73,5 +74,6 @@ tap.test(
       2,
       'Should have emitted 2 errors'
     );
+    checkForNegativeValues(t, report);
   }
 );
diff --git a/packages/artillery/test/helpers/expectations.js b/packages/artillery/test/helpers/expectations.js
@@ -0,0 +1,43 @@
+const _checkSummaries = (t, summaries, type) => {
+  for (const summaryMetric of Object.keys(summaries)) {
+    for (const aggregation of Object.keys(summaries[summaryMetric])) {
+      if (
+        summaries[summaryMetric][aggregation] < 0 ||
+        summaries[summaryMetric][aggregation] === null
+      ) {
+        t.fail(
+          `Found invalid value in ${type} summaries: ${summaryMetric}.${aggregation} = ${summaries[summaryMetric][aggregation]}`
+        );
+      }
+    }
+  }
+};
+
+const checkForNegativeValues = (t, report) => {
+  const aggregateSummaries = report.aggregate?.summaries;
+
+  if (!aggregateSummaries || Object.keys(aggregateSummaries).length === 0) {
+    t.fail('No aggregate summaries found in the report');
+  }
+
+  _checkSummaries(t, aggregateSummaries, 'aggregate');
+
+  if (!report.intermediate || report.intermediate.length === 0) {
+    t.fail('No intermediate summaries found in the report');
+  }
+
+  for (const intermediate of report.intermediate) {
+    const intermediateSummaries = intermediate.summaries;
+    if (
+      !intermediateSummaries ||
+      Object.keys(intermediateSummaries).length === 0
+    ) {
+      continue;
+    }
+    _checkSummaries(t, intermediateSummaries, 'intermediate');
+  }
+};
+
+module.exports = {
+  checkForNegativeValues
+};