Performance Tests: Only run 1 round of tests during PR commits

In the performance tests CI workflow we have been running every test
suite three times for each branch under test. The goal of this work,
introduced in #33710, was to reduce variation in the reported data
from the tests.

Unfortunately after measuring the data produced by our test runs, and
by running experiments that run the test suites thirty times over, the
overall variation is explained primarily by noise in the Github Actions
container running our jobs. If running the test suites three times each
reduces the variation in the results then it's not detectable above the
amount of variation introduced beyond our control.

Because these additional rounds extend the perf-test runtime by around
twenty minutes on each PR we're reducing the number of rounds to a
single pass for PR commits. This will free up compute resources and
remove the performance tests as a bottleneck in the PR workflow.

Additional work can and should be done to further remove variance in
the testing results, but this practical step will remove an obstacle
from developer iteration speed without reducing the quality of the
numbers being reported.

.github/workflows/performance.yml

@@ -61,7 +61,7 @@ jobs:
                   WP_VERSION=$(awk -F ': ' '/^Tested up to/{print $2}' readme.txt)
                   IFS=. read -ra WP_VERSION_ARRAY <<< "$WP_VERSION"
                   WP_MAJOR="${WP_VERSION_ARRAY[0]}.${WP_VERSION_ARRAY[1]}"
-                  ./bin/plugin/cli.js perf $GITHUB_SHA debd225d007f4e441ceec80fbd6fa96653f94737 --tests-branch $GITHUB_SHA  --wp-version "$WP_MAJOR"
+                  ./bin/plugin/cli.js perf $GITHUB_SHA debd225d007f4e441ceec80fbd6fa96653f94737 --tests-branch $GITHUB_SHA --wp-version "$WP_MAJOR"
 
             - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3
               if: github.event_name == 'push'

bin/plugin/cli.js

@@ -94,6 +94,10 @@ program
 	.command( 'performance-tests [branches...]' )
 	.alias( 'perf' )
 	.option( ...ciOption )
+	.option(
+		'--rounds <count>',
+		'Run each test suite this many times for each branch; results are summarized, default = 1'
+	)
 	.option(
 		'--tests-branch <branch>',
 		"Use this branch's performance test files"

bin/plugin/commands/performance.js

@@ -22,6 +22,7 @@ const config = require( '../config' );
  * @typedef WPPerformanceCommandOptions
  *
  * @property {boolean=} ci          Run on CI.
+ * @property {number=}  rounds      Run each test suite this many times for each branch.
  * @property {string=}  testsBranch The branch whose performance test files will be used for testing.
  * @property {string=}  wpVersion   The WordPress version to be used as the base install for testing.
  */
@@ -176,6 +177,7 @@ async function runTestSuite( testSuite, performanceTestDirectory ) {
  */
 async function runPerformanceTests( branches, options ) {
 	const runningInCI = !! process.env.CI || !! options.ci;
+	const TEST_ROUNDS = options.rounds || 1;
 
 	// The default value doesn't work because commander provides an array.
 	if ( branches.length === 0 ) {
@@ -330,7 +332,7 @@ async function runPerformanceTests( branches, options ) {
 		/** @type {Array<Record<string, WPPerformanceResults>>} */
 		const rawResults = [];
 		// Alternate three times between branches.
-		for ( let i = 0; i < 3; i++ ) {
+		for ( let i = 0; i < TEST_ROUNDS; i++ ) {
 			rawResults[ i ] = {};
 			for ( const branch of branches ) {
 				// @ts-ignore