From 4c9835ce86a5ae1b35989422a7b39845243193d4 Mon Sep 17 00:00:00 2001 From: Mateusz Baginski Date: Fri, 4 Oct 2024 07:11:52 +0200 Subject: [PATCH 1/3] Add retrying CI test support --- scripts/ci/check-unit-tests-for-package.mjs | 30 ++++++++++++++----- .../ci/generate-circleci-configuration.mjs | 1 + 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/ci/check-unit-tests-for-package.mjs b/scripts/ci/check-unit-tests-for-package.mjs index 462348d378b..2833f127e99 100644 --- a/scripts/ci/check-unit-tests-for-package.mjs +++ b/scripts/ci/check-unit-tests-for-package.mjs @@ -21,9 +21,9 @@ main() } ); async function main() { - const { packageName, checkCoverage, allowNonFullCoverage, coverageFile } = getOptions( process.argv.slice( 2 ) ); + const { packageName, checkCoverage, allowNonFullCoverage, coverageFile, retries } = getOptions( process.argv.slice( 2 ) ); - runTests( { packageName, checkCoverage } ); + runTests( { packageName, checkCoverage, retries } ); if ( checkCoverage && !allowNonFullCoverage ) { const exitCode = checkCodeCoverage(); @@ -46,8 +46,9 @@ async function main() { * @param {Object} options * @param {String} options.packageName * @param {Boolean} options.checkCoverage + * @param {Number} options.retries */ -function runTests( { packageName, checkCoverage } ) { +function runTests( { packageName, checkCoverage, retries = 3 } ) { const shortName = packageName.replace( /^ckeditor5?-/, '' ); const testCommand = [ @@ -59,10 +60,23 @@ function runTests( { packageName, checkCoverage } ) { checkCoverage ? '--coverage' : null ].filter( Boolean ); - execSync( testCommand.join( ' ' ), { - cwd: CKEDITOR5_ROOT_PATH, - stdio: 'inherit' - } ); + for ( let retry = 0; retry < retries; retry++ ) { + try { + execSync( testCommand.join( ' ' ), { + cwd: CKEDITOR5_ROOT_PATH, + stdio: 'inherit' + } ); + + break; + } catch ( err ) { + if ( retry === retries ) { + throw err; + } else { + console.error( err ); + console.log( `\n⚠️ Retry ${ retry + 1 } of ${ retries } for ${ packageName }!` ); + } + } + } } function checkCodeCoverage() { @@ -90,6 +104,7 @@ function checkCodeCoverage() { * @returns {Boolean} options.checkCoverage * @returns {Boolean} options.allowNonFullCoverage * @returns {String|null} options.coverageFile + * @returns {Number} options.retries */ function getOptions( argv ) { const options = minimist( argv, { @@ -107,6 +122,7 @@ function getOptions( argv ) { } } ); + options.retries = Number( options.retries ?? 3 ); options.packageName = options[ 'package-name' ]; options.coverageFile = options[ 'coverage-file' ]; options.checkCoverage = options[ 'check-coverage' ]; diff --git a/scripts/ci/generate-circleci-configuration.mjs b/scripts/ci/generate-circleci-configuration.mjs index 5f496938812..7b464cfe1f0 100755 --- a/scripts/ci/generate-circleci-configuration.mjs +++ b/scripts/ci/generate-circleci-configuration.mjs @@ -236,6 +236,7 @@ function generateTestSteps( packages, { checkCoverage, coverageFile = null } ) { 'node', 'scripts/ci/check-unit-tests-for-package.mjs', '--package-name', + '--retries 3', packageName, checkCoverage ? '--check-coverage' : null, allowNonFullCoverage ? '--allow-non-full-coverage' : null, From 5d29385a3dccfd15b3ae5950b0af418f240be174 Mon Sep 17 00:00:00 2001 From: Mateusz Baginski Date: Fri, 4 Oct 2024 07:17:48 +0200 Subject: [PATCH 2/3] Reorder args --- scripts/ci/generate-circleci-configuration.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/generate-circleci-configuration.mjs b/scripts/ci/generate-circleci-configuration.mjs index 7b464cfe1f0..dbad491dd9d 100755 --- a/scripts/ci/generate-circleci-configuration.mjs +++ b/scripts/ci/generate-circleci-configuration.mjs @@ -235,8 +235,8 @@ function generateTestSteps( packages, { checkCoverage, coverageFile = null } ) { const testCommand = [ 'node', 'scripts/ci/check-unit-tests-for-package.mjs', - '--package-name', '--retries 3', + '--package-name', packageName, checkCoverage ? '--check-coverage' : null, allowNonFullCoverage ? '--allow-non-full-coverage' : null, From f0907bbac3482fc14e05ab252ddd7a38dedcdc8b Mon Sep 17 00:00:00 2001 From: Kamil Piechaczek Date: Fri, 4 Oct 2024 08:50:45 +0200 Subject: [PATCH 3/3] Minor improvements in the retry mechanism. Now, it takes from a stack of available attempts. --- scripts/ci/check-unit-tests-for-package.mjs | 46 ++++++++++--------- .../ci/generate-circleci-configuration.mjs | 2 +- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/scripts/ci/check-unit-tests-for-package.mjs b/scripts/ci/check-unit-tests-for-package.mjs index 2833f127e99..cf5c87f0a4d 100644 --- a/scripts/ci/check-unit-tests-for-package.mjs +++ b/scripts/ci/check-unit-tests-for-package.mjs @@ -21,9 +21,9 @@ main() } ); async function main() { - const { packageName, checkCoverage, allowNonFullCoverage, coverageFile, retries } = getOptions( process.argv.slice( 2 ) ); + const { packageName, checkCoverage, allowNonFullCoverage, coverageFile, attempts } = getOptions( process.argv.slice( 2 ) ); - runTests( { packageName, checkCoverage, retries } ); + runTests( { packageName, checkCoverage, attempts } ); if ( checkCoverage && !allowNonFullCoverage ) { const exitCode = checkCodeCoverage(); @@ -46,9 +46,9 @@ async function main() { * @param {Object} options * @param {String} options.packageName * @param {Boolean} options.checkCoverage - * @param {Number} options.retries + * @param {Number} options.attempts */ -function runTests( { packageName, checkCoverage, retries = 3 } ) { +function runTests( { packageName, checkCoverage, attempts = 3 } ) { const shortName = packageName.replace( /^ckeditor5?-/, '' ); const testCommand = [ @@ -60,22 +60,23 @@ function runTests( { packageName, checkCoverage, retries = 3 } ) { checkCoverage ? '--coverage' : null ].filter( Boolean ); - for ( let retry = 0; retry < retries; retry++ ) { - try { - execSync( testCommand.join( ' ' ), { - cwd: CKEDITOR5_ROOT_PATH, - stdio: 'inherit' - } ); - - break; - } catch ( err ) { - if ( retry === retries ) { - throw err; - } else { - console.error( err ); - console.log( `\n⚠️ Retry ${ retry + 1 } of ${ retries } for ${ packageName }!` ); - } + try { + execSync( testCommand.join( ' ' ), { + cwd: CKEDITOR5_ROOT_PATH, + stdio: 'inherit' + } ); + } catch ( err ) { + if ( !attempts ) { + throw err; } + + console.log( `\n⚠️ Retry the test execution. Remaining attempts: ${ attempts - 1 }.` ); + + return runTests( { + packageName, + checkCoverage, + attempts: attempts - 1 + } ); } } @@ -104,13 +105,14 @@ function checkCodeCoverage() { * @returns {Boolean} options.checkCoverage * @returns {Boolean} options.allowNonFullCoverage * @returns {String|null} options.coverageFile - * @returns {Number} options.retries + * @returns {Number} options.attempts */ function getOptions( argv ) { const options = minimist( argv, { string: [ 'package-name', - 'coverage-file' + 'coverage-file', + 'attempts' ], boolean: [ 'check-coverage', @@ -122,7 +124,7 @@ function getOptions( argv ) { } } ); - options.retries = Number( options.retries ?? 3 ); + options.attempts = Number( options.attempts || 1 ); options.packageName = options[ 'package-name' ]; options.coverageFile = options[ 'coverage-file' ]; options.checkCoverage = options[ 'check-coverage' ]; diff --git a/scripts/ci/generate-circleci-configuration.mjs b/scripts/ci/generate-circleci-configuration.mjs index dbad491dd9d..6914d209549 100755 --- a/scripts/ci/generate-circleci-configuration.mjs +++ b/scripts/ci/generate-circleci-configuration.mjs @@ -235,7 +235,7 @@ function generateTestSteps( packages, { checkCoverage, coverageFile = null } ) { const testCommand = [ 'node', 'scripts/ci/check-unit-tests-for-package.mjs', - '--retries 3', + '--attempts 3', '--package-name', packageName, checkCoverage ? '--check-coverage' : null,