From dc7ad937859e14e812eb164c11a6c71cc08ed6d2 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 2 Feb 2023 23:53:16 +0800 Subject: [PATCH 001/100] feat: AsyncOperationQueue supports remote executing --- .../logic/operations/AsyncOperationQueue.ts | 87 ++++++++++++++++--- .../logic/operations/ConsoleTimelinePlugin.ts | 4 + .../src/logic/operations/OperationStatus.ts | 11 ++- .../test/AsyncOperationQueue.test.ts | 56 +++++++++++- .../AsyncOperationQueue.test.ts.snap | 2 +- 5 files changed, 147 insertions(+), 13 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 006df21d840..8dff124792f 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -5,7 +5,7 @@ import { OperationExecutionRecord } from './OperationExecutionRecord'; import { OperationStatus } from './OperationStatus'; /** - * Implmentation of the async iteration protocol for a collection of IOperation objects. + * Implementation of the async iteration protocol for a collection of IOperation objects. * The async iterator will wait for an operation to be ready for execution, or terminate if there are no more operations. * * @remarks @@ -18,6 +18,10 @@ export class AsyncOperationQueue { private readonly _queue: OperationExecutionRecord[]; private readonly _pendingIterators: ((result: IteratorResult) => void)[]; + private readonly _totalOperations: number; + + private _completedOperations: number; + private _isDone: boolean; /** * @param operations - The set of operations to be executed @@ -29,6 +33,9 @@ export class AsyncOperationQueue public constructor(operations: Iterable, sortFn: IOperationSortFunction) { this._queue = computeTopologyAndSort(operations, sortFn); this._pendingIterators = []; + this._totalOperations = this._queue.length; + this._isDone = false; + this._completedOperations = 0; } /** @@ -49,6 +56,17 @@ export class AsyncOperationQueue return promise; } + /** + * Set a callback to be invoked when one operation is completed. + * If all operations are completed, set the queue to done, resolve all pending iterators in next cycle. + */ + public complete(): void { + this._completedOperations++; + if (this._completedOperations === this._totalOperations) { + this._isDone = true; + } + } + /** * Routes ready operations with 0 dependencies to waiting iterators. Normally invoked as part of `next()`, but * if the caller does not update operation dependencies prior to calling `next()`, may need to be invoked manually. @@ -56,19 +74,42 @@ export class AsyncOperationQueue public assignOperations(): void { const { _queue: queue, _pendingIterators: waitingIterators } = this; + if (this._isDone) { + for (const resolveAsyncIterator of waitingIterators.splice(0)) { + resolveAsyncIterator({ + value: undefined, + done: true + }); + } + return; + } + // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; - if (operation.status === OperationStatus.Blocked) { + if ( + operation.status === OperationStatus.Blocked || + operation.status === OperationStatus.Success || + operation.status === OperationStatus.SuccessWithWarning || + operation.status === OperationStatus.FromCache || + operation.status === OperationStatus.NoOp || + operation.status === OperationStatus.Failure + ) { // It shouldn't be on the queue, remove it queue.splice(i, 1); + } else if ( + operation.status === OperationStatus.RemotePending || + operation.status === OperationStatus.RemoteExecuting + ) { + // This operation is not ready to execute yet, but it may become ready later + // next one plz :) + continue; } else if (operation.status !== OperationStatus.Ready) { // Sanity check throw new Error(`Unexpected status "${operation.status}" for queued operation: ${operation.name}`); } else if (operation.dependencies.size === 0) { // This task is ready to process, hand it to the iterator. - queue.splice(i, 1); // Needs to have queue semantics, otherwise tools that iterate it get confused waitingIterators.shift()!({ value: operation, @@ -78,13 +119,26 @@ export class AsyncOperationQueue // Otherwise operation is still waiting } - if (queue.length === 0) { - // Queue is empty, flush - for (const resolveAsyncIterator of waitingIterators.splice(0)) { - resolveAsyncIterator({ - value: undefined, - done: true - }); + if (waitingIterators.length > 0) { + // cycle through the queue again to find the next operation that is executed remotely + for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { + const operation: OperationExecutionRecord = queue[i]; + + if (operation.status === OperationStatus.RemoteExecuting) { + // try to attempt to get the lock again + waitingIterators.shift()!({ + value: operation, + done: false + }); + } + } + + if (waitingIterators.length > 0) { + // Queue is not empty, but no operations are ready to process + // Pause for a second and start over + setTimeout(() => { + this.assignOperations(); + }, 1000); } } } @@ -96,6 +150,19 @@ export class AsyncOperationQueue public [Symbol.asyncIterator](): AsyncIterator { return this; } + + /** + * Recursively sets the status of all operations that consume the specified operation. + */ + public static setOperationConsumersStatusRecursively( + operation: OperationExecutionRecord, + operationStatus: OperationStatus + ): void { + for (const consumer of operation.consumers) { + consumer.status = operationStatus; + AsyncOperationQueue.setOperationConsumersStatusRecursively(consumer, operationStatus); + } + } } export interface IOperationSortFunction { diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index aff052574b4..ac5cb13baa9 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -72,6 +72,8 @@ const TIMELINE_WIDTH: number = 109; const TIMELINE_CHART_SYMBOLS: Record = { [OperationStatus.Ready]: '?', [OperationStatus.Executing]: '?', + [OperationStatus.RemoteExecuting]: '?', + [OperationStatus.RemotePending]: '?', [OperationStatus.Success]: '#', [OperationStatus.SuccessWithWarning]: '!', [OperationStatus.Failure]: '!', @@ -87,6 +89,8 @@ const TIMELINE_CHART_SYMBOLS: Record = { const TIMELINE_CHART_COLORIZER: Record string> = { [OperationStatus.Ready]: colors.yellow, [OperationStatus.Executing]: colors.yellow, + [OperationStatus.RemoteExecuting]: colors.yellow, + [OperationStatus.RemotePending]: colors.yellow, [OperationStatus.Success]: colors.green, [OperationStatus.SuccessWithWarning]: colors.yellow, [OperationStatus.Failure]: colors.red, diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 3fdff28ba84..43047308cd1 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. /** @@ -14,6 +14,15 @@ export enum OperationStatus { * The Operation is currently executing */ Executing = 'EXECUTING', + /** + * The Operation is currently executing by a remote process + */ + RemoteExecuting = 'REMOTE EXECUTING', + /** + * The Operation is pending because one of the upstream operation is + * executing by a remote process + */ + RemotePending = 'REMOTE PENDING', /** * The Operation completed successfully and did not write to standard output */ diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 1270b19c262..3c030e619d4 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -5,6 +5,7 @@ import { Operation } from '../Operation'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; import { MockOperationRunner } from './MockOperationRunner'; import { AsyncOperationQueue, IOperationSortFunction } from '../AsyncOperationQueue'; +import { OperationStatus } from '../OperationStatus'; function addDependency(consumer: OperationExecutionRecord, dependency: OperationExecutionRecord): void { consumer.dependencies.add(dependency); @@ -40,6 +41,8 @@ describe(AsyncOperationQueue.name, () => { for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } + operation.status = OperationStatus.Success; + queue.complete(); } expect(actualOrder).toEqual(expectedOrder); @@ -68,7 +71,7 @@ describe(AsyncOperationQueue.name, () => { expect(actualOrder).toEqual(expectedOrder); }); - it('detects cyles', async () => { + it('detects cycles', async () => { const operations = [createRecord('a'), createRecord('b'), createRecord('c'), createRecord('d')]; addDependency(operations[0], operations[2]); @@ -124,6 +127,8 @@ describe(AsyncOperationQueue.name, () => { } --concurrency; + operation.status = OperationStatus.Success; + queue.complete(); } }) ); @@ -132,4 +137,53 @@ describe(AsyncOperationQueue.name, () => { expect(actualConcurrency.get(operation)).toEqual(operationConcurrency); } }); + + it('handles remote executed operations', async () => { + const operations = [ + createRecord('a'), + createRecord('b'), + createRecord('c'), + createRecord('d'), + createRecord('e') + ]; + + addDependency(operations[2], operations[1]); + addDependency(operations[3], operations[1]); + addDependency(operations[4], operations[1]); + addDependency(operations[3], operations[2]); + addDependency(operations[4], operations[3]); + + // b remote executing -> a -> b (remote executed) -> c -> d -> e + const expectedOrder: string[] = ['b', 'a', 'b', 'c', 'd', 'e']; + + const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); + + const actualOrder: string[] = []; + let remoteExecuted: boolean = false; + for await (const operation of queue) { + actualOrder.push(operation.name); + + if (operation === operations[1]) { + if (!remoteExecuted) { + operations[1].status = OperationStatus.RemoteExecuting; + AsyncOperationQueue.setOperationConsumersStatusRecursively( + operations[1], + OperationStatus.RemotePending + ); + // remote executed operation is finished later + remoteExecuted = true; + continue; + } else { + AsyncOperationQueue.setOperationConsumersStatusRecursively(operations[1], OperationStatus.Ready); + } + } + for (const consumer of operation.consumers) { + consumer.dependencies.delete(operation); + } + operation.status = OperationStatus.Success; + queue.complete(); + } + + expect(actualOrder).toEqual(expectedOrder); + }); }); diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap index 7a4c086efd4..c1642f68137 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/AsyncOperationQueue.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`AsyncOperationQueue detects cyles 1`] = ` +exports[`AsyncOperationQueue detects cycles 1`] = ` "A cyclic dependency was encountered: a -> c From 7b8a521472235ca8b1e17a2572d65d46dd63f5c7 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 9 Feb 2023 19:52:55 +0800 Subject: [PATCH 002/100] feat: cobuildlock & cobuildconfiguration --- common/reviews/api/rush-lib.api.md | 29 ++++ .../rush-lib/src/api/CobuildConfiguration.ts | 111 +++++++++++++ .../src/api/EnvironmentConfiguration.ts | 31 ++++ .../cli/scriptActions/PhasedScriptAction.ts | 8 + libraries/rush-lib/src/index.ts | 4 +- libraries/rush-lib/src/logic/RushConstants.ts | 11 ++ .../src/logic/buildCache/ProjectBuildCache.ts | 12 +- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 115 ++++++++++++++ .../src/logic/cobuild/ICobuildLockProvider.ts | 31 ++++ .../logic/operations/AsyncOperationQueue.ts | 26 ++- .../logic/operations/ConsoleTimelinePlugin.ts | 4 +- .../src/logic/operations/IOperationRunner.ts | 7 + .../operations/OperationExecutionManager.ts | 11 +- .../operations/OperationExecutionRecord.ts | 13 +- .../src/logic/operations/OperationStatus.ts | 11 +- .../src/logic/operations/RunnerWatcher.ts | 56 +++++++ .../logic/operations/ShellOperationRunner.ts | 148 +++++++++++++++--- .../operations/ShellOperationRunnerPlugin.ts | 2 + .../test/AsyncOperationQueue.test.ts | 6 - .../src/pluginFramework/PhasedCommandHooks.ts | 8 +- .../src/pluginFramework/RushSession.ts | 31 +++- .../rush-lib/src/schemas/cobuild.schema.json | 39 +++++ 22 files changed, 652 insertions(+), 62 deletions(-) create mode 100644 libraries/rush-lib/src/api/CobuildConfiguration.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/CobuildLock.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts create mode 100644 libraries/rush-lib/src/logic/operations/RunnerWatcher.ts create mode 100644 libraries/rush-lib/src/schemas/cobuild.schema.json diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9aa7b02ce99..242dffd798d 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -95,6 +95,23 @@ export class ChangeManager { // @beta (undocumented) export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) => ICloudBuildCacheProvider; +// @beta +export class CobuildConfiguration { + readonly cobuildEnabled: boolean; + // Warning: (ae-forgotten-export) The symbol "ICobuildLockProvider" needs to be exported by the entry point index.d.ts + readonly cobuildLockProvider: ICobuildLockProvider; + // (undocumented) + get contextId(): string; + // (undocumented) + static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string; + static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; +} + +// Warning: (ae-forgotten-export) The symbol "ICobuildJson" needs to be exported by the entry point index.d.ts +// +// @beta (undocumented) +export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; + // @public export class CommonVersionsConfiguration { readonly allowedAlternativeVersions: Map>; @@ -149,6 +166,7 @@ export class EnvironmentConfiguration { static get buildCacheCredential(): string | undefined; static get buildCacheEnabled(): boolean | undefined; static get buildCacheWriteAllowed(): boolean | undefined; + static get cobuildEnabled(): boolean | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // // @internal @@ -173,6 +191,7 @@ export enum EnvironmentVariableNames { RUSH_BUILD_CACHE_CREDENTIAL = "RUSH_BUILD_CACHE_CREDENTIAL", RUSH_BUILD_CACHE_ENABLED = "RUSH_BUILD_CACHE_ENABLED", RUSH_BUILD_CACHE_WRITE_ALLOWED = "RUSH_BUILD_CACHE_WRITE_ALLOWED", + RUSH_COBUILD_ENABLED = "RUSH_COBUILD_ENABLED", RUSH_DEPLOY_TARGET_FOLDER = "RUSH_DEPLOY_TARGET_FOLDER", RUSH_GIT_BINARY_PATH = "RUSH_GIT_BINARY_PATH", RUSH_GLOBAL_FOLDER = "RUSH_GLOBAL_FOLDER", @@ -258,6 +277,7 @@ export interface IConfigurationEnvironmentVariable { // @alpha export interface ICreateOperationsContext { readonly buildCacheConfiguration: BuildCacheConfiguration | undefined; + readonly cobuildConfiguration: CobuildConfiguration | undefined; readonly customParameters: ReadonlyMap; readonly isIncrementalBuildAllowed: boolean; readonly isInitial: boolean; @@ -432,6 +452,7 @@ export interface IOperationRunnerContext { // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; + status: OperationStatus; stdioSummarizer: StdioSummarizer; stopwatch: IStopwatchResult; } @@ -672,7 +693,9 @@ export enum OperationStatus { Failure = "FAILURE", FromCache = "FROM CACHE", NoOp = "NO OP", + Queued = "Queued", Ready = "READY", + RemoteExecuting = "REMOTE EXECUTING", Skipped = "SKIPPED", Success = "SUCCESS", SuccessWithWarning = "SUCCESS WITH WARNINGS" @@ -954,6 +977,8 @@ export class RushConstants { static readonly buildCommandName: string; static readonly bulkCommandKind: 'bulk'; static readonly changeFilesFolderName: string; + static readonly cobuildFilename: string; + static readonly cobuildLockVersion: number; static readonly commandLineFilename: string; static readonly commonFolderName: string; static readonly commonVersionsFilename: string; @@ -1026,12 +1051,16 @@ export class RushSession { // (undocumented) getCloudBuildCacheProviderFactory(cacheProviderName: string): CloudBuildCacheProviderFactory | undefined; // (undocumented) + getCobuildLockProviderFactory(cobuildLockProviderName: string): CobuildLockProviderFactory | undefined; + // (undocumented) getLogger(name: string): ILogger; // (undocumented) readonly hooks: RushLifecycleHooks; // (undocumented) registerCloudBuildCacheProviderFactory(cacheProviderName: string, factory: CloudBuildCacheProviderFactory): void; // (undocumented) + registerCobuildLockProviderFactory(cobuildLockProviderName: string, factory: CobuildLockProviderFactory): void; + // (undocumented) get terminalProvider(): ITerminalProvider; } diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts new file mode 100644 index 00000000000..3f9d2a12dac --- /dev/null +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as path from 'path'; +import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; +import schemaJson from '../schemas/cobuild.schema.json'; +import { EnvironmentConfiguration } from './EnvironmentConfiguration'; +import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; +import { RushConstants } from '../logic/RushConstants'; + +import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; +import type { RushConfiguration } from './RushConfiguration'; + +export interface ICobuildJson { + cobuildEnabled: boolean; + cobuildLockProvider: string; + cobuildContextIdPattern?: string; +} + +export interface ICobuildConfigurationOptions { + cobuildJson: ICobuildJson; + rushConfiguration: RushConfiguration; + rushSession: RushSession; +} + +/** + * Use this class to load and save the "common/config/rush/cobuild.json" config file. + * This file provides configuration options for the Rush Cobuild feature. + * @beta + */ +export class CobuildConfiguration { + private static _jsonSchema: JsonSchema = JsonSchema.fromLoadedObject(schemaJson); + + /** + * Indicates whether the cobuild feature is enabled. + * Typically it is enabled in the cobuild.json config file. + */ + public readonly cobuildEnabled: boolean; + /** + * Method to calculate the cobuild context id + * FIXME: + */ + // public readonly getCacheEntryId: GetCacheEntryIdFunction; + public readonly cobuildLockProvider: ICobuildLockProvider; + + private constructor(options: ICobuildConfigurationOptions) { + this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? options.cobuildJson.cobuildEnabled; + + const { cobuildJson } = options; + + const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = + options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); + if (!cobuildLockProviderFactory) { + throw new Error(`Unexpected cobuild lock provider: ${cobuildJson.cobuildLockProvider}`); + } + this.cobuildLockProvider = cobuildLockProviderFactory(cobuildJson); + } + + /** + * Attempts to load the cobuild.json data from the standard file path `common/config/rush/cobuild.json`. + * If the file has not been created yet, then undefined is returned. + */ + public static async tryLoadAsync( + terminal: ITerminal, + rushConfiguration: RushConfiguration, + rushSession: RushSession + ): Promise { + const jsonFilePath: string = CobuildConfiguration.getCobuildConfigFilePath(rushConfiguration); + if (!FileSystem.exists(jsonFilePath)) { + return undefined; + } + return await CobuildConfiguration._loadAsync(jsonFilePath, terminal, rushConfiguration, rushSession); + } + + public static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string { + return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.cobuildFilename); + } + private static async _loadAsync( + jsonFilePath: string, + terminal: ITerminal, + rushConfiguration: RushConfiguration, + rushSession: RushSession + ): Promise { + const cobuildJson: ICobuildJson = await JsonFile.loadAndValidateAsync( + jsonFilePath, + CobuildConfiguration._jsonSchema + ); + + // FIXME: + // let getCacheEntryId: GetCacheEntryIdFunction; + // try { + // getCacheEntryId = CacheEntryId.parsePattern(cobuildJson.cacheEntryNamePattern); + // } catch (e) { + // terminal.writeErrorLine( + // `Error parsing cache entry name pattern "${cobuildJson.cacheEntryNamePattern}": ${e}` + // ); + // throw new AlreadyReportedError(); + // } + + return new CobuildConfiguration({ + cobuildJson, + rushConfiguration, + rushSession + }); + } + + public get contextId(): string { + // FIXME: hardcode + return '123'; + } +} diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 3bd511504b7..269b571d983 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -143,6 +143,17 @@ export enum EnvironmentVariableNames { */ RUSH_BUILD_CACHE_WRITE_ALLOWED = 'RUSH_BUILD_CACHE_WRITE_ALLOWED', + /** + * Setting this environment variable overrides the value of `cobuildEnabled` in the `cobuild.json` + * configuration file. + * + * @remarks + * Specify `1` to enable the cobuild or `0` to disable it. + * + * If there is no build cache configured, then this environment variable is ignored. + */ + RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', + /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. */ @@ -196,6 +207,8 @@ export class EnvironmentConfiguration { private static _buildCacheWriteAllowed: boolean | undefined; + private static _cobuildEnabled: boolean | undefined; + private static _gitBinaryPath: string | undefined; private static _tarBinaryPath: string | undefined; @@ -293,6 +306,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._buildCacheWriteAllowed; } + /** + * If set, enables or disables the cobuild feature. + * See {@link EnvironmentVariableNames.RUSH_COBUILD_ENABLED} + */ + public static get cobuildEnabled(): boolean | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildEnabled; + } + /** * Allows the git binary path to be explicitly provided. * See {@link EnvironmentVariableNames.RUSH_GIT_BINARY_PATH} @@ -423,6 +445,15 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_ENABLED: { + EnvironmentConfiguration._cobuildEnabled = + EnvironmentConfiguration.parseBooleanEnvironmentVariable( + EnvironmentVariableNames.RUSH_COBUILD_ENABLED, + value + ); + break; + } + case EnvironmentVariableNames.RUSH_GIT_BINARY_PATH: { EnvironmentConfiguration._gitBinaryPath = value; break; diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 67c14ff22a5..99eb4c503b6 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -37,6 +37,7 @@ import { IExecutionResult } from '../../logic/operations/IOperationExecutionResu import { OperationResultSummarizerPlugin } from '../../logic/operations/OperationResultSummarizerPlugin'; import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; +import { CobuildConfiguration } from '../../api/CobuildConfiguration'; /** * Constructor parameters for PhasedScriptAction. @@ -299,12 +300,18 @@ export class PhasedScriptAction extends BaseScriptAction { const changedProjectsOnly: boolean = !!this._changedProjectsOnly?.value; let buildCacheConfiguration: BuildCacheConfiguration | undefined; + let cobuildConfiguration: CobuildConfiguration | undefined; if (!this._disableBuildCache) { buildCacheConfiguration = await BuildCacheConfiguration.tryLoadAsync( terminal, this.rushConfiguration, this.rushSession ); + cobuildConfiguration = await CobuildConfiguration.tryLoadAsync( + terminal, + this.rushConfiguration, + this.rushSession + ); } const projectSelection: Set = @@ -325,6 +332,7 @@ export class PhasedScriptAction extends BaseScriptAction { const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); const initialCreateOperationsContext: ICreateOperationsContext = { buildCacheConfiguration, + cobuildConfiguration, customParameters: customParametersByName, isIncrementalBuildAllowed: this._isIncrementalBuildAllowed, isInitial: true, diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index c98c9d20d56..18fbed045f0 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -31,6 +31,7 @@ export { } from './logic/pnpm/PnpmOptionsConfiguration'; export { BuildCacheConfiguration } from './api/BuildCacheConfiguration'; +export { CobuildConfiguration } from './api/CobuildConfiguration'; export { GetCacheEntryIdFunction, IGenerateCacheEntryIdOptions } from './logic/buildCache/CacheEntryId'; export { FileSystemBuildCacheProvider, @@ -98,7 +99,8 @@ export { OperationStatus } from './logic/operations/OperationStatus'; export { RushSession, IRushSessionOptions, - CloudBuildCacheProviderFactory + CloudBuildCacheProviderFactory, + CobuildLockProviderFactory } from './pluginFramework/RushSession'; export { diff --git a/libraries/rush-lib/src/logic/RushConstants.ts b/libraries/rush-lib/src/logic/RushConstants.ts index bdbdff0b3a7..04e5bcb40ae 100644 --- a/libraries/rush-lib/src/logic/RushConstants.ts +++ b/libraries/rush-lib/src/logic/RushConstants.ts @@ -182,6 +182,17 @@ export class RushConstants { */ public static readonly buildCacheVersion: number = 1; + /** + * Cobuild configuration file. + */ + public static readonly cobuildFilename: string = 'cobuild.json'; + + /** + * Cobuild version number, incremented when the logic to create cobuild lock changes. + * Changing this ensures that lock generated by an old version will no longer access as a cobuild lock. + */ + public static readonly cobuildLockVersion: number = 1; + /** * Per-project configuration filename. */ diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 658bd3217b0..f8a7ea21a07 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -74,6 +74,10 @@ export class ProjectBuildCache { return ProjectBuildCache._tarUtilityPromise; } + public get cacheId(): string | undefined { + return this._cacheId; + } + public static async tryGetProjectBuildCache( options: IProjectBuildCacheOptions ): Promise { @@ -133,8 +137,8 @@ export class ProjectBuildCache { } } - public async tryRestoreFromCacheAsync(terminal: ITerminal): Promise { - const cacheId: string | undefined = this._cacheId; + public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { + const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { terminal.writeWarningLine('Unable to get cache ID. Ensure Git is installed.'); return false; @@ -213,13 +217,13 @@ export class ProjectBuildCache { return restoreSuccess; } - public async trySetCacheEntryAsync(terminal: ITerminal): Promise { + public async trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { if (!this._cacheWriteEnabled) { // Skip writing local and cloud build caches, without any noise return true; } - const cacheId: string | undefined = this._cacheId; + const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { terminal.writeWarningLine('Unable to get cache ID. Ensure Git is installed.'); return false; diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts new file mode 100644 index 00000000000..6746c3d3b52 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { RushConstants } from '../RushConstants'; + +import type { ITerminal } from '@rushstack/node-core-library'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; +import { OperationStatus } from '../operations/OperationStatus'; + +export interface ICobuildLockOptions { + cobuildConfiguration: CobuildConfiguration; + projectBuildCache: ProjectBuildCache; + terminal: ITerminal; +} + +export interface ICobuildCompletedState { + status: OperationStatus.Success | OperationStatus.SuccessWithWarning | OperationStatus.Failure; + cacheId: string; +} + +const KEY_SEPARATOR: string = ':'; +const COMPLETED_STATE_SEPARATOR: string = ';'; + +export class CobuildLock { + public readonly options: ICobuildLockOptions; + public readonly lockKey: string; + public readonly completedKey: string; + + public readonly projectBuildCache: ProjectBuildCache; + public readonly cobuildConfiguration: CobuildConfiguration; + + public constructor(options: ICobuildLockOptions) { + this.options = options; + const { cobuildConfiguration, projectBuildCache } = options; + this.projectBuildCache = projectBuildCache; + this.cobuildConfiguration = cobuildConfiguration; + + const { contextId } = cobuildConfiguration; + const { cacheId } = projectBuildCache; + // Example: cobuild:v1:::lock + this.lockKey = ['cobuild', `v${RushConstants.cobuildLockVersion}`, contextId, cacheId, 'lock'].join( + KEY_SEPARATOR + ); + // Example: cobuild:v1:::completed + this.completedKey = [ + 'cobuild', + `v${RushConstants.cobuildLockVersion}`, + contextId, + cacheId, + 'completed' + ].join(KEY_SEPARATOR); + } + + public async setCompletedStateAsync(state: ICobuildCompletedState): Promise { + const { terminal } = this.options; + const serializedState: string = this._serializeCompletedState(state); + terminal.writeDebugLine(`Set completed state by key ${this.completedKey}: ${serializedState}`); + await this.cobuildConfiguration.cobuildLockProvider.setCompletedStateAsync({ + key: this.completedKey, + value: serializedState, + terminal + }); + } + + public async getCompletedStateAsync(): Promise { + const { terminal } = this.options; + const state: string | undefined = + await this.cobuildConfiguration.cobuildLockProvider.getCompletedStateAsync({ + key: this.completedKey, + terminal + }); + terminal.writeDebugLine(`Get completed state by key ${this.completedKey}: ${state}`); + if (!state) { + return; + } + return this._deserializeCompletedState(state); + } + + public async tryAcquireLockAsync(): Promise { + const { terminal } = this.options; + // const result: boolean = true; + // const result: boolean = false; + // const result: boolean = Math.random() > 0.5; + const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync({ + lockKey: this.lockKey, + terminal + }); + terminal.writeDebugLine(`Acquired lock for ${this.lockKey}, result: ${acquireLockResult}`); + return acquireLockResult; + } + + public async releaseLockAsync(): Promise { + const { terminal } = this.options; + terminal.writeDebugLine(`Released lock for ${this.lockKey}`); + return; + } + + public async renewLockAsync(): Promise { + const { terminal } = this.options; + terminal.writeDebugLine(`Renewed lock for ${this.lockKey}`); + return; + } + + private _serializeCompletedState(state: ICobuildCompletedState): string { + // Example: SUCCESS;1234567890 + // Example: FAILURE;1234567890 + return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; + } + + private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { + const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); + return { status: status as ICobuildCompletedState['status'], cacheId }; + } +} diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts new file mode 100644 index 00000000000..4735fc1542f --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { ITerminal } from '@rushstack/node-core-library'; + +export interface ILockOptions { + lockKey: string; + terminal: ITerminal; +} + +export interface IGetCompletedStateOptions { + key: string; + terminal: ITerminal; +} + +export interface ISetCompletedStateOptions { + key: string; + value: string; + terminal: ITerminal; +} + +/** + * @beta + */ +export interface ICobuildLockProvider { + acquireLockAsync(options: ILockOptions): Promise; + renewLockAsync(options: ILockOptions): Promise; + releaseLockAsync(options: ILockOptions): Promise; + setCompletedStateAsync(options: ISetCompletedStateOptions): Promise; + getCompletedStateAsync(options: IGetCompletedStateOptions): Promise; +} diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 8dff124792f..ffea8cc71e4 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -84,6 +84,10 @@ export class AsyncOperationQueue return; } + queue.forEach((q) => { + console.log(q.name, q.status); + }); + // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; @@ -99,9 +103,13 @@ export class AsyncOperationQueue // It shouldn't be on the queue, remove it queue.splice(i, 1); } else if ( - operation.status === OperationStatus.RemotePending || - operation.status === OperationStatus.RemoteExecuting + operation.status === OperationStatus.Queued || + operation.status === OperationStatus.Executing ) { + // This operation is currently executing + // next one plz :) + continue; + } else if (operation.status === OperationStatus.RemoteExecuting) { // This operation is not ready to execute yet, but it may become ready later // next one plz :) continue; @@ -111,6 +119,7 @@ export class AsyncOperationQueue } else if (operation.dependencies.size === 0) { // This task is ready to process, hand it to the iterator. // Needs to have queue semantics, otherwise tools that iterate it get confused + operation.status = OperationStatus.Queued; waitingIterators.shift()!({ value: operation, done: false @@ -150,19 +159,6 @@ export class AsyncOperationQueue public [Symbol.asyncIterator](): AsyncIterator { return this; } - - /** - * Recursively sets the status of all operations that consume the specified operation. - */ - public static setOperationConsumersStatusRecursively( - operation: OperationExecutionRecord, - operationStatus: OperationStatus - ): void { - for (const consumer of operation.consumers) { - consumer.status = operationStatus; - AsyncOperationQueue.setOperationConsumersStatusRecursively(consumer, operationStatus); - } - } } export interface IOperationSortFunction { diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index ac5cb13baa9..31c4b56a48e 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -71,9 +71,9 @@ const TIMELINE_WIDTH: number = 109; */ const TIMELINE_CHART_SYMBOLS: Record = { [OperationStatus.Ready]: '?', + [OperationStatus.Queued]: '?', [OperationStatus.Executing]: '?', [OperationStatus.RemoteExecuting]: '?', - [OperationStatus.RemotePending]: '?', [OperationStatus.Success]: '#', [OperationStatus.SuccessWithWarning]: '!', [OperationStatus.Failure]: '!', @@ -88,9 +88,9 @@ const TIMELINE_CHART_SYMBOLS: Record = { */ const TIMELINE_CHART_COLORIZER: Record string> = { [OperationStatus.Ready]: colors.yellow, + [OperationStatus.Queued]: colors.yellow, [OperationStatus.Executing]: colors.yellow, [OperationStatus.RemoteExecuting]: colors.yellow, - [OperationStatus.RemotePending]: colors.yellow, [OperationStatus.Success]: colors.green, [OperationStatus.SuccessWithWarning]: colors.yellow, [OperationStatus.Failure]: colors.red, diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 22062e82b71..5428e6c0c2a 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -40,6 +40,13 @@ export interface IOperationRunnerContext { * Object used to track elapsed time. */ stopwatch: IStopwatchResult; + /** + * The current execution status of an operation. Operations start in the 'ready' state, + * but can be 'blocked' if an upstream operation failed. It is 'executing' when + * the operation is executing. Once execution is complete, it is either 'success' or + * 'failure'. + */ + status: OperationStatus; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index a84a0cb67d6..d61c6c196fb 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -189,6 +189,11 @@ export class OperationExecutionManager { record: OperationExecutionRecord ) => { this._onOperationComplete(record); + + if (record.status !== OperationStatus.RemoteExecuting) { + // If the operation was not remote, then we can notify queue that it is complete + executionQueue.complete(); + } }; await Async.forEachAsync( @@ -328,8 +333,10 @@ export class OperationExecutionManager { item.runner.isSkipAllowed = false; } - // Remove this operation from the dependencies, to unblock the scheduler - item.dependencies.delete(record); + if (status !== OperationStatus.RemoteExecuting) { + // Remove this operation from the dependencies, to unblock the scheduler + item.dependencies.delete(record); + } } } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 11cd208e8bd..f20f05fe25f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -138,6 +138,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext { try { this.status = await this.runner.executeAsync(this); + + if (this.status === OperationStatus.RemoteExecuting) { + this.stopwatch.reset(); + } + // Delegate global state reporting onResult(this); } catch (error) { @@ -146,9 +151,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext { // Delegate global state reporting onResult(this); } finally { - this._collatedWriter?.close(); - this.stdioSummarizer.close(); - this.stopwatch.stop(); + if (this.status !== OperationStatus.RemoteExecuting) { + this._collatedWriter?.close(); + this.stdioSummarizer.close(); + this.stopwatch.stop(); + } } } } diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 43047308cd1..18e7204e78c 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. /** @@ -10,6 +10,10 @@ export enum OperationStatus { * The Operation is on the queue, ready to execute (but may be waiting for dependencies) */ Ready = 'READY', + /** + * The Operation is Queued + */ + Queued = 'Queued', /** * The Operation is currently executing */ @@ -18,11 +22,6 @@ export enum OperationStatus { * The Operation is currently executing by a remote process */ RemoteExecuting = 'REMOTE EXECUTING', - /** - * The Operation is pending because one of the upstream operation is - * executing by a remote process - */ - RemotePending = 'REMOTE PENDING', /** * The Operation completed successfully and did not write to standard output */ diff --git a/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts b/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts new file mode 100644 index 00000000000..e5823454cf6 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +export type ICallbackFn = () => Promise | void; + +export interface IRunnerWatcherOptions { + interval: number; +} + +/** + * A help class to run callbacks in a loop with a specified interval. + * + * @beta + */ +export class RunnerWatcher { + private _callbacks: ICallbackFn[]; + private _interval: number; + private _timeoutId: NodeJS.Timeout | undefined; + private _isRunning: boolean; + + public constructor(options: IRunnerWatcherOptions) { + this._callbacks = []; + this._interval = options.interval; + this._isRunning = false; + } + + public addCallback(callback: ICallbackFn): void { + if (this._isRunning) { + throw new Error('Can not add callback while watcher is running'); + } + this._callbacks.push(callback); + } + + public start(): void { + if (this._timeoutId) { + throw new Error('Watcher already started'); + } + if (this._callbacks.length === 0) { + return; + } + this._isRunning = true; + this._timeoutId = setTimeout(() => { + this._callbacks.forEach((callback) => callback()); + this._timeoutId = undefined; + this.start(); + }, this._interval); + } + + public stop(): void { + if (this._timeoutId) { + clearTimeout(this._timeoutId); + this._timeoutId = undefined; + this._isRunning = false; + } + } +} diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 2dca3e01b37..fedab5e5fe2 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -35,11 +35,14 @@ import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvid import { RushConstants } from '../RushConstants'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import { OperationMetadataManager } from './OperationMetadataManager'; +import { RunnerWatcher } from './RunnerWatcher'; +import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ProjectChangeAnalyzer, IRawRepoState } from '../ProjectChangeAnalyzer'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import type { IPhase } from '../../api/CommandLineConfiguration'; export interface IProjectDeps { @@ -51,6 +54,7 @@ export interface IOperationRunnerOptions { rushProject: RushConfigurationProject; rushConfiguration: RushConfiguration; buildCacheConfiguration: BuildCacheConfiguration | undefined; + cobuildConfiguration: CobuildConfiguration | undefined; commandToRun: string; isIncrementalBuildAllowed: boolean; projectChangeAnalyzer: ProjectChangeAnalyzer; @@ -95,6 +99,7 @@ export class ShellOperationRunner implements IOperationRunner { private readonly _phase: IPhase; private readonly _rushConfiguration: RushConfiguration; private readonly _buildCacheConfiguration: BuildCacheConfiguration | undefined; + private readonly _cobuildConfiguration: CobuildConfiguration | undefined; private readonly _commandName: string; private readonly _commandToRun: string; private readonly _isCacheReadAllowed: boolean; @@ -108,6 +113,7 @@ export class ShellOperationRunner implements IOperationRunner { * undefined === we didn't create one because the feature is not enabled */ private _projectBuildCache: ProjectBuildCache | undefined | UNINITIALIZED = UNINITIALIZED; + private _cobuildLock: CobuildLock | undefined | UNINITIALIZED = UNINITIALIZED; public constructor(options: IOperationRunnerOptions) { const { phase } = options; @@ -117,6 +123,7 @@ export class ShellOperationRunner implements IOperationRunner { this._phase = phase; this._rushConfiguration = options.rushConfiguration; this._buildCacheConfiguration = options.buildCacheConfiguration; + this._cobuildConfiguration = options.cobuildConfiguration; this._commandName = phase.name; this._commandToRun = options.commandToRun; this._isCacheReadAllowed = options.isIncrementalBuildAllowed; @@ -150,6 +157,10 @@ export class ShellOperationRunner implements IOperationRunner { context.collatedWriter.terminal, this._logFilenameIdentifier ); + const runnerWatcher: RunnerWatcher = new RunnerWatcher({ + interval: 10 * 1000 + // interval: 1000 + }); try { const removeColorsTransform: TextRewriterTransform = new TextRewriterTransform({ @@ -247,6 +258,16 @@ export class ShellOperationRunner implements IOperationRunner { }); } + // Try to acquire the cobuild lock + let cobuildLock: CobuildLock | undefined; + if (this._cobuildConfiguration?.cobuildEnabled) { + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( + terminal, + trackedFiles + ); + cobuildLock = await this._tryGetCobuildLockAsync(terminal, projectBuildCache); + } + // If possible, we want to skip this operation -- either by restoring it from the // cache, if caching is enabled, or determining that the project // is unchanged (using the older incremental execution logic). These two approaches, @@ -264,8 +285,30 @@ export class ShellOperationRunner implements IOperationRunner { // false if a dependency wasn't able to be skipped. // let buildCacheReadAttempted: boolean = false; - if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + if (cobuildLock) { + // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to + // the build cache once completed, but does not retrieve them (since the "incremental" + // flag is disabled). However, we still need a cobuild to be able to retrieve a finished + // build from another cobuild in this case. + const cobuildCompletedState: ICobuildCompletedState | undefined = + await cobuildLock.getCompletedStateAsync(); + if (cobuildCompletedState) { + const { status, cacheId } = cobuildCompletedState; + + const restoreFromCacheSuccess: boolean | undefined = + await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(terminal, cacheId); + + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await context._operationStateFile?.tryRestoreAsync(); + if (cobuildCompletedState) { + return cobuildCompletedState.status; + } + return status; + } + } + } else if (this._isCacheReadAllowed) { + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( terminal, trackedProjectFiles, operationMetadataManager: context._operationMetadataManager @@ -317,8 +360,25 @@ export class ShellOperationRunner implements IOperationRunner { return OperationStatus.Success; } + if (this.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + if (context.status === OperationStatus.RemoteExecuting) { + // This operation is used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; + } + runnerWatcher.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + } else { + // failed to acquire the lock, mark current operation to remote executing + return OperationStatus.RemoteExecuting; + } + } + // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); + runnerWatcher.start(); const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( this._commandToRun, @@ -366,6 +426,37 @@ export class ShellOperationRunner implements IOperationRunner { } ); + let setCompletedStatePromise: Promise | undefined; + let setCacheEntryPromise: Promise | undefined; + if (cobuildLock && this.isCacheWriteAllowed) { + const { projectBuildCache } = cobuildLock; + const cacheId: string | undefined = projectBuildCache.cacheId; + const contextId: string = cobuildLock.cobuildConfiguration.contextId; + + if (cacheId) { + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + setCompletedStatePromise = cobuildLock + .setCompletedStateAsync({ + status, + cacheId: finalCacheId + }) + .then(() => { + return cobuildLock?.releaseLockAsync(); + }); + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + terminal, + finalCacheId + ); + } + } + } + } + const taskIsSuccessful: boolean = status === OperationStatus.Success || (status === OperationStatus.SuccessWithWarning && @@ -373,9 +464,10 @@ export class ShellOperationRunner implements IOperationRunner { !!this._rushConfiguration.experimentsConfiguration.configuration .buildCacheWithAllowWarningsInSuccessfulBuild); + let writeProjectStatePromise: Promise | undefined; if (taskIsSuccessful && projectDeps) { // Write deps on success. - const writeProjectStatePromise: Promise = JsonFile.saveAsync(projectDeps, currentDepsPath, { + writeProjectStatePromise = JsonFile.saveAsync(projectDeps, currentDepsPath, { ensureFolderExists: true }); @@ -389,24 +481,23 @@ export class ShellOperationRunner implements IOperationRunner { // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. - const setCacheEntryPromise: Promise | undefined = this.isCacheWriteAllowed - ? ( - await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }) - )?.trySetCacheEntryAsync(terminal) - : undefined; - - const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); - - if (terminalProvider.hasErrors) { - status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false) { - status = OperationStatus.SuccessWithWarning; + if (!setCacheEntryPromise && this.isCacheWriteAllowed) { + setCacheEntryPromise = ( + await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles) + )?.trySetCacheEntryAsync(terminal); } } + const [, cacheWriteSuccess] = await Promise.all([ + writeProjectStatePromise, + setCacheEntryPromise, + setCompletedStatePromise + ]); + + if (terminalProvider.hasErrors) { + status = OperationStatus.Failure; + } else if (cacheWriteSuccess === false) { + status = OperationStatus.SuccessWithWarning; + } normalizeNewlineTransform.close(); @@ -419,6 +510,7 @@ export class ShellOperationRunner implements IOperationRunner { return status; } finally { projectLogWritable.close(); + runnerWatcher.stop(); } } @@ -513,6 +605,24 @@ export class ShellOperationRunner implements IOperationRunner { return this._projectBuildCache; } + + private async _tryGetCobuildLockAsync( + terminal: ITerminal, + projectBuildCache: ProjectBuildCache | undefined + ): Promise { + if (this._cobuildLock === UNINITIALIZED) { + this._cobuildLock = undefined; + + if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { + this._cobuildLock = new CobuildLock({ + cobuildConfiguration: this._cobuildConfiguration, + projectBuildCache: projectBuildCache, + terminal + }); + } + } + return this._cobuildLock; + } } /** diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index d070fc1588b..1d1a425feef 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -31,6 +31,7 @@ function createShellOperations( ): Set { const { buildCacheConfiguration, + cobuildConfiguration, isIncrementalBuildAllowed, phaseSelection: selectedPhases, projectChangeAnalyzer, @@ -79,6 +80,7 @@ function createShellOperations( if (commandToRun) { operation.runner = new ShellOperationRunner({ buildCacheConfiguration, + cobuildConfiguration, commandToRun: commandToRun || '', displayName, isIncrementalBuildAllowed, diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 3c030e619d4..41af93fe6ea 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -166,15 +166,9 @@ describe(AsyncOperationQueue.name, () => { if (operation === operations[1]) { if (!remoteExecuted) { operations[1].status = OperationStatus.RemoteExecuting; - AsyncOperationQueue.setOperationConsumersStatusRecursively( - operations[1], - OperationStatus.RemotePending - ); // remote executed operation is finished later remoteExecuted = true; continue; - } else { - AsyncOperationQueue.setOperationConsumersStatusRecursively(operations[1], OperationStatus.Ready); } } for (const consumer of operation.consumers) { diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 1c2e893b67d..237fcfd7eff 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -8,10 +8,10 @@ import type { BuildCacheConfiguration } from '../api/BuildCacheConfiguration'; import type { IPhase } from '../api/CommandLineConfiguration'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; - import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; -import { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; +import type { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; +import type { CobuildConfiguration } from '../api/CobuildConfiguration'; /** * A plugin that interacts with a phased commands. @@ -33,6 +33,10 @@ export interface ICreateOperationsContext { * The configuration for the build cache, if the feature is enabled. */ readonly buildCacheConfiguration: BuildCacheConfiguration | undefined; + /** + * The configuration for the cobuild, if cobuild feature and build cache feature are both enabled. + */ + readonly cobuildConfiguration: CobuildConfiguration | undefined; /** * The set of custom parameters for the executing command. * Maps from the `longName` field in command-line.json to the parser configuration in ts-command-line. diff --git a/libraries/rush-lib/src/pluginFramework/RushSession.ts b/libraries/rush-lib/src/pluginFramework/RushSession.ts index aa644ccc4d5..d01a16f533a 100644 --- a/libraries/rush-lib/src/pluginFramework/RushSession.ts +++ b/libraries/rush-lib/src/pluginFramework/RushSession.ts @@ -2,11 +2,14 @@ // See LICENSE in the project root for license information. import { InternalError, ITerminalProvider } from '@rushstack/node-core-library'; -import { IBuildCacheJson } from '../api/BuildCacheConfiguration'; -import { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider'; import { ILogger, ILoggerOptions, Logger } from './logging/Logger'; import { RushLifecycleHooks } from './RushLifeCycle'; +import type { IBuildCacheJson } from '../api/BuildCacheConfiguration'; +import type { ICloudBuildCacheProvider } from '../logic/buildCache/ICloudBuildCacheProvider'; +import type { ICobuildJson } from '../api/CobuildConfiguration'; +import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; + /** * @beta */ @@ -20,12 +23,18 @@ export interface IRushSessionOptions { */ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) => ICloudBuildCacheProvider; +/** + * @beta + */ +export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; + /** * @beta */ export class RushSession { private readonly _options: IRushSessionOptions; private readonly _cloudBuildCacheProviderFactories: Map = new Map(); + private readonly _cobuildLockProviderFactories: Map = new Map(); public readonly hooks: RushLifecycleHooks; @@ -68,4 +77,22 @@ export class RushSession { ): CloudBuildCacheProviderFactory | undefined { return this._cloudBuildCacheProviderFactories.get(cacheProviderName); } + + public registerCobuildLockProviderFactory( + cobuildLockProviderName: string, + factory: CobuildLockProviderFactory + ): void { + if (this._cobuildLockProviderFactories.has(cobuildLockProviderName)) { + throw new Error( + `A cobuild lock provider factory for ${cobuildLockProviderName} has already been registered` + ); + } + this._cobuildLockProviderFactories.set(cobuildLockProviderName, factory); + } + + public getCobuildLockProviderFactory( + cobuildLockProviderName: string + ): CobuildLockProviderFactory | undefined { + return this._cobuildLockProviderFactories.get(cobuildLockProviderName); + } } diff --git a/libraries/rush-lib/src/schemas/cobuild.schema.json b/libraries/rush-lib/src/schemas/cobuild.schema.json new file mode 100644 index 00000000000..5b13a4ec631 --- /dev/null +++ b/libraries/rush-lib/src/schemas/cobuild.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Configuration for Rush's cobuild.", + "description": "For use with the Rush tool, this file provides configuration options for cobuild feature. See http://rushjs.io for details.", + "definitions": { + "anything": { + "type": ["array", "boolean", "integer", "number", "object", "string"], + "items": { + "$ref": "#/definitions/anything" + } + } + }, + "type": "object", + "allOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["cobuildEnabled", "cobuildLockProvider"], + "properties": { + "$schema": { + "description": "Part of the JSON Schema standard, this optional keyword declares the URL of the schema that the file conforms to. Editors may download the schema and use it to perform syntax highlighting.", + "type": "string" + }, + "cobuildEnabled": { + "description": "Set this to true to enable the cobuild feature.", + "type": "boolean" + }, + "cobuildLockProvider": { + "description": "Specify the cobuild lock provider to use", + "type": "string" + }, + "cobuildContextIdPattern": { + "type": "string", + "description": "Setting this property overrides the cobuild context ID." + } + } + } + ] +} From 2382a94e5a75f53d7fcbdcb2df194c5882be9cff Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 10 Feb 2023 11:47:39 +0800 Subject: [PATCH 003/100] chore --- libraries/rush-lib/src/api/EnvironmentConfiguration.ts | 2 +- libraries/rush-lib/src/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 269b571d983..345f98f5fd1 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -150,7 +150,7 @@ export enum EnvironmentVariableNames { * @remarks * Specify `1` to enable the cobuild or `0` to disable it. * - * If there is no build cache configured, then this environment variable is ignored. + * If there is no cobuild configured, then this environment variable is ignored. */ RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 18fbed045f0..3b6022658c4 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -118,6 +118,7 @@ export { IRushPluginConfigurationBase as _IRushPluginConfigurationBase } from '. export { ILogger } from './pluginFramework/logging/Logger'; export { ICloudBuildCacheProvider } from './logic/buildCache/ICloudBuildCacheProvider'; +export { ICobuildLockProvider } from './logic/cobuild/ICobuildLockProvider'; export { ICredentialCacheOptions, ICredentialCacheEntry, CredentialCache } from './logic/CredentialCache'; From fedf61ad6e0d1bf1bcf255655d3b8e86bd3697e5 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 10 Feb 2023 23:45:52 +0800 Subject: [PATCH 004/100] fix: async queue --- .../rush-lib/src/logic/operations/AsyncOperationQueue.ts | 4 ---- .../src/logic/operations/test/AsyncOperationQueue.test.ts | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index ffea8cc71e4..8bd625da4e2 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -84,10 +84,6 @@ export class AsyncOperationQueue return; } - queue.forEach((q) => { - console.log(q.name, q.status); - }); - // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 41af93fe6ea..212c7c2edd3 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -66,6 +66,8 @@ describe(AsyncOperationQueue.name, () => { for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } + operation.status = OperationStatus.Success; + queue.complete(); } expect(actualOrder).toEqual(expectedOrder); From 1678c2c79d3b297a70c2022a5bfae68d3cacf385 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 11 Feb 2023 00:11:22 +0800 Subject: [PATCH 005/100] feat: registry of cobuild lock provider & redis cobuild plugin --- .../rush/nonbrowser-approved-packages.json | 4 + common/config/rush/pnpm-lock.yaml | 42 ++++++ common/config/rush/repo-state.json | 2 +- common/reviews/api/rush-lib.api.md | 34 ++++- .../api/rush-redis-cobuild-plugin.api.md | 29 +++++ libraries/rush-lib/src/index.ts | 6 +- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 91 ++++--------- .../src/logic/cobuild/ICobuildLockProvider.ts | 41 +++--- .../rush-redis-cobuild-plugin/.eslintrc.js | 10 ++ .../rush-redis-cobuild-plugin/.npmignore | 32 +++++ .../rush-redis-cobuild-plugin/LICENSE | 24 ++++ .../rush-redis-cobuild-plugin/README.md | 9 ++ .../config/api-extractor.json | 16 +++ .../config/jest.config.json | 3 + .../rush-redis-cobuild-plugin/config/rig.json | 7 + .../rush-redis-cobuild-plugin/package.json | 34 +++++ .../rush-plugin-manifest.json | 11 ++ .../src/RedisCobuildLockProvider.ts | 121 ++++++++++++++++++ .../src/RushRedisCobuildPlugin.ts | 40 ++++++ .../rush-redis-cobuild-plugin/src/index.ts | 7 + .../src/schemas/redis-config.schema.json | 70 ++++++++++ .../src/test/RedisCobuildLockProvider.test.ts | 93 ++++++++++++++ .../RedisCobuildLockProvider.test.ts.snap | 5 + .../rush-redis-cobuild-plugin/tsconfig.json | 8 ++ rush.json | 6 + 25 files changed, 659 insertions(+), 86 deletions(-) create mode 100644 common/reviews/api/rush-redis-cobuild-plugin.api.md create mode 100644 rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js create mode 100644 rush-plugins/rush-redis-cobuild-plugin/.npmignore create mode 100644 rush-plugins/rush-redis-cobuild-plugin/LICENSE create mode 100644 rush-plugins/rush-redis-cobuild-plugin/README.md create mode 100644 rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/config/rig.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/package.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/index.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts create mode 100644 rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap create mode 100644 rush-plugins/rush-redis-cobuild-plugin/tsconfig.json diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index d28a686b68e..0bc0e61f0eb 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -74,6 +74,10 @@ "name": "@pnpm/logger", "allowedCategories": [ "libraries" ] }, + { + "name": "@redis/client", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/debug-certificate-manager", "allowedCategories": [ "libraries" ] diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 2a1a2e1f46e..50f93c9f999 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -2416,6 +2416,29 @@ importers: '@types/heft-jest': 1.0.1 '@types/node': 14.18.36 + ../../rush-plugins/rush-redis-cobuild-plugin: + specifiers: + '@microsoft/rush-lib': workspace:* + '@redis/client': ~1.5.5 + '@rushstack/eslint-config': workspace:* + '@rushstack/heft': workspace:* + '@rushstack/heft-node-rig': workspace:* + '@rushstack/node-core-library': workspace:* + '@rushstack/rush-sdk': workspace:* + '@types/heft-jest': 1.0.1 + '@types/node': 14.18.36 + dependencies: + '@redis/client': 1.5.5 + '@rushstack/node-core-library': link:../../libraries/node-core-library + '@rushstack/rush-sdk': link:../../libraries/rush-sdk + devDependencies: + '@microsoft/rush-lib': link:../../libraries/rush-lib + '@rushstack/eslint-config': link:../../eslint/eslint-config + '@rushstack/heft': link:../../apps/heft + '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig + '@types/heft-jest': 1.0.1 + '@types/node': 14.18.36 + ../../rush-plugins/rush-serve-plugin: specifiers: '@rushstack/debug-certificate-manager': workspace:* @@ -6444,6 +6467,15 @@ packages: react: 16.13.1 dev: true + /@redis/client/1.5.5: + resolution: {integrity: sha512-fuMnpDYSjT5JXR9rrCW1YWA4L8N/9/uS4ImT3ZEC/hcaQRI1D/9FvwjriRj1UvepIgzZXthFVKMNRzP/LNL7BQ==} + engines: {node: '>=14'} + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + dev: false + /@reduxjs/toolkit/1.8.6_qfynotfwlyrsyq662adyrweaoe: resolution: {integrity: sha512-4Ia/Loc6WLmdSOzi7k5ff7dLK8CgG2b8aqpLsCAJhazAzGdp//YBUSaj0ceW6a3kDBDNRrq5CRwyCS0wBiL1ig==} peerDependencies: @@ -11236,6 +11268,11 @@ packages: engines: {node: '>=6'} dev: true + /cluster-key-slot/1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /cmd-extension/1.0.2: resolution: {integrity: sha512-iWDjmP8kvsMdBmLTHxFaqXikO8EdFRDfim7k6vUHglY/2xJ5jLrPsnQGijdfp4U+sr/BeecG0wKm02dSIAeQ1g==} engines: {node: '>=10'} @@ -14076,6 +14113,11 @@ packages: loader-utils: 1.4.2 dev: false + /generic-pool/3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + dev: false + /gensync/1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 9d0e76159ca..f31d8076457 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "b7c7c11ce97089924eb38eb913f8062e986ae06a", + "pnpmShrinkwrapHash": "b40972486600f4a66b11abdd57d7ff9a0032c357", "preferredVersionsHash": "5222ca779ae69ebfd201e39c17f48ce9eaf8c3c2" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 242dffd798d..cac66ad13c1 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -98,7 +98,6 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { readonly cobuildEnabled: boolean; - // Warning: (ae-forgotten-export) The symbol "ICobuildLockProvider" needs to be exported by the entry point index.d.ts readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) get contextId(): string; @@ -263,6 +262,39 @@ export interface ICloudBuildCacheProvider { updateCachedCredentialInteractiveAsync(terminal: ITerminal): Promise; } +// @beta (undocumented) +export interface ICobuildCompletedState { + cacheId: string; + // (undocumented) + status: OperationStatus.Success | OperationStatus.SuccessWithWarning | OperationStatus.Failure; +} + +// @beta (undocumented) +export interface ICobuildContext { + // (undocumented) + cacheId: string; + // (undocumented) + contextId: string; + // (undocumented) + terminal: ITerminal; + // (undocumented) + version: number; +} + +// @beta (undocumented) +export interface ICobuildLockProvider { + // (undocumented) + acquireLockAsync(context: ICobuildContext): Promise; + // (undocumented) + getCompletedStateAsync(context: ICobuildContext): Promise; + // (undocumented) + releaseLockAsync(context: ICobuildContext): Promise; + // (undocumented) + renewLockAsync(context: ICobuildContext): Promise; + // (undocumented) + setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; +} + // @public export interface IConfigurationEnvironment { [environmentVariableName: string]: IConfigurationEnvironmentVariable; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md new file mode 100644 index 00000000000..6a73c0c36bc --- /dev/null +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -0,0 +1,29 @@ +## API Report File for "@rushstack/rush-redis-cobuild-plugin" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import type { IRushPlugin } from '@rushstack/rush-sdk'; +import type { RedisClientOptions } from '@redis/client'; +import type { RushConfiguration } from '@rushstack/rush-sdk'; +import type { RushSession } from '@rushstack/rush-sdk'; + +// @public +export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { +} + +// @public (undocumented) +class RushRedisCobuildPlugin implements IRushPlugin { + // Warning: (ae-forgotten-export) The symbol "IRushRedisCobuildPluginOptions" needs to be exported by the entry point index.d.ts + constructor(options: IRushRedisCobuildPluginOptions); + // (undocumented) + apply(rushSession: RushSession, rushConfiguration: RushConfiguration): void; + // (undocumented) + pluginName: string; +} +export default RushRedisCobuildPlugin; + +// (No @packageDocumentation comment for this package) + +``` diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 3b6022658c4..55550568353 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -118,7 +118,11 @@ export { IRushPluginConfigurationBase as _IRushPluginConfigurationBase } from '. export { ILogger } from './pluginFramework/logging/Logger'; export { ICloudBuildCacheProvider } from './logic/buildCache/ICloudBuildCacheProvider'; -export { ICobuildLockProvider } from './logic/cobuild/ICobuildLockProvider'; +export { + ICobuildLockProvider, + ICobuildContext, + ICobuildCompletedState +} from './logic/cobuild/ICobuildLockProvider'; export { ICredentialCacheOptions, ICredentialCacheEntry, CredentialCache } from './logic/CredentialCache'; diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 6746c3d3b52..9343640b573 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -1,12 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { InternalError, ITerminal } from '@rushstack/node-core-library'; import { RushConstants } from '../RushConstants'; -import type { ITerminal } from '@rushstack/node-core-library'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { OperationStatus } from '../operations/OperationStatus'; +import type { OperationStatus } from '../operations/OperationStatus'; +import type { ICobuildContext } from './ICobuildLockProvider'; export interface ICobuildLockOptions { cobuildConfiguration: CobuildConfiguration; @@ -19,97 +20,55 @@ export interface ICobuildCompletedState { cacheId: string; } -const KEY_SEPARATOR: string = ':'; -const COMPLETED_STATE_SEPARATOR: string = ';'; - export class CobuildLock { - public readonly options: ICobuildLockOptions; - public readonly lockKey: string; - public readonly completedKey: string; - public readonly projectBuildCache: ProjectBuildCache; public readonly cobuildConfiguration: CobuildConfiguration; + private _cobuildContext: ICobuildContext; + public constructor(options: ICobuildLockOptions) { - this.options = options; - const { cobuildConfiguration, projectBuildCache } = options; + const { cobuildConfiguration, projectBuildCache, terminal } = options; this.projectBuildCache = projectBuildCache; this.cobuildConfiguration = cobuildConfiguration; const { contextId } = cobuildConfiguration; const { cacheId } = projectBuildCache; - // Example: cobuild:v1:::lock - this.lockKey = ['cobuild', `v${RushConstants.cobuildLockVersion}`, contextId, cacheId, 'lock'].join( - KEY_SEPARATOR - ); - // Example: cobuild:v1:::completed - this.completedKey = [ - 'cobuild', - `v${RushConstants.cobuildLockVersion}`, + + if (!cacheId) { + // This should never happen + throw new InternalError(`Cache id is require for cobuild lock`); + } + + this._cobuildContext = { + terminal, contextId, cacheId, - 'completed' - ].join(KEY_SEPARATOR); + version: RushConstants.cobuildLockVersion + }; } public async setCompletedStateAsync(state: ICobuildCompletedState): Promise { - const { terminal } = this.options; - const serializedState: string = this._serializeCompletedState(state); - terminal.writeDebugLine(`Set completed state by key ${this.completedKey}: ${serializedState}`); - await this.cobuildConfiguration.cobuildLockProvider.setCompletedStateAsync({ - key: this.completedKey, - value: serializedState, - terminal - }); + await this.cobuildConfiguration.cobuildLockProvider.setCompletedStateAsync(this._cobuildContext, state); } public async getCompletedStateAsync(): Promise { - const { terminal } = this.options; - const state: string | undefined = - await this.cobuildConfiguration.cobuildLockProvider.getCompletedStateAsync({ - key: this.completedKey, - terminal - }); - terminal.writeDebugLine(`Get completed state by key ${this.completedKey}: ${state}`); - if (!state) { - return; - } - return this._deserializeCompletedState(state); + const state: ICobuildCompletedState | undefined = + await this.cobuildConfiguration.cobuildLockProvider.getCompletedStateAsync(this._cobuildContext); + return state; } public async tryAcquireLockAsync(): Promise { - const { terminal } = this.options; - // const result: boolean = true; - // const result: boolean = false; - // const result: boolean = Math.random() > 0.5; - const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync({ - lockKey: this.lockKey, - terminal - }); - terminal.writeDebugLine(`Acquired lock for ${this.lockKey}, result: ${acquireLockResult}`); + const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync( + this._cobuildContext + ); return acquireLockResult; } public async releaseLockAsync(): Promise { - const { terminal } = this.options; - terminal.writeDebugLine(`Released lock for ${this.lockKey}`); - return; + await this.cobuildConfiguration.cobuildLockProvider.releaseLockAsync(this._cobuildContext); } public async renewLockAsync(): Promise { - const { terminal } = this.options; - terminal.writeDebugLine(`Renewed lock for ${this.lockKey}`); - return; - } - - private _serializeCompletedState(state: ICobuildCompletedState): string { - // Example: SUCCESS;1234567890 - // Example: FAILURE;1234567890 - return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; - } - - private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { - const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); - return { status: status as ICobuildCompletedState['status'], cacheId }; + await this.cobuildConfiguration.cobuildLockProvider.renewLockAsync(this._cobuildContext); } } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index 4735fc1542f..b95327373b6 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -1,31 +1,38 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { ITerminal } from '@rushstack/node-core-library'; +import type { ITerminal } from '@rushstack/node-core-library'; +import type { OperationStatus } from '../operations/OperationStatus'; -export interface ILockOptions { - lockKey: string; - terminal: ITerminal; -} - -export interface IGetCompletedStateOptions { - key: string; +/** + * @beta + */ +export interface ICobuildContext { + contextId: string; + cacheId: string; + version: number; terminal: ITerminal; } -export interface ISetCompletedStateOptions { - key: string; - value: string; - terminal: ITerminal; +/** + * @beta + */ +export interface ICobuildCompletedState { + status: OperationStatus.Success | OperationStatus.SuccessWithWarning | OperationStatus.Failure; + /** + * Completed state points to the cache id that was used to store the build cache. + * Note: Cache failed builds in a separate cache id + */ + cacheId: string; } /** * @beta */ export interface ICobuildLockProvider { - acquireLockAsync(options: ILockOptions): Promise; - renewLockAsync(options: ILockOptions): Promise; - releaseLockAsync(options: ILockOptions): Promise; - setCompletedStateAsync(options: ISetCompletedStateOptions): Promise; - getCompletedStateAsync(options: IGetCompletedStateOptions): Promise; + acquireLockAsync(context: ICobuildContext): Promise; + renewLockAsync(context: ICobuildContext): Promise; + releaseLockAsync(context: ICobuildContext): Promise; + setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; + getCompletedStateAsync(context: ICobuildContext): Promise; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js b/rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js new file mode 100644 index 00000000000..4c934799d67 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/.eslintrc.js @@ -0,0 +1,10 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('@rushstack/eslint-config/patch/modern-module-resolution'); + +module.exports = { + extends: [ + '@rushstack/eslint-config/profile/node-trusted-tool', + '@rushstack/eslint-config/mixins/friendly-locals' + ], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/rush-plugins/rush-redis-cobuild-plugin/.npmignore b/rush-plugins/rush-redis-cobuild-plugin/.npmignore new file mode 100644 index 00000000000..fcd991b60fa --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/.npmignore @@ -0,0 +1,32 @@ +# THIS IS A STANDARD TEMPLATE FOR .npmignore FILES IN THIS REPO. + +# Ignore all files by default, to avoid accidentally publishing unintended files. +* + +# Use negative patterns to bring back the specific things we want to publish. +!/bin/** +!/lib/** +!/lib-*/** +!/dist/** +!ThirdPartyNotice.txt + +# Ignore certain patterns that should not get published. +/dist/*.stats.* +/lib/**/test/ +/lib-*/**/test/ +*.test.js + +# NOTE: These don't need to be specified, because NPM includes them automatically. +# +# package.json +# README (and its variants) +# CHANGELOG (and its variants) +# LICENSE / LICENCE + +#-------------------------------------------- +# DO NOT MODIFY THE TEMPLATE ABOVE THIS LINE +#-------------------------------------------- + +# (Add your project-specific overrides here) +!/includes/** +!rush-plugin-manifest.json diff --git a/rush-plugins/rush-redis-cobuild-plugin/LICENSE b/rush-plugins/rush-redis-cobuild-plugin/LICENSE new file mode 100644 index 00000000000..0fbcd4e5857 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/LICENSE @@ -0,0 +1,24 @@ +@rushstack/rush-redis-cobuild-plugin + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rush-plugins/rush-redis-cobuild-plugin/README.md b/rush-plugins/rush-redis-cobuild-plugin/README.md new file mode 100644 index 00000000000..bfb2d49760b --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/README.md @@ -0,0 +1,9 @@ +# @rushstack/rush-amazon-s3-build-cache-plugin + +This is a Rush plugin for using Redis as cobuild lock provider during the "build" + +## Links + +- [CHANGELOG.md]( + https://github.com/microsoft/rushstack/blob/main/rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) - Find + out what's new in the latest version diff --git a/rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json b/rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json new file mode 100644 index 00000000000..74590d3c4f8 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/config/api-extractor.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + "mainEntryPointFilePath": "/lib/index.d.ts", + "apiReport": { + "enabled": true, + "reportFolder": "../../../common/reviews/api" + }, + "docModel": { + "enabled": false + }, + "dtsRollup": { + "enabled": true, + "betaTrimmedFilePath": "/dist/.d.ts" + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json b/rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json new file mode 100644 index 00000000000..4bb17bde3ee --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/config/jest.config.json @@ -0,0 +1,3 @@ +{ + "extends": "@rushstack/heft-node-rig/profiles/default/config/jest.config.json" +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/config/rig.json b/rush-plugins/rush-redis-cobuild-plugin/config/rig.json new file mode 100644 index 00000000000..6ac88a96368 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/config/rig.json @@ -0,0 +1,7 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + + "rigPackageName": "@rushstack/heft-node-rig" +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/package.json b/rush-plugins/rush-redis-cobuild-plugin/package.json new file mode 100644 index 00000000000..ce88b68a818 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/package.json @@ -0,0 +1,34 @@ +{ + "name": "@rushstack/rush-redis-cobuild-plugin", + "version": "5.88.2", + "description": "Rush plugin for Redis cobuild lock", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/rushstack", + "directory": "rush-plugins/rush-redis-cobuild-plugin" + }, + "homepage": "https://rushjs.io", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "license": "MIT", + "scripts": { + "build": "heft build --clean", + "start": "heft test --clean --watch", + "test": "heft test", + "_phase:build": "heft build --clean", + "_phase:test": "heft test --no-build" + }, + "dependencies": { + "@redis/client": "~1.5.5", + "@rushstack/node-core-library": "workspace:*", + "@rushstack/rush-sdk": "workspace:*" + }, + "devDependencies": { + "@microsoft/rush-lib": "workspace:*", + "@rushstack/eslint-config": "workspace:*", + "@rushstack/heft": "workspace:*", + "@rushstack/heft-node-rig": "workspace:*", + "@types/heft-jest": "1.0.1", + "@types/node": "14.18.36" + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json b/rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json new file mode 100644 index 00000000000..64ebf15d963 --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/rush-plugin-manifest.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json", + "plugins": [ + { + "pluginName": "rush-redis-cobuild-plugin", + "description": "Rush plugin for Redis cobuild lock", + "entryPoint": "lib/index.js", + "optionsSchema": "lib/schemas/redis-config.schema.json" + } + ] +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts new file mode 100644 index 00000000000..bef340d9abd --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { createClient } from '@redis/client'; + +import type { ICobuildLockProvider, ICobuildContext, ICobuildCompletedState } from '@rushstack/rush-sdk'; +import type { + RedisClientOptions, + RedisClientType, + RedisFunctions, + RedisModules, + RedisScripts +} from '@redis/client'; + +/** + * The redis client options + * @public + */ +export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} + +const KEY_SEPARATOR: string = ':'; +const COMPLETED_STATE_SEPARATOR: string = ';'; + +export class RedisCobuildLockProvider implements ICobuildLockProvider { + private readonly _options: IRedisCobuildLockProviderOptions; + + private _redisClient: RedisClientType; + private _lockKeyMap: WeakMap = new WeakMap(); + private _completedKeyMap: WeakMap = new WeakMap(); + + public constructor(options: IRedisCobuildLockProviderOptions) { + this._options = options; + this._redisClient = createClient(this._options); + } + + public async acquireLockAsync(context: ICobuildContext): Promise { + const { terminal } = context; + const lockKey: string = this.getLockKey(context); + const incrResult: number = await this._redisClient.incr(lockKey); + const result: boolean = incrResult === 1; + terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); + if (result) { + await this.renewLockAsync(context); + } + return result; + } + + public async renewLockAsync(context: ICobuildContext): Promise { + const { terminal } = context; + const lockKey: string = this.getLockKey(context); + await this._redisClient.expire(lockKey, 30); + terminal.writeDebugLine(`Renewed lock for ${lockKey}`); + } + + public async releaseLockAsync(context: ICobuildContext): Promise { + const { terminal } = context; + const lockKey: string = this.getLockKey(context); + await this._redisClient.set(lockKey, 0); + terminal.writeDebugLine(`Released lock for ${lockKey}`); + } + + public async setCompletedStateAsync( + context: ICobuildContext, + state: ICobuildCompletedState + ): Promise { + const { terminal } = context; + const key: string = this.getCompletedStateKey(context); + const value: string = this._serializeCompletedState(state); + await this._redisClient.set(key, value); + terminal.writeDebugLine(`Set completed state for ${key}: ${value}`); + } + + public async getCompletedStateAsync(context: ICobuildContext): Promise { + const key: string = this.getCompletedStateKey(context); + let state: ICobuildCompletedState | undefined; + const value: string | null = await this._redisClient.get(key); + if (value) { + state = this._deserializeCompletedState(value); + } + return state; + } + + /** + * Returns the lock key for the given context + * Example: cobuild:v1:::lock + */ + public getLockKey(context: ICobuildContext): string { + const { version, contextId, cacheId } = context; + let lockKey: string | undefined = this._lockKeyMap.get(context); + if (!lockKey) { + lockKey = ['cobuild', `v${version}`, contextId, cacheId, 'lock'].join(KEY_SEPARATOR); + this._completedKeyMap.set(context, lockKey); + } + return lockKey; + } + + /** + * Returns the completed key for the given context + * Example: cobuild:v1:::completed + */ + public getCompletedStateKey(context: ICobuildContext): string { + const { version, contextId, cacheId } = context; + let completedKey: string | undefined = this._completedKeyMap.get(context); + if (!completedKey) { + completedKey = ['cobuild', `v${version}`, contextId, cacheId, 'completed'].join(KEY_SEPARATOR); + this._completedKeyMap.set(context, completedKey); + } + return completedKey; + } + + private _serializeCompletedState(state: ICobuildCompletedState): string { + // Example: SUCCESS;1234567890 + // Example: FAILURE;1234567890 + return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; + } + + private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { + const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); + return { status: status as ICobuildCompletedState['status'], cacheId }; + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts new file mode 100644 index 00000000000..1d1d69fdaeb --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Import } from '@rushstack/node-core-library'; +import type { IRushPlugin, RushSession, RushConfiguration } from '@rushstack/rush-sdk'; +import type { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from './RedisCobuildLockProvider'; + +const RedisCobuildLockProviderModule: typeof import('./RedisCobuildLockProvider') = Import.lazy( + './RedisCobuildLockProvider', + require +); + +const PLUGIN_NAME: string = 'RedisCobuildPlugin'; + +/** + * @public + */ +export type IRushRedisCobuildPluginOptions = IRedisCobuildLockProviderOptions; + +/** + * @public + */ +export class RushRedisCobuildPlugin implements IRushPlugin { + public pluginName: string = PLUGIN_NAME; + + private _options: IRushRedisCobuildPluginOptions; + + public constructor(options: IRushRedisCobuildPluginOptions) { + this._options = options; + } + + public apply(rushSession: RushSession, rushConfiguration: RushConfiguration): void { + rushSession.hooks.initialize.tap(PLUGIN_NAME, () => { + rushSession.registerCobuildLockProviderFactory('redis', (): RedisCobuildLockProvider => { + const options: IRushRedisCobuildPluginOptions = this._options; + return new RedisCobuildLockProviderModule.RedisCobuildLockProvider(options); + }); + }); + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts new file mode 100644 index 00000000000..d4466255b4c --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; + +export default RushRedisCobuildPlugin; +export type { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json new file mode 100644 index 00000000000..4d984d81b3e --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json @@ -0,0 +1,70 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Configuration for cobuild lock with Redis configuration\n\nhttps://github.com/redis/node-redis/blob/master/docs/client-configuration.md", + "type": "object", + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "redis[s]://[[username][:password]@][host][:port][/db-number]\n\n See the following links for more information:\n\nredis: https://www.iana.org/assignments/uri-schemes/prov/redis\n\nrediss: https://www.iana.org/assignments/uri-schemes/prov/rediss" + }, + "socket": { + "type": "object", + "description": "Socket connection properties. Unlisted net.connect properties (and tls.connect) are also supported", + "properties": { + "port": { + "description": "Redis server port. Default value is 6379", + "type": "number" + }, + "host": { + "description": "Redis server host. Default value is localhost", + "type": "string" + }, + "family": { + "description": "IP Stack version (one of 4 | 6 | 0). Default value is 0", + "type": "number" + }, + "path": { + "description": "path to the UNIX Socket", + "type": "string" + }, + "connectTimeout": { + "description": "Connection timeout in milliseconds. Default value is 5000", + "type": "number" + }, + "noDelay": { + "description": "Toggle Nagle's algorithm. Default value is true", + "type": "boolean" + }, + "keepAlive": { + "description": "Toggle keep alive on the socket", + "type": "boolean" + } + } + }, + "username": { + "description": "ACL username", + "type": "string" + }, + "password": { + "description": "ACL password", + "type": "string" + }, + "name": { + "description": "Redis client name", + "type": "string" + }, + "database": { + "description": "Redis database number", + "type": "number" + }, + "legacyMode": { + "description": "Maintain some backwards compatibility", + "type": "boolean" + }, + "pingInterval": { + "description": "Send PING command at interval (in ms). Useful with \"Azure Cache for Redis\".", + "type": "number" + } + } +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts new file mode 100644 index 00000000000..90f0cbe0aea --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -0,0 +1,93 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { Terminal, ConsoleTerminalProvider } from '@rushstack/node-core-library'; +import { ICobuildCompletedState, ICobuildContext, OperationStatus } from '@rushstack/rush-sdk'; +import { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from '../RedisCobuildLockProvider'; + +import * as redisAPI from '@redis/client'; +import type { RedisClientType } from '@redis/client'; + +const terminal = new Terminal(new ConsoleTerminalProvider()); + +describe(RedisCobuildLockProvider.name, () => { + let storage: Record = {}; + beforeEach(() => { + jest.spyOn(redisAPI, 'createClient').mockImplementation(() => { + return { + incr: jest.fn().mockImplementation((key: string) => { + storage[key] = (Number(storage[key]) || 0) + 1; + return storage[key]; + }), + expire: jest.fn().mockResolvedValue(undefined), + set: jest.fn().mockImplementation((key: string, value: string) => { + storage[key] = value; + }), + get: jest.fn().mockImplementation((key: string) => { + return storage[key]; + }) + } as unknown as RedisClientType; + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + storage = {}; + }); + + function prepareSubject(): RedisCobuildLockProvider { + return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions); + } + const context: ICobuildContext = { + contextId: '123', + cacheId: 'abc', + version: 1, + terminal + }; + + it('getLockKey works', () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const lockKey: string = subject.getLockKey(context); + expect(lockKey).toMatchSnapshot(); + }); + + it('getCompletedStateKey works', () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const completedStateKey: string = subject.getCompletedStateKey(context); + expect(completedStateKey).toMatchSnapshot(); + }); + + it('acquires lock success', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const result: boolean = await subject.acquireLockAsync(context); + expect(result).toBe(true); + }); + + it('acquires lock fails at the second time', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const cobuildContext: ICobuildContext = { + ...context, + contextId: 'abc' + }; + const result1: boolean = await subject.acquireLockAsync(cobuildContext); + expect(result1).toBe(true); + const result2: boolean = await subject.acquireLockAsync(cobuildContext); + expect(result2).toBe(false); + }); + + it('releaseLockAsync works', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + expect(() => subject.releaseLockAsync(context)).not.toThrowError(); + }); + + it('set and get completedState works', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const cacheId: string = 'foo'; + const status: ICobuildCompletedState['status'] = OperationStatus.SuccessWithWarning; + expect(() => subject.setCompletedStateAsync(context, { status, cacheId })).not.toThrowError(); + const actualState: ICobuildCompletedState | undefined = await subject.getCompletedStateAsync(context); + expect(actualState?.cacheId).toBe(cacheId); + expect(actualState?.status).toBe(status); + }); +}); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap new file mode 100644 index 00000000000..33aa6130bbf --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`RedisCobuildLockProvider getCompletedStateKey works 1`] = `"cobuild:v1:123:abc:completed"`; + +exports[`RedisCobuildLockProvider getLockKey works 1`] = `"cobuild:v1:123:abc:lock"`; diff --git a/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json new file mode 100644 index 00000000000..b3d3ff2a64f --- /dev/null +++ b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json", + + "compilerOptions": { + "lib": ["DOM"], + "types": ["heft-jest", "node"] + } +} diff --git a/rush.json b/rush.json index 085a872ecf9..db2d4a025db 100644 --- a/rush.json +++ b/rush.json @@ -1026,6 +1026,12 @@ "reviewCategory": "libraries", "shouldPublish": false }, + { + "packageName": "@rushstack/rush-redis-cobuild-plugin", + "projectFolder": "rush-plugins/rush-redis-cobuild-plugin", + "reviewCategory": "libraries", + "versionPolicyName": "rush" + }, { "packageName": "@rushstack/rush-serve-plugin", "projectFolder": "rush-plugins/rush-serve-plugin", From c2bb3e3453987e8bb7394e4bd8bad30d8dd85895 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 11 Feb 2023 00:14:41 +0800 Subject: [PATCH 006/100] chore --- rush.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rush.json b/rush.json index db2d4a025db..8202810d639 100644 --- a/rush.json +++ b/rush.json @@ -1030,7 +1030,7 @@ "packageName": "@rushstack/rush-redis-cobuild-plugin", "projectFolder": "rush-plugins/rush-redis-cobuild-plugin", "reviewCategory": "libraries", - "versionPolicyName": "rush" + "shouldPublish": true }, { "packageName": "@rushstack/rush-serve-plugin", From e3839da56851a17ba8f00f9eac4f5adb856b20e7 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Feb 2023 10:55:43 +0800 Subject: [PATCH 007/100] feat: rush-redis-cobuild-plugin-integration-test --- .../.eslintrc.js | 7 +++ .../.gitignore | 1 + .../README.md | 18 +++++++ .../config/heft.json | 51 +++++++++++++++++++ .../config/rush-project.json | 8 +++ .../docker-compose.yml | 10 ++++ .../package.json | 24 +++++++++ .../src/testLockProvider.ts | 43 ++++++++++++++++ .../tsconfig.json | 25 +++++++++ .../rush/nonbrowser-approved-packages.json | 4 ++ common/config/rush/pnpm-lock.yaml | 24 +++++++++ common/reviews/api/rush-lib.api.md | 8 +++ .../api/rush-redis-cobuild-plugin.api.md | 26 +++++++++- .../rush-lib/src/api/CobuildConfiguration.ts | 9 ++++ .../cli/scriptActions/PhasedScriptAction.ts | 3 ++ .../src/logic/cobuild/ICobuildLockProvider.ts | 2 + .../src/RedisCobuildLockProvider.ts | 15 +++++- .../rush-redis-cobuild-plugin/src/index.ts | 1 + .../src/test/RedisCobuildLockProvider.test.ts | 1 + rush.json | 6 +++ 20 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/README.md create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js b/build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js new file mode 100644 index 00000000000..60160b354c4 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.eslintrc.js @@ -0,0 +1,7 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('@rushstack/eslint-config/patch/modern-module-resolution'); + +module.exports = { + extends: ['@rushstack/eslint-config/profile/node'], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore new file mode 100644 index 00000000000..16a0ee1e146 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore @@ -0,0 +1 @@ +redis-data/dump.rdb diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md new file mode 100644 index 00000000000..87b7b6e0af9 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -0,0 +1,18 @@ +# About +This package enables integration testing of the `RedisCobuildLockProvider` by connecting to an actual Redis created using an [redis](https://hub.docker.com/_/redis) docker image. + +# Prerequisites +Docker and docker compose must be installed + +# Start the Redis +In this folder run `docker-compose up -d` + +# Stop the Redis +In this folder run `docker-compose down` + +# Run the test +```sh +# start the docker container: docker-compose up -d +# build the code: rushx build +rushx test-lock-provider +``` diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json b/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json new file mode 100644 index 00000000000..99e058540fb --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json @@ -0,0 +1,51 @@ +/** + * Defines configuration used by core Heft. + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/heft/heft.schema.json", + + "eventActions": [ + { + /** + * The kind of built-in operation that should be performed. + * The "deleteGlobs" action deletes files or folders that match the + * specified glob patterns. + */ + "actionKind": "deleteGlobs", + + /** + * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json + * occur at the end of the stage of the Heft run. + */ + "heftEvent": "clean", + + /** + * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other + * configs. + */ + "actionId": "defaultClean", + + /** + * Glob patterns to be deleted. The paths are resolved relative to the project folder. + */ + "globsToDelete": ["dist", "lib", "temp"] + } + ], + + /** + * The list of Heft plugins to be loaded. + */ + "heftPlugins": [ + // { + // /** + // * The path to the plugin package. + // */ + // "plugin": "path/to/my-plugin", + // + // /** + // * An optional object that provides additional settings that may be defined by the plugin. + // */ + // // "options": { } + // } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json new file mode 100644 index 00000000000..247dc17187a --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/config/rush-project.json @@ -0,0 +1,8 @@ +{ + "operationSettings": [ + { + "operationName": "build", + "outputFolderNames": ["lib", "dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml new file mode 100644 index 00000000000..4514fcd8223 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.7' + +services: + redis: + image: redis:6.2.10-alpine + command: redis-server --save 20 1 --loglevel warning --requirepass redis123 + ports: + - '6379:6379' + volumes: + - ./redis-data:/data diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json new file mode 100644 index 00000000000..a667eec050e --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json @@ -0,0 +1,24 @@ +{ + "name": "rush-redis-cobuild-plugin-integration-test", + "description": "Tests connecting to an redis server", + "version": "1.0.0", + "private": true, + "license": "MIT", + "scripts": { + "build": "heft build --clean", + "_phase:build": "heft build --clean", + "test-lock-provider": "node ./lib/testLockProvider.js" + }, + "devDependencies": { + "@rushstack/eslint-config": "workspace:*", + "@rushstack/heft": "workspace:*", + "@microsoft/rush-lib": "workspace:*", + "@rushstack/rush-redis-cobuild-plugin": "workspace:*", + "@rushstack/node-core-library": "workspace:*", + "@types/node": "12.20.24", + "eslint": "~8.7.0", + "typescript": "~4.8.4", + "http-proxy": "~1.18.1", + "@types/http-proxy": "~1.17.8" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts new file mode 100644 index 00000000000..820532437f1 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -0,0 +1,43 @@ +import { + RedisCobuildLockProvider, + IRedisCobuildLockProviderOptions +} from '@rushstack/rush-redis-cobuild-plugin'; +import { ConsoleTerminalProvider, ITerminal, Terminal } from '@rushstack/node-core-library'; +import { OperationStatus, ICobuildContext } from '@microsoft/rush-lib'; + +const options: IRedisCobuildLockProviderOptions = { + url: 'redis://localhost:6379', + password: 'redis123' +}; + +const terminal: ITerminal = new Terminal( + new ConsoleTerminalProvider({ + verboseEnabled: true, + debugEnabled: true + }) +); + +async function main(): Promise { + const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options); + await lockProvider.connectAsync(); + const context: ICobuildContext = { + terminal, + contextId: 'test-context-id', + version: 1, + cacheId: 'test-cache-id' + }; + await lockProvider.acquireLockAsync(context); + await lockProvider.renewLockAsync(context); + await lockProvider.setCompletedStateAsync(context, { + status: OperationStatus.Success, + cacheId: 'test-cache-id' + }); + await lockProvider.releaseLockAsync(context); + const completedState = await lockProvider.getCompletedStateAsync(context); + console.log('Completed state: ', completedState); + await lockProvider.disconnectAsync(); +} +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json new file mode 100644 index 00000000000..6314e94a07d --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json.schemastore.org/tsconfig", + + "compilerOptions": { + "outDir": "lib", + "rootDir": "src", + + "forceConsistentCasingInFileNames": true, + "jsx": "react", + "declaration": true, + "sourceMap": true, + "declarationMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictNullChecks": true, + "noUnusedLocals": true, + "types": ["node"], + + "module": "commonjs", + "target": "es2017", + "lib": ["es2017", "DOM"] + }, + "include": ["src/**/*.ts", "src/**/*.tsx"], + "exclude": ["node_modules", "lib"] +} diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index 0bc0e61f0eb..3294b64fdae 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -170,6 +170,10 @@ "name": "@rushstack/package-deps-hash", "allowedCategories": [ "libraries" ] }, + { + "name": "@rushstack/rush-redis-cobuild-plugin", + "allowedCategories": [ "tests" ] + }, { "name": "@rushstack/rig-package", "allowedCategories": [ "libraries" ] diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 50f93c9f999..8d3c15d3bb0 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1346,6 +1346,30 @@ importers: '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig '@types/node': 14.18.36 + ../../build-tests/rush-redis-cobuild-plugin-integration-test: + specifiers: + '@microsoft/rush-lib': workspace:* + '@rushstack/eslint-config': workspace:* + '@rushstack/heft': workspace:* + '@rushstack/node-core-library': workspace:* + '@rushstack/rush-redis-cobuild-plugin': workspace:* + '@types/http-proxy': ~1.17.8 + '@types/node': 12.20.24 + eslint: ~8.7.0 + http-proxy: ~1.18.1 + typescript: ~4.8.4 + devDependencies: + '@microsoft/rush-lib': link:../../libraries/rush-lib + '@rushstack/eslint-config': link:../../eslint/eslint-config + '@rushstack/heft': link:../../apps/heft + '@rushstack/node-core-library': link:../../libraries/node-core-library + '@rushstack/rush-redis-cobuild-plugin': link:../../rush-plugins/rush-redis-cobuild-plugin + '@types/http-proxy': 1.17.9 + '@types/node': 12.20.24 + eslint: 8.7.0 + http-proxy: 1.18.1 + typescript: 4.8.4 + ../../build-tests/set-webpack-public-path-plugin-webpack4-test: specifiers: '@rushstack/eslint-config': workspace:* diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index cac66ad13c1..b967a5aeac1 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -100,8 +100,12 @@ export class CobuildConfiguration { readonly cobuildEnabled: boolean; readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) + connectLockProviderAsync(): Promise; + // (undocumented) get contextId(): string; // (undocumented) + disconnectLockProviderAsync(): Promise; + // (undocumented) static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string; static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; } @@ -286,6 +290,10 @@ export interface ICobuildLockProvider { // (undocumented) acquireLockAsync(context: ICobuildContext): Promise; // (undocumented) + connectAsync(): Promise; + // (undocumented) + disconnectAsync(): Promise; + // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; // (undocumented) releaseLockAsync(context: ICobuildContext): Promise; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 6a73c0c36bc..4a2f99398e4 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -4,15 +4,39 @@ ```ts +import type { ICobuildCompletedState } from '@rushstack/rush-sdk'; +import type { ICobuildContext } from '@rushstack/rush-sdk'; +import type { ICobuildLockProvider } from '@rushstack/rush-sdk'; import type { IRushPlugin } from '@rushstack/rush-sdk'; import type { RedisClientOptions } from '@redis/client'; import type { RushConfiguration } from '@rushstack/rush-sdk'; import type { RushSession } from '@rushstack/rush-sdk'; -// @public +// @beta export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { } +// @beta (undocumented) +export class RedisCobuildLockProvider implements ICobuildLockProvider { + constructor(options: IRedisCobuildLockProviderOptions); + // (undocumented) + acquireLockAsync(context: ICobuildContext): Promise; + // (undocumented) + connectAsync(): Promise; + // (undocumented) + disconnectAsync(): Promise; + // (undocumented) + getCompletedStateAsync(context: ICobuildContext): Promise; + getCompletedStateKey(context: ICobuildContext): string; + getLockKey(context: ICobuildContext): string; + // (undocumented) + releaseLockAsync(context: ICobuildContext): Promise; + // (undocumented) + renewLockAsync(context: ICobuildContext): Promise; + // (undocumented) + setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; +} + // @public (undocumented) class RushRedisCobuildPlugin implements IRushPlugin { // Warning: (ae-forgotten-export) The symbol "IRushRedisCobuildPluginOptions" needs to be exported by the entry point index.d.ts diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 3f9d2a12dac..1b4b636c390 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -75,6 +75,7 @@ export class CobuildConfiguration { public static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string { return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.cobuildFilename); } + private static async _loadAsync( jsonFilePath: string, terminal: ITerminal, @@ -108,4 +109,12 @@ export class CobuildConfiguration { // FIXME: hardcode return '123'; } + + public async connectLockProviderAsync(): Promise { + await this.cobuildLockProvider.connectAsync(); + } + + public async disconnectLockProviderAsync(): Promise { + await this.cobuildLockProvider.disconnectAsync(); + } } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 99eb4c503b6..8cb5064fa3c 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -312,6 +312,7 @@ export class PhasedScriptAction extends BaseScriptAction { this.rushConfiguration, this.rushSession ); + await cobuildConfiguration?.connectLockProviderAsync(); } const projectSelection: Set = @@ -376,6 +377,8 @@ export class PhasedScriptAction extends BaseScriptAction { await this._runWatchPhases(internalOptions); } + + await cobuildConfiguration?.disconnectLockProviderAsync(); } private async _runInitialPhases(options: IRunPhasesOptions): Promise { diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index b95327373b6..a36bc8ab702 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -30,6 +30,8 @@ export interface ICobuildCompletedState { * @beta */ export interface ICobuildLockProvider { + connectAsync(): Promise; + disconnectAsync(): Promise; acquireLockAsync(context: ICobuildContext): Promise; renewLockAsync(context: ICobuildContext): Promise; releaseLockAsync(context: ICobuildContext): Promise; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index bef340d9abd..3af20ccbb66 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -14,13 +14,16 @@ import type { /** * The redis client options - * @public + * @beta */ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} const KEY_SEPARATOR: string = ':'; const COMPLETED_STATE_SEPARATOR: string = ';'; +/** + * @beta + */ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; @@ -33,6 +36,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { this._redisClient = createClient(this._options); } + public async connectAsync(): Promise { + await this._redisClient.connect(); + } + + public async disconnectAsync(): Promise { + await this._redisClient.disconnect(); + } + public async acquireLockAsync(context: ICobuildContext): Promise { const { terminal } = context; const lockKey: string = this.getLockKey(context); @@ -89,7 +100,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { let lockKey: string | undefined = this._lockKeyMap.get(context); if (!lockKey) { lockKey = ['cobuild', `v${version}`, contextId, cacheId, 'lock'].join(KEY_SEPARATOR); - this._completedKeyMap.set(context, lockKey); + this._lockKeyMap.set(context, lockKey); } return lockKey; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts index d4466255b4c..f627507d614 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -4,4 +4,5 @@ import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; export default RushRedisCobuildPlugin; +export { RedisCobuildLockProvider } from './RedisCobuildLockProvider'; export type { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 90f0cbe0aea..5cd78099676 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -39,6 +39,7 @@ describe(RedisCobuildLockProvider.name, () => { function prepareSubject(): RedisCobuildLockProvider { return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions); } + const context: ICobuildContext = { contextId: '123', cacheId: 'abc', diff --git a/rush.json b/rush.json index 8202810d639..b3a91990c4f 100644 --- a/rush.json +++ b/rush.json @@ -796,6 +796,12 @@ "reviewCategory": "tests", "shouldPublish": false }, + { + "packageName": "rush-redis-cobuild-plugin-integration-test", + "projectFolder": "build-tests/rush-redis-cobuild-plugin-integration-test", + "reviewCategory": "tests", + "shouldPublish": false + }, { "packageName": "set-webpack-public-path-plugin-webpack4-test", "projectFolder": "build-tests/set-webpack-public-path-plugin-webpack4-test", From dbf758f282d475a5e8a66a5de2e249236f1e3051 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 16 Feb 2023 16:06:21 +0800 Subject: [PATCH 008/100] fix: rush redis cobuild --- .gitignore | 7 +- .prettierignore | 3 + .vscode/redis-cobuild.code-workspace | 91 +++ .../.gitignore | 2 +- .../docker-compose.yml | 2 +- .../package.json | 14 +- .../sandbox/repo/.gitignore | 4 + .../rush-redis-cobuild-plugin.json | 4 + .../repo/common/config/rush/build-cache.json | 92 +++ .../repo/common/config/rush/cobuild.json | 25 + .../repo/common/config/rush/command-line.json | 308 +++++++++ .../repo/common/config/rush/pnpm-lock.yaml | 34 + .../repo/common/config/rush/repo-state.json | 4 + .../common/scripts/install-run-rush-pnpm.js | 28 + .../repo/common/scripts/install-run-rush.js | 214 ++++++ .../repo/common/scripts/install-run-rushx.js | 28 + .../repo/common/scripts/install-run.js | 645 ++++++++++++++++++ .../repo/projects/a/config/rush-project.json | 12 + .../sandbox/repo/projects/a/package.json | 8 + .../repo/projects/b/config/rush-project.json | 12 + .../sandbox/repo/projects/b/package.json | 8 + .../sandbox/repo/projects/build.js | 12 + .../repo/projects/c/config/rush-project.json | 12 + .../sandbox/repo/projects/c/package.json | 11 + .../repo/projects/d/config/rush-project.json | 12 + .../sandbox/repo/projects/d/package.json | 12 + .../repo/projects/e/config/rush-project.json | 12 + .../sandbox/repo/projects/e/package.json | 12 + .../sandbox/repo/rush.json | 29 + .../src/paths.ts | 5 + .../src/runRush.ts | 37 + .../src/testLockProvider.ts | 18 +- .../tsconfig.json | 1 + common/config/rush/pnpm-lock.yaml | 4 +- common/reviews/api/rush-lib.api.md | 2 - .../api/rush-redis-cobuild-plugin.api.md | 2 +- .../rush-init/common/config/rush/cobuild.json | 25 + .../rush-lib/src/api/RushConfiguration.ts | 1 + .../rush-lib/src/cli/actions/InitAction.ts | 1 + .../rush-lib/src/logic/cobuild/CobuildLock.ts | 6 +- .../src/logic/cobuild/ICobuildLockProvider.ts | 2 - .../logic/operations/AsyncOperationQueue.ts | 1 + .../operations/OperationExecutionManager.ts | 15 +- .../logic/operations/ShellOperationRunner.ts | 38 +- .../src/RedisCobuildLockProvider.ts | 34 +- .../src/RushRedisCobuildPlugin.ts | 2 +- .../src/test/RedisCobuildLockProvider.test.ts | 14 +- 47 files changed, 1793 insertions(+), 72 deletions(-) create mode 100644 .vscode/redis-cobuild.code-workspace create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts create mode 100644 libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json diff --git a/.gitignore b/.gitignore index 3d02c0474a5..9e44f747b7b 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,12 @@ jspm_packages/ *.iml # Visual Studio Code -.vscode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!*.code-workspace # Rush temporary files common/deploy/ diff --git a/.prettierignore b/.prettierignore index f577e87f844..5355633673c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -104,5 +104,8 @@ libraries/rush-lib/assets/rush-init/ # These are intentionally invalid files libraries/heft-config-file/src/test/errorCases/invalidJson/config.json +# common scripts in sandbox repo +build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/ + # We'll consider enabling this later; Prettier reformats code blocks, which affects end-user content *.md diff --git a/.vscode/redis-cobuild.code-workspace b/.vscode/redis-cobuild.code-workspace new file mode 100644 index 00000000000..9ab11bb4b4b --- /dev/null +++ b/.vscode/redis-cobuild.code-workspace @@ -0,0 +1,91 @@ +{ + "folders": [ + { + "name": "rush-redis-cobuild-plugin-integration-test", + "path": "../build-tests/rush-redis-cobuild-plugin-integration-test" + }, + { + "name": "rush-redis-cobuild-plugin", + "path": "../rush-plugins/rush-redis-cobuild-plugin" + }, + { + "name": "rush-lib", + "path": "../libraries/rush-lib" + }, + { + "name": ".vscode", + "path": "../.vscode" + } + ], + "tasks": { + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "cobuild", + "dependsOrder": "sequence", + "dependsOn": ["update 1", "_cobuild"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "_cobuild", + "dependsOn": ["build 1", "build 2"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "update", + "command": "node ../../lib/runRush.js update", + "problemMatcher": [], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": false + }, + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + } + }, + { + "type": "shell", + "label": "build 1", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + }, + { + "type": "shell", + "label": "build 2", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + } + ] + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore index 16a0ee1e146..97e8499abcc 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.gitignore @@ -1 +1 @@ -redis-data/dump.rdb +redis-data/dump.rdb \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml index 4514fcd8223..2b9a3f3722b 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.7' services: redis: image: redis:6.2.10-alpine - command: redis-server --save 20 1 --loglevel warning --requirepass redis123 + command: redis-server --save "" --loglevel warning --requirepass redis123 ports: - '6379:6379' volumes: diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json index a667eec050e..52ffe8f3cfb 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json @@ -1,24 +1,24 @@ { "name": "rush-redis-cobuild-plugin-integration-test", - "description": "Tests connecting to an redis server", "version": "1.0.0", "private": true, + "description": "Tests connecting to an redis server", "license": "MIT", "scripts": { - "build": "heft build --clean", "_phase:build": "heft build --clean", + "build": "heft build --clean", "test-lock-provider": "node ./lib/testLockProvider.js" }, "devDependencies": { + "@microsoft/rush-lib": "workspace:*", "@rushstack/eslint-config": "workspace:*", "@rushstack/heft": "workspace:*", - "@microsoft/rush-lib": "workspace:*", - "@rushstack/rush-redis-cobuild-plugin": "workspace:*", "@rushstack/node-core-library": "workspace:*", - "@types/node": "12.20.24", + "@rushstack/rush-redis-cobuild-plugin": "workspace:*", + "@types/http-proxy": "~1.17.8", + "@types/node": "14.18.36", "eslint": "~8.7.0", - "typescript": "~4.8.4", "http-proxy": "~1.18.1", - "@types/http-proxy": "~1.17.8" + "typescript": "~4.8.4" } } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore new file mode 100644 index 00000000000..484950696c9 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore @@ -0,0 +1,4 @@ +# Rush temporary files +common/deploy/ +common/temp/ +common/autoinstallers/*/.npmrc \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json new file mode 100644 index 00000000000..625dff477fc --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json @@ -0,0 +1,4 @@ +{ + "url": "redis://localhost:6379", + "password": "redis123" +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json new file mode 100644 index 00000000000..d09eaa6a04c --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/build-cache.json @@ -0,0 +1,92 @@ +/** + * This configuration file manages Rush's build cache feature. + * More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/build-cache.schema.json", + + /** + * (Required) EXPERIMENTAL - Set this to true to enable the build cache feature. + * + * See https://rushjs.io/pages/maintainer/build_cache/ for details about this experimental feature. + */ + "buildCacheEnabled": true, + + /** + * (Required) Choose where project build outputs will be cached. + * + * Possible values: "local-only", "azure-blob-storage", "amazon-s3" + */ + "cacheProvider": "local-only", + + /** + * Setting this property overrides the cache entry ID. If this property is set, it must contain + * a [hash] token. + * + * Other available tokens: + * - [projectName] + * - [projectName:normalize] + * - [phaseName] + * - [phaseName:normalize] + * - [phaseName:trimPrefix] + */ + // "cacheEntryNamePattern": "[projectName:normalize]-[phaseName:normalize]-[hash]" + + /** + * Use this configuration with "cacheProvider"="azure-blob-storage" + */ + "azureBlobStorageConfiguration": { + /** + * (Required) The name of the the Azure storage account to use for build cache. + */ + // "storageAccountName": "example", + /** + * (Required) The name of the container in the Azure storage account to use for build cache. + */ + // "storageContainerName": "my-container", + /** + * The Azure environment the storage account exists in. Defaults to AzurePublicCloud. + * + * Possible values: "AzurePublicCloud", "AzureChina", "AzureGermany", "AzureGovernment" + */ + // "azureEnvironment": "AzurePublicCloud", + /** + * An optional prefix for cache item blob names. + */ + // "blobPrefix": "my-prefix", + /** + * If set to true, allow writing to the cache. Defaults to false. + */ + // "isCacheWriteAllowed": true + }, + + /** + * Use this configuration with "cacheProvider"="amazon-s3" + */ + "amazonS3Configuration": { + /** + * (Required unless s3Endpoint is specified) The name of the bucket to use for build cache. + * Example: "my-bucket" + */ + // "s3Bucket": "my-bucket", + /** + * (Required unless s3Bucket is specified) The Amazon S3 endpoint of the bucket to use for build cache. + * This should not include any path; use the s3Prefix to set the path. + * Examples: "my-bucket.s3.us-east-2.amazonaws.com" or "http://localhost:9000" + */ + // "s3Endpoint": "https://my-bucket.s3.us-east-2.amazonaws.com", + /** + * (Required) The Amazon S3 region of the bucket to use for build cache. + * Example: "us-east-1" + */ + // "s3Region": "us-east-1", + /** + * An optional prefix ("folder") for cache items. It should not start with "/". + */ + // "s3Prefix": "my-prefix", + /** + * If set to true, allow writing to the cache. Defaults to false. + */ + // "isCacheWriteAllowed": true + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json new file mode 100644 index 00000000000..29a49cbd86c --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json @@ -0,0 +1,25 @@ +/** + * This configuration file manages Rush's cobuild feature. + * More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/cobuild.schema.json", + + /** + * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + */ + "cobuildEnabled": true, + + /** + * (Required) Choose where cobuild lock will be acquired. + * + * The lock provider is registered by the rush plugins. + * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. + */ + "cobuildLockProvider": "redis" + + /** + * Setting this property overrides the cobuild context ID. + */ + // "cobuildContextIdPattern": "" +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json new file mode 100644 index 00000000000..1a8f7837fa3 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json @@ -0,0 +1,308 @@ +/** + * This configuration file defines custom commands for the "rush" command-line. + * More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json", + + /** + * Custom "commands" introduce new verbs for the command-line. To see the help for these + * example commands, try "rush --help", "rush my-bulk-command --help", or + * "rush my-global-command --help". + */ + "commands": [ + { + "commandKind": "bulk", + "summary": "Concurrent version of rush build", + "name": "cobuild", + "safeForSimultaneousRushProcesses": true, + "enableParallelism": true, + "incremental": true + } + + // { + // /** + // * (Required) Determines the type of custom command. + // * Rush's "bulk" commands are invoked separately for each project. Rush will look in + // * each project's package.json file for a "scripts" entry whose name matches the + // * command name. By default, the command will run for every project in the repo, + // * according to the dependency graph (similar to how "rush build" works). + // * The set of projects can be restricted e.g. using the "--to" or "--from" parameters. + // */ + // "commandKind": "bulk", + // + // /** + // * (Required) The name that will be typed as part of the command line. This is also the name + // * of the "scripts" hook in the project's package.json file. + // * The name should be comprised of lower case words separated by hyphens or colons. The name should include an + // * English verb (e.g. "deploy"). Use a hyphen to separate words (e.g. "upload-docs"). A group of related commands + // * can be prefixed with a colon (e.g. "docs:generate", "docs:deploy", "docs:serve", etc). + // * + // * Note that if the "rebuild" command is overridden here, it becomes separated from the "build" command + // * and will call the "rebuild" script instead of the "build" script. + // */ + // "name": "my-bulk-command", + // + // /** + // * (Required) A short summary of the custom command to be shown when printing command line + // * help, e.g. "rush --help". + // */ + // "summary": "Example bulk custom command", + // + // /** + // * A detailed description of the command to be shown when printing command line + // * help (e.g. "rush --help my-command"). + // * If omitted, the "summary" text will be shown instead. + // * + // * Whenever you introduce commands/parameters, taking a little time to write meaningful + // * documentation can make a big difference for the developer experience in your repo. + // */ + // "description": "This is an example custom command that runs separately for each project", + // + // /** + // * By default, Rush operations acquire a lock file which prevents multiple commands from executing simultaneously + // * in the same repo folder. (For example, it would be a mistake to run "rush install" and "rush build" at the + // * same time.) If your command makes sense to run concurrently with other operations, + // * set "safeForSimultaneousRushProcesses" to true to disable this protection. + // * + // * In particular, this is needed for custom scripts that invoke other Rush commands. + // */ + // "safeForSimultaneousRushProcesses": false, + // + // /** + // * (Required) If true, then this command is safe to be run in parallel, i.e. executed + // * simultaneously for multiple projects. Similar to "rush build", regardless of parallelism + // * projects will not start processing until their dependencies have completed processing. + // */ + // "enableParallelism": false, + // + // /** + // * Normally projects will be processed according to their dependency order: a given project will not start + // * processing the command until all of its dependencies have completed. This restriction doesn't apply for + // * certain operations, for example a "clean" task that deletes output files. In this case + // * you can set "ignoreDependencyOrder" to true to increase parallelism. + // */ + // "ignoreDependencyOrder": false, + // + // /** + // * Normally Rush requires that each project's package.json has a "scripts" entry matching + // * the custom command name. To disable this check, set "ignoreMissingScript" to true; + // * projects with a missing definition will be skipped. + // */ + // "ignoreMissingScript": false, + // + // /** + // * When invoking shell scripts, Rush uses a heuristic to distinguish errors from warnings: + // * - If the shell script returns a nonzero process exit code, Rush interprets this as "one or more errors". + // * Error output is displayed in red, and it prevents Rush from attempting to process any downstream projects. + // * - If the shell script returns a zero process exit code but writes something to its stderr stream, + // * Rush interprets this as "one or more warnings". Warning output is printed in yellow, but does NOT prevent + // * Rush from processing downstream projects. + // * + // * Thus, warnings do not interfere with local development, but they will cause a CI job to fail, because + // * the Rush process itself returns a nonzero exit code if there are any warnings or errors. This is by design. + // * In an active monorepo, we've found that if you allow any warnings in your main branch, it inadvertently + // * teaches developers to ignore warnings, which quickly leads to a situation where so many "expected" warnings + // * have accumulated that warnings no longer serve any useful purpose. + // * + // * Sometimes a poorly behaved task will write output to stderr even though its operation was successful. + // * In that case, it's strongly recommended to fix the task. However, as a workaround you can set + // * allowWarningsInSuccessfulBuild=true, which causes Rush to return a nonzero exit code for errors only. + // * + // * Note: The default value is false. In Rush 5.7.x and earlier, the default value was true. + // */ + // "allowWarningsInSuccessfulBuild": false, + // + // /** + // * If true then this command will be incremental like the built-in "build" command + // */ + // "incremental": false, + // + // /** + // * (EXPERIMENTAL) Normally Rush terminates after the command finishes. If this option is set to "true" Rush + // * will instead enter a loop where it watches the file system for changes to the selected projects. Whenever a + // * change is detected, the command will be invoked again for the changed project and any selected projects that + // * directly or indirectly depend on it. + // * + // * For details, refer to the website article "Using watch mode". + // */ + // "watchForChanges": false, + // + // /** + // * (EXPERIMENTAL) Disable cache for this action. This may be useful if this command affects state outside of + // * projects' own folders. + // */ + // "disableBuildCache": false + // }, + // + // { + // /** + // * (Required) Determines the type of custom command. + // * Rush's "global" commands are invoked once for the entire repo. + // */ + // "commandKind": "global", + // + // "name": "my-global-command", + // "summary": "Example global custom command", + // "description": "This is an example custom command that runs once for the entire repo", + // + // "safeForSimultaneousRushProcesses": false, + // + // /** + // * (Required) A script that will be invoked using the OS shell. The working directory will be + // * the folder that contains rush.json. If custom parameters are associated with this command, their + // * values will be appended to the end of this string. + // */ + // "shellCommand": "node common/scripts/my-global-command.js", + // + // /** + // * If your "shellCommand" script depends on NPM packages, the recommended best practice is + // * to make it into a regular Rush project that builds using your normal toolchain. In cases where + // * the command needs to work without first having to run "rush build", the recommended practice + // * is to publish the project to an NPM registry and use common/scripts/install-run.js to launch it. + // * + // * Autoinstallers offer another possibility: They are folders under "common/autoinstallers" with + // * a package.json file and shrinkwrap file. Rush will automatically invoke the package manager to + // * install these dependencies before an associated command is invoked. Autoinstallers have the + // * advantage that they work even in a branch where "rush install" is broken, which makes them a + // * good solution for Git hook scripts. But they have the disadvantages of not being buildable + // * projects, and of increasing the overall installation footprint for your monorepo. + // * + // * The "autoinstallerName" setting must not contain a path and must be a valid NPM package name. + // * For example, the name "my-task" would map to "common/autoinstallers/my-task/package.json", and + // * the "common/autoinstallers/my-task/node_modules/.bin" folder would be added to the shell PATH when + // * invoking the "shellCommand". + // */ + // // "autoinstallerName": "my-task" + // } + ], + + "phases": [], + + /** + * Custom "parameters" introduce new parameters for specified Rush command-line commands. + * For example, you might define a "--production" parameter for the "rush build" command. + */ + "parameters": [ + // { + // /** + // * (Required) Determines the type of custom parameter. + // * A "flag" is a custom command-line parameter whose presence acts as an on/off switch. + // */ + // "parameterKind": "flag", + // + // /** + // * (Required) The long name of the parameter. It must be lower-case and use dash delimiters. + // */ + // "longName": "--my-flag", + // + // /** + // * An optional alternative short name for the parameter. It must be a dash followed by a single + // * lower-case or upper-case letter, which is case-sensitive. + // * + // * NOTE: The Rush developers recommend that automation scripts should always use the long name + // * to improve readability. The short name is only intended as a convenience for humans. + // * The alphabet letters run out quickly, and are difficult to memorize, so *only* use + // * a short name if you expect the parameter to be needed very often in everyday operations. + // */ + // "shortName": "-m", + // + // /** + // * (Required) A long description to be shown in the command-line help. + // * + // * Whenever you introduce commands/parameters, taking a little time to write meaningful + // * documentation can make a big difference for the developer experience in your repo. + // */ + // "description": "A custom flag parameter that is passed to the scripts that are invoked when building projects", + // + // /** + // * (Required) A list of custom commands and/or built-in Rush commands that this parameter may + // * be used with. The parameter will be appended to the shell command that Rush invokes. + // */ + // "associatedCommands": ["build", "rebuild"] + // }, + // + // { + // /** + // * (Required) Determines the type of custom parameter. + // * A "string" is a custom command-line parameter whose value is a simple text string. + // */ + // "parameterKind": "string", + // "longName": "--my-string", + // "description": "A custom string parameter for the \"my-global-command\" custom command", + // + // "associatedCommands": ["my-global-command"], + // + // /** + // * The name of the argument, which will be shown in the command-line help. + // * + // * For example, if the parameter name is '--count" and the argument name is "NUMBER", + // * then the command-line help would display "--count NUMBER". The argument name must + // * be comprised of upper-case letters, numbers, and underscores. It should be kept short. + // */ + // "argumentName": "SOME_TEXT", + // + // /** + // * If true, this parameter must be included with the command. The default is false. + // */ + // "required": false + // }, + // + // { + // /** + // * (Required) Determines the type of custom parameter. + // * A "choice" is a custom command-line parameter whose argument must be chosen from a list of + // * allowable alternatives. + // */ + // "parameterKind": "choice", + // "longName": "--my-choice", + // "description": "A custom choice parameter for the \"my-global-command\" custom command", + // + // "associatedCommands": ["my-global-command"], + // + // /** + // * If true, this parameter must be included with the command. The default is false. + // */ + // "required": false, + // + // /** + // * Normally if a parameter is omitted from the command line, it will not be passed + // * to the shell command. this value will be inserted by default. Whereas if a "defaultValue" + // * is defined, the parameter will always be passed to the shell command, and will use the + // * default value if unspecified. The value must be one of the defined alternatives. + // */ + // "defaultValue": "vanilla", + // + // /** + // * (Required) A list of alternative argument values that can be chosen for this parameter. + // */ + // "alternatives": [ + // { + // /** + // * A token that is one of the alternatives that can be used with the choice parameter, + // * e.g. "vanilla" in "--flavor vanilla". + // */ + // "name": "vanilla", + // + // /** + // * A detailed description for the alternative that can be shown in the command-line help. + // * + // * Whenever you introduce commands/parameters, taking a little time to write meaningful + // * documentation can make a big difference for the developer experience in your repo. + // */ + // "description": "Use the vanilla flavor (the default)" + // }, + // + // { + // "name": "chocolate", + // "description": "Use the chocolate flavor" + // }, + // + // { + // "name": "strawberry", + // "description": "Use the strawberry flavor" + // } + // ] + // } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml new file mode 100644 index 00000000000..5918b7e8af7 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml @@ -0,0 +1,34 @@ +lockfileVersion: 5.4 + +importers: + + .: + specifiers: {} + + ../../projects/a: + specifiers: {} + + ../../projects/b: + specifiers: {} + + ../../projects/c: + specifiers: + b: workspace:* + dependencies: + b: link:../b + + ../../projects/d: + specifiers: + b: workspace:* + c: workspace:* + dependencies: + b: link:../b + c: link:../c + + ../../projects/e: + specifiers: + b: workspace:* + d: workspace:* + dependencies: + b: link:../b + d: link:../d diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json new file mode 100644 index 00000000000..0e7b144099d --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/repo-state.json @@ -0,0 +1,4 @@ +// DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. +{ + "preferredVersionsHash": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f" +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js new file mode 100644 index 00000000000..5c149955de6 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush-pnpm.js @@ -0,0 +1,28 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where the Rush command may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the version of Rush +// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the +// rush-pnpm command. +// +// An example usage would be: +// +// node common/scripts/install-run-rush-pnpm.js pnpm-command +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +var __webpack_exports__ = {}; +/*!*****************************************************!*\ + !*** ./lib-esnext/scripts/install-run-rush-pnpm.js ***! + \*****************************************************/ + +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. +require('./install-run-rush'); +//# sourceMappingURL=install-run-rush-pnpm.js.map +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run-rush-pnpm.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js new file mode 100644 index 00000000000..cada1eded21 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rush.js @@ -0,0 +1,214 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where the Rush command may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the version of Rush +// specified in the rush.json configuration file (if not already installed), and then pass a command-line to it. +// An example usage would be: +// +// node common/scripts/install-run-rush.js install +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 657147: +/*!*********************!*\ + !*** external "fs" ***! + \*********************/ +/***/ ((module) => { + +module.exports = require("fs"); + +/***/ }), + +/***/ 371017: +/*!***********************!*\ + !*** external "path" ***! + \***********************/ +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat get default export */ +/******/ (() => { +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = (module) => { +/******/ var getter = module && module.__esModule ? +/******/ () => (module['default']) : +/******/ () => (module); +/******/ __webpack_require__.d(getter, { a: getter }); +/******/ return getter; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/************************************************************************/ +var __webpack_exports__ = {}; +// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk. +(() => { +/*!************************************************!*\ + !*** ./lib-esnext/scripts/install-run-rush.js ***! + \************************************************/ +__webpack_require__.r(__webpack_exports__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! path */ 371017); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_0__); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! fs */ 657147); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(fs__WEBPACK_IMPORTED_MODULE_1__); +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. + + +const { installAndRun, findRushJsonFolder, RUSH_JSON_FILENAME, runWithErrorAndStatusCode } = require('./install-run'); +const PACKAGE_NAME = '@microsoft/rush'; +const RUSH_PREVIEW_VERSION = 'RUSH_PREVIEW_VERSION'; +const INSTALL_RUN_RUSH_LOCKFILE_PATH_VARIABLE = 'INSTALL_RUN_RUSH_LOCKFILE_PATH'; +function _getRushVersion(logger) { + const rushPreviewVersion = process.env[RUSH_PREVIEW_VERSION]; + if (rushPreviewVersion !== undefined) { + logger.info(`Using Rush version from environment variable ${RUSH_PREVIEW_VERSION}=${rushPreviewVersion}`); + return rushPreviewVersion; + } + const rushJsonFolder = findRushJsonFolder(); + const rushJsonPath = path__WEBPACK_IMPORTED_MODULE_0__.join(rushJsonFolder, RUSH_JSON_FILENAME); + try { + const rushJsonContents = fs__WEBPACK_IMPORTED_MODULE_1__.readFileSync(rushJsonPath, 'utf-8'); + // Use a regular expression to parse out the rushVersion value because rush.json supports comments, + // but JSON.parse does not and we don't want to pull in more dependencies than we need to in this script. + const rushJsonMatches = rushJsonContents.match(/\"rushVersion\"\s*\:\s*\"([0-9a-zA-Z.+\-]+)\"/); + return rushJsonMatches[1]; + } + catch (e) { + throw new Error(`Unable to determine the required version of Rush from rush.json (${rushJsonFolder}). ` + + "The 'rushVersion' field is either not assigned in rush.json or was specified " + + 'using an unexpected syntax.'); + } +} +function _getBin(scriptName) { + switch (scriptName.toLowerCase()) { + case 'install-run-rush-pnpm.js': + return 'rush-pnpm'; + case 'install-run-rushx.js': + return 'rushx'; + default: + return 'rush'; + } +} +function _run() { + const [nodePath /* Ex: /bin/node */, scriptPath /* /repo/common/scripts/install-run-rush.js */, ...packageBinArgs /* [build, --to, myproject] */] = process.argv; + // Detect if this script was directly invoked, or if the install-run-rushx script was invokved to select the + // appropriate binary inside the rush package to run + const scriptName = path__WEBPACK_IMPORTED_MODULE_0__.basename(scriptPath); + const bin = _getBin(scriptName); + if (!nodePath || !scriptPath) { + throw new Error('Unexpected exception: could not detect node path or script path'); + } + let commandFound = false; + let logger = { info: console.log, error: console.error }; + for (const arg of packageBinArgs) { + if (arg === '-q' || arg === '--quiet') { + // The -q/--quiet flag is supported by both `rush` and `rushx`, and will suppress + // any normal informational/diagnostic information printed during startup. + // + // To maintain the same user experience, the install-run* scripts pass along this + // flag but also use it to suppress any diagnostic information normally printed + // to stdout. + logger = { + info: () => { }, + error: console.error + }; + } + else if (!arg.startsWith('-') || arg === '-h' || arg === '--help') { + // We either found something that looks like a command (i.e. - doesn't start with a "-"), + // or we found the -h/--help flag, which can be run without a command + commandFound = true; + } + } + if (!commandFound) { + console.log(`Usage: ${scriptName} [args...]`); + if (scriptName === 'install-run-rush-pnpm.js') { + console.log(`Example: ${scriptName} pnpm-command`); + } + else if (scriptName === 'install-run-rush.js') { + console.log(`Example: ${scriptName} build --to myproject`); + } + else { + console.log(`Example: ${scriptName} custom-command`); + } + process.exit(1); + } + runWithErrorAndStatusCode(logger, () => { + const version = _getRushVersion(logger); + logger.info(`The rush.json configuration requests Rush version ${version}`); + const lockFilePath = process.env[INSTALL_RUN_RUSH_LOCKFILE_PATH_VARIABLE]; + if (lockFilePath) { + logger.info(`Found ${INSTALL_RUN_RUSH_LOCKFILE_PATH_VARIABLE}="${lockFilePath}", installing with lockfile.`); + } + return installAndRun(logger, PACKAGE_NAME, version, bin, packageBinArgs, lockFilePath); + }); +} +_run(); +//# sourceMappingURL=install-run-rush.js.map +})(); + +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run-rush.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js new file mode 100644 index 00000000000..b05df262bc2 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run-rushx.js @@ -0,0 +1,28 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where the Rush command may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the version of Rush +// specified in the rush.json configuration file (if not already installed), and then pass a command-line to the +// rushx command. +// +// An example usage would be: +// +// node common/scripts/install-run-rushx.js custom-command +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +var __webpack_exports__ = {}; +/*!*************************************************!*\ + !*** ./lib-esnext/scripts/install-run-rushx.js ***! + \*************************************************/ + +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. +require('./install-run-rush'); +//# sourceMappingURL=install-run-rushx.js.map +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run-rushx.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js new file mode 100644 index 00000000000..bcd982b369e --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js @@ -0,0 +1,645 @@ +// THIS FILE WAS GENERATED BY A TOOL. ANY MANUAL MODIFICATIONS WILL GET OVERWRITTEN WHENEVER RUSH IS UPGRADED. +// +// This script is intended for usage in an automated build environment where a Node tool may not have +// been preinstalled, or may have an unpredictable version. This script will automatically install the specified +// version of the specified tool (if not already installed), and then pass a command-line to it. +// An example usage would be: +// +// node common/scripts/install-run.js qrcode@1.2.2 qrcode https://rushjs.io +// +// For more information, see: https://rushjs.io/pages/maintainer/setup_new_repo/ + +/******/ (() => { // webpackBootstrap +/******/ "use strict"; +/******/ var __webpack_modules__ = ({ + +/***/ 679877: +/*!************************************************!*\ + !*** ./lib-esnext/utilities/npmrcUtilities.js ***! + \************************************************/ +/***/ ((__unused_webpack_module, __webpack_exports__, __webpack_require__) => { + +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "syncNpmrc": () => (/* binding */ syncNpmrc) +/* harmony export */ }); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! fs */ 657147); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(fs__WEBPACK_IMPORTED_MODULE_0__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! path */ 371017); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_1__); +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. +// IMPORTANT - do not use any non-built-in libraries in this file + + +/** + * As a workaround, copyAndTrimNpmrcFile() copies the .npmrc file to the target folder, and also trims + * unusable lines from the .npmrc file. + * + * Why are we trimming the .npmrc lines? NPM allows environment variables to be specified in + * the .npmrc file to provide different authentication tokens for different registry. + * However, if the environment variable is undefined, it expands to an empty string, which + * produces a valid-looking mapping with an invalid URL that causes an error. Instead, + * we'd prefer to skip that line and continue looking in other places such as the user's + * home directory. + * + * @returns + * The text of the the .npmrc. + */ +function _copyAndTrimNpmrcFile(logger, sourceNpmrcPath, targetNpmrcPath) { + logger.info(`Transforming ${sourceNpmrcPath}`); // Verbose + logger.info(` --> "${targetNpmrcPath}"`); + let npmrcFileLines = fs__WEBPACK_IMPORTED_MODULE_0__.readFileSync(sourceNpmrcPath).toString().split('\n'); + npmrcFileLines = npmrcFileLines.map((line) => (line || '').trim()); + const resultLines = []; + // This finds environment variable tokens that look like "${VAR_NAME}" + const expansionRegExp = /\$\{([^\}]+)\}/g; + // Comment lines start with "#" or ";" + const commentRegExp = /^\s*[#;]/; + // Trim out lines that reference environment variables that aren't defined + for (const line of npmrcFileLines) { + let lineShouldBeTrimmed = false; + // Ignore comment lines + if (!commentRegExp.test(line)) { + const environmentVariables = line.match(expansionRegExp); + if (environmentVariables) { + for (const token of environmentVariables) { + // Remove the leading "${" and the trailing "}" from the token + const environmentVariableName = token.substring(2, token.length - 1); + // Is the environment variable defined? + if (!process.env[environmentVariableName]) { + // No, so trim this line + lineShouldBeTrimmed = true; + break; + } + } + } + } + if (lineShouldBeTrimmed) { + // Example output: + // "; MISSING ENVIRONMENT VARIABLE: //my-registry.com/npm/:_authToken=${MY_AUTH_TOKEN}" + resultLines.push('; MISSING ENVIRONMENT VARIABLE: ' + line); + } + else { + resultLines.push(line); + } + } + const combinedNpmrc = resultLines.join('\n'); + fs__WEBPACK_IMPORTED_MODULE_0__.writeFileSync(targetNpmrcPath, combinedNpmrc); + return combinedNpmrc; +} +/** + * syncNpmrc() copies the .npmrc file to the target folder, and also trims unusable lines from the .npmrc file. + * If the source .npmrc file not exist, then syncNpmrc() will delete an .npmrc that is found in the target folder. + * + * IMPORTANT: THIS CODE SHOULD BE KEPT UP TO DATE WITH Utilities._syncNpmrc() + * + * @returns + * The text of the the synced .npmrc, if one exists. If one does not exist, then undefined is returned. + */ +function syncNpmrc(sourceNpmrcFolder, targetNpmrcFolder, useNpmrcPublish, logger = { + info: console.log, + error: console.error +}) { + const sourceNpmrcPath = path__WEBPACK_IMPORTED_MODULE_1__.join(sourceNpmrcFolder, !useNpmrcPublish ? '.npmrc' : '.npmrc-publish'); + const targetNpmrcPath = path__WEBPACK_IMPORTED_MODULE_1__.join(targetNpmrcFolder, '.npmrc'); + try { + if (fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(sourceNpmrcPath)) { + return _copyAndTrimNpmrcFile(logger, sourceNpmrcPath, targetNpmrcPath); + } + else if (fs__WEBPACK_IMPORTED_MODULE_0__.existsSync(targetNpmrcPath)) { + // If the source .npmrc doesn't exist and there is one in the target, delete the one in the target + logger.info(`Deleting ${targetNpmrcPath}`); // Verbose + fs__WEBPACK_IMPORTED_MODULE_0__.unlinkSync(targetNpmrcPath); + } + } + catch (e) { + throw new Error(`Error syncing .npmrc file: ${e}`); + } +} +//# sourceMappingURL=npmrcUtilities.js.map + +/***/ }), + +/***/ 532081: +/*!********************************!*\ + !*** external "child_process" ***! + \********************************/ +/***/ ((module) => { + +module.exports = require("child_process"); + +/***/ }), + +/***/ 657147: +/*!*********************!*\ + !*** external "fs" ***! + \*********************/ +/***/ ((module) => { + +module.exports = require("fs"); + +/***/ }), + +/***/ 822037: +/*!*********************!*\ + !*** external "os" ***! + \*********************/ +/***/ ((module) => { + +module.exports = require("os"); + +/***/ }), + +/***/ 371017: +/*!***********************!*\ + !*** external "path" ***! + \***********************/ +/***/ ((module) => { + +module.exports = require("path"); + +/***/ }) + +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __webpack_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ __webpack_modules__[moduleId](module, module.exports, __webpack_require__); +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat get default export */ +/******/ (() => { +/******/ // getDefaultExport function for compatibility with non-harmony modules +/******/ __webpack_require__.n = (module) => { +/******/ var getter = module && module.__esModule ? +/******/ () => (module['default']) : +/******/ () => (module); +/******/ __webpack_require__.d(getter, { a: getter }); +/******/ return getter; +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/define property getters */ +/******/ (() => { +/******/ // define getter functions for harmony exports +/******/ __webpack_require__.d = (exports, definition) => { +/******/ for(var key in definition) { +/******/ if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) { +/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] }); +/******/ } +/******/ } +/******/ }; +/******/ })(); +/******/ +/******/ /* webpack/runtime/hasOwnProperty shorthand */ +/******/ (() => { +/******/ __webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop)) +/******/ })(); +/******/ +/******/ /* webpack/runtime/make namespace object */ +/******/ (() => { +/******/ // define __esModule on exports +/******/ __webpack_require__.r = (exports) => { +/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { +/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); +/******/ } +/******/ Object.defineProperty(exports, '__esModule', { value: true }); +/******/ }; +/******/ })(); +/******/ +/************************************************************************/ +var __webpack_exports__ = {}; +// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk. +(() => { +/*!*******************************************!*\ + !*** ./lib-esnext/scripts/install-run.js ***! + \*******************************************/ +__webpack_require__.r(__webpack_exports__); +/* harmony export */ __webpack_require__.d(__webpack_exports__, { +/* harmony export */ "RUSH_JSON_FILENAME": () => (/* binding */ RUSH_JSON_FILENAME), +/* harmony export */ "getNpmPath": () => (/* binding */ getNpmPath), +/* harmony export */ "findRushJsonFolder": () => (/* binding */ findRushJsonFolder), +/* harmony export */ "installAndRun": () => (/* binding */ installAndRun), +/* harmony export */ "runWithErrorAndStatusCode": () => (/* binding */ runWithErrorAndStatusCode) +/* harmony export */ }); +/* harmony import */ var child_process__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(/*! child_process */ 532081); +/* harmony import */ var child_process__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(child_process__WEBPACK_IMPORTED_MODULE_0__); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(/*! fs */ 657147); +/* harmony import */ var fs__WEBPACK_IMPORTED_MODULE_1___default = /*#__PURE__*/__webpack_require__.n(fs__WEBPACK_IMPORTED_MODULE_1__); +/* harmony import */ var os__WEBPACK_IMPORTED_MODULE_2__ = __webpack_require__(/*! os */ 822037); +/* harmony import */ var os__WEBPACK_IMPORTED_MODULE_2___default = /*#__PURE__*/__webpack_require__.n(os__WEBPACK_IMPORTED_MODULE_2__); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_3__ = __webpack_require__(/*! path */ 371017); +/* harmony import */ var path__WEBPACK_IMPORTED_MODULE_3___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_3__); +/* harmony import */ var _utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__ = __webpack_require__(/*! ../utilities/npmrcUtilities */ 679877); +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See the @microsoft/rush package's LICENSE file for license information. + + + + + +const RUSH_JSON_FILENAME = 'rush.json'; +const RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME = 'RUSH_TEMP_FOLDER'; +const INSTALL_RUN_LOCKFILE_PATH_VARIABLE = 'INSTALL_RUN_LOCKFILE_PATH'; +const INSTALLED_FLAG_FILENAME = 'installed.flag'; +const NODE_MODULES_FOLDER_NAME = 'node_modules'; +const PACKAGE_JSON_FILENAME = 'package.json'; +/** + * Parse a package specifier (in the form of name\@version) into name and version parts. + */ +function _parsePackageSpecifier(rawPackageSpecifier) { + rawPackageSpecifier = (rawPackageSpecifier || '').trim(); + const separatorIndex = rawPackageSpecifier.lastIndexOf('@'); + let name; + let version = undefined; + if (separatorIndex === 0) { + // The specifier starts with a scope and doesn't have a version specified + name = rawPackageSpecifier; + } + else if (separatorIndex === -1) { + // The specifier doesn't have a version + name = rawPackageSpecifier; + } + else { + name = rawPackageSpecifier.substring(0, separatorIndex); + version = rawPackageSpecifier.substring(separatorIndex + 1); + } + if (!name) { + throw new Error(`Invalid package specifier: ${rawPackageSpecifier}`); + } + return { name, version }; +} +let _npmPath = undefined; +/** + * Get the absolute path to the npm executable + */ +function getNpmPath() { + if (!_npmPath) { + try { + if (os__WEBPACK_IMPORTED_MODULE_2__.platform() === 'win32') { + // We're on Windows + const whereOutput = child_process__WEBPACK_IMPORTED_MODULE_0__.execSync('where npm', { stdio: [] }).toString(); + const lines = whereOutput.split(os__WEBPACK_IMPORTED_MODULE_2__.EOL).filter((line) => !!line); + // take the last result, we are looking for a .cmd command + // see https://github.com/microsoft/rushstack/issues/759 + _npmPath = lines[lines.length - 1]; + } + else { + // We aren't on Windows - assume we're on *NIX or Darwin + _npmPath = child_process__WEBPACK_IMPORTED_MODULE_0__.execSync('command -v npm', { stdio: [] }).toString(); + } + } + catch (e) { + throw new Error(`Unable to determine the path to the NPM tool: ${e}`); + } + _npmPath = _npmPath.trim(); + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(_npmPath)) { + throw new Error('The NPM executable does not exist'); + } + } + return _npmPath; +} +function _ensureFolder(folderPath) { + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(folderPath)) { + const parentDir = path__WEBPACK_IMPORTED_MODULE_3__.dirname(folderPath); + _ensureFolder(parentDir); + fs__WEBPACK_IMPORTED_MODULE_1__.mkdirSync(folderPath); + } +} +/** + * Create missing directories under the specified base directory, and return the resolved directory. + * + * Does not support "." or ".." path segments. + * Assumes the baseFolder exists. + */ +function _ensureAndJoinPath(baseFolder, ...pathSegments) { + let joinedPath = baseFolder; + try { + for (let pathSegment of pathSegments) { + pathSegment = pathSegment.replace(/[\\\/]/g, '+'); + joinedPath = path__WEBPACK_IMPORTED_MODULE_3__.join(joinedPath, pathSegment); + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(joinedPath)) { + fs__WEBPACK_IMPORTED_MODULE_1__.mkdirSync(joinedPath); + } + } + } + catch (e) { + throw new Error(`Error building local installation folder (${path__WEBPACK_IMPORTED_MODULE_3__.join(baseFolder, ...pathSegments)}): ${e}`); + } + return joinedPath; +} +function _getRushTempFolder(rushCommonFolder) { + const rushTempFolder = process.env[RUSH_TEMP_FOLDER_ENV_VARIABLE_NAME]; + if (rushTempFolder !== undefined) { + _ensureFolder(rushTempFolder); + return rushTempFolder; + } + else { + return _ensureAndJoinPath(rushCommonFolder, 'temp'); + } +} +/** + * Resolve a package specifier to a static version + */ +function _resolvePackageVersion(logger, rushCommonFolder, { name, version }) { + if (!version) { + version = '*'; // If no version is specified, use the latest version + } + if (version.match(/^[a-zA-Z0-9\-\+\.]+$/)) { + // If the version contains only characters that we recognize to be used in static version specifiers, + // pass the version through + return version; + } + else { + // version resolves to + try { + const rushTempFolder = _getRushTempFolder(rushCommonFolder); + const sourceNpmrcFolder = path__WEBPACK_IMPORTED_MODULE_3__.join(rushCommonFolder, 'config', 'rush'); + (0,_utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__.syncNpmrc)(sourceNpmrcFolder, rushTempFolder, undefined, logger); + const npmPath = getNpmPath(); + // This returns something that looks like: + // @microsoft/rush@3.0.0 '3.0.0' + // @microsoft/rush@3.0.1 '3.0.1' + // ... + // @microsoft/rush@3.0.20 '3.0.20' + // + const npmVersionSpawnResult = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], { + cwd: rushTempFolder, + stdio: [] + }); + if (npmVersionSpawnResult.status !== 0) { + throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`); + } + const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString(); + const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line); + const latestVersion = versionLines[versionLines.length - 1]; + if (!latestVersion) { + throw new Error('No versions found for the specified version range.'); + } + const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/); + if (!versionMatches) { + throw new Error(`Invalid npm output ${latestVersion}`); + } + return versionMatches[1]; + } + catch (e) { + throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`); + } + } +} +let _rushJsonFolder; +/** + * Find the absolute path to the folder containing rush.json + */ +function findRushJsonFolder() { + if (!_rushJsonFolder) { + let basePath = __dirname; + let tempPath = __dirname; + do { + const testRushJsonPath = path__WEBPACK_IMPORTED_MODULE_3__.join(basePath, RUSH_JSON_FILENAME); + if (fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(testRushJsonPath)) { + _rushJsonFolder = basePath; + break; + } + else { + basePath = tempPath; + } + } while (basePath !== (tempPath = path__WEBPACK_IMPORTED_MODULE_3__.dirname(basePath))); // Exit the loop when we hit the disk root + if (!_rushJsonFolder) { + throw new Error('Unable to find rush.json.'); + } + } + return _rushJsonFolder; +} +/** + * Detects if the package in the specified directory is installed + */ +function _isPackageAlreadyInstalled(packageInstallFolder) { + try { + const flagFilePath = path__WEBPACK_IMPORTED_MODULE_3__.join(packageInstallFolder, INSTALLED_FLAG_FILENAME); + if (!fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(flagFilePath)) { + return false; + } + const fileContents = fs__WEBPACK_IMPORTED_MODULE_1__.readFileSync(flagFilePath).toString(); + return fileContents.trim() === process.version; + } + catch (e) { + return false; + } +} +/** + * Delete a file. Fail silently if it does not exist. + */ +function _deleteFile(file) { + try { + fs__WEBPACK_IMPORTED_MODULE_1__.unlinkSync(file); + } + catch (err) { + if (err.code !== 'ENOENT' && err.code !== 'ENOTDIR') { + throw err; + } + } +} +/** + * Removes the following files and directories under the specified folder path: + * - installed.flag + * - + * - node_modules + */ +function _cleanInstallFolder(rushTempFolder, packageInstallFolder, lockFilePath) { + try { + const flagFile = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, INSTALLED_FLAG_FILENAME); + _deleteFile(flagFile); + const packageLockFile = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, 'package-lock.json'); + if (lockFilePath) { + fs__WEBPACK_IMPORTED_MODULE_1__.copyFileSync(lockFilePath, packageLockFile); + } + else { + // Not running `npm ci`, so need to cleanup + _deleteFile(packageLockFile); + const nodeModulesFolder = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME); + if (fs__WEBPACK_IMPORTED_MODULE_1__.existsSync(nodeModulesFolder)) { + const rushRecyclerFolder = _ensureAndJoinPath(rushTempFolder, 'rush-recycler'); + fs__WEBPACK_IMPORTED_MODULE_1__.renameSync(nodeModulesFolder, path__WEBPACK_IMPORTED_MODULE_3__.join(rushRecyclerFolder, `install-run-${Date.now().toString()}`)); + } + } + } + catch (e) { + throw new Error(`Error cleaning the package install folder (${packageInstallFolder}): ${e}`); + } +} +function _createPackageJson(packageInstallFolder, name, version) { + try { + const packageJsonContents = { + name: 'ci-rush', + version: '0.0.0', + dependencies: { + [name]: version + }, + description: "DON'T WARN", + repository: "DON'T WARN", + license: 'MIT' + }; + const packageJsonPath = path__WEBPACK_IMPORTED_MODULE_3__.join(packageInstallFolder, PACKAGE_JSON_FILENAME); + fs__WEBPACK_IMPORTED_MODULE_1__.writeFileSync(packageJsonPath, JSON.stringify(packageJsonContents, undefined, 2)); + } + catch (e) { + throw new Error(`Unable to create package.json: ${e}`); + } +} +/** + * Run "npm install" in the package install folder. + */ +function _installPackage(logger, packageInstallFolder, name, version, command) { + try { + logger.info(`Installing ${name}...`); + const npmPath = getNpmPath(); + const result = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(npmPath, [command], { + stdio: 'inherit', + cwd: packageInstallFolder, + env: process.env + }); + if (result.status !== 0) { + throw new Error(`"npm ${command}" encountered an error`); + } + logger.info(`Successfully installed ${name}@${version}`); + } + catch (e) { + throw new Error(`Unable to install package: ${e}`); + } +} +/** + * Get the ".bin" path for the package. + */ +function _getBinPath(packageInstallFolder, binName) { + const binFolderPath = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin'); + const resolvedBinName = os__WEBPACK_IMPORTED_MODULE_2__.platform() === 'win32' ? `${binName}.cmd` : binName; + return path__WEBPACK_IMPORTED_MODULE_3__.resolve(binFolderPath, resolvedBinName); +} +/** + * Write a flag file to the package's install directory, signifying that the install was successful. + */ +function _writeFlagFile(packageInstallFolder) { + try { + const flagFilePath = path__WEBPACK_IMPORTED_MODULE_3__.join(packageInstallFolder, INSTALLED_FLAG_FILENAME); + fs__WEBPACK_IMPORTED_MODULE_1__.writeFileSync(flagFilePath, process.version); + } + catch (e) { + throw new Error(`Unable to create installed.flag file in ${packageInstallFolder}`); + } +} +function installAndRun(logger, packageName, packageVersion, packageBinName, packageBinArgs, lockFilePath = process.env[INSTALL_RUN_LOCKFILE_PATH_VARIABLE]) { + const rushJsonFolder = findRushJsonFolder(); + const rushCommonFolder = path__WEBPACK_IMPORTED_MODULE_3__.join(rushJsonFolder, 'common'); + const rushTempFolder = _getRushTempFolder(rushCommonFolder); + const packageInstallFolder = _ensureAndJoinPath(rushTempFolder, 'install-run', `${packageName}@${packageVersion}`); + if (!_isPackageAlreadyInstalled(packageInstallFolder)) { + // The package isn't already installed + _cleanInstallFolder(rushTempFolder, packageInstallFolder, lockFilePath); + const sourceNpmrcFolder = path__WEBPACK_IMPORTED_MODULE_3__.join(rushCommonFolder, 'config', 'rush'); + (0,_utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__.syncNpmrc)(sourceNpmrcFolder, packageInstallFolder, undefined, logger); + _createPackageJson(packageInstallFolder, packageName, packageVersion); + const command = lockFilePath ? 'ci' : 'install'; + _installPackage(logger, packageInstallFolder, packageName, packageVersion, command); + _writeFlagFile(packageInstallFolder); + } + const statusMessage = `Invoking "${packageBinName} ${packageBinArgs.join(' ')}"`; + const statusMessageLine = new Array(statusMessage.length + 1).join('-'); + logger.info('\n' + statusMessage + '\n' + statusMessageLine + '\n'); + const binPath = _getBinPath(packageInstallFolder, packageBinName); + const binFolderPath = path__WEBPACK_IMPORTED_MODULE_3__.resolve(packageInstallFolder, NODE_MODULES_FOLDER_NAME, '.bin'); + // Windows environment variables are case-insensitive. Instead of using SpawnSyncOptions.env, we need to + // assign via the process.env proxy to ensure that we append to the right PATH key. + const originalEnvPath = process.env.PATH || ''; + let result; + try { + // Node.js on Windows can not spawn a file when the path has a space on it + // unless the path gets wrapped in a cmd friendly way and shell mode is used + const shouldUseShell = binPath.includes(' ') && os__WEBPACK_IMPORTED_MODULE_2__.platform() === 'win32'; + const platformBinPath = shouldUseShell ? `"${binPath}"` : binPath; + process.env.PATH = [binFolderPath, originalEnvPath].join(path__WEBPACK_IMPORTED_MODULE_3__.delimiter); + result = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(platformBinPath, packageBinArgs, { + stdio: 'inherit', + windowsVerbatimArguments: false, + shell: shouldUseShell, + cwd: process.cwd(), + env: process.env + }); + } + finally { + process.env.PATH = originalEnvPath; + } + if (result.status !== null) { + return result.status; + } + else { + throw result.error || new Error('An unknown error occurred.'); + } +} +function runWithErrorAndStatusCode(logger, fn) { + process.exitCode = 1; + try { + const exitCode = fn(); + process.exitCode = exitCode; + } + catch (e) { + logger.error('\n\n' + e.toString() + '\n\n'); + } +} +function _run() { + const [nodePath /* Ex: /bin/node */, scriptPath /* /repo/common/scripts/install-run-rush.js */, rawPackageSpecifier /* qrcode@^1.2.0 */, packageBinName /* qrcode */, ...packageBinArgs /* [-f, myproject/lib] */] = process.argv; + if (!nodePath) { + throw new Error('Unexpected exception: could not detect node path'); + } + if (path__WEBPACK_IMPORTED_MODULE_3__.basename(scriptPath).toLowerCase() !== 'install-run.js') { + // If install-run.js wasn't directly invoked, don't execute the rest of this function. Return control + // to the script that (presumably) imported this file + return; + } + if (process.argv.length < 4) { + console.log('Usage: install-run.js @ [args...]'); + console.log('Example: install-run.js qrcode@1.2.2 qrcode https://rushjs.io'); + process.exit(1); + } + const logger = { info: console.log, error: console.error }; + runWithErrorAndStatusCode(logger, () => { + const rushJsonFolder = findRushJsonFolder(); + const rushCommonFolder = _ensureAndJoinPath(rushJsonFolder, 'common'); + const packageSpecifier = _parsePackageSpecifier(rawPackageSpecifier); + const name = packageSpecifier.name; + const version = _resolvePackageVersion(logger, rushCommonFolder, packageSpecifier); + if (packageSpecifier.version !== version) { + console.log(`Resolved to ${name}@${version}`); + } + return installAndRun(logger, name, version, packageBinName, packageBinArgs); + }); +} +_run(); +//# sourceMappingURL=install-run.js.map +})(); + +module.exports = __webpack_exports__; +/******/ })() +; +//# sourceMappingURL=install-run.js.map \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json new file mode 100644 index 00000000000..9472fdbd7e5 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -0,0 +1,8 @@ +{ + "name": "a", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json new file mode 100644 index 00000000000..a8cd24f8006 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json @@ -0,0 +1,8 @@ +{ + "name": "b", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js new file mode 100644 index 00000000000..14855ff8c72 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js @@ -0,0 +1,12 @@ +/* eslint-env es6 */ +const path = require('path'); +const { FileSystem } = require('@rushstack/node-core-library'); + +console.log('start'); +setTimeout(() => { + const outputFolder = path.resolve(process.cwd(), 'dist'); + const outputFile = path.resolve(outputFolder, 'output.txt'); + FileSystem.ensureFolder(outputFolder); + FileSystem.writeFile(outputFile, 'Hello world!'); + console.log('done'); +}, 5000); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json new file mode 100644 index 00000000000..b25880f2c84 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json @@ -0,0 +1,11 @@ +{ + "name": "c", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json new file mode 100644 index 00000000000..6580cb02700 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json @@ -0,0 +1,12 @@ +{ + "name": "d", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*", + "c": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json new file mode 100644 index 00000000000..f0036196559 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json @@ -0,0 +1,12 @@ +{ + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json new file mode 100644 index 00000000000..69ac8b1cc97 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json @@ -0,0 +1,12 @@ +{ + "name": "e", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*", + "d": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json new file mode 100644 index 00000000000..9f839d273ed --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json @@ -0,0 +1,29 @@ +{ + "rushVersion": "5.80.0", + "pnpmVersion": "7.13.0", + "pnpmOptions": { + "useWorkspaces": true + }, + "projects": [ + { + "packageName": "a", + "projectFolder": "projects/a" + }, + { + "packageName": "b", + "projectFolder": "projects/b" + }, + { + "packageName": "c", + "projectFolder": "projects/c" + }, + { + "packageName": "d", + "projectFolder": "projects/d" + }, + { + "packageName": "e", + "projectFolder": "projects/e" + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts new file mode 100644 index 00000000000..858c5f62257 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/paths.ts @@ -0,0 +1,5 @@ +import * as path from 'path'; + +const sandboxRepoFolder: string = path.resolve(__dirname, '../sandbox/repo'); + +export { sandboxRepoFolder }; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts new file mode 100644 index 00000000000..50c44d60968 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -0,0 +1,37 @@ +import { RushCommandLineParser } from '@microsoft/rush-lib/lib/cli/RushCommandLineParser'; +import * as rushLib from '@microsoft/rush-lib'; + +// Setup redis cobuild plugin +const builtInPluginConfigurations: rushLib._IBuiltInPluginConfiguration[] = []; + +const rushConfiguration: rushLib.RushConfiguration = rushLib.RushConfiguration.loadFromDefaultLocation({ + startingFolder: __dirname +}); +const project: rushLib.RushConfigurationProject | undefined = rushConfiguration.getProjectByName( + '@rushstack/rush-redis-cobuild-plugin' +); +if (!project) { + throw new Error('Project @rushstack/rush-redis-cobuild-plugin not found'); +} +builtInPluginConfigurations.push({ + packageName: '@rushstack/rush-redis-cobuild-plugin', + pluginName: 'rush-redis-cobuild-plugin', + pluginPackageFolder: project.projectFolder +}); + +async function rushRush(args: string[]): Promise { + const options: rushLib.ILaunchOptions = { + isManaged: false, + alreadyReportedNodeTooNewError: false, + builtInPluginConfigurations + }; + const parser: RushCommandLineParser = new RushCommandLineParser({ + alreadyReportedNodeTooNewError: options.alreadyReportedNodeTooNewError, + builtInPluginConfigurations: options.builtInPluginConfigurations + }); + console.log(`Executing: rush ${args.join(' ')}`); + await parser.execute(args).catch(console.error); // CommandLineParser.execute() should never reject the promise +} + +/* eslint-disable-next-line @typescript-eslint/no-floating-promises */ +rushRush(process.argv.slice(2)); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 820532437f1..474abae5bd9 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -2,26 +2,24 @@ import { RedisCobuildLockProvider, IRedisCobuildLockProviderOptions } from '@rushstack/rush-redis-cobuild-plugin'; -import { ConsoleTerminalProvider, ITerminal, Terminal } from '@rushstack/node-core-library'; -import { OperationStatus, ICobuildContext } from '@microsoft/rush-lib'; +import { ConsoleTerminalProvider } from '@rushstack/node-core-library'; +import { OperationStatus, ICobuildContext, RushSession } from '@microsoft/rush-lib'; const options: IRedisCobuildLockProviderOptions = { url: 'redis://localhost:6379', password: 'redis123' }; -const terminal: ITerminal = new Terminal( - new ConsoleTerminalProvider({ - verboseEnabled: true, - debugEnabled: true - }) -); +const rushSession: RushSession = new RushSession({ + terminalProvider: new ConsoleTerminalProvider(), + getIsDebugMode: () => true +}); async function main(): Promise { - const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options, rushSession as any); await lockProvider.connectAsync(); const context: ICobuildContext = { - terminal, contextId: 'test-context-id', version: 1, cacheId: 'test-cache-id' diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json index 6314e94a07d..599b3beb19e 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/tsconfig.json @@ -12,6 +12,7 @@ "declarationMap": true, "inlineSources": true, "experimentalDecorators": true, + "esModuleInterop": true, "strictNullChecks": true, "noUnusedLocals": true, "types": ["node"], diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 8d3c15d3bb0..585a7ec75b8 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1354,7 +1354,7 @@ importers: '@rushstack/node-core-library': workspace:* '@rushstack/rush-redis-cobuild-plugin': workspace:* '@types/http-proxy': ~1.17.8 - '@types/node': 12.20.24 + '@types/node': 14.18.36 eslint: ~8.7.0 http-proxy: ~1.18.1 typescript: ~4.8.4 @@ -1365,7 +1365,7 @@ importers: '@rushstack/node-core-library': link:../../libraries/node-core-library '@rushstack/rush-redis-cobuild-plugin': link:../../rush-plugins/rush-redis-cobuild-plugin '@types/http-proxy': 1.17.9 - '@types/node': 12.20.24 + '@types/node': 14.18.36 eslint: 8.7.0 http-proxy: 1.18.1 typescript: 4.8.4 diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index b967a5aeac1..09900023ba0 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -280,8 +280,6 @@ export interface ICobuildContext { // (undocumented) contextId: string; // (undocumented) - terminal: ITerminal; - // (undocumented) version: number; } diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 4a2f99398e4..b19ae9aa26b 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -18,7 +18,7 @@ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { // @beta (undocumented) export class RedisCobuildLockProvider implements ICobuildLockProvider { - constructor(options: IRedisCobuildLockProviderOptions); + constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession); // (undocumented) acquireLockAsync(context: ICobuildContext): Promise; // (undocumented) diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json new file mode 100644 index 00000000000..13cce367de6 --- /dev/null +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json @@ -0,0 +1,25 @@ +/** + * This configuration file manages Rush's cobuild feature. + * More documentation is available on the Rush website: https://rushjs.io + */ + { + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/cobuild.schema.json", + + /** + * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + */ + "cobuildEnabled": false, + + /** + * (Required) Choose where cobuild lock will be acquired. + * + * The lock provider is registered by the rush plugins. + * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. + */ + "cobuildLockProvider": "redis" + + /** + * Setting this property overrides the cobuild context ID. + */ + // "cobuildContextIdPattern": "" +} diff --git a/libraries/rush-lib/src/api/RushConfiguration.ts b/libraries/rush-lib/src/api/RushConfiguration.ts index 052e3e4ee29..c8892a9675c 100644 --- a/libraries/rush-lib/src/api/RushConfiguration.ts +++ b/libraries/rush-lib/src/api/RushConfiguration.ts @@ -57,6 +57,7 @@ const knownRushConfigFilenames: string[] = [ RushConstants.artifactoryFilename, RushConstants.browserApprovedPackagesFilename, RushConstants.buildCacheFilename, + RushConstants.cobuildFilename, RushConstants.commandLineFilename, RushConstants.commonVersionsFilename, RushConstants.experimentsFilename, diff --git a/libraries/rush-lib/src/cli/actions/InitAction.ts b/libraries/rush-lib/src/cli/actions/InitAction.ts index 8bb6f914ec9..3d3dc5b084f 100644 --- a/libraries/rush-lib/src/cli/actions/InitAction.ts +++ b/libraries/rush-lib/src/cli/actions/InitAction.ts @@ -156,6 +156,7 @@ export class InitAction extends BaseConfiglessRushAction { 'common/config/rush/[dot]npmrc-publish', 'common/config/rush/artifactory.json', 'common/config/rush/build-cache.json', + 'common/config/rush/cobuild.json', 'common/config/rush/command-line.json', 'common/config/rush/common-versions.json', 'common/config/rush/experiments.json', diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 9343640b573..aa7875ff687 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { InternalError, ITerminal } from '@rushstack/node-core-library'; +import { InternalError } from '@rushstack/node-core-library'; import { RushConstants } from '../RushConstants'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; @@ -12,7 +12,6 @@ import type { ICobuildContext } from './ICobuildLockProvider'; export interface ICobuildLockOptions { cobuildConfiguration: CobuildConfiguration; projectBuildCache: ProjectBuildCache; - terminal: ITerminal; } export interface ICobuildCompletedState { @@ -27,7 +26,7 @@ export class CobuildLock { private _cobuildContext: ICobuildContext; public constructor(options: ICobuildLockOptions) { - const { cobuildConfiguration, projectBuildCache, terminal } = options; + const { cobuildConfiguration, projectBuildCache } = options; this.projectBuildCache = projectBuildCache; this.cobuildConfiguration = cobuildConfiguration; @@ -40,7 +39,6 @@ export class CobuildLock { } this._cobuildContext = { - terminal, contextId, cacheId, version: RushConstants.cobuildLockVersion diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index a36bc8ab702..be00a297c11 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import type { ITerminal } from '@rushstack/node-core-library'; import type { OperationStatus } from '../operations/OperationStatus'; /** @@ -11,7 +10,6 @@ export interface ICobuildContext { contextId: string; cacheId: string; version: number; - terminal: ITerminal; } /** diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 8bd625da4e2..9e0cd9ef5eb 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -90,6 +90,7 @@ export class AsyncOperationQueue if ( operation.status === OperationStatus.Blocked || + operation.status === OperationStatus.Skipped || operation.status === OperationStatus.Success || operation.status === OperationStatus.SuccessWithWarning || operation.status === OperationStatus.FromCache || diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index d61c6c196fb..fb44dcf46cf 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -188,12 +188,7 @@ export class OperationExecutionManager { const onOperationComplete: (record: OperationExecutionRecord) => void = ( record: OperationExecutionRecord ) => { - this._onOperationComplete(record); - - if (record.status !== OperationStatus.RemoteExecuting) { - // If the operation was not remote, then we can notify queue that it is complete - executionQueue.complete(); - } + this._onOperationComplete(record, executionQueue); }; await Async.forEachAsync( @@ -221,7 +216,7 @@ export class OperationExecutionManager { /** * Handles the result of the operation and propagates any relevant effects. */ - private _onOperationComplete(record: OperationExecutionRecord): void { + private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { const { runner, name, status } = record; let blockCacheWrite: boolean = !runner.isCacheWriteAllowed; @@ -246,6 +241,7 @@ export class OperationExecutionManager { const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { + executionQueue.complete(); this._completedOperations++; // Now that we have the concept of architectural no-ops, we could implement this by replacing @@ -338,5 +334,10 @@ export class OperationExecutionManager { item.dependencies.delete(record); } } + + if (record.status !== OperationStatus.RemoteExecuting) { + // If the operation was not remote, then we can notify queue that it is complete + executionQueue.complete(); + } } } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index fedab5e5fe2..063b76c6e4a 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -265,7 +265,7 @@ export class ShellOperationRunner implements IOperationRunner { terminal, trackedFiles ); - cobuildLock = await this._tryGetCobuildLockAsync(terminal, projectBuildCache); + cobuildLock = await this._tryGetCobuildLockAsync(projectBuildCache); } // If possible, we want to skip this operation -- either by restoring it from the @@ -341,6 +341,22 @@ export class ShellOperationRunner implements IOperationRunner { } } + if (this.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + if (context.status === OperationStatus.RemoteExecuting) { + // This operation is used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; + } + runnerWatcher.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + } else { + // failed to acquire the lock, mark current operation to remote executing + return OperationStatus.RemoteExecuting; + } + } + // If the deps file exists, remove it before starting execution. FileSystem.deleteFile(currentDepsPath); @@ -360,22 +376,6 @@ export class ShellOperationRunner implements IOperationRunner { return OperationStatus.Success; } - if (this.isCacheWriteAllowed && cobuildLock) { - const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); - if (acquireSuccess) { - if (context.status === OperationStatus.RemoteExecuting) { - // This operation is used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - } - runnerWatcher.addCallback(async () => { - await cobuildLock?.renewLockAsync(); - }); - } else { - // failed to acquire the lock, mark current operation to remote executing - return OperationStatus.RemoteExecuting; - } - } - // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); runnerWatcher.start(); @@ -607,7 +607,6 @@ export class ShellOperationRunner implements IOperationRunner { } private async _tryGetCobuildLockAsync( - terminal: ITerminal, projectBuildCache: ProjectBuildCache | undefined ): Promise { if (this._cobuildLock === UNINITIALIZED) { @@ -616,8 +615,7 @@ export class ShellOperationRunner implements IOperationRunner { if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { this._cobuildLock = new CobuildLock({ cobuildConfiguration: this._cobuildConfiguration, - projectBuildCache: projectBuildCache, - terminal + projectBuildCache: projectBuildCache }); } } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 3af20ccbb66..d7d22d2898c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -3,7 +3,12 @@ import { createClient } from '@redis/client'; -import type { ICobuildLockProvider, ICobuildContext, ICobuildCompletedState } from '@rushstack/rush-sdk'; +import type { + ICobuildLockProvider, + ICobuildContext, + ICobuildCompletedState, + RushSession +} from '@rushstack/rush-sdk'; import type { RedisClientOptions, RedisClientType, @@ -11,6 +16,7 @@ import type { RedisModules, RedisScripts } from '@redis/client'; +import type { ITerminal } from '@rushstack/node-core-library'; /** * The redis client options @@ -26,18 +32,30 @@ const COMPLETED_STATE_SEPARATOR: string = ';'; */ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; + private _terminal: ITerminal; private _redisClient: RedisClientType; private _lockKeyMap: WeakMap = new WeakMap(); private _completedKeyMap: WeakMap = new WeakMap(); - public constructor(options: IRedisCobuildLockProviderOptions) { + public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = options; - this._redisClient = createClient(this._options); + this._terminal = rushSession.getLogger('RedisCobuildLockProvider').terminal; + try { + this._redisClient = createClient(this._options); + } catch (e) { + throw new Error(`Failed to create redis client: ${e.message}`); + } } public async connectAsync(): Promise { await this._redisClient.connect(); + // Check the connection works at early stage + try { + await this._redisClient.ping(); + } catch (e) { + throw new Error(`Failed to connect to redis server: ${e.message}`); + } } public async disconnectAsync(): Promise { @@ -45,7 +63,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async acquireLockAsync(context: ICobuildContext): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); const incrResult: number = await this._redisClient.incr(lockKey); const result: boolean = incrResult === 1; @@ -57,14 +75,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async renewLockAsync(context: ICobuildContext): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); await this._redisClient.expire(lockKey, 30); terminal.writeDebugLine(`Renewed lock for ${lockKey}`); } public async releaseLockAsync(context: ICobuildContext): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); await this._redisClient.set(lockKey, 0); terminal.writeDebugLine(`Released lock for ${lockKey}`); @@ -74,7 +92,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { context: ICobuildContext, state: ICobuildCompletedState ): Promise { - const { terminal } = context; + const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); const value: string = this._serializeCompletedState(state); await this._redisClient.set(key, value); @@ -82,12 +100,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async getCompletedStateAsync(context: ICobuildContext): Promise { + const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); let state: ICobuildCompletedState | undefined; const value: string | null = await this._redisClient.get(key); if (value) { state = this._deserializeCompletedState(value); } + terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); return state; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts index 1d1d69fdaeb..4e8f07d7f70 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RushRedisCobuildPlugin.ts @@ -33,7 +33,7 @@ export class RushRedisCobuildPlugin implements IRushPlugin { rushSession.hooks.initialize.tap(PLUGIN_NAME, () => { rushSession.registerCobuildLockProviderFactory('redis', (): RedisCobuildLockProvider => { const options: IRushRedisCobuildPluginOptions = this._options; - return new RedisCobuildLockProviderModule.RedisCobuildLockProvider(options); + return new RedisCobuildLockProviderModule.RedisCobuildLockProvider(options, rushSession); }); }); } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 5cd78099676..f908f762a1c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -2,14 +2,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { Terminal, ConsoleTerminalProvider } from '@rushstack/node-core-library'; -import { ICobuildCompletedState, ICobuildContext, OperationStatus } from '@rushstack/rush-sdk'; +import { ConsoleTerminalProvider } from '@rushstack/node-core-library'; +import { ICobuildCompletedState, ICobuildContext, OperationStatus, RushSession } from '@rushstack/rush-sdk'; import { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from '../RedisCobuildLockProvider'; import * as redisAPI from '@redis/client'; import type { RedisClientType } from '@redis/client'; -const terminal = new Terminal(new ConsoleTerminalProvider()); +const rushSession: RushSession = new RushSession({ + terminalProvider: new ConsoleTerminalProvider(), + getIsDebugMode: () => false +}); describe(RedisCobuildLockProvider.name, () => { let storage: Record = {}; @@ -37,14 +40,13 @@ describe(RedisCobuildLockProvider.name, () => { }); function prepareSubject(): RedisCobuildLockProvider { - return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions); + return new RedisCobuildLockProvider({} as IRedisCobuildLockProviderOptions, rushSession); } const context: ICobuildContext = { contextId: '123', cacheId: 'abc', - version: 1, - terminal + version: 1 }; it('getLockKey works', () => { From fe0a50c5064f8bbfc28aeaad12f45d1f7914c304 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 16 Feb 2023 16:57:05 +0800 Subject: [PATCH 009/100] fix: changes after rebase --- .../logic/operations/ShellOperationRunner.ts | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 063b76c6e4a..a5c9d960964 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -261,10 +261,11 @@ export class ShellOperationRunner implements IOperationRunner { // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; if (this._cobuildConfiguration?.cobuildEnabled) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ terminal, - trackedFiles - ); + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }); cobuildLock = await this._tryGetCobuildLockAsync(projectBuildCache); } @@ -300,7 +301,11 @@ export class ShellOperationRunner implements IOperationRunner { if (restoreFromCacheSuccess) { // Restore the original state of the operation without cache - await context._operationStateFile?.tryRestoreAsync(); + await context._operationMetadataManager?.tryRestoreAsync({ + terminal, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath + }); if (cobuildCompletedState) { return cobuildCompletedState.status; } @@ -308,7 +313,7 @@ export class ShellOperationRunner implements IOperationRunner { } } } else if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync( + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ terminal, trackedProjectFiles, operationMetadataManager: context._operationMetadataManager @@ -483,7 +488,11 @@ export class ShellOperationRunner implements IOperationRunner { // write a new cache entry. if (!setCacheEntryPromise && this.isCacheWriteAllowed) { setCacheEntryPromise = ( - await this._tryGetProjectBuildCacheAsync(terminal, trackedFiles) + await this._tryGetProjectBuildCacheAsync({ + terminal, + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }) )?.trySetCacheEntryAsync(terminal); } } From 44a1e802c871a7ee5ca4d79720e8a1a4bb63a69b Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 14:48:10 +0800 Subject: [PATCH 010/100] fix: cobuild integration test --- .vscode/redis-cobuild.code-workspace | 73 +------------------ .../.vscode/tasks.json | 71 ++++++++++++++++++ .../repo/common/scripts/install-run.js | 2 +- .../sandbox/repo/projects/a/package.json | 4 +- .../sandbox/repo/projects/build.js | 6 +- .../src/runRush.ts | 5 +- .../src/testLockProvider.ts | 1 - common/reviews/api/rush-lib.api.md | 2 - .../api/rush-redis-cobuild-plugin.api.md | 2 - .../rush-lib/src/cli/RushCommandLineParser.ts | 3 +- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 4 - .../src/logic/cobuild/ICobuildLockProvider.ts | 1 - .../logic/operations/AsyncOperationQueue.ts | 34 ++++----- .../logic/operations/OperationStateFile.ts | 2 +- .../logic/operations/ShellOperationRunner.ts | 50 ++++++------- .../src/RedisCobuildLockProvider.ts | 56 ++++++++------ .../src/test/RedisCobuildLockProvider.test.ts | 5 -- 17 files changed, 160 insertions(+), 161 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json diff --git a/.vscode/redis-cobuild.code-workspace b/.vscode/redis-cobuild.code-workspace index 9ab11bb4b4b..c51d9ed6da0 100644 --- a/.vscode/redis-cobuild.code-workspace +++ b/.vscode/redis-cobuild.code-workspace @@ -16,76 +16,5 @@ "name": ".vscode", "path": "../.vscode" } - ], - "tasks": { - "version": "2.0.0", - "tasks": [ - { - "type": "shell", - "label": "cobuild", - "dependsOrder": "sequence", - "dependsOn": ["update 1", "_cobuild"], - "problemMatcher": [] - }, - { - "type": "shell", - "label": "_cobuild", - "dependsOn": ["build 1", "build 2"], - "problemMatcher": [] - }, - { - "type": "shell", - "label": "update", - "command": "node ../../lib/runRush.js update", - "problemMatcher": [], - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "dedicated", - "showReuseMessage": true, - "clear": false - }, - "options": { - "cwd": "${workspaceFolder}/sandbox/repo" - } - }, - { - "type": "shell", - "label": "build 1", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", - "problemMatcher": [], - "options": { - "cwd": "${workspaceFolder}/sandbox/repo" - }, - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "dedicated", - "showReuseMessage": true, - "clear": true - }, - "group": "build" - }, - { - "type": "shell", - "label": "build 2", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", - "problemMatcher": [], - "options": { - "cwd": "${workspaceFolder}/sandbox/repo" - }, - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "dedicated", - "showReuseMessage": true, - "clear": true - }, - "group": "build" - } - ] - } + ] } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json new file mode 100644 index 00000000000..917c7ccf6c9 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -0,0 +1,71 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "cobuild", + "dependsOrder": "sequence", + "dependsOn": ["update 1", "_cobuild"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "_cobuild", + "dependsOn": ["build 1", "build 2"], + "problemMatcher": [] + }, + { + "type": "shell", + "label": "update", + "command": "node ../../lib/runRush.js update", + "problemMatcher": [], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": false + }, + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + } + }, + { + "type": "shell", + "label": "build 1", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + }, + { + "type": "shell", + "label": "build 2", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "problemMatcher": [], + "options": { + "cwd": "${workspaceFolder}/sandbox/repo" + }, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": true + }, + "group": "build" + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js index bcd982b369e..68b1b56fc58 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js @@ -238,8 +238,8 @@ var __webpack_exports__ = {}; __webpack_require__.r(__webpack_exports__); /* harmony export */ __webpack_require__.d(__webpack_exports__, { /* harmony export */ "RUSH_JSON_FILENAME": () => (/* binding */ RUSH_JSON_FILENAME), -/* harmony export */ "getNpmPath": () => (/* binding */ getNpmPath), /* harmony export */ "findRushJsonFolder": () => (/* binding */ findRushJsonFolder), +/* harmony export */ "getNpmPath": () => (/* binding */ getNpmPath), /* harmony export */ "installAndRun": () => (/* binding */ installAndRun), /* harmony export */ "runWithErrorAndStatusCode": () => (/* binding */ runWithErrorAndStatusCode) /* harmony export */ }); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json index 9472fdbd7e5..25b54c04e8d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -2,7 +2,7 @@ "name": "a", "version": "1.0.0", "scripts": { - "cobuild": "node ../build.js", - "build": "node ../build.js" + "cobuild": "node ../build.js a", + "build": "node ../build.js a" } } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js index 14855ff8c72..20f983ed146 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js @@ -2,11 +2,13 @@ const path = require('path'); const { FileSystem } = require('@rushstack/node-core-library'); -console.log('start'); +const args = process.argv.slice(2); + +console.log('start', args.join(' ')); setTimeout(() => { const outputFolder = path.resolve(process.cwd(), 'dist'); const outputFile = path.resolve(outputFolder, 'output.txt'); FileSystem.ensureFolder(outputFolder); - FileSystem.writeFile(outputFile, 'Hello world!'); + FileSystem.writeFile(outputFile, `Hello world! ${args.join(' ')}`); console.log('done'); }, 5000); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts index 50c44d60968..c758ec650a0 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -1,5 +1,6 @@ -import { RushCommandLineParser } from '@microsoft/rush-lib/lib/cli/RushCommandLineParser'; -import * as rushLib from '@microsoft/rush-lib'; +// Import from lib-commonjs for easy debugging +import { RushCommandLineParser } from '@microsoft/rush-lib/lib-commonjs/cli/RushCommandLineParser'; +import * as rushLib from '@microsoft/rush-lib/lib-commonjs'; // Setup redis cobuild plugin const builtInPluginConfigurations: rushLib._IBuiltInPluginConfiguration[] = []; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 474abae5bd9..859135e6eda 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -30,7 +30,6 @@ async function main(): Promise { status: OperationStatus.Success, cacheId: 'test-cache-id' }); - await lockProvider.releaseLockAsync(context); const completedState = await lockProvider.getCompletedStateAsync(context); console.log('Completed state: ', completedState); await lockProvider.disconnectAsync(); diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 09900023ba0..debe1604379 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -294,8 +294,6 @@ export interface ICobuildLockProvider { // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; // (undocumented) - releaseLockAsync(context: ICobuildContext): Promise; - // (undocumented) renewLockAsync(context: ICobuildContext): Promise; // (undocumented) setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index b19ae9aa26b..93da87162d0 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -30,8 +30,6 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { getCompletedStateKey(context: ICobuildContext): string; getLockKey(context: ICobuildContext): string; // (undocumented) - releaseLockAsync(context: ICobuildContext): Promise; - // (undocumented) renewLockAsync(context: ICobuildContext): Promise; // (undocumented) setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; diff --git a/libraries/rush-lib/src/cli/RushCommandLineParser.ts b/libraries/rush-lib/src/cli/RushCommandLineParser.ts index 2f027d1f09e..d5ba58362ca 100644 --- a/libraries/rush-lib/src/cli/RushCommandLineParser.ts +++ b/libraries/rush-lib/src/cli/RushCommandLineParser.ts @@ -187,7 +187,8 @@ export class RushCommandLineParser extends CommandLineParser { } public async execute(args?: string[]): Promise { - this._terminalProvider.verboseEnabled = this.isDebug; + // debugParameter will be correctly parsed during super.execute(), so manually parse here. + this._terminalProvider.debugEnabled = process.argv.indexOf('--debug') >= 0; await this.pluginManager.tryInitializeUnassociatedPluginsAsync(); diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index aa7875ff687..22d1f7818d3 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -62,10 +62,6 @@ export class CobuildLock { return acquireLockResult; } - public async releaseLockAsync(): Promise { - await this.cobuildConfiguration.cobuildLockProvider.releaseLockAsync(this._cobuildContext); - } - public async renewLockAsync(): Promise { await this.cobuildConfiguration.cobuildLockProvider.renewLockAsync(this._cobuildContext); } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index be00a297c11..73092467805 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -32,7 +32,6 @@ export interface ICobuildLockProvider { disconnectAsync(): Promise; acquireLockAsync(context: ICobuildContext): Promise; renewLockAsync(context: ICobuildContext): Promise; - releaseLockAsync(context: ICobuildContext): Promise; setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; getCompletedStateAsync(context: ICobuildContext): Promise; } diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 9e0cd9ef5eb..c4bd76ba542 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -126,26 +126,26 @@ export class AsyncOperationQueue } if (waitingIterators.length > 0) { - // cycle through the queue again to find the next operation that is executed remotely - for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { - const operation: OperationExecutionRecord = queue[i]; - - if (operation.status === OperationStatus.RemoteExecuting) { - // try to attempt to get the lock again - waitingIterators.shift()!({ - value: operation, - done: false - }); + // Pause for a few time + setTimeout(() => { + // cycle through the queue again to find the next operation that is executed remotely + for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { + const operation: OperationExecutionRecord = queue[i]; + + if (operation.status === OperationStatus.RemoteExecuting) { + // try to attempt to get the lock again + waitingIterators.shift()!({ + value: operation, + done: false + }); + } } - } - if (waitingIterators.length > 0) { - // Queue is not empty, but no operations are ready to process - // Pause for a second and start over - setTimeout(() => { + if (waitingIterators.length > 0) { + // Queue is not empty, but no operations are ready to process, start over this.assignOperations(); - }, 1000); - } + } + }, 5000); } } diff --git a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts index de8b6ed7321..021c66aca60 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts @@ -53,7 +53,7 @@ export class OperationStateFile { } public async writeAsync(json: IOperationStateJson): Promise { - await JsonFile.saveAsync(json, this.filepath, { ensureFolderExists: true, updateExistingFile: true }); + await JsonFile.saveAsync(json, this.filepath, { ensureFolderExists: true }); this._state = json; } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index a5c9d960964..cc452034a81 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -431,7 +431,22 @@ export class ShellOperationRunner implements IOperationRunner { } ); - let setCompletedStatePromise: Promise | undefined; + const taskIsSuccessful: boolean = + status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + this.warningsAreAllowed && + !!this._rushConfiguration.experimentsConfiguration.configuration + .buildCacheWithAllowWarningsInSuccessfulBuild); + + // Save the metadata to disk + const { duration: durationInSeconds } = context.stopwatch; + await context._operationMetadataManager?.saveAsync({ + durationInSeconds, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath + }); + + let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; let setCacheEntryPromise: Promise | undefined; if (cobuildLock && this.isCacheWriteAllowed) { const { projectBuildCache } = cobuildLock; @@ -445,14 +460,13 @@ export class ShellOperationRunner implements IOperationRunner { case OperationStatus.SuccessWithWarning: case OperationStatus.Success: case OperationStatus.Failure: { - setCompletedStatePromise = cobuildLock - .setCompletedStateAsync({ - status, + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, cacheId: finalCacheId - }) - .then(() => { - return cobuildLock?.releaseLockAsync(); }); + }; setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( terminal, finalCacheId @@ -462,13 +476,6 @@ export class ShellOperationRunner implements IOperationRunner { } } - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - this.warningsAreAllowed && - !!this._rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - let writeProjectStatePromise: Promise | undefined; if (taskIsSuccessful && projectDeps) { // Write deps on success. @@ -476,14 +483,6 @@ export class ShellOperationRunner implements IOperationRunner { ensureFolderExists: true }); - // If the operation without cache was successful, we can save the metadata to disk - const { duration: durationInSeconds } = context.stopwatch; - await context._operationMetadataManager?.saveAsync({ - durationInSeconds, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }); - // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. if (!setCacheEntryPromise && this.isCacheWriteAllowed) { @@ -496,11 +495,8 @@ export class ShellOperationRunner implements IOperationRunner { )?.trySetCacheEntryAsync(terminal); } } - const [, cacheWriteSuccess] = await Promise.all([ - writeProjectStatePromise, - setCacheEntryPromise, - setCompletedStatePromise - ]); + const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); + await setCompletedStatePromiseFunction?.(); if (terminalProvider.hasErrors) { status = OperationStatus.Failure; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index d7d22d2898c..c6ae989db55 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -49,9 +49,9 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async connectAsync(): Promise { - await this._redisClient.connect(); - // Check the connection works at early stage try { + await this._redisClient.connect(); + // Check the connection works at early stage await this._redisClient.ping(); } catch (e) { throw new Error(`Failed to connect to redis server: ${e.message}`); @@ -59,17 +59,26 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } public async disconnectAsync(): Promise { - await this._redisClient.disconnect(); + try { + await this._redisClient.disconnect(); + } catch (e) { + throw new Error(`Failed to disconnect to redis server: ${e.message}`); + } } public async acquireLockAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); - const incrResult: number = await this._redisClient.incr(lockKey); - const result: boolean = incrResult === 1; - terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); - if (result) { - await this.renewLockAsync(context); + let result: boolean = false; + try { + const incrResult: number = await this._redisClient.incr(lockKey); + result = incrResult === 1; + terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); + if (result) { + await this.renewLockAsync(context); + } + } catch (e) { + throw new Error(`Failed to acquire lock for ${lockKey}: ${e.message}`); } return result; } @@ -77,17 +86,14 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { public async renewLockAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; const lockKey: string = this.getLockKey(context); - await this._redisClient.expire(lockKey, 30); + try { + await this._redisClient.expire(lockKey, 30); + } catch (e) { + throw new Error(`Failed to renew lock for ${lockKey}: ${e.message}`); + } terminal.writeDebugLine(`Renewed lock for ${lockKey}`); } - public async releaseLockAsync(context: ICobuildContext): Promise { - const { _terminal: terminal } = this; - const lockKey: string = this.getLockKey(context); - await this._redisClient.set(lockKey, 0); - terminal.writeDebugLine(`Released lock for ${lockKey}`); - } - public async setCompletedStateAsync( context: ICobuildContext, state: ICobuildCompletedState @@ -95,7 +101,11 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); const value: string = this._serializeCompletedState(state); - await this._redisClient.set(key, value); + try { + await this._redisClient.set(key, value); + } catch (e) { + throw new Error(`Failed to set completed state for ${key}: ${e.message}`); + } terminal.writeDebugLine(`Set completed state for ${key}: ${value}`); } @@ -103,11 +113,15 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { const { _terminal: terminal } = this; const key: string = this.getCompletedStateKey(context); let state: ICobuildCompletedState | undefined; - const value: string | null = await this._redisClient.get(key); - if (value) { - state = this._deserializeCompletedState(value); + try { + const value: string | null = await this._redisClient.get(key); + if (value) { + state = this._deserializeCompletedState(value); + } + terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); + } catch (e) { + throw new Error(`Failed to get completed state for ${key}: ${e.message}`); } - terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); return state; } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index f908f762a1c..3425e83c31a 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -79,11 +79,6 @@ describe(RedisCobuildLockProvider.name, () => { expect(result2).toBe(false); }); - it('releaseLockAsync works', async () => { - const subject: RedisCobuildLockProvider = prepareSubject(); - expect(() => subject.releaseLockAsync(context)).not.toThrowError(); - }); - it('set and get completedState works', async () => { const subject: RedisCobuildLockProvider = prepareSubject(); const cacheId: string = 'foo'; From 662b5efa45750a9843b92bc1a3785dd35b851564 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 14:56:32 +0800 Subject: [PATCH 011/100] :memo: update readme --- .../README.md | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index 87b7b6e0af9..b2bbdd801cf 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -10,9 +10,27 @@ In this folder run `docker-compose up -d` # Stop the Redis In this folder run `docker-compose down` -# Run the test +# Install and build the integration test code + +```sh +rush update +rush build -t rush-redis-cobuild-plugin-integration-test +``` + +# Run the test for lock provider + ```sh # start the docker container: docker-compose up -d # build the code: rushx build rushx test-lock-provider ``` + +# Testing cobuild + +> Note: This test requires Visual Studio Code to be installed. + +1. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. + +2. Open Command Palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. + +3. Two new terminal windows will open. Both running `rush cobuild` command under sandbox repo. From 1a5085f80f73cf24558e137860907c84cdb2f5bf Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 15:02:18 +0800 Subject: [PATCH 012/100] chore: rush change --- .../@microsoft/rush/feat-cobuild_2023-02-17-07-02.json | 10 ++++++++++ .../feat-cobuild_2023-02-17-07-02.json | 10 ++++++++++ rush-plugins/rush-redis-cobuild-plugin/package.json | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json create mode 100644 common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json diff --git a/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json new file mode 100644 index 00000000000..c569757d07d --- /dev/null +++ b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "(EXPERIMENTAL) Add a cheap way to get distributed builds called \"cobuild\"", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json b/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json new file mode 100644 index 00000000000..77f64cbf75a --- /dev/null +++ b/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-redis-cobuild-plugin", + "comment": "Implement a redis lock provider for cobuild feature", + "type": "minor" + } + ], + "packageName": "@rushstack/rush-redis-cobuild-plugin" +} \ No newline at end of file diff --git a/rush-plugins/rush-redis-cobuild-plugin/package.json b/rush-plugins/rush-redis-cobuild-plugin/package.json index ce88b68a818..19caebf3bcd 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/package.json +++ b/rush-plugins/rush-redis-cobuild-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@rushstack/rush-redis-cobuild-plugin", - "version": "5.88.2", + "version": "0.0.0", "description": "Rush plugin for Redis cobuild lock", "repository": { "type": "git", From d0871f4b803ebd6788a3958057672fb11cda4d6c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 15:21:14 +0800 Subject: [PATCH 013/100] chore: tasks.json --- .../.vscode/tasks.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 917c7ccf6c9..8e1982bd3f9 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -5,7 +5,7 @@ "type": "shell", "label": "cobuild", "dependsOrder": "sequence", - "dependsOn": ["update 1", "_cobuild"], + "dependsOn": ["update", "_cobuild"], "problemMatcher": [] }, { @@ -34,7 +34,7 @@ { "type": "shell", "label": "build 1", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { "cwd": "${workspaceFolder}/sandbox/repo" @@ -52,7 +52,7 @@ { "type": "shell", "label": "build 2", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { "cwd": "${workspaceFolder}/sandbox/repo" From c40bfc93553b7432572f6ff712f94565b84dc7c2 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 17:34:13 +0800 Subject: [PATCH 014/100] feat: allow failing build to be cached when cobuilding --- .../sandbox/repo/projects/a/package.json | 1 + .../src/logic/operations/ShellOperationRunner.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json index 25b54c04e8d..f8b84111f98 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js a", + "_cobuild": "sleep 5 && exit 1", "build": "node ../build.js a" } } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index cc452034a81..5b9dea9446c 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -418,7 +418,13 @@ export class ShellOperationRunner implements IOperationRunner { subProcess.on('close', (code: number) => { try { if (code !== 0) { - reject(new OperationError('error', `Returned error code: ${code}`)); + if (cobuildLock) { + // In order to preventing the worst case that all cobuild tasks go through the same failure, + // allowing a failing build to be cached and retrieved + resolve(OperationStatus.Failure); + } else { + reject(new OperationError('error', `Returned error code: ${code}`)); + } } else if (hasWarningOrError) { resolve(OperationStatus.SuccessWithWarning); } else { @@ -500,7 +506,7 @@ export class ShellOperationRunner implements IOperationRunner { if (terminalProvider.hasErrors) { status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false) { + } else if (cacheWriteSuccess === false && status === OperationStatus.Success) { status = OperationStatus.SuccessWithWarning; } From 17754b493799b5b9798efd1a7042b8b92c22454f Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 19:24:10 +0800 Subject: [PATCH 015/100] feat: cobuild context id --- common/reviews/api/rush-lib.api.md | 2 + .../rush-lib/src/api/CobuildConfiguration.ts | 42 ++++++++----- .../src/logic/cobuild/CobuildContextId.ts | 59 +++++++++++++++++++ .../rush-lib/src/logic/cobuild/CobuildLock.ts | 4 ++ .../cobuild/test/CobuildContextId.test.ts | 37 ++++++++++++ .../logic/cobuild/test/CobuildLock.test.ts | 28 +++++++++ .../CobuildContextId.test.ts.snap | 5 ++ 7 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index debe1604379..e6bb418cb7f 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -97,7 +97,9 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { + readonly cobuildContextId: string; readonly cobuildEnabled: boolean; + // (undocumented) readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) connectLockProviderAsync(): Promise; diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 1b4b636c390..f571feec6a8 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -2,7 +2,13 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; +import { + AlreadyReportedError, + FileSystem, + ITerminal, + JsonFile, + JsonSchema +} from '@rushstack/node-core-library'; import schemaJson from '../schemas/cobuild.schema.json'; import { EnvironmentConfiguration } from './EnvironmentConfiguration'; import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; @@ -10,6 +16,7 @@ import { RushConstants } from '../logic/RushConstants'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; +import { CobuildContextId, GetCobuildContextIdFunction } from '../logic/cobuild/CobuildContextId'; export interface ICobuildJson { cobuildEnabled: boolean; @@ -19,6 +26,7 @@ export interface ICobuildJson { export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; + getCobuildContextId: GetCobuildContextIdFunction; rushConfiguration: RushConfiguration; rushSession: RushSession; } @@ -38,15 +46,18 @@ export class CobuildConfiguration { public readonly cobuildEnabled: boolean; /** * Method to calculate the cobuild context id - * FIXME: */ - // public readonly getCacheEntryId: GetCacheEntryIdFunction; + public readonly cobuildContextId: string; public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? options.cobuildJson.cobuildEnabled; - const { cobuildJson } = options; + const { cobuildJson, getCobuildContextId } = options; + + this.cobuildContextId = getCobuildContextId({ + environment: process.env + }); const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); @@ -87,27 +98,26 @@ export class CobuildConfiguration { CobuildConfiguration._jsonSchema ); - // FIXME: - // let getCacheEntryId: GetCacheEntryIdFunction; - // try { - // getCacheEntryId = CacheEntryId.parsePattern(cobuildJson.cacheEntryNamePattern); - // } catch (e) { - // terminal.writeErrorLine( - // `Error parsing cache entry name pattern "${cobuildJson.cacheEntryNamePattern}": ${e}` - // ); - // throw new AlreadyReportedError(); - // } + let getCobuildContextId: GetCobuildContextIdFunction; + try { + getCobuildContextId = CobuildContextId.parsePattern(cobuildJson.cobuildContextIdPattern); + } catch (e) { + terminal.writeErrorLine( + `Error parsing cobuild context id pattern "${cobuildJson.cobuildContextIdPattern}": ${e}` + ); + throw new AlreadyReportedError(); + } return new CobuildConfiguration({ cobuildJson, + getCobuildContextId, rushConfiguration, rushSession }); } public get contextId(): string { - // FIXME: hardcode - return '123'; + return this.cobuildContextId; } public async connectLockProviderAsync(): Promise { diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts b/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts new file mode 100644 index 00000000000..2c191eca242 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { RushConstants } from '../RushConstants'; + +export interface IGenerateCobuildContextIdOptions { + environment: NodeJS.ProcessEnv; +} + +/** + * Calculates the cache entry id string for an operation. + * @beta + */ +export type GetCobuildContextIdFunction = (options: IGenerateCobuildContextIdOptions) => string; + +export class CobuildContextId { + private constructor() {} + + public static parsePattern(pattern?: string): GetCobuildContextIdFunction { + if (!pattern) { + return () => ''; + } else { + const resolvedPattern: string = pattern.trim(); + + return (options: IGenerateCobuildContextIdOptions) => { + const { environment } = options; + return this._expandWithEnvironmentVariables(resolvedPattern, environment); + }; + } + } + + private static _expandWithEnvironmentVariables(pattern: string, environment: NodeJS.ProcessEnv): string { + const missingEnvironmentVariables: Set = new Set(); + const expandedPattern: string = pattern.replace( + /\$\{([^\}]+)\}/g, + (match: string, variableName: string): string => { + const variable: string | undefined = + variableName in environment ? environment[variableName] : undefined; + if (variable !== undefined) { + return variable; + } else { + missingEnvironmentVariables.add(variableName); + return match; + } + } + ); + if (missingEnvironmentVariables.size) { + throw new Error( + `The "cobuildContextIdPattern" value in ${ + RushConstants.cobuildFilename + } contains missing environment variable${ + missingEnvironmentVariables.size > 1 ? 's' : '' + }: ${Array.from(missingEnvironmentVariables).join(', ')}` + ); + } + + return expandedPattern; + } +} diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 22d1f7818d3..9b656a9877d 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -65,4 +65,8 @@ export class CobuildLock { public async renewLockAsync(): Promise { await this.cobuildConfiguration.cobuildLockProvider.renewLockAsync(this._cobuildContext); } + + public get cobuildContext(): ICobuildContext { + return this._cobuildContext; + } } diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts new file mode 100644 index 00000000000..ea18a0e9242 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { CobuildContextId } from '../CobuildContextId'; + +describe(CobuildContextId.name, () => { + describe('Valid pattern names', () => { + it('expands a environment variable', () => { + const contextId: string = CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ + environment: { + MR_ID: '123', + AUTHOR_NAME: 'Mr.example' + } + }); + expect(contextId).toEqual('context-123-Mr.example'); + }); + }); + + describe('Invalid pattern names', () => { + it('throws an error if a environment variable is missing', () => { + expect(() => + CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ + environment: { + MR_ID: '123' + } + }) + ).toThrowErrorMatchingSnapshot(); + }); + it('throws an error if multiple environment variables are missing', () => { + expect(() => + CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ + environment: {} + }) + ).toThrowErrorMatchingSnapshot(); + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts new file mode 100644 index 00000000000..2603dbafb93 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { CobuildConfiguration } from '../../../api/CobuildConfiguration'; +import { ProjectBuildCache } from '../../buildCache/ProjectBuildCache'; +import { CobuildLock } from '../CobuildLock'; + +describe(CobuildLock.name, () => { + function prepareSubject(): CobuildLock { + const subject: CobuildLock = new CobuildLock({ + cobuildConfiguration: { + contextId: 'foo' + } as unknown as CobuildConfiguration, + projectBuildCache: { + cacheId: 'bar' + } as unknown as ProjectBuildCache + }); + return subject; + } + it('returns cobuild context', () => { + const subject: CobuildLock = prepareSubject(); + expect(subject.cobuildContext).toEqual({ + contextId: 'foo', + cacheId: 'bar', + version: 1 + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap b/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap new file mode 100644 index 00000000000..c2495bc8537 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap @@ -0,0 +1,5 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CobuildContextId Invalid pattern names throws an error if a environment variable is missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variable: AUTHOR_NAME"`; + +exports[`CobuildContextId Invalid pattern names throws an error if multiple environment variables are missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variables: MR_ID, AUTHOR_NAME"`; From 7c2ec9eff372c1a362f4ebd8cc9fd8f6bfdb16aa Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 21:45:07 +0800 Subject: [PATCH 016/100] feat: add RUSH_COBUILD_CONTEXT_ID environment variable --- .../rush-lib/src/api/CobuildConfiguration.ts | 8 +++--- .../src/api/EnvironmentConfiguration.ts | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index f571feec6a8..214e61c0a03 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -55,9 +55,11 @@ export class CobuildConfiguration { const { cobuildJson, getCobuildContextId } = options; - this.cobuildContextId = getCobuildContextId({ - environment: process.env - }); + this.cobuildContextId = + EnvironmentConfiguration.cobuildContextId ?? + getCobuildContextId({ + environment: process.env + }); const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 345f98f5fd1..e2224ce3d7f 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -154,6 +154,15 @@ export enum EnvironmentVariableNames { */ RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', + /** + * Setting this environment variable overrides the value of `cobuildContextId` calculated by + * `cobuildContextIdPattern` in the `cobuild.json` configuration file. + * + * @remarks + * If there is no cobuild configured, then this environment variable is ignored. + */ + RUSH_COBUILD_CONTEXT_ID = 'RUSH_COBUILD_CONTEXT_ID', + /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. */ @@ -209,6 +218,8 @@ export class EnvironmentConfiguration { private static _cobuildEnabled: boolean | undefined; + private static _cobuildContextId: string | undefined; + private static _gitBinaryPath: string | undefined; private static _tarBinaryPath: string | undefined; @@ -315,6 +326,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._cobuildEnabled; } + /** + * Provides a determined cobuild context id if configured + * See {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID} + */ + public static get cobuildContextId(): string | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildContextId; + } + /** * Allows the git binary path to be explicitly provided. * See {@link EnvironmentVariableNames.RUSH_GIT_BINARY_PATH} @@ -454,6 +474,11 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID: { + EnvironmentConfiguration._cobuildContextId = value; + break; + } + case EnvironmentVariableNames.RUSH_GIT_BINARY_PATH: { EnvironmentConfiguration._gitBinaryPath = value; break; From 64fd7a4b45ab1b636c50a6c2590922aab530c708 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 22:31:23 +0800 Subject: [PATCH 017/100] :memo: --- .../README.md | 106 +++++++++++++++++- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index b2bbdd801cf..5a72f9010b5 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -1,13 +1,17 @@ # About + This package enables integration testing of the `RedisCobuildLockProvider` by connecting to an actual Redis created using an [redis](https://hub.docker.com/_/redis) docker image. # Prerequisites + Docker and docker compose must be installed # Start the Redis + In this folder run `docker-compose up -d` # Stop the Redis + In this folder run `docker-compose down` # Install and build the integration test code @@ -25,12 +29,106 @@ rush build -t rush-redis-cobuild-plugin-integration-test rushx test-lock-provider ``` -# Testing cobuild +# Integration test in sandbox repo + +Sandbox repo folder: **build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo** + +```sh +cd sandbox/repo +rush update +``` + +## Case 1: Disable cobuild by setting `RUSH_COBUILD_ENABLED=0` + +```sh +rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is disabled. Run command successfully. + +```sh +RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. + +## Case 2: Cobuild enabled, run one cobuild command only + +1. Clear redis server + +```sh +(cd ../.. && docker compose down && docker compose up -d) +``` + +2. Run `rush cobuild` command + +```sh +rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is enabled. Run command successfully. +You can also see cobuild related logs in the terminal. + +```sh +Get completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null +Acquired lock for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success +Set completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 +``` + +## Case 3: Cobuild enabled, run two cobuild commands in parallel > Note: This test requires Visual Studio Code to be installed. -1. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. +1. Clear redis server + +```sh +(cd ../.. && docker compose down && docker compose up -d) +``` + +2. Clear build cache + +```sh +rm -rf common/temp/build-cache +``` + +3. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. + +4. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. + +> In this step, two dedicated terminal windows will open. Running `rush cobuild` command under sandbox repo respectively. + +Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. + +## Case 4: Cobuild enabled, run two cobuild commands in parallel, one of them failed + +> Note: This test requires Visual Studio Code to be installed. + +1. Making the cobuild command of project "A" fails + +**sandbox/repo/projects/a/package.json** + +```diff + "scripts": { +- "cobuild": "node ../build.js a", ++ "cobuild": "sleep 5 && exit 1", + "build": "node ../build.js a" + } +``` + +2. Clear redis server + +```sh +(cd ../.. && docker compose down && docker compose up -d) +``` + +3. Clear build cache + +```sh +rm -rf common/temp/build-cache +``` + +4. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. -2. Open Command Palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. +5. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. -3. Two new terminal windows will open. Both running `rush cobuild` command under sandbox repo. +Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. These two cobuild commands fail because of the failing build of project "A". And, one of them restored the failing build cache created by the other one. From e4cfabcc00784ace92430fa6a264b2970ff1df2b Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Feb 2023 23:28:46 +0800 Subject: [PATCH 018/100] fix: test --- common/reviews/api/rush-lib.api.md | 2 ++ libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore | 1 + .../src/cli/actions/test/removeRepo/common/temp/rush#90625.lock | 1 - .../src/logic/operations/test/AsyncOperationQueue.test.ts | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore delete mode 100644 libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index e6bb418cb7f..fa04eb4266a 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -171,6 +171,7 @@ export class EnvironmentConfiguration { static get buildCacheCredential(): string | undefined; static get buildCacheEnabled(): boolean | undefined; static get buildCacheWriteAllowed(): boolean | undefined; + static get cobuildContextId(): string | undefined; static get cobuildEnabled(): boolean | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // @@ -196,6 +197,7 @@ export enum EnvironmentVariableNames { RUSH_BUILD_CACHE_CREDENTIAL = "RUSH_BUILD_CACHE_CREDENTIAL", RUSH_BUILD_CACHE_ENABLED = "RUSH_BUILD_CACHE_ENABLED", RUSH_BUILD_CACHE_WRITE_ALLOWED = "RUSH_BUILD_CACHE_WRITE_ALLOWED", + RUSH_COBUILD_CONTEXT_ID = "RUSH_COBUILD_CONTEXT_ID", RUSH_COBUILD_ENABLED = "RUSH_COBUILD_ENABLED", RUSH_DEPLOY_TARGET_FOLDER = "RUSH_DEPLOY_TARGET_FOLDER", RUSH_GIT_BINARY_PATH = "RUSH_GIT_BINARY_PATH", diff --git a/libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore b/libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore new file mode 100644 index 00000000000..b37486fa4a9 --- /dev/null +++ b/libraries/rush-lib/src/cli/actions/test/removeRepo/.gitignore @@ -0,0 +1 @@ +common/temp \ No newline at end of file diff --git a/libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock b/libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock deleted file mode 100644 index b2b0d4002b8..00000000000 --- a/libraries/rush-lib/src/cli/actions/test/removeRepo/common/temp/rush#90625.lock +++ /dev/null @@ -1 +0,0 @@ -Fri Jan 27 01:50:33 2023 \ No newline at end of file diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 212c7c2edd3..e0cd272a802 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -181,5 +181,5 @@ describe(AsyncOperationQueue.name, () => { } expect(actualOrder).toEqual(expectedOrder); - }); + }, 6000); }); From e2353fc4f084b94297d1573aa1baad824b9d3045 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Feb 2023 14:51:53 +0800 Subject: [PATCH 019/100] chore --- libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 5b9dea9446c..678325448e5 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -159,7 +159,6 @@ export class ShellOperationRunner implements IOperationRunner { ); const runnerWatcher: RunnerWatcher = new RunnerWatcher({ interval: 10 * 1000 - // interval: 1000 }); try { From 27f78f0a23a441f37b266aee18415daddfe31868 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 22 Feb 2023 16:51:32 +0800 Subject: [PATCH 020/100] chore --- rush-plugins/rush-redis-cobuild-plugin/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rush-plugins/rush-redis-cobuild-plugin/README.md b/rush-plugins/rush-redis-cobuild-plugin/README.md index bfb2d49760b..9ffa0231a46 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/README.md +++ b/rush-plugins/rush-redis-cobuild-plugin/README.md @@ -1,4 +1,4 @@ -# @rushstack/rush-amazon-s3-build-cache-plugin +# @rushstack/rush-redis-cobuild-plugin This is a Rush plugin for using Redis as cobuild lock provider during the "build" From afd26fc814a33318f2d0a35b074033d6ca446d6c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 28 Feb 2023 19:19:15 +0800 Subject: [PATCH 021/100] refact(ShellOperationRunner): extract build cache related logic to plugin --- common/reviews/api/rush-lib.api.md | 6 +- .../CacheableOperationRunnerPlugin.ts | 449 ++++++++++++++++++ .../src/logic/operations/IOperationRunner.ts | 20 +- .../operations/IOperationRunnerPlugin.ts | 14 + .../operations/OperationExecutionManager.ts | 26 +- .../operations/OperationExecutionRecord.ts | 5 - .../logic/operations/OperationLifecycle.ts | 60 +++ .../{RunnerWatcher.ts => PeriodicCallback.ts} | 6 +- .../logic/operations/ShellOperationRunner.ts | 404 +++------------- .../operations/ShellOperationRunnerPlugin.ts | 26 +- 10 files changed, 647 insertions(+), 369 deletions(-) create mode 100644 libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts create mode 100644 libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts create mode 100644 libraries/rush-lib/src/logic/operations/OperationLifecycle.ts rename libraries/rush-lib/src/logic/operations/{RunnerWatcher.ts => PeriodicCallback.ts} (90%) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index fa04eb4266a..901545dfa60 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -477,8 +477,6 @@ export interface IOperationOptions { // @beta export interface IOperationRunner { executeAsync(context: IOperationRunnerContext): Promise; - isCacheWriteAllowed: boolean; - isSkipAllowed: boolean; readonly name: string; reportTiming: boolean; silent: boolean; @@ -489,12 +487,14 @@ export interface IOperationRunner { export interface IOperationRunnerContext { collatedWriter: CollatedWriter; debugMode: boolean; + error?: Error; // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; status: OperationStatus; stdioSummarizer: StdioSummarizer; - stopwatch: IStopwatchResult; + // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts + stopwatch: Stopwatch; } // @internal (undocumented) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts new file mode 100644 index 00000000000..4005e754d27 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts @@ -0,0 +1,449 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; +import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; +import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import { OperationStatus } from './OperationStatus'; +import { ColorValue, InternalError, ITerminal, JsonObject } from '@rushstack/node-core-library'; +import { RushConstants } from '../RushConstants'; +import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; +import { PrintUtilities } from '@rushstack/terminal'; + +import type { IOperationRunnerPlugin } from './IOperationRunnerPlugin'; +import type { + IOperationRunnerAfterExecuteContext, + IOperationRunnerBeforeExecuteContext, + OperationRunnerLifecycleHooks +} from './OperationLifecycle'; +import type { OperationMetadataManager } from './OperationMetadataManager'; +import type { IOperationRunner } from './IOperationRunner'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import type { IPhase } from '../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; + +const PLUGIN_NAME: 'CacheableOperationRunnerPlugin' = 'CacheableOperationRunnerPlugin'; + +export interface ICacheableOperationRunnerPluginOptions { + buildCacheConfiguration: BuildCacheConfiguration; + cobuildConfiguration: CobuildConfiguration | undefined; + isIncrementalBuildAllowed: boolean; +} + +export interface IOperationBuildCacheContext { + isCacheWriteAllowed: boolean; + isCacheReadAllowed: boolean; + isSkipAllowed: boolean; + projectBuildCache: ProjectBuildCache | undefined; + cobuildLock: CobuildLock | undefined; +} + +export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { + private static _runnerBuildCacheContextMap: Map = new Map< + IOperationRunner, + IOperationBuildCacheContext + >(); + private readonly _buildCacheConfiguration: BuildCacheConfiguration; + private readonly _cobuildConfiguration: CobuildConfiguration | undefined; + + public constructor(options: ICacheableOperationRunnerPluginOptions) { + this._buildCacheConfiguration = options.buildCacheConfiguration; + this._cobuildConfiguration = options.cobuildConfiguration; + } + + public static getBuildCacheContextByRunner( + runner: IOperationRunner + ): IOperationBuildCacheContext | undefined { + const buildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.get(runner); + return buildCacheContext; + } + + public static getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { + const buildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); + if (!buildCacheContext) { + // This should not happen + throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); + } + return buildCacheContext; + } + + public static setBuildCacheContextByRunner( + runner: IOperationRunner, + buildCacheContext: IOperationBuildCacheContext + ): void { + CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.set(runner, buildCacheContext); + } + + public static clearAllBuildCacheContexts(): void { + CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.clear(); + } + + public apply(hooks: OperationRunnerLifecycleHooks): void { + hooks.beforeExecute.tapPromise( + PLUGIN_NAME, + async (beforeExecuteContext: IOperationRunnerBeforeExecuteContext) => { + const earlyReturnStatus: OperationStatus | undefined = await (async () => { + const { + context, + runner, + terminal, + lastProjectDeps, + projectDeps, + trackedProjectFiles, + logPath, + errorLogPath, + rushProject, + phase, + selectedPhases, + projectChangeAnalyzer, + commandName, + commandToRun, + earlyReturnStatus + } = beforeExecuteContext; + if (earlyReturnStatus) { + // If there is existing early return status, we don't need to do anything + return earlyReturnStatus; + } + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + + if (!projectDeps && buildCacheContext.isSkipAllowed) { + // To test this code path: + // Remove the `.git` folder then run "rush build --verbose" + terminal.writeLine({ + text: PrintUtilities.wrapWords( + 'This workspace does not appear to be tracked by Git. ' + + 'Rush will proceed without incremental execution, caching, and change detection.' + ), + foregroundColor: ColorValue.Cyan + }); + } + + const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + runner, + rushProject, + phase, + selectedPhases, + projectChangeAnalyzer, + commandName, + commandToRun, + terminal, + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }); + buildCacheContext.projectBuildCache = projectBuildCache; + + // Try to acquire the cobuild lock + let cobuildLock: CobuildLock | undefined; + if (this._cobuildConfiguration?.cobuildEnabled) { + cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache }); + } + buildCacheContext.cobuildLock = cobuildLock; + + // If possible, we want to skip this operation -- either by restoring it from the + // cache, if caching is enabled, or determining that the project + // is unchanged (using the older incremental execution logic). These two approaches, + // "caching" and "skipping", are incompatible, so only one applies. + // + // Note that "caching" and "skipping" take two different approaches + // to tracking dependents: + // + // - For caching, "isCacheReadAllowed" is set if a project supports + // incremental builds, and determining whether this project or a dependent + // has changed happens inside the hashing logic. + // + // - For skipping, "isSkipAllowed" is set to true initially, and during + // the process of running dependents, it will be changed by OperationExecutionManager to + // false if a dependency wasn't able to be skipped. + // + let buildCacheReadAttempted: boolean = false; + + if (cobuildLock) { + // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to + // the build cache once completed, but does not retrieve them (since the "incremental" + // flag is disabled). However, we still need a cobuild to be able to retrieve a finished + // build from another cobuild in this case. + const cobuildCompletedState: ICobuildCompletedState | undefined = + await cobuildLock.getCompletedStateAsync(); + if (cobuildCompletedState) { + const { status, cacheId } = cobuildCompletedState; + + const restoreFromCacheSuccess: boolean | undefined = + await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(terminal, cacheId); + + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await context._operationMetadataManager?.tryRestoreAsync({ + terminal, + logPath, + errorLogPath + }); + if (cobuildCompletedState) { + return cobuildCompletedState.status; + } + return status; + } + } + } else if (buildCacheContext.isCacheReadAllowed) { + buildCacheReadAttempted = !!projectBuildCache; + const restoreFromCacheSuccess: boolean | undefined = + await projectBuildCache?.tryRestoreFromCacheAsync(terminal); + + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await context._operationMetadataManager?.tryRestoreAsync({ + terminal, + logPath, + errorLogPath + }); + return OperationStatus.FromCache; + } + } + if (buildCacheContext.isSkipAllowed && !buildCacheReadAttempted) { + const isPackageUnchanged: boolean = !!( + lastProjectDeps && + projectDeps && + projectDeps.arguments === lastProjectDeps.arguments && + _areShallowEqual(projectDeps.files, lastProjectDeps.files) + ); + + if (isPackageUnchanged) { + return OperationStatus.Skipped; + } + } + + if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + if (context.status === OperationStatus.RemoteExecuting) { + // This operation is used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; + } + runner.periodicCallback.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + } else { + // failed to acquire the lock, mark current operation to remote executing + context.stopwatch.reset(); + return OperationStatus.RemoteExecuting; + } + } + })(); + if (earlyReturnStatus) { + beforeExecuteContext.earlyReturnStatus = earlyReturnStatus; + } + return beforeExecuteContext; + } + ); + + hooks.afterExecute.tapPromise( + PLUGIN_NAME, + async (afterExecuteContext: IOperationRunnerAfterExecuteContext) => { + const { context, runner, terminal, status, taskIsSuccessful } = afterExecuteContext; + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + + const { cobuildLock, projectBuildCache, isCacheWriteAllowed } = buildCacheContext; + + let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; + let setCacheEntryPromise: Promise | undefined; + if (cobuildLock && isCacheWriteAllowed) { + if (context.error) { + // In order to preventing the worst case that all cobuild tasks go through the same failure, + // allowing a failing build to be cached and retrieved, print the error message to the terminal + // and clear the error in context. + const message: string | undefined = context.error?.message; + if (message) { + context.collatedWriter.terminal.writeStderrLine(message); + } + context.error = undefined; + } + const cacheId: string | undefined = cobuildLock.projectBuildCache.cacheId; + const contextId: string = cobuildLock.cobuildConfiguration.contextId; + + if (cacheId) { + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, + cacheId: finalCacheId + }); + }; + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + terminal, + finalCacheId + ); + } + } + } + } + + // If the command is successful, we can calculate project hash, and no dependencies were skipped, + // write a new cache entry. + if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && projectBuildCache) { + setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(terminal); + } + const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise; + await setCompletedStatePromiseFunction?.(); + + if (cacheWriteSuccess === false && afterExecuteContext.status === OperationStatus.Success) { + afterExecuteContext.status = OperationStatus.SuccessWithWarning; + } + + return afterExecuteContext; + } + ); + } + + private async _tryGetProjectBuildCacheAsync({ + runner, + rushProject, + phase, + selectedPhases, + projectChangeAnalyzer, + commandName, + commandToRun, + terminal, + trackedProjectFiles, + operationMetadataManager + }: { + runner: IOperationRunner; + rushProject: RushConfigurationProject; + phase: IPhase; + selectedPhases: Iterable; + projectChangeAnalyzer: ProjectChangeAnalyzer; + commandName: string; + commandToRun: string; + terminal: ITerminal; + trackedProjectFiles: string[] | undefined; + operationMetadataManager: OperationMetadataManager | undefined; + }): Promise { + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + if (!buildCacheContext.projectBuildCache) { + if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { + // Disable legacy skip logic if the build cache is in play + buildCacheContext.isSkipAllowed = false; + + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); + if (projectConfiguration) { + projectConfiguration.validatePhaseConfiguration(selectedPhases, terminal); + if (projectConfiguration.disableBuildCacheForProject) { + terminal.writeVerboseLine('Caching has been disabled for this project.'); + } else { + const operationSettings: IOperationSettings | undefined = + projectConfiguration.operationSettingsByOperationName.get(commandName); + if (!operationSettings) { + terminal.writeVerboseLine( + `This project does not define the caching behavior of the "${commandName}" command, so caching has been disabled.` + ); + } else if (operationSettings.disableBuildCacheForOperation) { + terminal.writeVerboseLine( + `Caching has been disabled for this project's "${commandName}" command.` + ); + } else { + const projectOutputFolderNames: ReadonlyArray = + operationSettings.outputFolderNames || []; + const additionalProjectOutputFilePaths: ReadonlyArray = [ + ...(operationMetadataManager?.relativeFilepaths || []) + ]; + const additionalContext: Record = {}; + if (operationSettings.dependsOnEnvVars) { + for (const varName of operationSettings.dependsOnEnvVars) { + additionalContext['$' + varName] = process.env[varName] || ''; + } + } + + if (operationSettings.dependsOnAdditionalFiles) { + const repoState: IRawRepoState | undefined = + await projectChangeAnalyzer._ensureInitializedAsync(terminal); + + const additionalFiles: Map = await getHashesForGlobsAsync( + operationSettings.dependsOnAdditionalFiles, + rushProject.projectFolder, + repoState + ); + + terminal.writeDebugLine( + `Including additional files to calculate build cache hash:\n ${Array.from( + additionalFiles.keys() + ).join('\n ')} ` + ); + + for (const [filePath, fileHash] of additionalFiles) { + additionalContext['file://' + filePath] = fileHash; + } + } + buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ + projectConfiguration, + projectOutputFolderNames, + additionalProjectOutputFilePaths, + additionalContext, + buildCacheConfiguration: this._buildCacheConfiguration, + terminal, + command: commandToRun, + trackedProjectFiles: trackedProjectFiles, + projectChangeAnalyzer: projectChangeAnalyzer, + phaseName: phase.name + }); + } + } + } else { + terminal.writeVerboseLine( + `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + + 'or one provided by a rig, so it does not support caching.' + ); + } + } + } + + return buildCacheContext.projectBuildCache; + } + + private async _tryGetCobuildLockAsync({ + runner, + projectBuildCache + }: { + runner: IOperationRunner; + projectBuildCache: ProjectBuildCache | undefined; + }): Promise { + const buildCacheContext: IOperationBuildCacheContext = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + if (!buildCacheContext.cobuildLock) { + buildCacheContext.cobuildLock = undefined; + + if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { + buildCacheContext.cobuildLock = new CobuildLock({ + cobuildConfiguration: this._cobuildConfiguration, + projectBuildCache: projectBuildCache + }); + } + } + return buildCacheContext.cobuildLock; + } +} + +function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { + for (const n in object1) { + if (!(n in object2) || object1[n] !== object2[n]) { + return false; + } + } + for (const n in object2) { + if (!(n in object1)) { + return false; + } + } + return true; +} diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 5428e6c0c2a..d9d6c02cf38 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -6,7 +6,7 @@ import type { CollatedWriter } from '@rushstack/stream-collator'; import type { OperationStatus } from './OperationStatus'; import type { OperationMetadataManager } from './OperationMetadataManager'; -import type { IStopwatchResult } from '../../utilities/Stopwatch'; +import type { Stopwatch } from '../../utilities/Stopwatch'; /** * Information passed to the executing `IOperationRunner` @@ -39,7 +39,7 @@ export interface IOperationRunnerContext { /** * Object used to track elapsed time. */ - stopwatch: IStopwatchResult; + stopwatch: Stopwatch; /** * The current execution status of an operation. Operations start in the 'ready' state, * but can be 'blocked' if an upstream operation failed. It is 'executing' when @@ -47,6 +47,12 @@ export interface IOperationRunnerContext { * 'failure'. */ status: OperationStatus; + + /** + * Error which occurred while executing this operation, this is stored in case we need + * it later (for example to re-print errors at end of execution). + */ + error?: Error; } /** @@ -62,11 +68,6 @@ export interface IOperationRunner { */ readonly name: string; - /** - * This flag determines if the operation is allowed to be skipped if up to date. - */ - isSkipAllowed: boolean; - /** * Indicates that this runner's duration has meaning. */ @@ -83,11 +84,6 @@ export interface IOperationRunner { */ warningsAreAllowed: boolean; - /** - * Indicates if the output of this operation may be written to the cache - */ - isCacheWriteAllowed: boolean; - /** * Method to be executed for the operation. */ diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts new file mode 100644 index 00000000000..31580484ce6 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { OperationRunnerLifecycleHooks } from './OperationLifecycle'; + +/** + * A plugin tht interacts with a operation runner + */ +export interface IOperationRunnerPlugin { + /** + * Applies this plugin. + */ + apply(hooks: OperationRunnerLifecycleHooks): void; +} diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index fb44dcf46cf..8c25bd1925e 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -11,6 +11,10 @@ import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; +import { + CacheableOperationRunnerPlugin, + IOperationBuildCacheContext +} from './CacheableOperationRunnerPlugin'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -219,8 +223,11 @@ export class OperationExecutionManager { private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { const { runner, name, status } = record; - let blockCacheWrite: boolean = !runner.isCacheWriteAllowed; - let blockSkip: boolean = !runner.isSkipAllowed; + const buildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); + + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; const silent: boolean = runner.silent; @@ -321,12 +328,15 @@ export class OperationExecutionManager { // Apply status changes to direct dependents for (const item of record.consumers) { - if (blockCacheWrite) { - item.runner.isCacheWriteAllowed = false; - } - - if (blockSkip) { - item.runner.isSkipAllowed = false; + const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = + CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(item.runner); + if (itemRunnerBuildCacheContext) { + if (blockCacheWrite) { + itemRunnerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + itemRunnerBuildCacheContext.isSkipAllowed = false; + } } if (status !== OperationStatus.RemoteExecuting) { diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index f20f05fe25f..127aeb59252 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -138,11 +138,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext { try { this.status = await this.runner.executeAsync(this); - - if (this.status === OperationStatus.RemoteExecuting) { - this.stopwatch.reset(); - } - // Delegate global state reporting onResult(this); } catch (error) { diff --git a/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts b/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts new file mode 100644 index 00000000000..fbec2c25ef7 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { AsyncSeriesWaterfallHook } from 'tapable'; + +import type { ITerminal } from '@rushstack/node-core-library'; +import type { IOperationRunnerContext } from './IOperationRunner'; +import type { OperationStatus } from './OperationStatus'; +import type { IProjectDeps, ShellOperationRunner } from './ShellOperationRunner'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { IPhase } from '../../api/CommandLineConfiguration'; +import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; + +export interface IOperationRunnerBeforeExecuteContext { + context: IOperationRunnerContext; + runner: ShellOperationRunner; + earlyReturnStatus: OperationStatus | undefined; + terminal: ITerminal; + projectDeps: IProjectDeps | undefined; + lastProjectDeps: IProjectDeps | undefined; + trackedProjectFiles: string[] | undefined; + logPath: string; + errorLogPath: string; + rushProject: RushConfigurationProject; + phase: IPhase; + selectedPhases: Iterable; + projectChangeAnalyzer: ProjectChangeAnalyzer; + commandName: string; + commandToRun: string; +} + +export interface IOperationRunnerAfterExecuteContext { + context: IOperationRunnerContext; + runner: ShellOperationRunner; + terminal: ITerminal; + /** + * Exit code of the operation command + */ + exitCode: number; + status: OperationStatus; + taskIsSuccessful: boolean; +} + +/** + * Hooks into the lifecycle of the operation runner + * + */ +export class OperationRunnerLifecycleHooks { + public beforeExecute: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook( + ['beforeExecuteContext'], + 'beforeExecute' + ); + + public afterExecute: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook( + ['afterExecuteContext'], + 'afterExecute' + ); +} diff --git a/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts similarity index 90% rename from libraries/rush-lib/src/logic/operations/RunnerWatcher.ts rename to libraries/rush-lib/src/logic/operations/PeriodicCallback.ts index e5823454cf6..f3cc9f1e141 100644 --- a/libraries/rush-lib/src/logic/operations/RunnerWatcher.ts +++ b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts @@ -3,7 +3,7 @@ export type ICallbackFn = () => Promise | void; -export interface IRunnerWatcherOptions { +export interface IPeriodicCallbackOptions { interval: number; } @@ -12,13 +12,13 @@ export interface IRunnerWatcherOptions { * * @beta */ -export class RunnerWatcher { +export class PeriodicCallback { private _callbacks: ICallbackFn[]; private _interval: number; private _timeoutId: NodeJS.Timeout | undefined; private _isRunning: boolean; - public constructor(options: IRunnerWatcherOptions) { + public constructor(options: IPeriodicCallbackOptions) { this._callbacks = []; this._interval = options.interval; this._isRunning = false; diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 678325448e5..e6871be2fd6 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -7,10 +7,8 @@ import { JsonFile, Text, FileSystem, - JsonObject, NewlineKind, InternalError, - ITerminal, Terminal, ColorValue } from '@rushstack/node-core-library'; @@ -19,30 +17,26 @@ import { TextRewriterTransform, StderrLineTransform, SplitterTransform, - DiscardStdoutTransform, - PrintUtilities + DiscardStdoutTransform } from '@rushstack/terminal'; import { CollatedTerminal } from '@rushstack/stream-collator'; -import { Utilities, UNINITIALIZED } from '../../utilities/Utilities'; +import { Utilities } from '../../utilities/Utilities'; import { OperationStatus } from './OperationStatus'; import { OperationError } from './OperationError'; import { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import { ProjectLogWritable } from './ProjectLogWritable'; -import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; -import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import { RushConstants } from '../RushConstants'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import { OperationMetadataManager } from './OperationMetadataManager'; -import { RunnerWatcher } from './RunnerWatcher'; -import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; +import { PeriodicCallback } from './PeriodicCallback'; +import { + IOperationRunnerAfterExecuteContext, + IOperationRunnerBeforeExecuteContext, + OperationRunnerLifecycleHooks +} from './OperationLifecycle'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { ProjectChangeAnalyzer, IRawRepoState } from '../ProjectChangeAnalyzer'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { IPhase } from '../../api/CommandLineConfiguration'; export interface IProjectDeps { @@ -53,10 +47,7 @@ export interface IProjectDeps { export interface IOperationRunnerOptions { rushProject: RushConfigurationProject; rushConfiguration: RushConfiguration; - buildCacheConfiguration: BuildCacheConfiguration | undefined; - cobuildConfiguration: CobuildConfiguration | undefined; commandToRun: string; - isIncrementalBuildAllowed: boolean; projectChangeAnalyzer: ProjectChangeAnalyzer; displayName: string; phase: IPhase; @@ -66,20 +57,6 @@ export interface IOperationRunnerOptions { selectedPhases: Iterable; } -function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { - for (const n in object1) { - if (!(n in object2) || object1[n] !== object2[n]) { - return false; - } - } - for (const n in object2) { - if (!(n in object1)) { - return false; - } - } - return true; -} - /** * An `IOperationRunner` subclass that performs an operation via a shell command. * Currently contains the build cache logic, pending extraction as separate operations. @@ -88,33 +65,23 @@ function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { export class ShellOperationRunner implements IOperationRunner { public readonly name: string; - // This runner supports cache writes by default. - public isCacheWriteAllowed: boolean = true; - public isSkipAllowed: boolean; public readonly reportTiming: boolean = true; public readonly silent: boolean = false; public readonly warningsAreAllowed: boolean; + public readonly hooks: OperationRunnerLifecycleHooks; + public readonly periodicCallback: PeriodicCallback; + private readonly _rushProject: RushConfigurationProject; private readonly _phase: IPhase; private readonly _rushConfiguration: RushConfiguration; - private readonly _buildCacheConfiguration: BuildCacheConfiguration | undefined; - private readonly _cobuildConfiguration: CobuildConfiguration | undefined; private readonly _commandName: string; private readonly _commandToRun: string; - private readonly _isCacheReadAllowed: boolean; private readonly _projectChangeAnalyzer: ProjectChangeAnalyzer; private readonly _packageDepsFilename: string; private readonly _logFilenameIdentifier: string; private readonly _selectedPhases: Iterable; - /** - * UNINITIALIZED === we haven't tried to initialize yet - * undefined === we didn't create one because the feature is not enabled - */ - private _projectBuildCache: ProjectBuildCache | undefined | UNINITIALIZED = UNINITIALIZED; - private _cobuildLock: CobuildLock | undefined | UNINITIALIZED = UNINITIALIZED; - public constructor(options: IOperationRunnerOptions) { const { phase } = options; @@ -122,18 +89,19 @@ export class ShellOperationRunner implements IOperationRunner { this._rushProject = options.rushProject; this._phase = phase; this._rushConfiguration = options.rushConfiguration; - this._buildCacheConfiguration = options.buildCacheConfiguration; - this._cobuildConfiguration = options.cobuildConfiguration; this._commandName = phase.name; this._commandToRun = options.commandToRun; - this._isCacheReadAllowed = options.isIncrementalBuildAllowed; - this.isSkipAllowed = options.isIncrementalBuildAllowed; this._projectChangeAnalyzer = options.projectChangeAnalyzer; this._packageDepsFilename = `package-deps_${phase.logFilenameIdentifier}.json`; this.warningsAreAllowed = EnvironmentConfiguration.allowWarningsInSuccessfulBuild || phase.allowWarningsOnSuccess || false; this._logFilenameIdentifier = phase.logFilenameIdentifier; this._selectedPhases = options.selectedPhases; + + this.hooks = new OperationRunnerLifecycleHooks(); + this.periodicCallback = new PeriodicCallback({ + interval: 10 * 1000 + }); } public async executeAsync(context: IOperationRunnerContext): Promise { @@ -157,9 +125,6 @@ export class ShellOperationRunner implements IOperationRunner { context.collatedWriter.terminal, this._logFilenameIdentifier ); - const runnerWatcher: RunnerWatcher = new RunnerWatcher({ - interval: 10 * 1000 - }); try { const removeColorsTransform: TextRewriterTransform = new TextRewriterTransform({ @@ -236,16 +201,6 @@ export class ShellOperationRunner implements IOperationRunner { files, arguments: this._commandToRun }; - } else if (this.isSkipAllowed) { - // To test this code path: - // Remove the `.git` folder then run "rush build --verbose" - terminal.writeLine({ - text: PrintUtilities.wrapWords( - 'This workspace does not appear to be tracked by Git. ' + - 'Rush will proceed without incremental execution, caching, and change detection.' - ), - foregroundColor: ColorValue.Cyan - }); } } catch (error) { // To test this code path: @@ -257,108 +212,28 @@ export class ShellOperationRunner implements IOperationRunner { }); } - // Try to acquire the cobuild lock - let cobuildLock: CobuildLock | undefined; - if (this._cobuildConfiguration?.cobuildEnabled) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }); - cobuildLock = await this._tryGetCobuildLockAsync(projectBuildCache); - } - - // If possible, we want to skip this operation -- either by restoring it from the - // cache, if caching is enabled, or determining that the project - // is unchanged (using the older incremental execution logic). These two approaches, - // "caching" and "skipping", are incompatible, so only one applies. - // - // Note that "caching" and "skipping" take two different approaches - // to tracking dependents: - // - // - For caching, "isCacheReadAllowed" is set if a project supports - // incremental builds, and determining whether this project or a dependent - // has changed happens inside the hashing logic. - // - // - For skipping, "isSkipAllowed" is set to true initially, and during - // the process of running dependents, it will be changed by OperationExecutionManager to - // false if a dependency wasn't able to be skipped. - // - let buildCacheReadAttempted: boolean = false; - if (cobuildLock) { - // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to - // the build cache once completed, but does not retrieve them (since the "incremental" - // flag is disabled). However, we still need a cobuild to be able to retrieve a finished - // build from another cobuild in this case. - const cobuildCompletedState: ICobuildCompletedState | undefined = - await cobuildLock.getCompletedStateAsync(); - if (cobuildCompletedState) { - const { status, cacheId } = cobuildCompletedState; - - const restoreFromCacheSuccess: boolean | undefined = - await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(terminal, cacheId); - - if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await context._operationMetadataManager?.tryRestoreAsync({ - terminal, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }); - if (cobuildCompletedState) { - return cobuildCompletedState.status; - } - return status; - } - } - } else if (this._isCacheReadAllowed) { - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }); - - buildCacheReadAttempted = !!projectBuildCache; - const restoreFromCacheSuccess: boolean | undefined = - await projectBuildCache?.tryRestoreFromCacheAsync(terminal); - - if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await context._operationMetadataManager?.tryRestoreAsync({ - terminal, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }); - return OperationStatus.FromCache; - } - } - if (this.isSkipAllowed && !buildCacheReadAttempted) { - const isPackageUnchanged: boolean = !!( - lastProjectDeps && - projectDeps && - projectDeps.arguments === lastProjectDeps.arguments && - _areShallowEqual(projectDeps.files, lastProjectDeps.files) - ); - - if (isPackageUnchanged) { - return OperationStatus.Skipped; - } - } - - if (this.isCacheWriteAllowed && cobuildLock) { - const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); - if (acquireSuccess) { - if (context.status === OperationStatus.RemoteExecuting) { - // This operation is used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - } - runnerWatcher.addCallback(async () => { - await cobuildLock?.renewLockAsync(); - }); - } else { - // failed to acquire the lock, mark current operation to remote executing - return OperationStatus.RemoteExecuting; - } + const beforeExecuteContext: IOperationRunnerBeforeExecuteContext = { + context, + runner: this, + terminal, + projectDeps, + lastProjectDeps, + trackedProjectFiles, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath, + rushProject: this._rushProject, + phase: this._phase, + selectedPhases: this._selectedPhases, + projectChangeAnalyzer: this._projectChangeAnalyzer, + commandName: this._commandName, + commandToRun: this._commandToRun, + earlyReturnStatus: undefined + }; + + await this.hooks.beforeExecute.promise(beforeExecuteContext); + + if (beforeExecuteContext.earlyReturnStatus) { + return beforeExecuteContext.earlyReturnStatus; } // If the deps file exists, remove it before starting execution. @@ -382,7 +257,7 @@ export class ShellOperationRunner implements IOperationRunner { // Run the operation terminal.writeLine('Invoking: ' + this._commandToRun); - runnerWatcher.start(); + this.periodicCallback.start(); const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( this._commandToRun, @@ -412,18 +287,16 @@ export class ShellOperationRunner implements IOperationRunner { }); } + let exitCode: number = 1; let status: OperationStatus = await new Promise( (resolve: (status: OperationStatus) => void, reject: (error: OperationError) => void) => { subProcess.on('close', (code: number) => { + exitCode = code; try { if (code !== 0) { - if (cobuildLock) { - // In order to preventing the worst case that all cobuild tasks go through the same failure, - // allowing a failing build to be cached and retrieved - resolve(OperationStatus.Failure); - } else { - reject(new OperationError('error', `Returned error code: ${code}`)); - } + // Do NOT reject here immediately, give a chance for hooks to suppress the error + context.error = new OperationError('error', `Returned error code: ${code}`); + resolve(OperationStatus.Failure); } else if (hasWarningOrError) { resolve(OperationStatus.SuccessWithWarning); } else { @@ -436,13 +309,6 @@ export class ShellOperationRunner implements IOperationRunner { } ); - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - this.warningsAreAllowed && - !!this._rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - // Save the metadata to disk const { duration: durationInSeconds } = context.stopwatch; await context._operationMetadataManager?.saveAsync({ @@ -451,62 +317,40 @@ export class ShellOperationRunner implements IOperationRunner { errorLogPath: projectLogWritable.errorLogPath }); - let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; - let setCacheEntryPromise: Promise | undefined; - if (cobuildLock && this.isCacheWriteAllowed) { - const { projectBuildCache } = cobuildLock; - const cacheId: string | undefined = projectBuildCache.cacheId; - const contextId: string = cobuildLock.cobuildConfiguration.contextId; - - if (cacheId) { - const finalCacheId: string = - status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; - switch (status) { - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: - case OperationStatus.Failure: { - const currentStatus: ICobuildCompletedState['status'] = status; - setCompletedStatePromiseFunction = () => { - return cobuildLock?.setCompletedStateAsync({ - status: currentStatus, - cacheId: finalCacheId - }); - }; - setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( - terminal, - finalCacheId - ); - } - } - } - } + const taskIsSuccessful: boolean = + status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + this.warningsAreAllowed && + !!this._rushConfiguration.experimentsConfiguration.configuration + .buildCacheWithAllowWarningsInSuccessfulBuild); - let writeProjectStatePromise: Promise | undefined; if (taskIsSuccessful && projectDeps) { // Write deps on success. - writeProjectStatePromise = JsonFile.saveAsync(projectDeps, currentDepsPath, { + await JsonFile.saveAsync(projectDeps, currentDepsPath, { ensureFolderExists: true }); + } - // If the command is successful, we can calculate project hash, and no dependencies were skipped, - // write a new cache entry. - if (!setCacheEntryPromise && this.isCacheWriteAllowed) { - setCacheEntryPromise = ( - await this._tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager - }) - )?.trySetCacheEntryAsync(terminal); - } + const afterExecuteContext: IOperationRunnerAfterExecuteContext = { + context, + runner: this, + terminal, + exitCode, + status, + taskIsSuccessful + }; + + await this.hooks.afterExecute.promise(afterExecuteContext); + + if (context.error) { + throw context.error; } - const [, cacheWriteSuccess] = await Promise.all([writeProjectStatePromise, setCacheEntryPromise]); - await setCompletedStatePromiseFunction?.(); + + // Sync the status in case it was changed by the hook + status = afterExecuteContext.status; if (terminalProvider.hasErrors) { status = OperationStatus.Failure; - } else if (cacheWriteSuccess === false && status === OperationStatus.Success) { - status = OperationStatus.SuccessWithWarning; } normalizeNewlineTransform.close(); @@ -520,116 +364,8 @@ export class ShellOperationRunner implements IOperationRunner { return status; } finally { projectLogWritable.close(); - runnerWatcher.stop(); - } - } - - private async _tryGetProjectBuildCacheAsync({ - terminal, - trackedProjectFiles, - operationMetadataManager - }: { - terminal: ITerminal; - trackedProjectFiles: string[] | undefined; - operationMetadataManager: OperationMetadataManager | undefined; - }): Promise { - if (this._projectBuildCache === UNINITIALIZED) { - this._projectBuildCache = undefined; - - if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { - // Disable legacy skip logic if the build cache is in play - this.isSkipAllowed = false; - - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(this._rushProject, terminal); - if (projectConfiguration) { - projectConfiguration.validatePhaseConfiguration(this._selectedPhases, terminal); - if (projectConfiguration.disableBuildCacheForProject) { - terminal.writeVerboseLine('Caching has been disabled for this project.'); - } else { - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(this._commandName); - if (!operationSettings) { - terminal.writeVerboseLine( - `This project does not define the caching behavior of the "${this._commandName}" command, so caching has been disabled.` - ); - } else if (operationSettings.disableBuildCacheForOperation) { - terminal.writeVerboseLine( - `Caching has been disabled for this project's "${this._commandName}" command.` - ); - } else { - const projectOutputFolderNames: ReadonlyArray = - operationSettings.outputFolderNames || []; - const additionalProjectOutputFilePaths: ReadonlyArray = [ - ...(operationMetadataManager?.relativeFilepaths || []) - ]; - const additionalContext: Record = {}; - if (operationSettings.dependsOnEnvVars) { - for (const varName of operationSettings.dependsOnEnvVars) { - additionalContext['$' + varName] = process.env[varName] || ''; - } - } - - if (operationSettings.dependsOnAdditionalFiles) { - const repoState: IRawRepoState | undefined = - await this._projectChangeAnalyzer._ensureInitializedAsync(terminal); - - const additionalFiles: Map = await getHashesForGlobsAsync( - operationSettings.dependsOnAdditionalFiles, - this._rushProject.projectFolder, - repoState - ); - - terminal.writeDebugLine( - `Including additional files to calculate build cache hash:\n ${Array.from( - additionalFiles.keys() - ).join('\n ')} ` - ); - - for (const [filePath, fileHash] of additionalFiles) { - additionalContext['file://' + filePath] = fileHash; - } - } - this._projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ - projectConfiguration, - projectOutputFolderNames, - additionalProjectOutputFilePaths, - additionalContext, - buildCacheConfiguration: this._buildCacheConfiguration, - terminal, - command: this._commandToRun, - trackedProjectFiles: trackedProjectFiles, - projectChangeAnalyzer: this._projectChangeAnalyzer, - phaseName: this._phase.name - }); - } - } - } else { - terminal.writeVerboseLine( - `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + - 'or one provided by a rig, so it does not support caching.' - ); - } - } - } - - return this._projectBuildCache; - } - - private async _tryGetCobuildLockAsync( - projectBuildCache: ProjectBuildCache | undefined - ): Promise { - if (this._cobuildLock === UNINITIALIZED) { - this._cobuildLock = undefined; - - if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { - this._cobuildLock = new CobuildLock({ - cobuildConfiguration: this._cobuildConfiguration, - projectBuildCache: projectBuildCache - }); - } + this.periodicCallback.stop(); } - return this._cobuildLock; } } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index 1d1a425feef..f00e7951f5c 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -13,6 +13,7 @@ import type { PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import { Operation } from './Operation'; +import { CacheableOperationRunnerPlugin } from './CacheableOperationRunnerPlugin'; const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; @@ -22,6 +23,9 @@ const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; export class ShellOperationRunnerPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { hooks.createOperations.tap(PLUGIN_NAME, createShellOperations); + hooks.afterExecuteOperations.tap(PLUGIN_NAME, () => { + CacheableOperationRunnerPlugin.clearAllBuildCacheContexts(); + }); } } @@ -78,18 +82,32 @@ function createShellOperations( const displayName: string = getDisplayName(phase, project); if (commandToRun) { - operation.runner = new ShellOperationRunner({ - buildCacheConfiguration, - cobuildConfiguration, + const shellOperationRunner: ShellOperationRunner = new ShellOperationRunner({ commandToRun: commandToRun || '', displayName, - isIncrementalBuildAllowed, phase, projectChangeAnalyzer, rushConfiguration, rushProject: project, selectedPhases }); + + if (buildCacheConfiguration) { + new CacheableOperationRunnerPlugin({ + buildCacheConfiguration, + cobuildConfiguration, + isIncrementalBuildAllowed + }).apply(shellOperationRunner.hooks); + CacheableOperationRunnerPlugin.setBuildCacheContextByRunner(shellOperationRunner, { + // This runner supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined + }); + } + operation.runner = shellOperationRunner; } else { // Empty build script indicates a no-op, so use a no-op runner operation.runner = new NullOperationRunner({ From 1daf2bbad3238307e4a31cfd7f3299a232bc819d Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 14:49:34 +0800 Subject: [PATCH 022/100] chore: improve onComplete in AsyncOperationQueue --- .../src/logic/operations/AsyncOperationQueue.ts | 10 +++++----- .../src/logic/operations/OperationExecutionManager.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index c4bd76ba542..bf2e1a34442 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -19,8 +19,8 @@ export class AsyncOperationQueue private readonly _queue: OperationExecutionRecord[]; private readonly _pendingIterators: ((result: IteratorResult) => void)[]; private readonly _totalOperations: number; + private readonly _completedOperations: Set; - private _completedOperations: number; private _isDone: boolean; /** @@ -35,7 +35,7 @@ export class AsyncOperationQueue this._pendingIterators = []; this._totalOperations = this._queue.length; this._isDone = false; - this._completedOperations = 0; + this._completedOperations = new Set(); } /** @@ -60,9 +60,9 @@ export class AsyncOperationQueue * Set a callback to be invoked when one operation is completed. * If all operations are completed, set the queue to done, resolve all pending iterators in next cycle. */ - public complete(): void { - this._completedOperations++; - if (this._completedOperations === this._totalOperations) { + public complete(record: OperationExecutionRecord): void { + this._completedOperations.add(record); + if (this._completedOperations.size === this._totalOperations) { this._isDone = true; } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 8c25bd1925e..0b433b8dfcc 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -248,7 +248,7 @@ export class OperationExecutionManager { const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { - executionQueue.complete(); + executionQueue.complete(blockedRecord); this._completedOperations++; // Now that we have the concept of architectural no-ops, we could implement this by replacing @@ -347,7 +347,7 @@ export class OperationExecutionManager { if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete - executionQueue.complete(); + executionQueue.complete(record); } } } From 6d250a428b2184174786485654a05ea7d6655d96 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 14:52:44 +0800 Subject: [PATCH 023/100] chore: use setInterval in PeriodicCallback --- .../src/logic/operations/PeriodicCallback.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts index f3cc9f1e141..26aa1814f55 100644 --- a/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts +++ b/libraries/rush-lib/src/logic/operations/PeriodicCallback.ts @@ -15,7 +15,7 @@ export interface IPeriodicCallbackOptions { export class PeriodicCallback { private _callbacks: ICallbackFn[]; private _interval: number; - private _timeoutId: NodeJS.Timeout | undefined; + private _intervalId: NodeJS.Timeout | undefined; private _isRunning: boolean; public constructor(options: IPeriodicCallbackOptions) { @@ -32,24 +32,22 @@ export class PeriodicCallback { } public start(): void { - if (this._timeoutId) { + if (this._intervalId) { throw new Error('Watcher already started'); } if (this._callbacks.length === 0) { return; } this._isRunning = true; - this._timeoutId = setTimeout(() => { + this._intervalId = setInterval(() => { this._callbacks.forEach((callback) => callback()); - this._timeoutId = undefined; - this.start(); }, this._interval); } public stop(): void { - if (this._timeoutId) { - clearTimeout(this._timeoutId); - this._timeoutId = undefined; + if (this._intervalId) { + clearInterval(this._intervalId); + this._intervalId = undefined; this._isRunning = false; } } From 00b87cbfded73cf5c366d29356a5585204da3077 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 19:44:55 +0800 Subject: [PATCH 024/100] refact: CacheableOperationPlugin --- common/reviews/api/rush-lib.api.md | 10 +- .../cli/scriptActions/PhasedScriptAction.ts | 4 + ...rPlugin.ts => CacheableOperationPlugin.ts} | 214 ++++++++++++------ .../operations/IOperationRunnerPlugin.ts | 14 -- .../operations/OperationExecutionManager.ts | 38 +--- .../operations/OperationExecutionRecord.ts | 10 +- ...onLifecycle.ts => OperationRunnerHooks.ts} | 13 +- .../logic/operations/PhasedOperationHooks.ts | 29 +++ .../logic/operations/ShellOperationRunner.ts | 9 +- .../operations/ShellOperationRunnerPlugin.ts | 29 +-- .../test/AsyncOperationQueue.test.ts | 8 +- .../src/pluginFramework/PhasedCommandHooks.ts | 13 ++ 12 files changed, 234 insertions(+), 157 deletions(-) rename libraries/rush-lib/src/logic/operations/{CacheableOperationRunnerPlugin.ts => CacheableOperationPlugin.ts} (72%) delete mode 100644 libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts rename libraries/rush-lib/src/logic/operations/{OperationLifecycle.ts => OperationRunnerHooks.ts} (90%) create mode 100644 libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 901545dfa60..fb32629c02f 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -9,7 +9,7 @@ import { AsyncParallelHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; -import type { CollatedWriter } from '@rushstack/stream-collator'; +import { CollatedWriter } from '@rushstack/stream-collator'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import { HookMap } from 'tapable'; import { IPackageJson } from '@rushstack/node-core-library'; @@ -17,9 +17,11 @@ import { ITerminal } from '@rushstack/node-core-library'; import { ITerminalProvider } from '@rushstack/node-core-library'; import { JsonObject } from '@rushstack/node-core-library'; import { PackageNameParser } from '@rushstack/node-core-library'; -import type { StdioSummarizer } from '@rushstack/terminal'; +import { StdioSummarizer } from '@rushstack/terminal'; +import { StreamCollator } from '@rushstack/stream-collator'; import { SyncHook } from 'tapable'; import { Terminal } from '@rushstack/node-core-library'; +import { TerminalWritable } from '@rushstack/terminal'; // @public export class ApprovedPackagesConfiguration { @@ -807,6 +809,10 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage export class PhasedCommandHooks { readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]>; + // Warning: (ae-forgotten-export) The symbol "OperationExecutionManager" needs to be exported by the entry point index.d.ts + // + // @internal + readonly operationExecutionManager: AsyncSeriesHook; readonly waitingForChanges: SyncHook; } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 8cb5064fa3c..883c3a5916c 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -38,6 +38,7 @@ import { OperationResultSummarizerPlugin } from '../../logic/operations/Operatio import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; /** * Constructor parameters for PhasedScriptAction. @@ -141,6 +142,8 @@ export class PhasedScriptAction extends BaseScriptAction { new PhasedOperationPlugin().apply(this.hooks); // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(this.hooks); + // Applies the build cache related logic to the selected operations + new CacheableOperationPlugin().apply(this.hooks); if (this._enableParallelism) { this._parallelismParameter = this.defineStringParameter({ @@ -508,6 +511,7 @@ export class PhasedScriptAction extends BaseScriptAction { operations, executionManagerOptions ); + await this.hooks.operationExecutionManager.promise(executionManager); const { isInitial, isWatch } = options.createOperationsContext; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts similarity index 72% rename from libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts rename to libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 4005e754d27..3b9dde61605 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -1,36 +1,37 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { ColorValue, InternalError, ITerminal, JsonObject } from '@rushstack/node-core-library'; +import { ShellOperationRunner } from './ShellOperationRunner'; +import { OperationStatus } from './OperationStatus'; import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; -import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import { OperationStatus } from './OperationStatus'; -import { ColorValue, InternalError, ITerminal, JsonObject } from '@rushstack/node-core-library'; +import { PrintUtilities } from '@rushstack/terminal'; import { RushConstants } from '../RushConstants'; +import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; -import { PrintUtilities } from '@rushstack/terminal'; -import type { IOperationRunnerPlugin } from './IOperationRunnerPlugin'; +import type { Operation } from './Operation'; +import type { OperationExecutionManager } from './OperationExecutionManager'; +import type { OperationExecutionRecord } from './OperationExecutionRecord'; import type { IOperationRunnerAfterExecuteContext, - IOperationRunnerBeforeExecuteContext, - OperationRunnerLifecycleHooks -} from './OperationLifecycle'; -import type { OperationMetadataManager } from './OperationMetadataManager'; + IOperationRunnerBeforeExecuteContext +} from './OperationRunnerHooks'; import type { IOperationRunner } from './IOperationRunner'; -import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; -import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { + ICreateOperationsContext, + IPhasedCommandPlugin, + PhasedCommandHooks +} from '../../pluginFramework/PhasedCommandHooks'; +import type { IPhase } from '../../api/CommandLineConfiguration'; import type { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +import type { OperationMetadataManager } from './OperationMetadataManager'; +import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; +import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; -const PLUGIN_NAME: 'CacheableOperationRunnerPlugin' = 'CacheableOperationRunnerPlugin'; - -export interface ICacheableOperationRunnerPluginOptions { - buildCacheConfiguration: BuildCacheConfiguration; - cobuildConfiguration: CobuildConfiguration | undefined; - isIncrementalBuildAllowed: boolean; -} +const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; export interface IOperationBuildCacheContext { isCacheWriteAllowed: boolean; @@ -40,49 +41,102 @@ export interface IOperationBuildCacheContext { cobuildLock: CobuildLock | undefined; } -export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { - private static _runnerBuildCacheContextMap: Map = new Map< +export class CacheableOperationPlugin implements IPhasedCommandPlugin { + private _buildCacheContextByOperationRunner: Map = new Map< IOperationRunner, IOperationBuildCacheContext >(); - private readonly _buildCacheConfiguration: BuildCacheConfiguration; - private readonly _cobuildConfiguration: CobuildConfiguration | undefined; - public constructor(options: ICacheableOperationRunnerPluginOptions) { - this._buildCacheConfiguration = options.buildCacheConfiguration; - this._cobuildConfiguration = options.cobuildConfiguration; - } + public apply(hooks: PhasedCommandHooks): void { + hooks.createOperations.tapPromise( + PLUGIN_NAME, + async (operations: Set, context: ICreateOperationsContext): Promise> => { + const { buildCacheConfiguration, isIncrementalBuildAllowed } = context; + if (!buildCacheConfiguration) { + return operations; + } - public static getBuildCacheContextByRunner( - runner: IOperationRunner - ): IOperationBuildCacheContext | undefined { - const buildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.get(runner); - return buildCacheContext; - } + for (const operation of operations) { + if (operation.runner) { + if (operation.runner instanceof ShellOperationRunner) { + const buildCacheContext: IOperationBuildCacheContext = { + // ShellOperationRunner supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperationRunner.set(operation.runner, buildCacheContext); + + this._applyOperationRunner(operation.runner, context); + } + } + } - public static getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { - const buildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); - if (!buildCacheContext) { - // This should not happen - throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); - } - return buildCacheContext; - } + return operations; + } + ); - public static setBuildCacheContextByRunner( - runner: IOperationRunner, - buildCacheContext: IOperationBuildCacheContext - ): void { - CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.set(runner, buildCacheContext); - } + hooks.operationExecutionManager.tap( + PLUGIN_NAME, + (operationExecutionManager: OperationExecutionManager) => { + operationExecutionManager.hooks.afterExecuteOperation.tapPromise( + PLUGIN_NAME, + async (operation: OperationExecutionRecord): Promise => { + const { runner, status, consumers } = operation; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(runner); + + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; + + switch (status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; + } + + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: { + // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. + blockSkip ||= !operationExecutionManager.changedProjectsOnly; + break; + } + } - public static clearAllBuildCacheContexts(): void { - CacheableOperationRunnerPlugin._runnerBuildCacheContextMap.clear(); + // Apply status changes to direct dependents + for (const item of consumers) { + const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(item.runner); + if (itemRunnerBuildCacheContext) { + if (blockCacheWrite) { + itemRunnerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + itemRunnerBuildCacheContext.isSkipAllowed = false; + } + } + } + return operation; + } + ); + } + ); + + hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { + this._buildCacheContextByOperationRunner.clear(); + }); } - public apply(hooks: OperationRunnerLifecycleHooks): void { + private _applyOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { + const { buildCacheConfiguration, cobuildConfiguration } = context; + const { hooks } = runner; + + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + hooks.beforeExecute.tapPromise( PLUGIN_NAME, async (beforeExecuteContext: IOperationRunnerBeforeExecuteContext) => { @@ -108,8 +162,6 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { // If there is existing early return status, we don't need to do anything return earlyReturnStatus; } - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); if (!projectDeps && buildCacheContext.isSkipAllowed) { // To test this code path: @@ -124,6 +176,7 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { } const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + buildCacheConfiguration, runner, rushProject, phase, @@ -135,13 +188,20 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { trackedProjectFiles, operationMetadataManager: context._operationMetadataManager }); + // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally buildCacheContext.projectBuildCache = projectBuildCache; // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; - if (this._cobuildConfiguration?.cobuildEnabled) { - cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache }); + if (cobuildConfiguration?.cobuildEnabled) { + cobuildLock = await this._tryGetCobuildLockAsync({ + runner, + projectBuildCache, + cobuildConfiguration + }); } + + // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally buildCacheContext.cobuildLock = cobuildLock; // If possible, we want to skip this operation -- either by restoring it from the @@ -240,12 +300,10 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { } ); - hooks.afterExecute.tapPromise( + runner.hooks.afterExecute.tapPromise( PLUGIN_NAME, async (afterExecuteContext: IOperationRunnerAfterExecuteContext) => { - const { context, runner, terminal, status, taskIsSuccessful } = afterExecuteContext; - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + const { context, terminal, status, taskIsSuccessful } = afterExecuteContext; const { cobuildLock, projectBuildCache, isCacheWriteAllowed } = buildCacheContext; @@ -305,7 +363,24 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { ); } + private _getBuildCacheContextByRunner(runner: IOperationRunner): IOperationBuildCacheContext | undefined { + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._buildCacheContextByOperationRunner.get(runner); + return buildCacheContext; + } + + private _getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(runner); + if (!buildCacheContext) { + // This should not happen + throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); + } + return buildCacheContext; + } + private async _tryGetProjectBuildCacheAsync({ + buildCacheConfiguration, runner, rushProject, phase, @@ -317,6 +392,7 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { trackedProjectFiles, operationMetadataManager }: { + buildCacheConfiguration: BuildCacheConfiguration | undefined; runner: IOperationRunner; rushProject: RushConfigurationProject; phase: IPhase; @@ -328,10 +404,9 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { trackedProjectFiles: string[] | undefined; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.projectBuildCache) { - if (this._buildCacheConfiguration && this._buildCacheConfiguration.buildCacheEnabled) { + if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { // Disable legacy skip logic if the build cache is in play buildCacheContext.isSkipAllowed = false; @@ -390,7 +465,7 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { projectOutputFolderNames, additionalProjectOutputFilePaths, additionalContext, - buildCacheConfiguration: this._buildCacheConfiguration, + buildCacheConfiguration, terminal, command: commandToRun, trackedProjectFiles: trackedProjectFiles, @@ -412,21 +487,22 @@ export class CacheableOperationRunnerPlugin implements IOperationRunnerPlugin { } private async _tryGetCobuildLockAsync({ + cobuildConfiguration, runner, projectBuildCache }: { + cobuildConfiguration: CobuildConfiguration | undefined; runner: IOperationRunner; projectBuildCache: ProjectBuildCache | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.cobuildLock) { buildCacheContext.cobuildLock = undefined; - if (projectBuildCache && this._cobuildConfiguration && this._cobuildConfiguration.cobuildEnabled) { + if (projectBuildCache && cobuildConfiguration && cobuildConfiguration.cobuildEnabled) { buildCacheContext.cobuildLock = new CobuildLock({ - cobuildConfiguration: this._cobuildConfiguration, - projectBuildCache: projectBuildCache + cobuildConfiguration, + projectBuildCache }); } } diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts deleted file mode 100644 index 31580484ce6..00000000000 --- a/libraries/rush-lib/src/logic/operations/IOperationRunnerPlugin.ts +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import type { OperationRunnerLifecycleHooks } from './OperationLifecycle'; - -/** - * A plugin tht interacts with a operation runner - */ -export interface IOperationRunnerPlugin { - /** - * Applies this plugin. - */ - apply(hooks: OperationRunnerLifecycleHooks): void; -} diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 0b433b8dfcc..4cce3332824 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -11,10 +11,7 @@ import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; -import { - CacheableOperationRunnerPlugin, - IOperationBuildCacheContext -} from './CacheableOperationRunnerPlugin'; +import { PhasedOperationHooks } from './PhasedOperationHooks'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -36,7 +33,7 @@ const ASCII_HEADER_WIDTH: number = 79; * tasks are complete, or prematurely fails if any of the tasks fail. */ export class OperationExecutionManager { - private readonly _changedProjectsOnly: boolean; + public readonly changedProjectsOnly: boolean; private readonly _executionRecords: Map; private readonly _quietMode: boolean; private readonly _parallelism: number; @@ -53,13 +50,15 @@ export class OperationExecutionManager { private _hasAnyNonAllowedWarnings: boolean; private _completedOperations: number; + public readonly hooks: PhasedOperationHooks = new PhasedOperationHooks(); + public constructor(operations: Set, options: IOperationExecutionManagerOptions) { const { quietMode, debugMode, parallelism, changedProjectsOnly } = options; this._completedOperations = 0; this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; - this._changedProjectsOnly = changedProjectsOnly; + this.changedProjectsOnly = changedProjectsOnly; this._parallelism = parallelism; // TERMINAL PIPELINE: @@ -189,15 +188,17 @@ export class OperationExecutionManager { // This function is a callback because it may write to the collatedWriter before // operation.executeAsync returns (and cleans up the writer) - const onOperationComplete: (record: OperationExecutionRecord) => void = ( + const onOperationComplete: (record: OperationExecutionRecord) => Promise = async ( record: OperationExecutionRecord ) => { this._onOperationComplete(record, executionQueue); + await this.hooks.afterExecuteOperation.promise(record); }; await Async.forEachAsync( executionQueue, async (operation: OperationExecutionRecord) => { + await this.hooks.beforeExecuteOperation.promise(operation); await operation.executeAsync(onOperationComplete); }, { @@ -223,12 +224,6 @@ export class OperationExecutionManager { private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { const { runner, name, status } = record; - const buildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(runner); - - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; - const silent: boolean = runner.silent; switch (status) { @@ -287,8 +282,6 @@ export class OperationExecutionManager { if (!silent) { record.collatedWriter.terminal.writeStdoutLine(colors.green(`"${name}" was skipped.`)); } - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; break; } @@ -308,8 +301,6 @@ export class OperationExecutionManager { colors.green(`"${name}" completed successfully in ${record.stopwatch.toString()}.`) ); } - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !this._changedProjectsOnly; break; } @@ -319,8 +310,6 @@ export class OperationExecutionManager { colors.yellow(`"${name}" completed with warnings in ${record.stopwatch.toString()}.`) ); } - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !this._changedProjectsOnly; this._hasAnyNonAllowedWarnings = this._hasAnyNonAllowedWarnings || !runner.warningsAreAllowed; break; } @@ -328,17 +317,6 @@ export class OperationExecutionManager { // Apply status changes to direct dependents for (const item of record.consumers) { - const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = - CacheableOperationRunnerPlugin.getBuildCacheContextByRunner(item.runner); - if (itemRunnerBuildCacheContext) { - if (blockCacheWrite) { - itemRunnerBuildCacheContext.isCacheWriteAllowed = false; - } - if (blockSkip) { - itemRunnerBuildCacheContext.isSkipAllowed = false; - } - } - if (status !== OperationStatus.RemoteExecuting) { // Remove this operation from the dependencies, to unblock the scheduler item.dependencies.delete(record); diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 127aeb59252..1fe67a79859 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -20,6 +20,8 @@ export interface IOperationExecutionRecordContext { /** * Internal class representing everything about executing an operation + * + * @internal */ export class OperationExecutionRecord implements IOperationRunnerContext { /** @@ -46,6 +48,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { * operation to execute, the operation with the highest criticalPathLength is chosen. * * Example: + * ``` * (0) A * \ * (1) B C (0) (applications) @@ -62,6 +65,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { * X has a score of 1, since the only package which depends on it is A * Z has a score of 2, since only X depends on it, and X has a score of 1 * Y has a score of 2, since the chain Y->X->C is longer than Y->C + * ``` * * The algorithm is implemented in AsyncOperationQueue.ts as calculateCriticalPathLength() */ @@ -132,19 +136,19 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._operationMetadataManager?.stateFile.state?.nonCachedDurationMs; } - public async executeAsync(onResult: (record: OperationExecutionRecord) => void): Promise { + public async executeAsync(onResult: (record: OperationExecutionRecord) => Promise): Promise { this.status = OperationStatus.Executing; this.stopwatch.start(); try { this.status = await this.runner.executeAsync(this); // Delegate global state reporting - onResult(this); + await onResult(this); } catch (error) { this.status = OperationStatus.Failure; this.error = error; // Delegate global state reporting - onResult(this); + await onResult(this); } finally { if (this.status !== OperationStatus.RemoteExecuting) { this._collatedWriter?.close(); diff --git a/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts similarity index 90% rename from libraries/rush-lib/src/logic/operations/OperationLifecycle.ts rename to libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index fbec2c25ef7..813e310283f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationLifecycle.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -11,6 +11,16 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec import type { IPhase } from '../../api/CommandLineConfiguration'; import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +/** + * A plugin tht interacts with a operation runner + */ +export interface IOperationRunnerPlugin { + /** + * Applies this plugin. + */ + apply(hooks: OperationRunnerHooks): void; +} + export interface IOperationRunnerBeforeExecuteContext { context: IOperationRunnerContext; runner: ShellOperationRunner; @@ -31,7 +41,6 @@ export interface IOperationRunnerBeforeExecuteContext { export interface IOperationRunnerAfterExecuteContext { context: IOperationRunnerContext; - runner: ShellOperationRunner; terminal: ITerminal; /** * Exit code of the operation command @@ -45,7 +54,7 @@ export interface IOperationRunnerAfterExecuteContext { * Hooks into the lifecycle of the operation runner * */ -export class OperationRunnerLifecycleHooks { +export class OperationRunnerHooks { public beforeExecute: AsyncSeriesWaterfallHook = new AsyncSeriesWaterfallHook( ['beforeExecuteContext'], diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts new file mode 100644 index 00000000000..05e67ac3c48 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { AsyncSeriesWaterfallHook } from 'tapable'; + +import type { OperationExecutionRecord } from './OperationExecutionRecord'; + +/** + * A plugin that interacts with a phased commands. + * @alpha + */ +export interface IPhasedOperationPlugin { + /** + * Applies this plugin. + */ + apply(hooks: PhasedOperationHooks): void; +} + +/** + * Hooks into the execution process for phased operation + * @alpha + */ +export class PhasedOperationHooks { + public beforeExecuteOperation: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook(['operation'], 'beforeExecuteOperation'); + + public afterExecuteOperation: AsyncSeriesWaterfallHook = + new AsyncSeriesWaterfallHook(['operation'], 'afterExecuteOperation'); +} diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index e6871be2fd6..b29e68e4f2c 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -31,8 +31,8 @@ import { PeriodicCallback } from './PeriodicCallback'; import { IOperationRunnerAfterExecuteContext, IOperationRunnerBeforeExecuteContext, - OperationRunnerLifecycleHooks -} from './OperationLifecycle'; + OperationRunnerHooks +} from './OperationRunnerHooks'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; @@ -69,7 +69,7 @@ export class ShellOperationRunner implements IOperationRunner { public readonly silent: boolean = false; public readonly warningsAreAllowed: boolean; - public readonly hooks: OperationRunnerLifecycleHooks; + public readonly hooks: OperationRunnerHooks; public readonly periodicCallback: PeriodicCallback; private readonly _rushProject: RushConfigurationProject; @@ -98,7 +98,7 @@ export class ShellOperationRunner implements IOperationRunner { this._logFilenameIdentifier = phase.logFilenameIdentifier; this._selectedPhases = options.selectedPhases; - this.hooks = new OperationRunnerLifecycleHooks(); + this.hooks = new OperationRunnerHooks(); this.periodicCallback = new PeriodicCallback({ interval: 10 * 1000 }); @@ -333,7 +333,6 @@ export class ShellOperationRunner implements IOperationRunner { const afterExecuteContext: IOperationRunnerAfterExecuteContext = { context, - runner: this, terminal, exitCode, status, diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index f00e7951f5c..3e711cbf6bf 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -13,7 +13,6 @@ import type { PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import { Operation } from './Operation'; -import { CacheableOperationRunnerPlugin } from './CacheableOperationRunnerPlugin'; const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; @@ -23,9 +22,6 @@ const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPlugin'; export class ShellOperationRunnerPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { hooks.createOperations.tap(PLUGIN_NAME, createShellOperations); - hooks.afterExecuteOperations.tap(PLUGIN_NAME, () => { - CacheableOperationRunnerPlugin.clearAllBuildCacheContexts(); - }); } } @@ -33,14 +29,7 @@ function createShellOperations( operations: Set, context: ICreateOperationsContext ): Set { - const { - buildCacheConfiguration, - cobuildConfiguration, - isIncrementalBuildAllowed, - phaseSelection: selectedPhases, - projectChangeAnalyzer, - rushConfiguration - } = context; + const { phaseSelection: selectedPhases, projectChangeAnalyzer, rushConfiguration } = context; const customParametersByPhase: Map = new Map(); @@ -91,22 +80,6 @@ function createShellOperations( rushProject: project, selectedPhases }); - - if (buildCacheConfiguration) { - new CacheableOperationRunnerPlugin({ - buildCacheConfiguration, - cobuildConfiguration, - isIncrementalBuildAllowed - }).apply(shellOperationRunner.hooks); - CacheableOperationRunnerPlugin.setBuildCacheContextByRunner(shellOperationRunner, { - // This runner supports cache writes by default. - isCacheWriteAllowed: true, - isCacheReadAllowed: isIncrementalBuildAllowed, - isSkipAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - cobuildLock: undefined - }); - } operation.runner = shellOperationRunner; } else { // Empty build script indicates a no-op, so use a no-op runner diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index e0cd272a802..e245ced36b6 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -42,7 +42,7 @@ describe(AsyncOperationQueue.name, () => { consumer.dependencies.delete(operation); } operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } expect(actualOrder).toEqual(expectedOrder); @@ -67,7 +67,7 @@ describe(AsyncOperationQueue.name, () => { consumer.dependencies.delete(operation); } operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } expect(actualOrder).toEqual(expectedOrder); @@ -130,7 +130,7 @@ describe(AsyncOperationQueue.name, () => { --concurrency; operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } }) ); @@ -177,7 +177,7 @@ describe(AsyncOperationQueue.name, () => { consumer.dependencies.delete(operation); } operation.status = OperationStatus.Success; - queue.complete(); + queue.complete(operation); } expect(actualOrder).toEqual(expectedOrder); diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 237fcfd7eff..e1914f4cc6b 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -12,6 +12,7 @@ import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; import type { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; import type { CobuildConfiguration } from '../api/CobuildConfiguration'; +import type { OperationExecutionManager } from '../logic/operations/OperationExecutionManager'; /** * A plugin that interacts with a phased commands. @@ -99,6 +100,18 @@ export class PhasedCommandHooks { public readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]> = new AsyncSeriesHook(['results', 'context']); + /** + * Hook invoked after the operationExecutionManager has been created. + * Maybe used to tap into the lifecycle of operation execution process. + * + * @internal + */ + public readonly operationExecutionManager: AsyncSeriesHook = + new AsyncSeriesHook( + ['operationExecutionManager'], + 'operationExecutionManager' + ); + /** * Hook invoked after a run has finished and the command is watching for changes. * May be used to display additional relevant data to the user. From 67b3c454e2b376270d00f912868c805c528e270c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 20:37:45 +0800 Subject: [PATCH 025/100] chore: store executionQueue in OperationExecutionManager --- .../operations/OperationExecutionManager.ts | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 4cce3332824..f470ae6312d 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -26,6 +26,13 @@ export interface IOperationExecutionManagerOptions { */ const ASCII_HEADER_WIDTH: number = 79; +const prioritySort: IOperationSortFunction = ( + a: OperationExecutionRecord, + b: OperationExecutionRecord +): number => { + return a.criticalPathLength! - b.criticalPathLength!; +}; + /** * A class which manages the execution of a set of tasks with interdependencies. * Initially, and at the end of each task execution, all unblocked tasks @@ -49,6 +56,7 @@ export class OperationExecutionManager { private _hasAnyFailures: boolean; private _hasAnyNonAllowedWarnings: boolean; private _completedOperations: number; + private _executionQueue: AsyncOperationQueue; public readonly hooks: PhasedOperationHooks = new PhasedOperationHooks(); @@ -112,6 +120,12 @@ export class OperationExecutionManager { dependencyRecord.consumers.add(consumer); } } + + const executionQueue: AsyncOperationQueue = new AsyncOperationQueue( + this._executionRecords.values(), + prioritySort + ); + this._executionQueue = executionQueue; } private _streamCollator_onWriterActive = (writer: CollatedWriter | undefined): void => { @@ -175,28 +189,18 @@ export class OperationExecutionManager { this._terminal.writeStdoutLine(`Executing a maximum of ${this._parallelism} simultaneous processes...`); const maxParallelism: number = Math.min(totalOperations, this._parallelism); - const prioritySort: IOperationSortFunction = ( - a: OperationExecutionRecord, - b: OperationExecutionRecord - ): number => { - return a.criticalPathLength! - b.criticalPathLength!; - }; - const executionQueue: AsyncOperationQueue = new AsyncOperationQueue( - this._executionRecords.values(), - prioritySort - ); // This function is a callback because it may write to the collatedWriter before // operation.executeAsync returns (and cleans up the writer) const onOperationComplete: (record: OperationExecutionRecord) => Promise = async ( record: OperationExecutionRecord ) => { - this._onOperationComplete(record, executionQueue); + this._onOperationComplete(record); await this.hooks.afterExecuteOperation.promise(record); }; await Async.forEachAsync( - executionQueue, + this._executionQueue, async (operation: OperationExecutionRecord) => { await this.hooks.beforeExecuteOperation.promise(operation); await operation.executeAsync(onOperationComplete); @@ -221,7 +225,7 @@ export class OperationExecutionManager { /** * Handles the result of the operation and propagates any relevant effects. */ - private _onOperationComplete(record: OperationExecutionRecord, executionQueue: AsyncOperationQueue): void { + private _onOperationComplete(record: OperationExecutionRecord): void { const { runner, name, status } = record; const silent: boolean = runner.silent; @@ -243,7 +247,7 @@ export class OperationExecutionManager { const blockedQueue: Set = new Set(record.consumers); for (const blockedRecord of blockedQueue) { if (blockedRecord.status === OperationStatus.Ready) { - executionQueue.complete(blockedRecord); + this._executionQueue.complete(blockedRecord); this._completedOperations++; // Now that we have the concept of architectural no-ops, we could implement this by replacing @@ -325,7 +329,7 @@ export class OperationExecutionManager { if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete - executionQueue.complete(record); + this._executionQueue.complete(record); } } } From 3649b5252d68eb6d9244ea2dd3f6e32e57d2128c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 22:57:12 +0800 Subject: [PATCH 026/100] feat: add an unassigned operation in AsyncOperationQueue --- .../logic/operations/AsyncOperationQueue.ts | 60 +++++++++++-------- .../operations/OperationExecutionManager.ts | 27 +++++++-- .../test/AsyncOperationQueue.test.ts | 37 +++++++++--- 3 files changed, 86 insertions(+), 38 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index bf2e1a34442..5fd7d34984d 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -4,6 +4,16 @@ import { OperationExecutionRecord } from './OperationExecutionRecord'; import { OperationStatus } from './OperationStatus'; +/** + * When the queue returns an unassigned operation, it means there is no workable operation at the time, + * and the caller has a chance to make a decision synchronously or asynchronously: + * 1. Manually invoke `tryGetRemoteExecutingOperation()` to get a remote executing operation. + * 2. Or, return in callback or continue the for-loop, which internally invoke `assignOperations()` to assign new operations. + */ +export const UNASSIGNED_OPERATION: 'UNASSIGNED_OPERATION' = 'UNASSIGNED_OPERATION'; + +export type IOperationIteratorResult = OperationExecutionRecord | typeof UNASSIGNED_OPERATION; + /** * Implementation of the async iteration protocol for a collection of IOperation objects. * The async iterator will wait for an operation to be ready for execution, or terminate if there are no more operations. @@ -14,10 +24,10 @@ import { OperationStatus } from './OperationStatus'; * stall until another operations completes. */ export class AsyncOperationQueue - implements AsyncIterable, AsyncIterator + implements AsyncIterable, AsyncIterator { private readonly _queue: OperationExecutionRecord[]; - private readonly _pendingIterators: ((result: IteratorResult) => void)[]; + private readonly _pendingIterators: ((result: IteratorResult) => void)[]; private readonly _totalOperations: number; private readonly _completedOperations: Set; @@ -42,11 +52,11 @@ export class AsyncOperationQueue * For use with `for await (const operation of taskQueue)` * @see {AsyncIterator} */ - public next(): Promise> { + public next(): Promise> { const { _pendingIterators: waitingIterators } = this; - const promise: Promise> = new Promise( - (resolve: (result: IteratorResult) => void) => { + const promise: Promise> = new Promise( + (resolve: (result: IteratorResult) => void) => { waitingIterators.push(resolve); } ); @@ -126,34 +136,32 @@ export class AsyncOperationQueue } if (waitingIterators.length > 0) { - // Pause for a few time - setTimeout(() => { - // cycle through the queue again to find the next operation that is executed remotely - for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { - const operation: OperationExecutionRecord = queue[i]; - - if (operation.status === OperationStatus.RemoteExecuting) { - // try to attempt to get the lock again - waitingIterators.shift()!({ - value: operation, - done: false - }); - } - } - - if (waitingIterators.length > 0) { - // Queue is not empty, but no operations are ready to process, start over - this.assignOperations(); - } - }, 5000); + // Queue is not empty, but no operations are ready to process, returns a unassigned operation to let caller decide + waitingIterators.shift()!({ + value: UNASSIGNED_OPERATION, + done: false + }); + } + } + + public tryGetRemoteExecutingOperation(): OperationExecutionRecord | undefined { + const { _queue: queue } = this; + // cycle through the queue to find the next operation that is executed remotely + for (let i: number = queue.length - 1; i >= 0; i--) { + const operation: OperationExecutionRecord = queue[i]; + + if (operation.status === OperationStatus.RemoteExecuting) { + return operation; + } } + return undefined; } /** * Returns this queue as an async iterator, such that multiple functions iterating this object concurrently * receive distinct iteration results. */ - public [Symbol.asyncIterator](): AsyncIterator { + public [Symbol.asyncIterator](): AsyncIterator { return this; } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index f470ae6312d..1f7ee8c323f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -6,7 +6,12 @@ import { TerminalWritable, StdioWritable, TextRewriterTransform } from '@rushsta import { StreamCollator, CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; import { NewlineKind, Async } from '@rushstack/node-core-library'; -import { AsyncOperationQueue, IOperationSortFunction } from './AsyncOperationQueue'; +import { + AsyncOperationQueue, + IOperationIteratorResult, + IOperationSortFunction, + UNASSIGNED_OPERATION +} from './AsyncOperationQueue'; import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; @@ -201,9 +206,23 @@ export class OperationExecutionManager { await Async.forEachAsync( this._executionQueue, - async (operation: OperationExecutionRecord) => { - await this.hooks.beforeExecuteOperation.promise(operation); - await operation.executeAsync(onOperationComplete); + async (operation: IOperationIteratorResult) => { + let record: OperationExecutionRecord | undefined; + if (operation === UNASSIGNED_OPERATION) { + // Pause for a few time + await Async.sleep(5000); + record = this._executionQueue.tryGetRemoteExecutingOperation(); + } else { + record = operation; + } + + if (!record) { + // Fail to assign a operation, start over again + return; + } else { + await this.hooks.beforeExecuteOperation.promise(record); + await record.executeAsync(onOperationComplete); + } }, { concurrency: maxParallelism diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index e245ced36b6..9436b0c73f4 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -4,8 +4,9 @@ import { Operation } from '../Operation'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; import { MockOperationRunner } from './MockOperationRunner'; -import { AsyncOperationQueue, IOperationSortFunction } from '../AsyncOperationQueue'; +import { AsyncOperationQueue, IOperationSortFunction, UNASSIGNED_OPERATION } from '../AsyncOperationQueue'; import { OperationStatus } from '../OperationStatus'; +import { Async } from '@rushstack/node-core-library'; function addDependency(consumer: OperationExecutionRecord, dependency: OperationExecutionRecord): void { consumer.dependencies.add(dependency); @@ -38,6 +39,9 @@ describe(AsyncOperationQueue.name, () => { const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); for await (const operation of queue) { actualOrder.push(operation); + if (operation === UNASSIGNED_OPERATION) { + continue; + } for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } @@ -63,6 +67,9 @@ describe(AsyncOperationQueue.name, () => { const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, customSort); for await (const operation of queue) { actualOrder.push(operation); + if (operation === UNASSIGNED_OPERATION) { + continue; + } for (const consumer of operation.consumers) { consumer.dependencies.delete(operation); } @@ -117,6 +124,9 @@ describe(AsyncOperationQueue.name, () => { await Promise.all( Array.from({ length: 3 }, async () => { for await (const operation of queue) { + if (operation === UNASSIGNED_OPERATION) { + continue; + } ++concurrency; await Promise.resolve(); @@ -163,9 +173,20 @@ describe(AsyncOperationQueue.name, () => { const actualOrder: string[] = []; let remoteExecuted: boolean = false; for await (const operation of queue) { - actualOrder.push(operation.name); + let record: OperationExecutionRecord | undefined; + if (operation === UNASSIGNED_OPERATION) { + await Async.sleep(100); + record = queue.tryGetRemoteExecutingOperation(); + } else { + record = operation; + } + if (!record) { + continue; + } + + actualOrder.push(record.name); - if (operation === operations[1]) { + if (record === operations[1]) { if (!remoteExecuted) { operations[1].status = OperationStatus.RemoteExecuting; // remote executed operation is finished later @@ -173,13 +194,13 @@ describe(AsyncOperationQueue.name, () => { continue; } } - for (const consumer of operation.consumers) { - consumer.dependencies.delete(operation); + for (const consumer of record.consumers) { + consumer.dependencies.delete(record); } - operation.status = OperationStatus.Success; - queue.complete(operation); + record.status = OperationStatus.Success; + queue.complete(record); } expect(actualOrder).toEqual(expectedOrder); - }, 6000); + }); }); From c62b646e1c4c61b7828c82fa72004883913eed58 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 23:39:12 +0800 Subject: [PATCH 027/100] feat: expand redis-cobuild-plugin configuration with env vars --- .../api/rush-redis-cobuild-plugin.api.md | 4 ++ .../src/RedisCobuildLockProvider.ts | 39 ++++++++++++++++++- .../src/test/RedisCobuildLockProvider.test.ts | 30 ++++++++++++++ .../RedisCobuildLockProvider.test.ts.snap | 5 +++ 4 files changed, 77 insertions(+), 1 deletion(-) diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 93da87162d0..fa3242d63c9 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -4,6 +4,8 @@ ```ts +/// + import type { ICobuildCompletedState } from '@rushstack/rush-sdk'; import type { ICobuildContext } from '@rushstack/rush-sdk'; import type { ICobuildLockProvider } from '@rushstack/rush-sdk'; @@ -26,6 +28,8 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { // (undocumented) disconnectAsync(): Promise; // (undocumented) + static expandOptionsWithEnvironmentVariables(options: IRedisCobuildLockProviderOptions, environment?: NodeJS.ProcessEnv): IRedisCobuildLockProviderOptions; + // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; getCompletedStateKey(context: ICobuildContext): string; getLockKey(context: ICobuildContext): string; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index c6ae989db55..ca39dc5bf7f 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -39,7 +39,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { private _completedKeyMap: WeakMap = new WeakMap(); public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { - this._options = options; + this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); this._terminal = rushSession.getLogger('RedisCobuildLockProvider').terminal; try { this._redisClient = createClient(this._options); @@ -48,6 +48,43 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } } + public static expandOptionsWithEnvironmentVariables( + options: IRedisCobuildLockProviderOptions, + environment: NodeJS.ProcessEnv = process.env + ): IRedisCobuildLockProviderOptions { + const finalOptions: IRedisCobuildLockProviderOptions = { ...options }; + const missingEnvironmentVariables: Set = new Set(); + for (const [key, value] of Object.entries(finalOptions)) { + if (typeof value === 'string') { + const expandedValue: string = value.replace( + /\$\{([^\}]+)\}/g, + (match: string, variableName: string): string => { + const variable: string | undefined = + variableName in environment ? environment[variableName] : undefined; + if (variable !== undefined) { + return variable; + } else { + missingEnvironmentVariables.add(variableName); + return match; + } + } + ); + (finalOptions as Record)[key] = expandedValue; + } + } + + if (missingEnvironmentVariables.size) { + throw new Error( + `The "RedisCobuildLockProvider" tries to access missing environment variable${ + missingEnvironmentVariables.size > 1 ? 's' : '' + }: ${Array.from(missingEnvironmentVariables).join( + ', ' + )}\nPlease check the configuration in rush-redis-cobuild-plugin.json file` + ); + } + return finalOptions; + } + public async connectAsync(): Promise { try { await this._redisClient.connect(); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 3425e83c31a..8aab4c72db2 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -49,6 +49,36 @@ describe(RedisCobuildLockProvider.name, () => { version: 1 }; + it('expands options with environment variables', () => { + const expectedOptions = { + username: 'redisuser', + password: 'redis123' + }; + const actualOptions = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( + { + username: '${REDIS_USERNAME}', + password: '${REDIS_PASS}' + }, + { + REDIS_USERNAME: 'redisuser', + REDIS_PASS: 'redis123' + } + ); + expect(actualOptions).toEqual(expectedOptions); + }); + + it('throws error with missing environment variables', () => { + expect(() => { + RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( + { + username: '${REDIS_USERNAME}', + password: '${REDIS_PASS}' + }, + {} + ); + }).toThrowErrorMatchingSnapshot(); + }); + it('getLockKey works', () => { const subject: RedisCobuildLockProvider = prepareSubject(); const lockKey: string = subject.getLockKey(context); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap index 33aa6130bbf..be0ade872fd 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap @@ -3,3 +3,8 @@ exports[`RedisCobuildLockProvider getCompletedStateKey works 1`] = `"cobuild:v1:123:abc:completed"`; exports[`RedisCobuildLockProvider getLockKey works 1`] = `"cobuild:v1:123:abc:lock"`; + +exports[`RedisCobuildLockProvider throws error with missing environment variables 1`] = ` +"The \\"RedisCobuildLockProvider\\" tries to access missing environment variables: REDIS_USERNAME, REDIS_PASS +Please check the configuration in rush-redis-cobuild-plugin.json file" +`; From 6f2991078c9c0a1db50b4f244c00307a43913a90 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 14 Mar 2023 23:57:54 +0800 Subject: [PATCH 028/100] feat: RUSH_COBUILD_CONTEXT_ID is required to opt into running with cobuilds --- .../.vscode/tasks.json | 10 +++- .../repo/common/config/rush/cobuild.json | 7 +-- common/reviews/api/rush-lib.api.md | 4 +- .../rush-init/common/config/rush/cobuild.json | 7 +-- .../rush-lib/src/api/CobuildConfiguration.ts | 42 ++++++------- .../src/api/EnvironmentConfiguration.ts | 3 +- .../src/logic/cobuild/CobuildContextId.ts | 59 ------------------- .../rush-lib/src/logic/cobuild/CobuildLock.ts | 5 ++ .../cobuild/test/CobuildContextId.test.ts | 37 ------------ .../CobuildContextId.test.ts.snap | 5 -- .../operations/CacheableOperationPlugin.ts | 41 ++++++------- 11 files changed, 56 insertions(+), 164 deletions(-) delete mode 100644 libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts delete mode 100644 libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts delete mode 100644 libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 8e1982bd3f9..2f17191e324 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -37,7 +37,10 @@ "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { - "cwd": "${workspaceFolder}/sandbox/repo" + "cwd": "${workspaceFolder}/sandbox/repo", + "env": { + "RUSH_COBUILD_CONTEXT_ID": "integration-test" + } }, "presentation": { "echo": true, @@ -55,7 +58,10 @@ "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", "problemMatcher": [], "options": { - "cwd": "${workspaceFolder}/sandbox/repo" + "cwd": "${workspaceFolder}/sandbox/repo", + "env": { + "RUSH_COBUILD_CONTEXT_ID": "integration-test" + } }, "presentation": { "echo": true, diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json index 29a49cbd86c..f16a120b1a6 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json @@ -7,6 +7,8 @@ /** * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + * RUSH_COBUILD_CONTEXT_ID should always be specified as an environment variable with an non-empty string, + * otherwise the cobuild feature will be disabled. */ "cobuildEnabled": true, @@ -17,9 +19,4 @@ * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. */ "cobuildLockProvider": "redis" - - /** - * Setting this property overrides the cobuild context ID. - */ - // "cobuildContextIdPattern": "" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index fb32629c02f..f13e1c954e5 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -99,14 +99,14 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { - readonly cobuildContextId: string; + readonly cobuildContextId: string | undefined; readonly cobuildEnabled: boolean; // (undocumented) readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) connectLockProviderAsync(): Promise; // (undocumented) - get contextId(): string; + get contextId(): string | undefined; // (undocumented) disconnectLockProviderAsync(): Promise; // (undocumented) diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json index 13cce367de6..15a874bebb4 100644 --- a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json @@ -7,6 +7,8 @@ /** * (Required) EXPERIMENTAL - Set this to true to enable the cobuild feature. + * RUSH_COBUILD_CONTEXT_ID should always be specified as an environment variable with an non-empty string, + * otherwise the cobuild feature will be disabled. */ "cobuildEnabled": false, @@ -17,9 +19,4 @@ * For example, @rushstack/rush-redis-cobuild-plugin registers the "redis" lock provider. */ "cobuildLockProvider": "redis" - - /** - * Setting this property overrides the cobuild context ID. - */ - // "cobuildContextIdPattern": "" } diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 214e61c0a03..d5814bf19c0 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -16,17 +16,14 @@ import { RushConstants } from '../logic/RushConstants'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; -import { CobuildContextId, GetCobuildContextIdFunction } from '../logic/cobuild/CobuildContextId'; export interface ICobuildJson { cobuildEnabled: boolean; cobuildLockProvider: string; - cobuildContextIdPattern?: string; } export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; - getCobuildContextId: GetCobuildContextIdFunction; rushConfiguration: RushConfiguration; rushSession: RushSession; } @@ -42,24 +39,30 @@ export class CobuildConfiguration { /** * Indicates whether the cobuild feature is enabled. * Typically it is enabled in the cobuild.json config file. + * + * Note: The orchestrator (or local users) should always have to opt into running with cobuilds by + * providing a cobuild context id. Even if cobuilds are "enabled" as a feature, they don't + * actually turn on for that particular build unless the cobuild context id is provided as an + * non-empty string. */ public readonly cobuildEnabled: boolean; /** - * Method to calculate the cobuild context id + * Cobuild context id + * + * @remark + * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ - public readonly cobuildContextId: string; + public readonly cobuildContextId: string | undefined; public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { - this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? options.cobuildJson.cobuildEnabled; + const { cobuildJson } = options; - const { cobuildJson, getCobuildContextId } = options; - - this.cobuildContextId = - EnvironmentConfiguration.cobuildContextId ?? - getCobuildContextId({ - environment: process.env - }); + this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; + this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; + if (!this.cobuildContextId) { + this.cobuildEnabled = false; + } const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); @@ -100,25 +103,14 @@ export class CobuildConfiguration { CobuildConfiguration._jsonSchema ); - let getCobuildContextId: GetCobuildContextIdFunction; - try { - getCobuildContextId = CobuildContextId.parsePattern(cobuildJson.cobuildContextIdPattern); - } catch (e) { - terminal.writeErrorLine( - `Error parsing cobuild context id pattern "${cobuildJson.cobuildContextIdPattern}": ${e}` - ); - throw new AlreadyReportedError(); - } - return new CobuildConfiguration({ cobuildJson, - getCobuildContextId, rushConfiguration, rushSession }); } - public get contextId(): string { + public get contextId(): string | undefined { return this.cobuildContextId; } diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index e2224ce3d7f..e53dfa97927 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -155,8 +155,7 @@ export enum EnvironmentVariableNames { RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', /** - * Setting this environment variable overrides the value of `cobuildContextId` calculated by - * `cobuildContextIdPattern` in the `cobuild.json` configuration file. + * Setting this environment variable opt into running with cobuilds. * * @remarks * If there is no cobuild configured, then this environment variable is ignored. diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts b/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts deleted file mode 100644 index 2c191eca242..00000000000 --- a/libraries/rush-lib/src/logic/cobuild/CobuildContextId.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { RushConstants } from '../RushConstants'; - -export interface IGenerateCobuildContextIdOptions { - environment: NodeJS.ProcessEnv; -} - -/** - * Calculates the cache entry id string for an operation. - * @beta - */ -export type GetCobuildContextIdFunction = (options: IGenerateCobuildContextIdOptions) => string; - -export class CobuildContextId { - private constructor() {} - - public static parsePattern(pattern?: string): GetCobuildContextIdFunction { - if (!pattern) { - return () => ''; - } else { - const resolvedPattern: string = pattern.trim(); - - return (options: IGenerateCobuildContextIdOptions) => { - const { environment } = options; - return this._expandWithEnvironmentVariables(resolvedPattern, environment); - }; - } - } - - private static _expandWithEnvironmentVariables(pattern: string, environment: NodeJS.ProcessEnv): string { - const missingEnvironmentVariables: Set = new Set(); - const expandedPattern: string = pattern.replace( - /\$\{([^\}]+)\}/g, - (match: string, variableName: string): string => { - const variable: string | undefined = - variableName in environment ? environment[variableName] : undefined; - if (variable !== undefined) { - return variable; - } else { - missingEnvironmentVariables.add(variableName); - return match; - } - } - ); - if (missingEnvironmentVariables.size) { - throw new Error( - `The "cobuildContextIdPattern" value in ${ - RushConstants.cobuildFilename - } contains missing environment variable${ - missingEnvironmentVariables.size > 1 ? 's' : '' - }: ${Array.from(missingEnvironmentVariables).join(', ')}` - ); - } - - return expandedPattern; - } -} diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 9b656a9877d..2b36c66f777 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -38,6 +38,11 @@ export class CobuildLock { throw new InternalError(`Cache id is require for cobuild lock`); } + if (!contextId) { + // This should never happen + throw new InternalError(`Cobuild context id is require for cobuild lock`); + } + this._cobuildContext = { contextId, cacheId, diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts deleted file mode 100644 index ea18a0e9242..00000000000 --- a/libraries/rush-lib/src/logic/cobuild/test/CobuildContextId.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { CobuildContextId } from '../CobuildContextId'; - -describe(CobuildContextId.name, () => { - describe('Valid pattern names', () => { - it('expands a environment variable', () => { - const contextId: string = CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ - environment: { - MR_ID: '123', - AUTHOR_NAME: 'Mr.example' - } - }); - expect(contextId).toEqual('context-123-Mr.example'); - }); - }); - - describe('Invalid pattern names', () => { - it('throws an error if a environment variable is missing', () => { - expect(() => - CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ - environment: { - MR_ID: '123' - } - }) - ).toThrowErrorMatchingSnapshot(); - }); - it('throws an error if multiple environment variables are missing', () => { - expect(() => - CobuildContextId.parsePattern('context-${MR_ID}-${AUTHOR_NAME}')({ - environment: {} - }) - ).toThrowErrorMatchingSnapshot(); - }); - }); -}); diff --git a/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap b/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap deleted file mode 100644 index c2495bc8537..00000000000 --- a/libraries/rush-lib/src/logic/cobuild/test/__snapshots__/CobuildContextId.test.ts.snap +++ /dev/null @@ -1,5 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`CobuildContextId Invalid pattern names throws an error if a environment variable is missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variable: AUTHOR_NAME"`; - -exports[`CobuildContextId Invalid pattern names throws an error if multiple environment variables are missing 1`] = `"The \\"cobuildContextIdPattern\\" value in cobuild.json contains missing environment variables: MR_ID, AUTHOR_NAME"`; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 3b9dde61605..cb35690d8e3 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -320,28 +320,25 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } context.error = undefined; } - const cacheId: string | undefined = cobuildLock.projectBuildCache.cacheId; - const contextId: string = cobuildLock.cobuildConfiguration.contextId; - - if (cacheId) { - const finalCacheId: string = - status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; - switch (status) { - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: - case OperationStatus.Failure: { - const currentStatus: ICobuildCompletedState['status'] = status; - setCompletedStatePromiseFunction = () => { - return cobuildLock?.setCompletedStateAsync({ - status: currentStatus, - cacheId: finalCacheId - }); - }; - setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( - terminal, - finalCacheId - ); - } + const { cacheId, contextId } = cobuildLock.cobuildContext; + + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, + cacheId: finalCacheId + }); + }; + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + terminal, + finalCacheId + ); } } } From 8b2933a4cf59cffa68260d29237f2dbce33d4bbb Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Mar 2023 00:04:14 +0800 Subject: [PATCH 029/100] :memo: --- .../README.md | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index 5a72f9010b5..a8466abf387 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -52,7 +52,17 @@ RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. -## Case 2: Cobuild enabled, run one cobuild command only +## Case 2: Cobuild enabled without specifying RUSH_COBUILD_CONTEXT_ID + +Run `rush cobuild` command without specifying cobuild context id. + +```sh +rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +``` + +Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. + +## Case 3: Cobuild enabled, run one cobuild command only 1. Clear redis server @@ -63,19 +73,19 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success 2. Run `rush cobuild` command ```sh -rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is enabled. Run command successfully. You can also see cobuild related logs in the terminal. ```sh -Get completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null -Acquired lock for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success -Set completed state for cobuild:v1::c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 +Get completed state for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null +Acquired lock for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success +Set completed state for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 ``` -## Case 3: Cobuild enabled, run two cobuild commands in parallel +## Case 4: Cobuild enabled, run two cobuild commands in parallel > Note: This test requires Visual Studio Code to be installed. @@ -99,7 +109,7 @@ rm -rf common/temp/build-cache Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. -## Case 4: Cobuild enabled, run two cobuild commands in parallel, one of them failed +## Case 5: Cobuild enabled, run two cobuild commands in parallel, one of them failed > Note: This test requires Visual Studio Code to be installed. From bb6c51af5f16d5ba2ce010dccfda07f81c131c9f Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Mar 2023 18:02:36 +0800 Subject: [PATCH 030/100] chore --- libraries/rush-lib/src/api/CobuildConfiguration.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index d5814bf19c0..953b330e090 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -2,13 +2,7 @@ // See LICENSE in the project root for license information. import * as path from 'path'; -import { - AlreadyReportedError, - FileSystem, - ITerminal, - JsonFile, - JsonSchema -} from '@rushstack/node-core-library'; +import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; import schemaJson from '../schemas/cobuild.schema.json'; import { EnvironmentConfiguration } from './EnvironmentConfiguration'; import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; @@ -49,7 +43,7 @@ export class CobuildConfiguration { /** * Cobuild context id * - * @remark + * @remarks * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ public readonly cobuildContextId: string | undefined; From 6014f5db781baeac0bb83d2b5635df7ca3c6af6c Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Wed, 15 Mar 2023 20:21:53 +0800 Subject: [PATCH 031/100] chore: update snapshots --- libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap index 36bce37042e..4758750c3b3 100644 --- a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap +++ b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap @@ -13,6 +13,7 @@ Loaded @microsoft/rush-lib from process.env._RUSH_LIB_PATH 'BuildCacheConfiguration', 'BumpType', 'ChangeManager', + 'CobuildConfiguration', 'CommonVersionsConfiguration', 'CredentialCache', 'DependencyType', From e3ce661bc754774daf115b5b403d81d789e3fd64 Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 17 Mar 2023 14:59:28 +0800 Subject: [PATCH 032/100] Apply suggestions from code review Co-authored-by: David Michon --- .../src/logic/operations/OperationStatus.ts | 2 +- .../src/RedisCobuildLockProvider.ts | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 18e7204e78c..38a6955296c 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -13,7 +13,7 @@ export enum OperationStatus { /** * The Operation is Queued */ - Queued = 'Queued', + Queued = 'QUEUED', /** * The Operation is currently executing */ diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index ca39dc5bf7f..0ab229b7be0 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -24,19 +24,19 @@ import type { ITerminal } from '@rushstack/node-core-library'; */ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} -const KEY_SEPARATOR: string = ':'; -const COMPLETED_STATE_SEPARATOR: string = ';'; +const KEY_SEPARATOR: ':' = ':'; +const COMPLETED_STATE_SEPARATOR: ';' = ';'; /** * @beta */ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; - private _terminal: ITerminal; + private readonly _terminal: ITerminal; - private _redisClient: RedisClientType; - private _lockKeyMap: WeakMap = new WeakMap(); - private _completedKeyMap: WeakMap = new WeakMap(); + private readonly _redisClient: RedisClientType; + private readonly _lockKeyMap: WeakMap = new WeakMap(); + private readonly _completedKeyMap: WeakMap = new WeakMap(); public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); @@ -59,8 +59,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { const expandedValue: string = value.replace( /\$\{([^\}]+)\}/g, (match: string, variableName: string): string => { - const variable: string | undefined = - variableName in environment ? environment[variableName] : undefined; + const variable: string | undefined = environment[variableName]; if (variable !== undefined) { return variable; } else { From 7e8d7936fad0fb05d793ee9ef501b92023f18c42 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 14:48:59 +0800 Subject: [PATCH 033/100] chore: add catch and reanble no-floating-promise rule --- .../rush-redis-cobuild-plugin-integration-test/src/runRush.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts index c758ec650a0..2547cf233fc 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -34,5 +34,4 @@ async function rushRush(args: string[]): Promise { await parser.execute(args).catch(console.error); // CommandLineParser.execute() should never reject the promise } -/* eslint-disable-next-line @typescript-eslint/no-floating-promises */ -rushRush(process.argv.slice(2)); +rushRush(process.argv.slice(2)).catch(console.error); From 3f40c36e43b908bdc4891d84e9725cd67cfe2c40 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 14:57:27 +0800 Subject: [PATCH 034/100] chore: remove DOM lib in tsconfig.json --- rush-plugins/rush-redis-cobuild-plugin/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json index b3d3ff2a64f..fbc2f5c0a6c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json +++ b/rush-plugins/rush-redis-cobuild-plugin/tsconfig.json @@ -2,7 +2,6 @@ "extends": "./node_modules/@rushstack/heft-node-rig/profiles/default/tsconfig-base.json", "compilerOptions": { - "lib": ["DOM"], "types": ["heft-jest", "node"] } } From 1cdf9a57b513cfe3c33917fff50b899ed6a73ef2 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 15:00:19 +0800 Subject: [PATCH 035/100] chore: remove context id pattern property in Schema --- libraries/rush-lib/src/schemas/cobuild.schema.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/rush-lib/src/schemas/cobuild.schema.json b/libraries/rush-lib/src/schemas/cobuild.schema.json index 5b13a4ec631..1a3b4720d42 100644 --- a/libraries/rush-lib/src/schemas/cobuild.schema.json +++ b/libraries/rush-lib/src/schemas/cobuild.schema.json @@ -28,10 +28,6 @@ "cobuildLockProvider": { "description": "Specify the cobuild lock provider to use", "type": "string" - }, - "cobuildContextIdPattern": { - "type": "string", - "description": "Setting this property overrides the cobuild context ID." } } } From 130bb5572a8c4ed94d09e49074648022ee29cec4 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Fri, 17 Mar 2023 16:58:56 +0800 Subject: [PATCH 036/100] chore: comments the usage of unassigned operation --- .../src/logic/operations/OperationExecutionManager.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 1f7ee8c323f..ae59877605f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -208,6 +208,11 @@ export class OperationExecutionManager { this._executionQueue, async (operation: IOperationIteratorResult) => { let record: OperationExecutionRecord | undefined; + /** + * If the operation is UNASSIGNED_OPERATION, it means that the queue is not able to assign a operation. + * This happens when some operations run remotely. So, we should try to get a remote executing operation + * from the queue manually here. + */ if (operation === UNASSIGNED_OPERATION) { // Pause for a few time await Async.sleep(5000); From 796aa5d413c104912b0153f93649acfa682f573e Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 16:14:25 +0800 Subject: [PATCH 037/100] chore: get projectChangeAnalyzer and selectedPhases from createContext --- common/reviews/api/rush-lib.api.md | 2 +- .../src/logic/operations/CacheableOperationPlugin.ts | 9 ++++++--- .../src/logic/operations/OperationRunnerHooks.ts | 3 --- .../src/logic/operations/ShellOperationRunner.ts | 2 -- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index fb6cd7788f3..1e890bc34fd 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -736,7 +736,7 @@ export enum OperationStatus { Failure = "FAILURE", FromCache = "FROM CACHE", NoOp = "NO OP", - Queued = "Queued", + Queued = "QUEUED", Ready = "READY", RemoteExecuting = "REMOTE EXECUTING", Skipped = "SKIPPED", diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index cb35690d8e3..32f3a4842e4 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -132,7 +132,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } private _applyOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { - const { buildCacheConfiguration, cobuildConfiguration } = context; + const { + buildCacheConfiguration, + cobuildConfiguration, + phaseSelection: selectedPhases, + projectChangeAnalyzer + } = context; const { hooks } = runner; const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); @@ -152,8 +157,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { errorLogPath, rushProject, phase, - selectedPhases, - projectChangeAnalyzer, commandName, commandToRun, earlyReturnStatus diff --git a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index 813e310283f..a2b32af9759 100644 --- a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -9,7 +9,6 @@ import type { OperationStatus } from './OperationStatus'; import type { IProjectDeps, ShellOperationRunner } from './ShellOperationRunner'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; /** * A plugin tht interacts with a operation runner @@ -33,8 +32,6 @@ export interface IOperationRunnerBeforeExecuteContext { errorLogPath: string; rushProject: RushConfigurationProject; phase: IPhase; - selectedPhases: Iterable; - projectChangeAnalyzer: ProjectChangeAnalyzer; commandName: string; commandToRun: string; } diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index b29e68e4f2c..42b833a7480 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -223,8 +223,6 @@ export class ShellOperationRunner implements IOperationRunner { errorLogPath: projectLogWritable.errorLogPath, rushProject: this._rushProject, phase: this._phase, - selectedPhases: this._selectedPhases, - projectChangeAnalyzer: this._projectChangeAnalyzer, commandName: this._commandName, commandToRun: this._commandToRun, earlyReturnStatus: undefined From e0781cd27e92e0c1de15b848f31df1159ca5cd25 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 16:36:45 +0800 Subject: [PATCH 038/100] feat(rush-redis-cobuild-plugin): passwordEnvrionmentVariable --- .../.vscode/tasks.json | 6 ++-- .../README.md | 8 ++--- .../rush-redis-cobuild-plugin.json | 2 +- .../api/rush-redis-cobuild-plugin.api.md | 1 + .../src/RedisCobuildLockProvider.ts | 35 ++++++++++--------- .../src/schemas/redis-config.schema.json | 4 +-- .../src/test/RedisCobuildLockProvider.test.ts | 8 ++--- .../RedisCobuildLockProvider.test.ts.snap | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 2f17191e324..e55c4f0e872 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -39,7 +39,8 @@ "options": { "cwd": "${workspaceFolder}/sandbox/repo", "env": { - "RUSH_COBUILD_CONTEXT_ID": "integration-test" + "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "REDIS_PASS": "redis123" } }, "presentation": { @@ -60,7 +61,8 @@ "options": { "cwd": "${workspaceFolder}/sandbox/repo", "env": { - "RUSH_COBUILD_CONTEXT_ID": "integration-test" + "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "REDIS_PASS": "redis123" } }, "presentation": { diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index a8466abf387..2cb561c5168 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -41,13 +41,13 @@ rush update ## Case 1: Disable cobuild by setting `RUSH_COBUILD_ENABLED=0` ```sh -rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is disabled. Run command successfully. ```sh -RUSH_COBUILD_ENABLED=0 node ../../lib/runRush.js --debug cobuild +RUSH_COBUILD_ENABLED=0 REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. @@ -57,7 +57,7 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success Run `rush cobuild` command without specifying cobuild context id. ```sh -rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. @@ -73,7 +73,7 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success 2. Run `rush cobuild` command ```sh -rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is enabled. Run command successfully. diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json index 625dff477fc..c27270adc35 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush-plugins/rush-redis-cobuild-plugin.json @@ -1,4 +1,4 @@ { "url": "redis://localhost:6379", - "password": "redis123" + "passwordEnvironmentVariable": "REDIS_PASS" } diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index fa3242d63c9..743f0b95b65 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -16,6 +16,7 @@ import type { RushSession } from '@rushstack/rush-sdk'; // @beta export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { + passwordEnvironmentVariable?: string; } // @beta (undocumented) diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 0ab229b7be0..62810471855 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -22,7 +22,12 @@ import type { ITerminal } from '@rushstack/node-core-library'; * The redis client options * @beta */ -export interface IRedisCobuildLockProviderOptions extends RedisClientOptions {} +export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { + /** + * The environment variable name for the redis password + */ + passwordEnvironmentVariable?: string; +} const KEY_SEPARATOR: ':' = ':'; const COMPLETED_STATE_SEPARATOR: ';' = ';'; @@ -36,7 +41,10 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _redisClient: RedisClientType; private readonly _lockKeyMap: WeakMap = new WeakMap(); - private readonly _completedKeyMap: WeakMap = new WeakMap(); + private readonly _completedKeyMap: WeakMap = new WeakMap< + ICobuildContext, + string + >(); public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); @@ -54,22 +62,15 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { ): IRedisCobuildLockProviderOptions { const finalOptions: IRedisCobuildLockProviderOptions = { ...options }; const missingEnvironmentVariables: Set = new Set(); - for (const [key, value] of Object.entries(finalOptions)) { - if (typeof value === 'string') { - const expandedValue: string = value.replace( - /\$\{([^\}]+)\}/g, - (match: string, variableName: string): string => { - const variable: string | undefined = environment[variableName]; - if (variable !== undefined) { - return variable; - } else { - missingEnvironmentVariables.add(variableName); - return match; - } - } - ); - (finalOptions as Record)[key] = expandedValue; + + if (finalOptions.passwordEnvironmentVariable) { + const password: string | undefined = environment[finalOptions.passwordEnvironmentVariable]; + if (password !== undefined) { + finalOptions.password = password; + } else { + missingEnvironmentVariables.add(finalOptions.passwordEnvironmentVariable); } + delete finalOptions.passwordEnvironmentVariable; } if (missingEnvironmentVariables.size) { diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json index 4d984d81b3e..d4283ba7be2 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json +++ b/rush-plugins/rush-redis-cobuild-plugin/src/schemas/redis-config.schema.json @@ -46,8 +46,8 @@ "description": "ACL username", "type": "string" }, - "password": { - "description": "ACL password", + "passwordEnvironmentVariable": { + "description": "The environment variable used to get the ACL password", "type": "string" }, "name": { diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 8aab4c72db2..8b275b9875a 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -51,16 +51,13 @@ describe(RedisCobuildLockProvider.name, () => { it('expands options with environment variables', () => { const expectedOptions = { - username: 'redisuser', password: 'redis123' }; const actualOptions = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( { - username: '${REDIS_USERNAME}', - password: '${REDIS_PASS}' + passwordEnvironmentVariable: 'REDIS_PASS' }, { - REDIS_USERNAME: 'redisuser', REDIS_PASS: 'redis123' } ); @@ -71,8 +68,7 @@ describe(RedisCobuildLockProvider.name, () => { expect(() => { RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables( { - username: '${REDIS_USERNAME}', - password: '${REDIS_PASS}' + passwordEnvironmentVariable: 'REDIS_PASS' }, {} ); diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap index be0ade872fd..f2e6e72eb4c 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/__snapshots__/RedisCobuildLockProvider.test.ts.snap @@ -5,6 +5,6 @@ exports[`RedisCobuildLockProvider getCompletedStateKey works 1`] = `"cobuild:v1: exports[`RedisCobuildLockProvider getLockKey works 1`] = `"cobuild:v1:123:abc:lock"`; exports[`RedisCobuildLockProvider throws error with missing environment variables 1`] = ` -"The \\"RedisCobuildLockProvider\\" tries to access missing environment variables: REDIS_USERNAME, REDIS_PASS +"The \\"RedisCobuildLockProvider\\" tries to access missing environment variable: REDIS_PASS Please check the configuration in rush-redis-cobuild-plugin.json file" `; From 0903effd2dfbde9bb2d97290023f54b603977456 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 16:42:59 +0800 Subject: [PATCH 039/100] feat: always define build cache context, whether shell operation or not --- .../operations/CacheableOperationPlugin.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 32f3a4842e4..66307bcf070 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -58,19 +58,19 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const operation of operations) { if (operation.runner) { - if (operation.runner instanceof ShellOperationRunner) { - const buildCacheContext: IOperationBuildCacheContext = { - // ShellOperationRunner supports cache writes by default. - isCacheWriteAllowed: true, - isCacheReadAllowed: isIncrementalBuildAllowed, - isSkipAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - cobuildLock: undefined - }; - // Upstream runners may mutate the property of build cache context for downstream runners - this._buildCacheContextByOperationRunner.set(operation.runner, buildCacheContext); + const buildCacheContext: IOperationBuildCacheContext = { + // ShellOperationRunner supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperationRunner.set(operation.runner, buildCacheContext); - this._applyOperationRunner(operation.runner, context); + if (operation.runner instanceof ShellOperationRunner) { + this._applyShellOperationRunner(operation.runner, context); } } } @@ -131,7 +131,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } - private _applyOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { + private _applyShellOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { const { buildCacheConfiguration, cobuildConfiguration, From cce3342eee7866567f71d8e8295ef5951cf87a96 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 20 Mar 2023 17:56:13 +0800 Subject: [PATCH 040/100] refact: before/afterExecuteOperaiton hook --- common/reviews/api/rush-lib.api.md | 15 ++-- .../cli/scriptActions/PhasedScriptAction.ts | 10 ++- .../operations/CacheableOperationPlugin.ts | 76 +++++++++---------- .../src/logic/operations/IOperationRunner.ts | 18 +++++ .../operations/OperationExecutionManager.ts | 29 +++++-- .../operations/OperationExecutionRecord.ts | 5 ++ .../logic/operations/PhasedOperationHooks.ts | 29 ------- .../src/pluginFramework/PhasedCommandHooks.ts | 22 +++--- 8 files changed, 106 insertions(+), 98 deletions(-) delete mode 100644 libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 1e890bc34fd..62631fd47ed 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -9,7 +9,7 @@ import { AsyncParallelHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; -import { CollatedWriter } from '@rushstack/stream-collator'; +import type { CollatedWriter } from '@rushstack/stream-collator'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import { HookMap } from 'tapable'; import { IPackageJson } from '@rushstack/node-core-library'; @@ -17,11 +17,9 @@ import { ITerminal } from '@rushstack/node-core-library'; import { ITerminalProvider } from '@rushstack/node-core-library'; import { JsonObject } from '@rushstack/node-core-library'; import { PackageNameParser } from '@rushstack/node-core-library'; -import { StdioSummarizer } from '@rushstack/terminal'; -import { StreamCollator } from '@rushstack/stream-collator'; +import type { StdioSummarizer } from '@rushstack/terminal'; import { SyncHook } from 'tapable'; import { Terminal } from '@rushstack/node-core-library'; -import { TerminalWritable } from '@rushstack/terminal'; // @public export class ApprovedPackagesConfiguration { @@ -488,12 +486,15 @@ export interface IOperationRunner { // @beta export interface IOperationRunnerContext { + readonly changedProjectsOnly: boolean; collatedWriter: CollatedWriter; + readonly consumers: Set; debugMode: boolean; error?: Error; // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; + readonly runner: IOperationRunner; status: OperationStatus; stdioSummarizer: StdioSummarizer; // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts @@ -808,12 +809,10 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage // @alpha export class PhasedCommandHooks { + readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; + readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]>; - // Warning: (ae-forgotten-export) The symbol "OperationExecutionManager" needs to be exported by the entry point index.d.ts - // - // @internal - readonly operationExecutionManager: AsyncSeriesHook; readonly waitingForChanges: SyncHook; } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 883c3a5916c..fdda90d3238 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -39,6 +39,7 @@ import type { ITelemetryOperationResult } from '../../logic/Telemetry'; import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; +import type { IOperationRunnerContext } from '../../logic/operations/IOperationRunner'; /** * Constructor parameters for PhasedScriptAction. @@ -352,7 +353,13 @@ export class PhasedScriptAction extends BaseScriptAction { quietMode: isQuietMode, debugMode: this.parser.isDebug, parallelism, - changedProjectsOnly + changedProjectsOnly, + beforeExecuteOperation: async (record: IOperationRunnerContext) => { + await this.hooks.beforeExecuteOperation.promise(record); + }, + afterExecuteOperation: async (record: IOperationRunnerContext) => { + await this.hooks.afterExecuteOperation.promise(record); + } }; const internalOptions: IRunPhasesOptions = { @@ -511,7 +518,6 @@ export class PhasedScriptAction extends BaseScriptAction { operations, executionManagerOptions ); - await this.hooks.operationExecutionManager.promise(executionManager); const { isInitial, isWatch } = options.createOperationsContext; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 66307bcf070..ea08dd56189 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -12,13 +12,11 @@ import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProj import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; import type { Operation } from './Operation'; -import type { OperationExecutionManager } from './OperationExecutionManager'; -import type { OperationExecutionRecord } from './OperationExecutionRecord'; import type { IOperationRunnerAfterExecuteContext, IOperationRunnerBeforeExecuteContext } from './OperationRunnerHooks'; -import type { IOperationRunner } from './IOperationRunner'; +import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ICreateOperationsContext, @@ -79,50 +77,44 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } ); - hooks.operationExecutionManager.tap( + hooks.afterExecuteOperation.tapPromise( PLUGIN_NAME, - (operationExecutionManager: OperationExecutionManager) => { - operationExecutionManager.hooks.afterExecuteOperation.tapPromise( - PLUGIN_NAME, - async (operation: OperationExecutionRecord): Promise => { - const { runner, status, consumers } = operation; - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(runner); - - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; - - switch (status) { - case OperationStatus.Skipped: { - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; - break; - } + async (runnerContext: IOperationRunnerContext): Promise => { + const { runner, status, consumers, changedProjectsOnly } = runnerContext; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(runner); + + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; + + switch (status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; + } - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: { - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !operationExecutionManager.changedProjectsOnly; - break; - } - } + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: { + // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. + blockSkip ||= !changedProjectsOnly; + break; + } + } - // Apply status changes to direct dependents - for (const item of consumers) { - const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(item.runner); - if (itemRunnerBuildCacheContext) { - if (blockCacheWrite) { - itemRunnerBuildCacheContext.isCacheWriteAllowed = false; - } - if (blockSkip) { - itemRunnerBuildCacheContext.isSkipAllowed = false; - } - } + // Apply status changes to direct dependents + for (const item of consumers) { + const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByRunner(item.runner); + if (itemRunnerBuildCacheContext) { + if (blockCacheWrite) { + itemRunnerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + itemRunnerBuildCacheContext.isSkipAllowed = false; } - return operation; } - ); + } } ); diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index d9d6c02cf38..49ac8f109a8 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -53,6 +53,24 @@ export interface IOperationRunnerContext { * it later (for example to re-print errors at end of execution). */ error?: Error; + + /** + * The set of operations that depend on this operation. + */ + readonly consumers: Set; + + /** + * The operation runner that is executing this operation. + */ + readonly runner: IOperationRunner; + + /** + * Normally the incremental build logic will rebuild changed projects as well as + * any projects that directly or indirectly depend on a changed project. + * If true, then the incremental build logic will only rebuild changed projects and + * ignore dependent projects. + */ + readonly changedProjectsOnly: boolean; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index ae59877605f..e1ef4102a81 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -16,7 +16,6 @@ import { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; import { IExecutionResult } from './IOperationExecutionResult'; -import { PhasedOperationHooks } from './PhasedOperationHooks'; export interface IOperationExecutionManagerOptions { quietMode: boolean; @@ -24,6 +23,8 @@ export interface IOperationExecutionManagerOptions { parallelism: number; changedProjectsOnly: boolean; destination?: TerminalWritable; + beforeExecuteOperation?: (operation: OperationExecutionRecord) => Promise; + afterExecuteOperation?: (operation: OperationExecutionRecord) => Promise; } /** @@ -50,6 +51,12 @@ export class OperationExecutionManager { private readonly _quietMode: boolean; private readonly _parallelism: number; private readonly _totalOperations: number; + private readonly _beforeExecuteOperation: + | ((operation: OperationExecutionRecord) => Promise) + | undefined; + private readonly _afterExecuteOperation: + | ((operation: OperationExecutionRecord) => Promise) + | undefined; private readonly _outputWritable: TerminalWritable; private readonly _colorsNewlinesTransform: TextRewriterTransform; @@ -63,16 +70,23 @@ export class OperationExecutionManager { private _completedOperations: number; private _executionQueue: AsyncOperationQueue; - public readonly hooks: PhasedOperationHooks = new PhasedOperationHooks(); - public constructor(operations: Set, options: IOperationExecutionManagerOptions) { - const { quietMode, debugMode, parallelism, changedProjectsOnly } = options; + const { + quietMode, + debugMode, + parallelism, + changedProjectsOnly, + beforeExecuteOperation, + afterExecuteOperation + } = options; this._completedOperations = 0; this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; this.changedProjectsOnly = changedProjectsOnly; this._parallelism = parallelism; + this._beforeExecuteOperation = beforeExecuteOperation; + this._afterExecuteOperation = afterExecuteOperation; // TERMINAL PIPELINE: // @@ -94,7 +108,8 @@ export class OperationExecutionManager { const executionRecordContext: IOperationExecutionRecordContext = { streamCollator: this._streamCollator, debugMode, - quietMode + quietMode, + changedProjectsOnly }; let totalOperations: number = 0; @@ -201,7 +216,7 @@ export class OperationExecutionManager { record: OperationExecutionRecord ) => { this._onOperationComplete(record); - await this.hooks.afterExecuteOperation.promise(record); + await this._afterExecuteOperation?.(record); }; await Async.forEachAsync( @@ -225,7 +240,7 @@ export class OperationExecutionManager { // Fail to assign a operation, start over again return; } else { - await this.hooks.beforeExecuteOperation.promise(record); + await this._beforeExecuteOperation?.(record); await record.executeAsync(onOperationComplete); } }, diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 1fe67a79859..16590abeef8 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -16,6 +16,7 @@ export interface IOperationExecutionRecordContext { debugMode: boolean; quietMode: boolean; + changedProjectsOnly: boolean; } /** @@ -123,6 +124,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._context.quietMode; } + public get changedProjectsOnly(): boolean { + return this._context.changedProjectsOnly; + } + public get collatedWriter(): CollatedWriter { // Lazy instantiate because the registerTask() call affects display ordering if (!this._collatedWriter) { diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts deleted file mode 100644 index 05e67ac3c48..00000000000 --- a/libraries/rush-lib/src/logic/operations/PhasedOperationHooks.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { AsyncSeriesWaterfallHook } from 'tapable'; - -import type { OperationExecutionRecord } from './OperationExecutionRecord'; - -/** - * A plugin that interacts with a phased commands. - * @alpha - */ -export interface IPhasedOperationPlugin { - /** - * Applies this plugin. - */ - apply(hooks: PhasedOperationHooks): void; -} - -/** - * Hooks into the execution process for phased operation - * @alpha - */ -export class PhasedOperationHooks { - public beforeExecuteOperation: AsyncSeriesWaterfallHook = - new AsyncSeriesWaterfallHook(['operation'], 'beforeExecuteOperation'); - - public afterExecuteOperation: AsyncSeriesWaterfallHook = - new AsyncSeriesWaterfallHook(['operation'], 'afterExecuteOperation'); -} diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index e1914f4cc6b..9bc403b4d74 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -12,7 +12,7 @@ import type { Operation } from '../logic/operations/Operation'; import type { ProjectChangeAnalyzer } from '../logic/ProjectChangeAnalyzer'; import type { IExecutionResult } from '../logic/operations/IOperationExecutionResult'; import type { CobuildConfiguration } from '../api/CobuildConfiguration'; -import type { OperationExecutionManager } from '../logic/operations/OperationExecutionManager'; +import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; /** * A plugin that interacts with a phased commands. @@ -101,16 +101,18 @@ export class PhasedCommandHooks { new AsyncSeriesHook(['results', 'context']); /** - * Hook invoked after the operationExecutionManager has been created. - * Maybe used to tap into the lifecycle of operation execution process. - * - * @internal + * Hook invoked before executing a operation. */ - public readonly operationExecutionManager: AsyncSeriesHook = - new AsyncSeriesHook( - ['operationExecutionManager'], - 'operationExecutionManager' - ); + public readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]> = new AsyncSeriesHook< + [IOperationRunnerContext] + >(['runnerContext'], 'beforeExecuteOperation'); + + /** + * Hook invoked after executing a operation. + */ + public readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]> = new AsyncSeriesHook< + [IOperationRunnerContext] + >(['runnerContext'], 'afterExecuteOperation'); /** * Hook invoked after a run has finished and the command is watching for changes. From 10456808992d795d9ec1a8dce83109dbcc261e97 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 23 Mar 2023 16:18:23 +0800 Subject: [PATCH 041/100] fix: async operation queue in non cobuild --- .../logic/operations/AsyncOperationQueue.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 5fd7d34984d..423fbb4a8c7 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -5,10 +5,12 @@ import { OperationExecutionRecord } from './OperationExecutionRecord'; import { OperationStatus } from './OperationStatus'; /** - * When the queue returns an unassigned operation, it means there is no workable operation at the time, - * and the caller has a chance to make a decision synchronously or asynchronously: - * 1. Manually invoke `tryGetRemoteExecutingOperation()` to get a remote executing operation. - * 2. Or, return in callback or continue the for-loop, which internally invoke `assignOperations()` to assign new operations. + * When the queue returns an unassigned operation, it means there is at least one remote executing operation, + * at this time, the caller has a chance to make a decision: + * 1. Manually invoke `tryGetRemoteExecutingOperation()` to get the remote executing operation. + * 2. If there is no remote executing operation available, wait for some time and return in callback, which + * internally invoke `assignOperations()` to assign new operations. + * NOTE: the caller must wait for some time to avoid busy loop and burn CPU cycles. */ export const UNASSIGNED_OPERATION: 'UNASSIGNED_OPERATION' = 'UNASSIGNED_OPERATION'; @@ -136,11 +138,14 @@ export class AsyncOperationQueue } if (waitingIterators.length > 0) { - // Queue is not empty, but no operations are ready to process, returns a unassigned operation to let caller decide - waitingIterators.shift()!({ - value: UNASSIGNED_OPERATION, - done: false - }); + // returns an unassigned operation to let caller decide when there is at least one + // remote executing operation which is not ready to process. + if (queue.some((operation) => operation.status === OperationStatus.RemoteExecuting)) { + waitingIterators.shift()!({ + value: UNASSIGNED_OPERATION, + done: false + }); + } } } From b2c2e6ce290f5343399e1d4e7631068795c68d91 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 27 Mar 2023 11:43:15 +0800 Subject: [PATCH 042/100] :memo: --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5f5e0ec89f5..38fa3f5acbb 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/rigs/heft-web-rig](./rigs/heft-web-rig/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig.svg)](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig) | [changelog](./rigs/heft-web-rig/CHANGELOG.md) | [@rushstack/heft-web-rig](https://www.npmjs.com/package/@rushstack/heft-web-rig) | | [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) | | [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) | +| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-serve-plugin](./rush-plugins/rush-serve-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin) | [changelog](./rush-plugins/rush-serve-plugin/CHANGELOG.md) | [@rushstack/rush-serve-plugin](https://www.npmjs.com/package/@rushstack/rush-serve-plugin) | | [/webpack/hashed-folder-copy-plugin](./webpack/hashed-folder-copy-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin) | [changelog](./webpack/hashed-folder-copy-plugin/CHANGELOG.md) | [@rushstack/hashed-folder-copy-plugin](https://www.npmjs.com/package/@rushstack/hashed-folder-copy-plugin) | | [/webpack/loader-load-themed-styles](./webpack/loader-load-themed-styles/) | [![npm version](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles.svg)](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles) | [changelog](./webpack/loader-load-themed-styles/CHANGELOG.md) | [@microsoft/loader-load-themed-styles](https://www.npmjs.com/package/@microsoft/loader-load-themed-styles) | @@ -151,6 +152,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/build-tests/rush-amazon-s3-build-cache-plugin-integration-test](./build-tests/rush-amazon-s3-build-cache-plugin-integration-test/) | Tests connecting to an amazon S3 endpoint | | [/build-tests/rush-lib-declaration-paths-test](./build-tests/rush-lib-declaration-paths-test/) | This project ensures all of the paths in rush-lib/lib/... have imports that resolve correctly. If this project builds, all `lib/**/*.d.ts` files in the `@microsoft/rush-lib` package are valid. | | [/build-tests/rush-project-change-analyzer-test](./build-tests/rush-project-change-analyzer-test/) | This is an example project that uses rush-lib's ProjectChangeAnalyzer to | +| [/build-tests/rush-redis-cobuild-plugin-integration-test](./build-tests/rush-redis-cobuild-plugin-integration-test/) | Tests connecting to an redis server | | [/build-tests/set-webpack-public-path-plugin-webpack4-test](./build-tests/set-webpack-public-path-plugin-webpack4-test/) | Building this project tests the set-webpack-public-path-plugin using Webpack 4 | | [/build-tests/ts-command-line-test](./build-tests/ts-command-line-test/) | Building this project is a regression test for ts-command-line | | [/libraries/rush-themed-ui](./libraries/rush-themed-ui/) | Rush Component Library: a set of themed components for rush projects | From 51bef482504f09436f70e4674831ac6dec18a5ba Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 3 Apr 2023 16:43:04 +0800 Subject: [PATCH 043/100] chore: fix missing exports --- common/reviews/api/rush-lib.api.md | 10 ++++++++-- libraries/rush-lib/src/api/CobuildConfiguration.ts | 6 ++++++ libraries/rush-lib/src/index.ts | 2 +- rush-plugins/rush-redis-cobuild-plugin/src/index.ts | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 786d2eeed48..d37a0d4f881 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -112,8 +112,6 @@ export class CobuildConfiguration { static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; } -// Warning: (ae-forgotten-export) The symbol "ICobuildJson" needs to be exported by the entry point index.d.ts -// // @beta (undocumented) export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; @@ -288,6 +286,14 @@ export interface ICobuildContext { version: number; } +// @beta (undocumented) +export interface ICobuildJson { + // (undocumented) + cobuildEnabled: boolean; + // (undocumented) + cobuildLockProvider: string; +} + // @beta (undocumented) export interface ICobuildLockProvider { // (undocumented) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 953b330e090..e9ae1101647 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -11,11 +11,17 @@ import { RushConstants } from '../logic/RushConstants'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; +/** + * @beta + */ export interface ICobuildJson { cobuildEnabled: boolean; cobuildLockProvider: string; } +/** + * @beta + */ export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; rushConfiguration: RushConfiguration; diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 8c1b574c850..226cbfe7a75 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -31,7 +31,7 @@ export { } from './logic/pnpm/PnpmOptionsConfiguration'; export { BuildCacheConfiguration } from './api/BuildCacheConfiguration'; -export { CobuildConfiguration } from './api/CobuildConfiguration'; +export { CobuildConfiguration, ICobuildJson } from './api/CobuildConfiguration'; export { GetCacheEntryIdFunction, IGenerateCacheEntryIdOptions } from './logic/buildCache/CacheEntryId'; export { FileSystemBuildCacheProvider, diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts index f627507d614..b07442ac86d 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -5,4 +5,4 @@ import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; export default RushRedisCobuildPlugin; export { RedisCobuildLockProvider } from './RedisCobuildLockProvider'; -export type { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; +export { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; From 4eadd4b6cd5555689412a5440fa9ace7749e8042 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Mon, 3 Apr 2023 17:27:51 +0800 Subject: [PATCH 044/100] chore: code changes according to code review --- common/reviews/api/rush-lib.api.md | 3 --- .../src/logic/cobuild/ICobuildLockProvider.ts | 14 ++++++++++++++ .../operations/CacheableOperationPlugin.ts | 6 ++---- .../operations/OperationExecutionManager.ts | 18 ++++++++---------- .../test/AsyncOperationQueue.test.ts | 14 ++++++++++++++ .../src/RedisCobuildLockProvider.ts | 2 +- 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index d37a0d4f881..edb4710ed7c 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -278,11 +278,8 @@ export interface ICobuildCompletedState { // @beta (undocumented) export interface ICobuildContext { - // (undocumented) cacheId: string; - // (undocumented) contextId: string; - // (undocumented) version: number; } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index 73092467805..817dd719dcd 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -7,8 +7,22 @@ import type { OperationStatus } from '../operations/OperationStatus'; * @beta */ export interface ICobuildContext { + /** + * The contextId is provided by the monorepo maintainer, it reads from environment variable {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID}. + * It ensure only the builds from the same given contextId cooperated. If user was more permissive, + * and wanted all PR and CI builds building anything with the same contextId to cooperate, then just + * set it to a static value. + */ contextId: string; + /** + * The id of cache. It should be keep same as the normal cacheId from ProjectBuildCache. + * Otherwise, there is a discrepancy in the success case then turning on cobuilds will + * fail to populate the normal build cache. + */ cacheId: string; + /** + * {@inheritdoc RushConstants.cobuildLockVersion} + */ version: number; } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index ea08dd56189..4aa87c9bc8f 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -274,10 +274,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); if (acquireSuccess) { - if (context.status === OperationStatus.RemoteExecuting) { - // This operation is used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - } + // The operation may be used to marked remote executing, now change it to executing + context.status = OperationStatus.Executing; runner.periodicCallback.addCallback(async () => { await cobuildLock?.renewLockAsync(); }); diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 66c43c3b689..85450514ac2 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -49,7 +49,7 @@ const prioritySort: IOperationSortFunction = ( * tasks are complete, or prematurely fails if any of the tasks fail. */ export class OperationExecutionManager { - public readonly changedProjectsOnly: boolean; + private readonly _changedProjectsOnly: boolean; private readonly _executionRecords: Map; private readonly _quietMode: boolean; private readonly _parallelism: number; @@ -89,7 +89,7 @@ export class OperationExecutionManager { this._quietMode = quietMode; this._hasAnyFailures = false; this._hasAnyNonAllowedWarnings = false; - this.changedProjectsOnly = changedProjectsOnly; + this._changedProjectsOnly = changedProjectsOnly; this._parallelism = parallelism; this._beforeExecuteOperation = beforeExecuteOperation; @@ -371,17 +371,15 @@ export class OperationExecutionManager { } } - // Apply status changes to direct dependents - for (const item of record.consumers) { - if (status !== OperationStatus.RemoteExecuting) { - // Remove this operation from the dependencies, to unblock the scheduler - item.dependencies.delete(record); - } - } - if (record.status !== OperationStatus.RemoteExecuting) { // If the operation was not remote, then we can notify queue that it is complete this._executionQueue.complete(record); + + // Apply status changes to direct dependents + for (const item of record.consumers) { + // Remove this operation from the dependencies, to unblock the scheduler + item.dependencies.delete(record); + } } } } diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 9436b0c73f4..3ba19c3d7b7 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -36,10 +36,13 @@ describe(AsyncOperationQueue.name, () => { const expectedOrder = [operations[2], operations[0], operations[1], operations[3]]; const actualOrder = []; + // Nothing sets the RemoteExecuting status, this should be a error if it happens + let hasUnassignedOperation: boolean = false; const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); for await (const operation of queue) { actualOrder.push(operation); if (operation === UNASSIGNED_OPERATION) { + hasUnassignedOperation = true; continue; } for (const consumer of operation.consumers) { @@ -50,6 +53,7 @@ describe(AsyncOperationQueue.name, () => { } expect(actualOrder).toEqual(expectedOrder); + expect(hasUnassignedOperation).toEqual(false); }); it('respects the sort predicate', async () => { @@ -63,11 +67,14 @@ describe(AsyncOperationQueue.name, () => { ): number => { return expectedOrder.indexOf(b) - expectedOrder.indexOf(a); }; + // Nothing sets the RemoteExecuting status, this should be a error if it happens + let hasUnassignedOperation: boolean = false; const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, customSort); for await (const operation of queue) { actualOrder.push(operation); if (operation === UNASSIGNED_OPERATION) { + hasUnassignedOperation = true; continue; } for (const consumer of operation.consumers) { @@ -78,6 +85,8 @@ describe(AsyncOperationQueue.name, () => { } expect(actualOrder).toEqual(expectedOrder); + + expect(hasUnassignedOperation).toEqual(false); }); it('detects cycles', async () => { @@ -119,12 +128,15 @@ describe(AsyncOperationQueue.name, () => { const actualConcurrency: Map = new Map(); const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); let concurrency: number = 0; + // Nothing sets the RemoteExecuting status, this should be a error if it happens + let hasUnassignedOperation: boolean = false; // Use 3 concurrent iterators to verify that it handles having more than the operation concurrency await Promise.all( Array.from({ length: 3 }, async () => { for await (const operation of queue) { if (operation === UNASSIGNED_OPERATION) { + hasUnassignedOperation = true; continue; } ++concurrency; @@ -148,6 +160,8 @@ describe(AsyncOperationQueue.name, () => { for (const [operation, operationConcurrency] of expectedConcurrency) { expect(actualConcurrency.get(operation)).toEqual(operationConcurrency); } + + expect(hasUnassignedOperation).toEqual(false); }); it('handles remote executed operations', async () => { diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 62810471855..f6d6cc297b4 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -70,7 +70,7 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } else { missingEnvironmentVariables.add(finalOptions.passwordEnvironmentVariable); } - delete finalOptions.passwordEnvironmentVariable; + finalOptions.passwordEnvironmentVariable = undefined; } if (missingEnvironmentVariables.size) { From a8950a0801065afe8f19945fe63bffbeabf53e6f Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 8 Apr 2023 22:48:46 +0800 Subject: [PATCH 045/100] feat: cobuild leaf project log only allowed --- .../.vscode/tasks.json | 2 + .../sandbox/repo/.gitignore | 5 +- .../repo/common/config/rush/pnpm-lock.yaml | 12 ++ .../repo/projects/f/config/rush-project.json | 13 ++ .../sandbox/repo/projects/f/package.json | 11 ++ .../sandbox/repo/projects/g/package.json | 11 ++ .../sandbox/repo/rush.json | 8 ++ common/reviews/api/rush-lib.api.md | 3 + .../rush-lib/src/api/CobuildConfiguration.ts | 9 ++ .../src/api/EnvironmentConfiguration.ts | 27 ++++ .../src/logic/buildCache/ProjectBuildCache.ts | 32 ++--- .../buildCache/test/ProjectBuildCache.test.ts | 14 +- .../operations/CacheableOperationPlugin.ts | 122 +++++++++++++++++- 13 files changed, 234 insertions(+), 35 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index e55c4f0e872..1029c807a58 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -40,6 +40,7 @@ "cwd": "${workspaceFolder}/sandbox/repo", "env": { "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1", "REDIS_PASS": "redis123" } }, @@ -62,6 +63,7 @@ "cwd": "${workspaceFolder}/sandbox/repo", "env": { "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1", "REDIS_PASS": "redis123" } }, diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore index 484950696c9..6200fef459b 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/.gitignore @@ -1,4 +1,7 @@ # Rush temporary files common/deploy/ common/temp/ -common/autoinstallers/*/.npmrc \ No newline at end of file +common/autoinstallers/*/.npmrc +projects/*/dist/ +*.log +node_modules/ \ No newline at end of file diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml index 5918b7e8af7..36f86a80a56 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml @@ -32,3 +32,15 @@ importers: dependencies: b: link:../b d: link:../d + + ../../projects/f: + specifiers: + b: workspace:* + dependencies: + b: link:../b + + ../../projects/g: + specifiers: + b: workspace:* + dependencies: + b: link:../b diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json new file mode 100644 index 00000000000..23e6a93085e --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json @@ -0,0 +1,13 @@ +{ + "disableBuildCacheForProject": true, + "operationSettings": [ + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json new file mode 100644 index 00000000000..f703b70f3b2 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json @@ -0,0 +1,11 @@ +{ + "name": "f", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json new file mode 100644 index 00000000000..14c42344694 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json @@ -0,0 +1,11 @@ +{ + "name": "g", + "version": "1.0.0", + "scripts": { + "cobuild": "node ../build.js", + "build": "node ../build.js" + }, + "dependencies": { + "b": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json index 9f839d273ed..70c6c4890de 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json @@ -24,6 +24,14 @@ { "packageName": "e", "projectFolder": "projects/e" + }, + { + "packageName": "f", + "projectFolder": "projects/f" + }, + { + "packageName": "g", + "projectFolder": "projects/g" } ] } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 7d1584e2286..1a2397f29c1 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -99,6 +99,7 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = export class CobuildConfiguration { readonly cobuildContextId: string | undefined; readonly cobuildEnabled: boolean; + readonly cobuildLeafProjectLogOnlyAllowed: boolean; // (undocumented) readonly cobuildLockProvider: ICobuildLockProvider; // (undocumented) @@ -171,6 +172,7 @@ export class EnvironmentConfiguration { static get buildCacheWriteAllowed(): boolean | undefined; static get cobuildContextId(): string | undefined; static get cobuildEnabled(): boolean | undefined; + static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // // @internal @@ -197,6 +199,7 @@ export enum EnvironmentVariableNames { RUSH_BUILD_CACHE_WRITE_ALLOWED = "RUSH_BUILD_CACHE_WRITE_ALLOWED", RUSH_COBUILD_CONTEXT_ID = "RUSH_COBUILD_CONTEXT_ID", RUSH_COBUILD_ENABLED = "RUSH_COBUILD_ENABLED", + RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED = "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED", RUSH_DEPLOY_TARGET_FOLDER = "RUSH_DEPLOY_TARGET_FOLDER", RUSH_GIT_BINARY_PATH = "RUSH_GIT_BINARY_PATH", RUSH_GLOBAL_FOLDER = "RUSH_GLOBAL_FOLDER", diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index e9ae1101647..7c50e649299 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -53,6 +53,13 @@ export class CobuildConfiguration { * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ public readonly cobuildContextId: string | undefined; + /** + * If true, Rush will automatically handle the leaf project with build cache "disabled" by writing + * to the cache in a special "log files only mode". This is useful when you want to use Cobuilds + * to improve the performance in CI validations and the leaf projects have not enabled cache. + */ + public readonly cobuildLeafProjectLogOnlyAllowed: boolean; + public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { @@ -60,6 +67,8 @@ export class CobuildConfiguration { this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; + this.cobuildLeafProjectLogOnlyAllowed = + EnvironmentConfiguration.cobuildLeafProjectLogOnlyAllowed ?? false; if (!this.cobuildContextId) { this.cobuildEnabled = false; } diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index f7a6167c8e8..b34d7649d4a 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -162,6 +162,13 @@ export enum EnvironmentVariableNames { */ RUSH_COBUILD_CONTEXT_ID = 'RUSH_COBUILD_CONTEXT_ID', + /** + * If this variable is set to "1", When getting distributed builds, Rush will automatically handle the leaf project + * with build cache "disabled" by writing to the cache in a special "log files only mode". This is useful when you + * want to use Cobuilds to improve the performance in CI validations and the leaf projects have not enabled cache. + */ + RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED = 'RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED', + /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. */ @@ -225,6 +232,8 @@ export class EnvironmentConfiguration { private static _cobuildContextId: string | undefined; + private static _cobuildLeafProjectLogOnlyAllowed: boolean | undefined; + private static _gitBinaryPath: string | undefined; private static _tarBinaryPath: string | undefined; @@ -340,6 +349,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._cobuildContextId; } + /** + * If set, enables or disables the cobuild leaf project log only feature. + * See {@link EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED} + */ + public static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildLeafProjectLogOnlyAllowed; + } + /** * Allows the git binary path to be explicitly provided. * See {@link EnvironmentVariableNames.RUSH_GIT_BINARY_PATH} @@ -484,6 +502,15 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: { + EnvironmentConfiguration._cobuildLeafProjectLogOnlyAllowed = + EnvironmentConfiguration.parseBooleanEnvironmentVariable( + EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED, + value + ); + break; + } + case EnvironmentVariableNames.RUSH_GIT_BINARY_PATH: { EnvironmentConfiguration._gitBinaryPath = value; break; diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index f8a7ea21a07..b8c224e53df 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -7,7 +7,6 @@ import { FileSystem, Path, ITerminal, FolderItem, InternalError, Async } from '@ import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; -import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { RushConstants } from '../RushConstants'; import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; @@ -17,7 +16,7 @@ import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; export interface IProjectBuildCacheOptions { buildCacheConfiguration: BuildCacheConfiguration; - projectConfiguration: RushProjectConfiguration; + project: RushConfigurationProject; projectOutputFolderNames: ReadonlyArray; additionalProjectOutputFilePaths?: ReadonlyArray; additionalContext?: Record; @@ -50,13 +49,9 @@ export class ProjectBuildCache { private _cacheId: string | undefined; private constructor(cacheId: string | undefined, options: IProjectBuildCacheOptions) { - const { - buildCacheConfiguration, - projectConfiguration, - projectOutputFolderNames, - additionalProjectOutputFilePaths - } = options; - this._project = projectConfiguration.project; + const { buildCacheConfiguration, project, projectOutputFolderNames, additionalProjectOutputFilePaths } = + options; + this._project = project; this._localBuildCacheProvider = buildCacheConfiguration.localCacheProvider; this._cloudBuildCacheProvider = buildCacheConfiguration.cloudCacheProvider; this._buildCacheEnabled = buildCacheConfiguration.buildCacheEnabled; @@ -81,18 +76,13 @@ export class ProjectBuildCache { public static async tryGetProjectBuildCache( options: IProjectBuildCacheOptions ): Promise { - const { terminal, projectConfiguration, projectOutputFolderNames, trackedProjectFiles } = options; + const { terminal, project, projectOutputFolderNames, trackedProjectFiles } = options; if (!trackedProjectFiles) { return undefined; } if ( - !ProjectBuildCache._validateProject( - terminal, - projectConfiguration, - projectOutputFolderNames, - trackedProjectFiles - ) + !ProjectBuildCache._validateProject(terminal, project, projectOutputFolderNames, trackedProjectFiles) ) { return undefined; } @@ -103,13 +93,11 @@ export class ProjectBuildCache { private static _validateProject( terminal: ITerminal, - projectConfiguration: RushProjectConfiguration, + project: RushConfigurationProject, projectOutputFolderNames: ReadonlyArray, trackedProjectFiles: string[] ): boolean { - const normalizedProjectRelativeFolder: string = Path.convertToSlashes( - projectConfiguration.project.projectRelativeFolder - ); + const normalizedProjectRelativeFolder: string = Path.convertToSlashes(project.projectRelativeFolder); const outputFolders: string[] = []; if (projectOutputFolderNames) { for (const outputFolderName of projectOutputFolderNames) { @@ -434,7 +422,7 @@ export class ProjectBuildCache { const projectStates: string[] = []; const projectsThatHaveBeenProcessed: Set = new Set(); let projectsToProcess: Set = new Set(); - projectsToProcess.add(options.projectConfiguration.project); + projectsToProcess.add(options.project); while (projectsToProcess.size > 0) { const newProjectsToProcess: Set = new Set(); @@ -491,7 +479,7 @@ export class ProjectBuildCache { const projectStateHash: string = hash.digest('hex'); return options.buildCacheConfiguration.getCacheEntryId({ - projectName: options.projectConfiguration.project.packageName, + projectName: options.project.packageName, projectStateHash, phaseName: options.phaseName }); diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts index c690607a45a..afbae532838 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts @@ -3,7 +3,7 @@ import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library'; import { BuildCacheConfiguration } from '../../../api/BuildCacheConfiguration'; -import { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; +import { RushConfigurationProject } from '../../../api/RushConfigurationProject'; import { ProjectChangeAnalyzer } from '../../ProjectChangeAnalyzer'; import { IGenerateCacheEntryIdOptions } from '../CacheEntryId'; import { FileSystemBuildCacheProvider } from '../FileSystemBuildCacheProvider'; @@ -36,13 +36,11 @@ describe(ProjectBuildCache.name, () => { } } as unknown as BuildCacheConfiguration, projectOutputFolderNames: ['dist'], - projectConfiguration: { - project: { - packageName: 'acme-wizard', - projectRelativeFolder: 'apps/acme-wizard', - dependencyProjects: [] - } - } as unknown as RushProjectConfiguration, + project: { + packageName: 'acme-wizard', + projectRelativeFolder: 'apps/acme-wizard', + dependencyProjects: [] + } as unknown as RushConfigurationProject, command: 'build', trackedProjectFiles: options.hasOwnProperty('trackedProjectFiles') ? options.trackedProjectFiles : [], projectChangeAnalyzer, diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 4aa87c9bc8f..eddcdd9ad0f 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -170,7 +170,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } - const projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ buildCacheConfiguration, runner, rushProject, @@ -183,12 +183,40 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { trackedProjectFiles, operationMetadataManager: context._operationMetadataManager }); - // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally - buildCacheContext.projectBuildCache = projectBuildCache; // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; if (cobuildConfiguration?.cobuildEnabled) { + if ( + cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && + rushProject.consumingProjects.size === 0 && + !projectBuildCache + ) { + // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get + // a log files only project build cache + projectBuildCache = await this._tryGetLogOnlyProjectBuildCacheAsync({ + buildCacheConfiguration, + runner, + rushProject, + phase, + projectChangeAnalyzer, + commandName, + commandToRun, + terminal, + trackedProjectFiles, + operationMetadataManager: context._operationMetadataManager + }); + if (projectBuildCache) { + terminal.writeVerboseLine( + `Log files only build cache is enabled for the project "${rushProject.packageName}" because the cobuild leaf project log only is allowed` + ); + } else { + terminal.writeWarningLine( + `Failed to get log files only build cache for the project "${rushProject.packageName}"` + ); + } + } + cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache, @@ -451,7 +479,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ - projectConfiguration, + project: rushProject, projectOutputFolderNames, additionalProjectOutputFilePaths, additionalContext, @@ -476,6 +504,92 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext.projectBuildCache; } + private async _tryGetLogOnlyProjectBuildCacheAsync({ + runner, + rushProject, + terminal, + commandName, + commandToRun, + buildCacheConfiguration, + phase, + trackedProjectFiles, + projectChangeAnalyzer, + operationMetadataManager + }: { + buildCacheConfiguration: BuildCacheConfiguration | undefined; + runner: IOperationRunner; + rushProject: RushConfigurationProject; + phase: IPhase; + commandToRun: string; + commandName: string; + terminal: ITerminal; + trackedProjectFiles: string[] | undefined; + projectChangeAnalyzer: ProjectChangeAnalyzer; + operationMetadataManager: OperationMetadataManager | undefined; + }): Promise { + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { + // Disable legacy skip logic if the build cache is in play + buildCacheContext.isSkipAllowed = false; + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); + + let projectOutputFolderNames: ReadonlyArray = []; + const additionalProjectOutputFilePaths: ReadonlyArray = [ + ...(operationMetadataManager?.relativeFilepaths || []) + ]; + const additionalContext: Record = { + // Force the cache to be a log files only cache + logFilesOnly: '1' + }; + if (projectConfiguration) { + const operationSettings: IOperationSettings | undefined = + projectConfiguration.operationSettingsByOperationName.get(commandName); + if (operationSettings) { + if (operationSettings.outputFolderNames) { + projectOutputFolderNames = operationSettings.outputFolderNames; + } + if (operationSettings.dependsOnEnvVars) { + for (const varName of operationSettings.dependsOnEnvVars) { + additionalContext['$' + varName] = process.env[varName] || ''; + } + } + + if (operationSettings.dependsOnAdditionalFiles) { + const repoState: IRawRepoState | undefined = await projectChangeAnalyzer._ensureInitializedAsync( + terminal + ); + + const additionalFiles: Map = await getHashesForGlobsAsync( + operationSettings.dependsOnAdditionalFiles, + rushProject.projectFolder, + repoState + ); + + for (const [filePath, fileHash] of additionalFiles) { + additionalContext['file://' + filePath] = fileHash; + } + } + } + } + const projectBuildCache: ProjectBuildCache | undefined = + await ProjectBuildCache.tryGetProjectBuildCache({ + project: rushProject, + projectOutputFolderNames, + additionalProjectOutputFilePaths, + additionalContext, + buildCacheConfiguration, + terminal, + command: commandToRun, + trackedProjectFiles, + projectChangeAnalyzer: projectChangeAnalyzer, + phaseName: phase.name + }); + buildCacheContext.projectBuildCache = projectBuildCache; + return projectBuildCache; + } + } + private async _tryGetCobuildLockAsync({ cobuildConfiguration, runner, From 082f3503af8883ac4421a4d9eb02c5ae8bfc65a6 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Tue, 11 Apr 2023 11:39:22 +0800 Subject: [PATCH 046/100] chore --- libraries/rush-lib/src/api/EnvironmentConfiguration.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index f802d05cfa2..327cf1aae74 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -153,7 +153,7 @@ export const EnvironmentVariableNames = { * * If there is no cobuild configured, then this environment variable is ignored. */ - RUSH_COBUILD_ENABLED = 'RUSH_COBUILD_ENABLED', + RUSH_COBUILD_ENABLED: 'RUSH_COBUILD_ENABLED', /** * Setting this environment variable opt into running with cobuilds. @@ -161,14 +161,14 @@ export const EnvironmentVariableNames = { * @remarks * If there is no cobuild configured, then this environment variable is ignored. */ - RUSH_COBUILD_CONTEXT_ID = 'RUSH_COBUILD_CONTEXT_ID', + RUSH_COBUILD_CONTEXT_ID: 'RUSH_COBUILD_CONTEXT_ID', /** * If this variable is set to "1", When getting distributed builds, Rush will automatically handle the leaf project * with build cache "disabled" by writing to the cache in a special "log files only mode". This is useful when you * want to use Cobuilds to improve the performance in CI validations and the leaf projects have not enabled cache. */ - RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED = 'RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED', + RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: 'RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED', /** * Explicitly specifies the path for the Git binary that is invoked by certain Rush operations. From 820aa3c33d9318ed7b71194a8e98c04136b2503d Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Fri, 21 Apr 2023 19:50:34 -0700 Subject: [PATCH 047/100] Fix merge conflict --- README.md | 2 +- common/config/rush/repo-state.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 164584ff6a2..d9a5c9b3f8b 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,8 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/rigs/heft-web-rig](./rigs/heft-web-rig/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig.svg)](https://badge.fury.io/js/%40rushstack%2Fheft-web-rig) | [changelog](./rigs/heft-web-rig/CHANGELOG.md) | [@rushstack/heft-web-rig](https://www.npmjs.com/package/@rushstack/heft-web-rig) | | [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) | | [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) | -| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-http-build-cache-plugin](./rush-plugins/rush-http-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin) | | [@rushstack/rush-http-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-http-build-cache-plugin) | +| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-serve-plugin](./rush-plugins/rush-serve-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin) | [changelog](./rush-plugins/rush-serve-plugin/CHANGELOG.md) | [@rushstack/rush-serve-plugin](https://www.npmjs.com/package/@rushstack/rush-serve-plugin) | | [/webpack/hashed-folder-copy-plugin](./webpack/hashed-folder-copy-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin) | [changelog](./webpack/hashed-folder-copy-plugin/CHANGELOG.md) | [@rushstack/hashed-folder-copy-plugin](https://www.npmjs.com/package/@rushstack/hashed-folder-copy-plugin) | | [/webpack/loader-load-themed-styles](./webpack/loader-load-themed-styles/) | [![npm version](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles.svg)](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles) | [changelog](./webpack/loader-load-themed-styles/CHANGELOG.md) | [@microsoft/loader-load-themed-styles](https://www.npmjs.com/package/@microsoft/loader-load-themed-styles) | diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 8bf829eaecf..3792953e049 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "a1681a4c0e1d3c156fa5e501a94688a520955940", + "pnpmShrinkwrapHash": "ec82d5af16b25e9021d898f5e7a1f5ee34317770", "preferredVersionsHash": "5222ca779ae69ebfd201e39c17f48ce9eaf8c3c2" } From 9c32bf8aac1bb3490c78406085377b453264e43b Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Sat, 22 Apr 2023 21:00:35 +0800 Subject: [PATCH 048/100] chore: connect/disconnect lock provider only cobuild enabled --- libraries/rush-lib/src/api/CobuildConfiguration.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 7c50e649299..a5025e61d8a 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -124,10 +124,14 @@ export class CobuildConfiguration { } public async connectLockProviderAsync(): Promise { - await this.cobuildLockProvider.connectAsync(); + if (this.cobuildEnabled) { + await this.cobuildLockProvider.connectAsync(); + } } public async disconnectLockProviderAsync(): Promise { - await this.cobuildLockProvider.disconnectAsync(); + if (this.cobuildEnabled) { + await this.cobuildLockProvider.disconnectAsync(); + } } } From 6358549153fcef8d97119548b0a142d75e09bc19 Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 15 May 2023 14:48:14 +0800 Subject: [PATCH 049/100] feat: cobuild lock provider factory can return promise --- common/reviews/api/rush-lib.api.md | 2 +- .../rush-lib/src/api/CobuildConfiguration.ts | 21 ++++++++++++------- .../src/pluginFramework/RushSession.ts | 4 +++- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index afbd3fea55c..8e68d4ab3a8 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -114,7 +114,7 @@ export class CobuildConfiguration { } // @beta (undocumented) -export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; +export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider | Promise; // @public export class CommonVersionsConfiguration { diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index a5025e61d8a..1fefe757340 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -26,6 +26,7 @@ export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; rushConfiguration: RushConfiguration; rushSession: RushSession; + cobuildLockProvider: ICobuildLockProvider; } /** @@ -63,7 +64,7 @@ export class CobuildConfiguration { public readonly cobuildLockProvider: ICobuildLockProvider; private constructor(options: ICobuildConfigurationOptions) { - const { cobuildJson } = options; + const { cobuildJson, cobuildLockProvider } = options; this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; @@ -73,12 +74,7 @@ export class CobuildConfiguration { this.cobuildEnabled = false; } - const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = - options.rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); - if (!cobuildLockProviderFactory) { - throw new Error(`Unexpected cobuild lock provider: ${cobuildJson.cobuildLockProvider}`); - } - this.cobuildLockProvider = cobuildLockProviderFactory(cobuildJson); + this.cobuildLockProvider = cobuildLockProvider; } /** @@ -112,10 +108,19 @@ export class CobuildConfiguration { CobuildConfiguration._jsonSchema ); + const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = + rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); + if (!cobuildLockProviderFactory) { + throw new Error(`Unexpected cobuild lock provider: ${cobuildJson.cobuildLockProvider}`); + } + + const cobuildLockProvider: ICobuildLockProvider = await cobuildLockProviderFactory(cobuildJson); + return new CobuildConfiguration({ cobuildJson, rushConfiguration, - rushSession + rushSession, + cobuildLockProvider }); } diff --git a/libraries/rush-lib/src/pluginFramework/RushSession.ts b/libraries/rush-lib/src/pluginFramework/RushSession.ts index f90137efa93..374960f9530 100644 --- a/libraries/rush-lib/src/pluginFramework/RushSession.ts +++ b/libraries/rush-lib/src/pluginFramework/RushSession.ts @@ -28,7 +28,9 @@ export type CloudBuildCacheProviderFactory = ( /** * @beta */ -export type CobuildLockProviderFactory = (cobuildJson: ICobuildJson) => ICobuildLockProvider; +export type CobuildLockProviderFactory = ( + cobuildJson: ICobuildJson +) => ICobuildLockProvider | Promise; /** * @beta From 98aeff37dad9e98ea9fa93d2e879bbe0ef101226 Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 15 May 2023 19:30:19 +0800 Subject: [PATCH 050/100] feat: only create cobuild lock provider if needed --- common/reviews/api/rush-lib.api.md | 8 ++--- .../rush-lib/src/api/CobuildConfiguration.ts | 34 +++++++++++++------ .../cli/scriptActions/PhasedScriptAction.ts | 4 +-- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 8e68d4ab3a8..055faa85362 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -101,13 +101,13 @@ export class CobuildConfiguration { readonly cobuildEnabled: boolean; readonly cobuildLeafProjectLogOnlyAllowed: boolean; // (undocumented) - readonly cobuildLockProvider: ICobuildLockProvider; - // (undocumented) - connectLockProviderAsync(): Promise; + get cobuildLockProvider(): ICobuildLockProvider; // (undocumented) get contextId(): string | undefined; // (undocumented) - disconnectLockProviderAsync(): Promise; + createLockProviderAsync(): Promise; + // (undocumented) + destroyLockProviderAsync(): Promise; // (undocumented) static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string; static tryLoadAsync(terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession): Promise; diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 1fefe757340..dc5df932c8e 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -26,7 +26,7 @@ export interface ICobuildConfigurationOptions { cobuildJson: ICobuildJson; rushConfiguration: RushConfiguration; rushSession: RushSession; - cobuildLockProvider: ICobuildLockProvider; + cobuildLockProviderFactory: CobuildLockProviderFactory; } /** @@ -61,10 +61,12 @@ export class CobuildConfiguration { */ public readonly cobuildLeafProjectLogOnlyAllowed: boolean; - public readonly cobuildLockProvider: ICobuildLockProvider; + private _cobuildLockProvider: ICobuildLockProvider | undefined; + private readonly _cobuildLockProviderFactory: CobuildLockProviderFactory; + private readonly _cobuildJson: ICobuildJson; private constructor(options: ICobuildConfigurationOptions) { - const { cobuildJson, cobuildLockProvider } = options; + const { cobuildJson, cobuildLockProviderFactory } = options; this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; @@ -74,7 +76,8 @@ export class CobuildConfiguration { this.cobuildEnabled = false; } - this.cobuildLockProvider = cobuildLockProvider; + this._cobuildLockProviderFactory = cobuildLockProviderFactory; + this._cobuildJson = cobuildJson; } /** @@ -114,13 +117,11 @@ export class CobuildConfiguration { throw new Error(`Unexpected cobuild lock provider: ${cobuildJson.cobuildLockProvider}`); } - const cobuildLockProvider: ICobuildLockProvider = await cobuildLockProviderFactory(cobuildJson); - return new CobuildConfiguration({ cobuildJson, rushConfiguration, rushSession, - cobuildLockProvider + cobuildLockProviderFactory }); } @@ -128,15 +129,26 @@ export class CobuildConfiguration { return this.cobuildContextId; } - public async connectLockProviderAsync(): Promise { + public async createLockProviderAsync(): Promise { if (this.cobuildEnabled) { - await this.cobuildLockProvider.connectAsync(); + const cobuildLockProvider: ICobuildLockProvider = await this._cobuildLockProviderFactory( + this._cobuildJson + ); + this._cobuildLockProvider = cobuildLockProvider; + await this._cobuildLockProvider.connectAsync(); } } - public async disconnectLockProviderAsync(): Promise { + public async destroyLockProviderAsync(): Promise { if (this.cobuildEnabled) { - await this.cobuildLockProvider.disconnectAsync(); + await this._cobuildLockProvider?.disconnectAsync(); + } + } + + public get cobuildLockProvider(): ICobuildLockProvider { + if (!this._cobuildLockProvider) { + throw new Error(`Cobuild lock provider has not been created`); } + return this._cobuildLockProvider; } } diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 690ef91097c..a8f2e0bafc7 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -320,7 +320,7 @@ export class PhasedScriptAction extends BaseScriptAction { this.rushConfiguration, this.rushSession ); - await cobuildConfiguration?.connectLockProviderAsync(); + await cobuildConfiguration?.createLockProviderAsync(); } const projectSelection: Set = @@ -399,7 +399,7 @@ export class PhasedScriptAction extends BaseScriptAction { await this._runWatchPhases(internalOptions); } - await cobuildConfiguration?.disconnectLockProviderAsync(); + await cobuildConfiguration?.destroyLockProviderAsync(); } private async _runInitialPhases(options: IRunPhasesOptions): Promise { From b0f97004acac775cd9586429bd172c5c34c37d67 Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 26 May 2023 16:29:45 +0800 Subject: [PATCH 051/100] feat: cobuild runner id --- .../.vscode/tasks.json | 4 +- .../rush/nonbrowser-approved-packages.json | 12 ++-- common/config/rush/pnpm-lock.yaml | 16 +++-- common/config/rush/repo-state.json | 2 +- common/reviews/api/rush-lib.api.md | 16 ++++- libraries/rush-lib/package.json | 4 +- .../rush-lib/src/api/CobuildConfiguration.ts | 11 +++- .../src/api/EnvironmentConfiguration.ts | 28 +++++++++ .../cli/scriptActions/PhasedScriptAction.ts | 2 +- .../operations/CacheableOperationPlugin.ts | 12 +++- .../logic/operations/ConsoleTimelinePlugin.ts | 61 +++++++++++++++---- .../operations/IOperationExecutionResult.ts | 4 ++ .../operations/OperationExecutionRecord.ts | 5 ++ .../operations/OperationMetadataManager.ts | 14 ++++- .../logic/operations/OperationRunnerHooks.ts | 2 + .../logic/operations/OperationStateFile.ts | 2 + .../logic/operations/ShellOperationRunner.ts | 4 +- .../test/OperationExecutionManager.test.ts | 2 +- 18 files changed, 169 insertions(+), 32 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index 1029c807a58..bcb42db3f62 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -34,7 +34,7 @@ { "type": "shell", "label": "build 1", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "command": "node ../../lib/runRush.js --debug cobuild --timeline --parallelism 1 --verbose", "problemMatcher": [], "options": { "cwd": "${workspaceFolder}/sandbox/repo", @@ -57,7 +57,7 @@ { "type": "shell", "label": "build 2", - "command": "node ../../lib/runRush.js --debug cobuild --parallelism 1 --verbose", + "command": "node ../../lib/runRush.js --debug cobuild --timeline --parallelism 1 --verbose", "problemMatcher": [], "options": { "cwd": "${workspaceFolder}/sandbox/repo", diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index 04bfca90017..ee6f5805c40 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -174,10 +174,6 @@ "name": "@rushstack/package-extractor", "allowedCategories": [ "libraries" ] }, - { - "name": "@rushstack/rush-redis-cobuild-plugin", - "allowedCategories": [ "tests" ] - }, { "name": "@rushstack/rig-package", "allowedCategories": [ "libraries" ] @@ -194,6 +190,10 @@ "name": "@rushstack/rush-http-build-cache-plugin", "allowedCategories": [ "libraries" ] }, + { + "name": "@rushstack/rush-redis-cobuild-plugin", + "allowedCategories": [ "tests" ] + }, { "name": "@rushstack/rush-sdk", "allowedCategories": [ "libraries", "tests" ] @@ -738,6 +738,10 @@ "name": "url-loader", "allowedCategories": [ "libraries" ] }, + { + "name": "uuid", + "allowedCategories": [ "libraries" ] + }, { "name": "webpack", "allowedCategories": [ "libraries", "tests" ] diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index ae79b76107c..076ee821f0a 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -2068,6 +2068,7 @@ importers: '@types/ssri': ~7.1.0 '@types/strict-uri-encode': 2.0.0 '@types/tar': 6.1.1 + '@types/uuid': ~8.3.4 '@types/webpack-env': 1.18.0 '@yarnpkg/lockfile': ~1.0.2 builtin-modules: ~3.1.0 @@ -2094,6 +2095,7 @@ importers: tapable: 2.2.1 tar: ~6.1.11 true-case-path: ~2.2.1 + uuid: ~8.3.2 webpack: ~5.80.0 dependencies: '@pnpm/link-bins': 5.3.25 @@ -2131,6 +2133,7 @@ importers: tapable: 2.2.1 tar: 6.1.13 true-case-path: 2.2.1 + uuid: 8.3.2 devDependencies: '@pnpm/logger': 4.0.0 '@rushstack/eslint-config': link:../../eslint/eslint-config @@ -2152,6 +2155,7 @@ importers: '@types/ssri': 7.1.1 '@types/strict-uri-encode': 2.0.0 '@types/tar': 6.1.1 + '@types/uuid': 8.3.4 '@types/webpack-env': 1.18.0 webpack: 5.80.0 @@ -6827,7 +6831,7 @@ packages: js-string-escape: 1.0.1 loader-utils: 2.0.4 lodash: 4.17.21 - nanoid: 3.3.4 + nanoid: 3.3.6 p-limit: 3.1.0 prettier: 2.3.0 prop-types: 15.8.1 @@ -8870,6 +8874,10 @@ packages: resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==} dev: false + /@types/uuid/8.3.4: + resolution: {integrity: sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==} + dev: true + /@types/webpack-env/1.18.0: resolution: {integrity: sha512-56/MAlX5WMsPVbOg7tAxnYvNYMMWr/QJiIp6BxVSW3JJXUVzzOn64qW8TzQyMSqSUFM2+PVI4aUHcHOzIz/1tg==} @@ -16914,8 +16922,8 @@ packages: requiresBuild: true optional: true - /nanoid/3.3.4: - resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==} + /nanoid/3.3.6: + resolution: {integrity: sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -18274,7 +18282,7 @@ packages: resolution: {integrity: sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==} engines: {node: ^10 || ^12 || >=14} dependencies: - nanoid: 3.3.4 + nanoid: 3.3.6 picocolors: 1.0.0 source-map-js: 1.0.2 diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 8cd74bfd40a..58baca5a418 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "e42ca74053e55c066f8ce4291d494a77ebd367fa", + "pnpmShrinkwrapHash": "03f6c90980f46e8036baf7f321d98c778148cb22", "preferredVersionsHash": "1926a5b12ac8f4ab41e76503a0d1d0dccc9c0e06" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 64715e3b19a..1873f637fd6 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -102,10 +102,11 @@ export class CobuildConfiguration { readonly cobuildLeafProjectLogOnlyAllowed: boolean; // (undocumented) get cobuildLockProvider(): ICobuildLockProvider; + readonly cobuildRunnerId: string; // (undocumented) get contextId(): string | undefined; // (undocumented) - createLockProviderAsync(): Promise; + createLockProviderAsync(terminal: ITerminal): Promise; // (undocumented) destroyLockProviderAsync(): Promise; // (undocumented) @@ -173,6 +174,7 @@ export class EnvironmentConfiguration { static get cobuildContextId(): string | undefined; static get cobuildEnabled(): boolean | undefined; static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined; + static get cobuildRunnerId(): string | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts // // @internal @@ -207,6 +209,7 @@ export const EnvironmentVariableNames: { readonly RUSH_BUILD_CACHE_WRITE_ALLOWED: "RUSH_BUILD_CACHE_WRITE_ALLOWED"; readonly RUSH_COBUILD_ENABLED: "RUSH_COBUILD_ENABLED"; readonly RUSH_COBUILD_CONTEXT_ID: "RUSH_COBUILD_CONTEXT_ID"; + readonly RUSH_COBUILD_RUNNER_ID: "RUSH_COBUILD_RUNNER_ID"; readonly RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED"; readonly RUSH_GIT_BINARY_PATH: "RUSH_GIT_BINARY_PATH"; readonly RUSH_TAR_BINARY_PATH: "RUSH_TAR_BINARY_PATH"; @@ -452,6 +455,7 @@ export interface _INpmOptionsJson extends IPackageManagerOptionsJsonBase { // @alpha export interface IOperationExecutionResult { + readonly cobuildRunnerId: string | undefined; readonly error: Error | undefined; readonly nonCachedDurationMs: number | undefined; readonly status: OperationStatus; @@ -461,6 +465,10 @@ export interface IOperationExecutionResult { // @internal (undocumented) export interface _IOperationMetadata { + // (undocumented) + cobuildContextId: string | undefined; + // (undocumented) + cobuildRunnerId: string | undefined; // (undocumented) durationInSeconds: number; // (undocumented) @@ -520,6 +528,10 @@ export interface _IOperationStateFileOptions { // @internal (undocumented) export interface _IOperationStateJson { + // (undocumented) + cobuildContextId: string | undefined; + // (undocumented) + cobuildRunnerId: string | undefined; // (undocumented) nonCachedDurationMs: number; } @@ -725,7 +737,7 @@ export class _OperationMetadataManager { constructor(options: _IOperationMetadataManagerOptions); get relativeFilepaths(): string[]; // (undocumented) - saveAsync({ durationInSeconds, logPath, errorLogPath }: _IOperationMetadata): Promise; + saveAsync({ durationInSeconds, cobuildContextId, cobuildRunnerId, logPath, errorLogPath }: _IOperationMetadata): Promise; // (undocumented) readonly stateFile: _OperationStateFile; // (undocumented) diff --git a/libraries/rush-lib/package.json b/libraries/rush-lib/package.json index 2598766edd8..398a38bab18 100644 --- a/libraries/rush-lib/package.json +++ b/libraries/rush-lib/package.json @@ -56,7 +56,8 @@ "strict-uri-encode": "~2.0.0", "tapable": "2.2.1", "tar": "~6.1.11", - "true-case-path": "~2.2.1" + "true-case-path": "~2.2.1", + "uuid": "~8.3.2" }, "devDependencies": { "@pnpm/logger": "4.0.0", @@ -79,6 +80,7 @@ "@types/ssri": "~7.1.0", "@types/strict-uri-encode": "2.0.0", "@types/tar": "6.1.1", + "@types/uuid": "~8.3.4", "@types/webpack-env": "1.18.0", "webpack": "~5.80.0" }, diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index dc5df932c8e..3c014c8462a 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -7,6 +7,7 @@ import schemaJson from '../schemas/cobuild.schema.json'; import { EnvironmentConfiguration } from './EnvironmentConfiguration'; import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; import { RushConstants } from '../logic/RushConstants'; +import { v4 as uuidv4 } from 'uuid'; import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; @@ -54,6 +55,12 @@ export class CobuildConfiguration { * The cobuild feature won't be enabled until the context id is provided as an non-empty string. */ public readonly cobuildContextId: string | undefined; + + /** + * This is a name of the participating cobuild runner. It can be specified by the environment variable + * RUSH_COBUILD_RUNNER_ID. If it is not provided, a random id will be generated to identify the runner. + */ + public readonly cobuildRunnerId: string; /** * If true, Rush will automatically handle the leaf project with build cache "disabled" by writing * to the cache in a special "log files only mode". This is useful when you want to use Cobuilds @@ -70,6 +77,7 @@ export class CobuildConfiguration { this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; + this.cobuildRunnerId = EnvironmentConfiguration.cobuildRunnerId || uuidv4(); this.cobuildLeafProjectLogOnlyAllowed = EnvironmentConfiguration.cobuildLeafProjectLogOnlyAllowed ?? false; if (!this.cobuildContextId) { @@ -129,8 +137,9 @@ export class CobuildConfiguration { return this.cobuildContextId; } - public async createLockProviderAsync(): Promise { + public async createLockProviderAsync(terminal: ITerminal): Promise { if (this.cobuildEnabled) { + terminal.writeLine(`Running cobuild (runner ${this.cobuildContextId}/${this.cobuildRunnerId})`); const cobuildLockProvider: ICobuildLockProvider = await this._cobuildLockProviderFactory( this._cobuildJson ); diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 327cf1aae74..37fa392ea36 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -163,6 +163,18 @@ export const EnvironmentVariableNames = { */ RUSH_COBUILD_CONTEXT_ID: 'RUSH_COBUILD_CONTEXT_ID', + /** + * Explicitly specifies a name for each participating cobuild runner. + * + * Setting this environment variable opt into running with cobuilds. + * + * @remarks + * This environment variable is optional, if it is not provided, a random id is used. + * + * If there is no cobuild configured, then this environment variable is ignored. + */ + RUSH_COBUILD_RUNNER_ID: 'RUSH_COBUILD_RUNNER_ID', + /** * If this variable is set to "1", When getting distributed builds, Rush will automatically handle the leaf project * with build cache "disabled" by writing to the cache in a special "log files only mode". This is useful when you @@ -233,6 +245,8 @@ export class EnvironmentConfiguration { private static _cobuildContextId: string | undefined; + private static _cobuildRunnerId: string | undefined; + private static _cobuildLeafProjectLogOnlyAllowed: boolean | undefined; private static _gitBinaryPath: string | undefined; @@ -350,6 +364,15 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._cobuildContextId; } + /** + * Provides a determined cobuild runner id if configured + * See {@link EnvironmentVariableNames.RUSH_COBUILD_RUNNER_ID} + */ + public static get cobuildRunnerId(): string | undefined { + EnvironmentConfiguration._ensureValidated(); + return EnvironmentConfiguration._cobuildRunnerId; + } + /** * If set, enables or disables the cobuild leaf project log only feature. * See {@link EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED} @@ -503,6 +526,11 @@ export class EnvironmentConfiguration { break; } + case EnvironmentVariableNames.RUSH_COBUILD_RUNNER_ID: { + EnvironmentConfiguration._cobuildRunnerId = value; + break; + } + case EnvironmentVariableNames.RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: { EnvironmentConfiguration._cobuildLeafProjectLogOnlyAllowed = EnvironmentConfiguration.parseBooleanEnvironmentVariable( diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index d3e39df0ba0..bcffbeeaf4a 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -320,7 +320,7 @@ export class PhasedScriptAction extends BaseScriptAction { this.rushConfiguration, this.rushSession ); - await cobuildConfiguration?.createLockProviderAsync(); + await cobuildConfiguration?.createLockProviderAsync(terminal); } const projectSelection: Set = diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 49eab52065f..5dd25da90db 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -339,10 +339,20 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { runner.hooks.afterExecute.tapPromise( PLUGIN_NAME, async (afterExecuteContext: IOperationRunnerAfterExecuteContext) => { - const { context, status, taskIsSuccessful } = afterExecuteContext; + const { context, status, taskIsSuccessful, logPath, errorLogPath } = afterExecuteContext; const { cobuildLock, projectBuildCache, isCacheWriteAllowed, buildCacheTerminal } = buildCacheContext; + // Save the metadata to disk + const { duration: durationInSeconds } = context.stopwatch; + await context._operationMetadataManager?.saveAsync({ + durationInSeconds, + cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, + cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, + logPath, + errorLogPath + }); + if (!buildCacheTerminal) { // This should not happen throw new InternalError(`Build Cache Terminal is not created`); diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index 31c4b56a48e..27a73a78b5c 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -12,6 +12,7 @@ import { } from '../../pluginFramework/PhasedCommandHooks'; import { IExecutionResult } from './IOperationExecutionResult'; import { OperationStatus } from './OperationStatus'; +import { CobuildConfiguration } from '../../api/CobuildConfiguration'; const PLUGIN_NAME: 'ConsoleTimelinePlugin' = 'ConsoleTimelinePlugin'; @@ -55,7 +56,11 @@ export class ConsoleTimelinePlugin implements IPhasedCommandPlugin { hooks.afterExecuteOperations.tap( PLUGIN_NAME, (result: IExecutionResult, context: ICreateOperationsContext): void => { - _printTimeline(this._terminal, result); + _printTimeline({ + terminal: this._terminal, + result, + cobuildConfiguration: context.cobuildConfiguration + }); } ); } @@ -106,12 +111,23 @@ interface ITimelineRecord { durationString: string; name: string; status: OperationStatus; + isExecuteByOtherCobuildRunner: boolean; } + +/** + * @internal + */ +export interface IPrintTimelineParameters { + terminal: ITerminal; + result: IExecutionResult; + cobuildConfiguration: CobuildConfiguration | undefined; +} + /** * Print a more detailed timeline and analysis of CPU usage for the build. * @internal */ -export function _printTimeline(terminal: ITerminal, result: IExecutionResult): void { +export function _printTimeline({ terminal, result, cobuildConfiguration }: IPrintTimelineParameters): void { // // Gather the operation records we'll be displaying. Do some inline max() // finding to reduce the number of times we need to loop through operations. @@ -167,7 +183,10 @@ export function _printTimeline(terminal: ITerminal, result: IExecutionResult): v endTime, durationString, name: operation.name!, - status: operationResult.status + status: operationResult.status, + isExecuteByOtherCobuildRunner: + !!operationResult.cobuildRunnerId && + operationResult.cobuildRunnerId !== cobuildConfiguration?.cobuildRunnerId }); } } @@ -207,7 +226,19 @@ export function _printTimeline(terminal: ITerminal, result: IExecutionResult): v terminal.writeLine(''); terminal.writeLine('='.repeat(maxWidth)); - for (const { startTime, endTime, durationString, name, status } of data) { + let hasCobuildSymbol: boolean = false; + + function getChartSymbol(record: ITimelineRecord): string { + const { isExecuteByOtherCobuildRunner, status } = record; + if (isExecuteByOtherCobuildRunner) { + hasCobuildSymbol = true; + return 'C'; + } + return TIMELINE_CHART_SYMBOLS[status]; + } + + for (const record of data) { + const { startTime, endTime, durationString, name, status } = record; // Track busy CPUs const openCpu: number = getOpenCPU(startTime); busyCpus[openCpu] = endTime; @@ -219,7 +250,7 @@ export function _printTimeline(terminal: ITerminal, result: IExecutionResult): v const chart: string = colors.gray('-'.repeat(startIdx)) + - TIMELINE_CHART_COLORIZER[status](TIMELINE_CHART_SYMBOLS[status].repeat(length)) + + TIMELINE_CHART_COLORIZER[status](getChartSymbol(record).repeat(length)) + colors.gray('-'.repeat(chartWidth - endIdx)); terminal.writeLine( `${colors.cyan(name.padStart(longestNameLength))} ${chart} ${colors.white( @@ -236,19 +267,27 @@ export function _printTimeline(terminal: ITerminal, result: IExecutionResult): v const usedCpus: number = busyCpus.length; - const legend: string[] = ['LEGEND:', ' [#] Success [!] Failed/warnings [%] Skipped/cached/no-op']; + const legend: string[] = [ + 'LEGEND:', + ' [#] Success [!] Failed/warnings [%] Skipped/cached/no-op', + '', + '' + ]; + if (hasCobuildSymbol) { + legend[2] = ' [C] Cobuild'; + } const summary: string[] = [ `Total Work: ${workDuration.toFixed(1)}s`, - `Wall Clock: ${allDurationSeconds.toFixed(1)}s` + `Wall Clock: ${allDurationSeconds.toFixed(1)}s`, + `Max Parallelism Used: ${usedCpus}`, + `Avg Parallelism Used: ${(workDuration / allDurationSeconds).toFixed(1)}` ]; terminal.writeLine(legend[0] + summary[0].padStart(maxWidth - legend[0].length)); terminal.writeLine(legend[1] + summary[1].padStart(maxWidth - legend[1].length)); - terminal.writeLine(`Max Parallelism Used: ${usedCpus}`.padStart(maxWidth)); - terminal.writeLine( - `Avg Parallelism Used: ${(workDuration / allDurationSeconds).toFixed(1)}`.padStart(maxWidth) - ); + terminal.writeLine(legend[2] + summary[2].padStart(maxWidth - legend[2].length)); + terminal.writeLine(legend[3] + summary[3].padStart(maxWidth - legend[3].length)); // // Include time-by-phase, if phases are enabled diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index ebb3e892cc2..a7bb902a3d0 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -35,6 +35,10 @@ export interface IOperationExecutionResult { * The value indicates the duration of the same operation without cache hit. */ readonly nonCachedDurationMs: number | undefined; + /** + * The id of the runner which actually runs the building process in cobuild mode. + */ + readonly cobuildRunnerId: string | undefined; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 2f1d0eb4b90..d75ed3f968e 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -142,6 +142,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._operationMetadataManager?.stateFile.state?.nonCachedDurationMs; } + public get cobuildRunnerId(): string | undefined { + // Lazy calculated because the state file is created/restored later on + return this._operationMetadataManager?.stateFile.state?.cobuildRunnerId; + } + public async executeAsync(onResult: (record: OperationExecutionRecord) => Promise): Promise { this.status = OperationStatus.Executing; this.stopwatch.start(); diff --git a/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts b/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts index 90c7caa22ff..2c2008f084b 100644 --- a/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts @@ -26,6 +26,8 @@ export interface IOperationMetaData { durationInSeconds: number; logPath: string; errorLogPath: string; + cobuildContextId: string | undefined; + cobuildRunnerId: string | undefined; } /** @@ -70,9 +72,17 @@ export class OperationMetadataManager { return [this.stateFile.relativeFilepath, this._relativeLogPath, this._relativeErrorLogPath]; } - public async saveAsync({ durationInSeconds, logPath, errorLogPath }: IOperationMetaData): Promise { + public async saveAsync({ + durationInSeconds, + cobuildContextId, + cobuildRunnerId, + logPath, + errorLogPath + }: IOperationMetaData): Promise { const state: IOperationStateJson = { - nonCachedDurationMs: durationInSeconds * 1000 + nonCachedDurationMs: durationInSeconds * 1000, + cobuildContextId, + cobuildRunnerId }; await this.stateFile.writeAsync(state); diff --git a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index a2b32af9759..66415c697af 100644 --- a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -45,6 +45,8 @@ export interface IOperationRunnerAfterExecuteContext { exitCode: number; status: OperationStatus; taskIsSuccessful: boolean; + logPath: string; + errorLogPath: string; } /** diff --git a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts index 021c66aca60..0a88dad27de 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts @@ -16,6 +16,8 @@ export interface IOperationStateFileOptions { */ export interface IOperationStateJson { nonCachedDurationMs: number; + cobuildContextId: string | undefined; + cobuildRunnerId: string | undefined; } /** diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 9c523f492e1..42e230fb3f2 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -335,7 +335,9 @@ export class ShellOperationRunner implements IOperationRunner { terminal, exitCode, status, - taskIsSuccessful + taskIsSuccessful, + logPath: projectLogWritable.logPath, + errorLogPath: projectLogWritable.errorLogPath }; await this.hooks.afterExecute.promise(afterExecuteContext); diff --git a/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts index 378175310d6..5818fce6721 100644 --- a/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts @@ -212,7 +212,7 @@ describe(OperationExecutionManager.name, () => { ); const result: IExecutionResult = await executionManager.executeAsync(); - _printTimeline(mockTerminal, result); + _printTimeline({ terminal: mockTerminal, result, cobuildConfiguration: undefined }); _printOperationStatus(mockTerminal, result); const allMessages: string = mockWritable.getAllOutput(); expect(allMessages).toContain('Build step 1'); From 6f16f273a7693b50689236b21ce17e5d8526015d Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 26 May 2023 17:34:24 +0800 Subject: [PATCH 052/100] fix: test --- libraries/rush-lib/src/logic/operations/OperationStateFile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts index 0a88dad27de..b05cf70b57f 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStateFile.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStateFile.ts @@ -55,7 +55,7 @@ export class OperationStateFile { } public async writeAsync(json: IOperationStateJson): Promise { - await JsonFile.saveAsync(json, this.filepath, { ensureFolderExists: true }); + await JsonFile.saveAsync(json, this.filepath, { ensureFolderExists: true, ignoreUndefinedValues: true }); this._state = json; } From b1178be5e644e62664f83323e7154047f694228b Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 28 Jun 2023 09:54:37 +0800 Subject: [PATCH 053/100] feat: disjoint-set --- .../rush-lib/src/logic/cobuild/DisjointSet.ts | 87 +++++++++++++++++++ .../logic/cobuild/test/DisjointSet.test.ts | 82 +++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 libraries/rush-lib/src/logic/cobuild/DisjointSet.ts create mode 100644 libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts diff --git a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts new file mode 100644 index 00000000000..9ebe2c7973d --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { InternalError } from '@rushstack/node-core-library'; + +/** + * A disjoint set data structure + */ +export class DisjointSet { + private _forest: WeakSet; + private _parentMap: WeakMap; + private _sizeMap: WeakMap; + + public constructor() { + this._forest = new WeakSet(); + this._parentMap = new WeakMap(); + this._sizeMap = new WeakMap(); + } + + /** + * Adds a new set containing specific object + */ + public add(x: T): void { + if (this._forest.has(x)) { + return; + } + + this._forest.add(x); + this._parentMap.set(x, x); + this._sizeMap.set(x, 1); + } + + /** + * Unions the sets that contain two objects + */ + public union(a: T, b: T): void { + let x: T = this._find(a); + let y: T = this._find(b); + + if (x === y) { + // x and y are already in the same set + return; + } + + if (this._getSize(x) < this._getSize(y)) { + const t: T = x; + x = y; + y = t; + } + this._parentMap.set(y, x); + this._sizeMap.set(x, this._getSize(x) + this._getSize(y)); + } + + /** + * Returns true if x and y are in the same set + */ + public isConnected(x: T, y: T): boolean { + return this._find(x) === this._find(y); + } + + private _find(a: T): T { + let x: T = a; + while (this._getParent(x) !== x) { + this._parentMap.set(x, this._getParent(this._getParent(x))); + x = this._getParent(x); + } + return x; + } + + private _getParent(x: T): T { + const parent: T | undefined = this._parentMap.get(x); + if (parent === undefined) { + // This should not happen + throw new InternalError(`Can not find parent`); + } + return parent; + } + + private _getSize(x: T): number { + const size: number | undefined = this._sizeMap.get(x); + if (size === undefined) { + // This should not happen + throw new InternalError(`Can not get size`); + } + return size; + } +} diff --git a/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts b/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts new file mode 100644 index 00000000000..df507c9d524 --- /dev/null +++ b/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts @@ -0,0 +1,82 @@ +import { DisjointSet } from '../DisjointSet'; + +describe(DisjointSet.name, () => { + it('can disjoint two sets', () => { + const disjointSet = new DisjointSet<{ id: number }>(); + const obj1 = { id: 1 }; + const obj2 = { id: 2 }; + disjointSet.add(obj1); + disjointSet.add(obj2); + + expect(disjointSet.isConnected(obj1, obj2)).toBe(false); + }); + + it('can disjoint multiple sets', () => { + const disjointSet = new DisjointSet<{ id: number }>(); + const obj1 = { id: 1 }; + const obj2 = { id: 2 }; + const obj3 = { id: 3 }; + const obj4 = { id: 4 }; + disjointSet.add(obj1); + disjointSet.add(obj2); + disjointSet.add(obj3); + disjointSet.add(obj4); + + expect(disjointSet.isConnected(obj1, obj2)).toBe(false); + expect(disjointSet.isConnected(obj1, obj3)).toBe(false); + expect(disjointSet.isConnected(obj1, obj4)).toBe(false); + }); + + it('can union two sets', () => { + const disjointSet = new DisjointSet<{ id: number }>(); + const obj1 = { id: 1 }; + const obj2 = { id: 2 }; + disjointSet.add(obj1); + disjointSet.add(obj2); + expect(disjointSet.isConnected(obj1, obj2)).toBe(false); + + disjointSet.union(obj1, obj2); + expect(disjointSet.isConnected(obj1, obj2)).toBe(true); + }); + + it('can union two sets transitively', () => { + const disjointSet = new DisjointSet<{ id: number }>(); + const obj1 = { id: 1 }; + const obj2 = { id: 2 }; + const obj3 = { id: 3 }; + disjointSet.add(obj1); + disjointSet.add(obj2); + disjointSet.add(obj3); + + disjointSet.union(obj1, obj2); + expect(disjointSet.isConnected(obj1, obj2)).toBe(true); + expect(disjointSet.isConnected(obj1, obj3)).toBe(false); + expect(disjointSet.isConnected(obj2, obj3)).toBe(false); + + disjointSet.union(obj1, obj3); + expect(disjointSet.isConnected(obj1, obj2)).toBe(true); + expect(disjointSet.isConnected(obj2, obj3)).toBe(true); + expect(disjointSet.isConnected(obj1, obj3)).toBe(true); + }); + + it('can union and disjoint sets', () => { + const disjointSet = new DisjointSet<{ id: number }>(); + const obj1 = { id: 1 }; + const obj2 = { id: 2 }; + const obj3 = { id: 3 }; + const obj4 = { id: 4 }; + disjointSet.add(obj1); + disjointSet.add(obj2); + disjointSet.add(obj3); + disjointSet.add(obj4); + + expect(disjointSet.isConnected(obj1, obj2)).toBe(false); + expect(disjointSet.isConnected(obj1, obj3)).toBe(false); + expect(disjointSet.isConnected(obj1, obj4)).toBe(false); + + disjointSet.union(obj1, obj2); + expect(disjointSet.isConnected(obj1, obj2)).toBe(true); + expect(disjointSet.isConnected(obj1, obj3)).toBe(false); + expect(disjointSet.isConnected(obj1, obj4)).toBe(false); + }); +}); From 9a8fc06f3497ff26e40e2c21c027445365528a5d Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 10 Jul 2023 19:50:31 +0800 Subject: [PATCH 054/100] feat: disjointSet getAllSets --- .../rush-lib/src/logic/cobuild/DisjointSet.ts | 40 ++++++++++++++++--- .../logic/cobuild/test/DisjointSet.test.ts | 23 +++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts index 9ebe2c7973d..cf704865cd0 100644 --- a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts +++ b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts @@ -7,14 +7,23 @@ import { InternalError } from '@rushstack/node-core-library'; * A disjoint set data structure */ export class DisjointSet { - private _forest: WeakSet; - private _parentMap: WeakMap; - private _sizeMap: WeakMap; + private _forest: Set; + private _parentMap: Map; + private _sizeMap: Map; + private _setByElement: Map> | undefined; public constructor() { - this._forest = new WeakSet(); - this._parentMap = new WeakMap(); - this._sizeMap = new WeakMap(); + this._forest = new Set(); + this._parentMap = new Map(); + this._sizeMap = new Map(); + this._setByElement = new Map>(); + } + + public destroy(): void { + this._forest.clear(); + this._parentMap.clear(); + this._sizeMap.clear(); + this._setByElement?.clear(); } /** @@ -28,6 +37,7 @@ export class DisjointSet { this._forest.add(x); this._parentMap.set(x, x); this._sizeMap.set(x, 1); + this._setByElement = undefined; } /** @@ -49,6 +59,24 @@ export class DisjointSet { } this._parentMap.set(y, x); this._sizeMap.set(x, this._getSize(x) + this._getSize(y)); + this._setByElement = undefined; + } + + public getAllSets(): Iterable> { + if (this._setByElement === undefined) { + this._setByElement = new Map>(); + + for (const element of this._forest) { + const root: T = this._find(element); + let set: Set | undefined = this._setByElement.get(root); + if (set === undefined) { + set = new Set(); + this._setByElement.set(root, set); + } + set.add(element); + } + } + return this._setByElement.values(); } /** diff --git a/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts b/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts index df507c9d524..255f0a426e7 100644 --- a/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts +++ b/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts @@ -79,4 +79,27 @@ describe(DisjointSet.name, () => { expect(disjointSet.isConnected(obj1, obj3)).toBe(false); expect(disjointSet.isConnected(obj1, obj4)).toBe(false); }); + + it('can get all sets', () => { + const disjointSet = new DisjointSet<{ id: number }>(); + const obj1 = { id: 1 }; + const obj2 = { id: 2 }; + const obj3 = { id: 3 }; + disjointSet.add(obj1); + disjointSet.add(obj2); + disjointSet.add(obj3); + + disjointSet.union(obj1, obj2); + + const allSets: Iterable> = disjointSet.getAllSets(); + + const allSetList: Array> = []; + for (const set of allSets) { + allSetList.push(set); + } + + expect(allSetList.length).toBe(2); + expect(Array.from(allSetList[0]).map((x) => x.id)).toEqual(expect.arrayContaining([1, 2])); + expect(Array.from(allSetList[1]).map((x) => x.id)).toEqual(expect.arrayContaining([3])); + }); }); From e3f116c702c4f6bbfd0b1e945f21d4caafc3a8c3 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 11 Jul 2023 22:12:26 +0800 Subject: [PATCH 055/100] refact: clustering operations in cobuilds --- .../.vscode/tasks.json | 2 + .../config/heft.json | 47 +----- .../package.json | 1 + .../src/testLockProvider.ts | 14 +- common/config/rush/pnpm-lock.yaml | 40 +++++- common/config/rush/repo-state.json | 2 +- common/reviews/api/rush-lib.api.md | 25 ++-- .../api/rush-redis-cobuild-plugin.api.md | 11 +- .../rush-lib/src/api/CobuildConfiguration.ts | 4 - .../src/api/EnvironmentConfiguration.ts | 5 +- libraries/rush-lib/src/logic/RushConstants.ts | 6 - .../rush-lib/src/logic/cobuild/CobuildLock.ts | 62 ++++++-- .../src/logic/cobuild/ICobuildLockProvider.ts | 71 +++++++-- .../logic/cobuild/test/CobuildLock.test.ts | 41 ++++-- .../operations/CacheableOperationPlugin.ts | 136 ++++++++++++++++-- .../logic/operations/ShellOperationRunner.ts | 3 +- .../rush-redis-cobuild-plugin/package.json | 8 +- .../src/RedisCobuildLockProvider.ts | 121 +++++++++------- .../src/test/RedisCobuildLockProvider.test.ts | 80 +++++++---- 19 files changed, 466 insertions(+), 213 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json index bcb42db3f62..93aa001729c 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/.vscode/tasks.json @@ -40,6 +40,7 @@ "cwd": "${workspaceFolder}/sandbox/repo", "env": { "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "RUSH_COBUILD_RUNNER_ID": "runner1", "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1", "REDIS_PASS": "redis123" } @@ -63,6 +64,7 @@ "cwd": "${workspaceFolder}/sandbox/repo", "env": { "RUSH_COBUILD_CONTEXT_ID": "integration-test", + "RUSH_COBUILD_RUNNER_ID": "runner2", "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED": "1", "REDIS_PASS": "redis123" } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json b/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json index 99e058540fb..f290ba09665 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/config/heft.json @@ -2,50 +2,11 @@ * Defines configuration used by core Heft. */ { - "$schema": "https://developer.microsoft.com/json-schemas/heft/heft.schema.json", - - "eventActions": [ - { - /** - * The kind of built-in operation that should be performed. - * The "deleteGlobs" action deletes files or folders that match the - * specified glob patterns. - */ - "actionKind": "deleteGlobs", - - /** - * The stage of the Heft run during which this action should occur. Note that actions specified in heft.json - * occur at the end of the stage of the Heft run. - */ - "heftEvent": "clean", - - /** - * A user-defined tag whose purpose is to allow configs to replace/delete handlers that were added by other - * configs. - */ - "actionId": "defaultClean", - - /** - * Glob patterns to be deleted. The paths are resolved relative to the project folder. - */ - "globsToDelete": ["dist", "lib", "temp"] - } - ], + "$schema": "https://developer.microsoft.com/json-schemas/heft/v0/heft.schema.json", /** - * The list of Heft plugins to be loaded. + * Optionally specifies another JSON config file that this file extends from. This provides a way for standard + * settings to be shared across multiple projects. */ - "heftPlugins": [ - // { - // /** - // * The path to the plugin package. - // */ - // "plugin": "path/to/my-plugin", - // - // /** - // * An optional object that provides additional settings that may be defined by the plugin. - // */ - // // "options": { } - // } - ] + "extends": "@rushstack/heft-node-rig/profiles/default/config/heft.json" } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json index 9953d6d84dd..68eb95a20df 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/package.json @@ -13,6 +13,7 @@ "@microsoft/rush-lib": "workspace:*", "@rushstack/eslint-config": "workspace:*", "@rushstack/heft": "workspace:*", + "@rushstack/heft-node-rig": "workspace:*", "@rushstack/node-core-library": "workspace:*", "@rushstack/rush-redis-cobuild-plugin": "workspace:*", "@types/http-proxy": "~1.17.8", diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 859135e6eda..9eaa09bef34 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -20,15 +20,21 @@ async function main(): Promise { const lockProvider: RedisCobuildLockProvider = new RedisCobuildLockProvider(options, rushSession as any); await lockProvider.connectAsync(); const context: ICobuildContext = { - contextId: 'test-context-id', - version: 1, - cacheId: 'test-cache-id' + contextId: 'context_id', + cacheId: 'cache_id', + lockKey: 'lock_key', + lockExpireTimeInSeconds: 30, + completedStateKey: 'completed_state_key', + clusterId: 'cluster_id', + runnerId: 'runner_id', + packageName: 'package_name', + phaseName: 'phase_name' }; await lockProvider.acquireLockAsync(context); await lockProvider.renewLockAsync(context); await lockProvider.setCompletedStateAsync(context, { status: OperationStatus.Success, - cacheId: 'test-cache-id' + cacheId: 'cache_id' }); const completedState = await lockProvider.getCompletedStateAsync(context); console.log('Completed state: ', completedState); diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 5f1ceb3a5ea..acfe173d629 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -1266,8 +1266,8 @@ importers: '@rushstack/heft-jest-plugin': link:../../heft-plugins/heft-jest-plugin '@rushstack/heft-lint-plugin': link:../../heft-plugins/heft-lint-plugin '@rushstack/heft-typescript-plugin': link:../../heft-plugins/heft-typescript-plugin - '@types/jest': 29.5.2 - '@types/node': 20.3.2 + '@types/jest': 29.5.3 + '@types/node': 20.4.1 eslint: 8.7.0 tslint: 5.20.1_typescript@4.9.5 tslint-microsoft-contrib: 6.2.0_uwqr5pcif4g7c56scrk6kqzf7i @@ -1512,6 +1512,7 @@ importers: '@microsoft/rush-lib': workspace:* '@rushstack/eslint-config': workspace:* '@rushstack/heft': workspace:* + '@rushstack/heft-node-rig': workspace:* '@rushstack/node-core-library': workspace:* '@rushstack/rush-redis-cobuild-plugin': workspace:* '@types/http-proxy': ~1.17.8 @@ -1523,6 +1524,7 @@ importers: '@microsoft/rush-lib': link:../../libraries/rush-lib '@rushstack/eslint-config': link:../../eslint/eslint-config '@rushstack/heft': link:../../apps/heft + '@rushstack/heft-node-rig': link:../../rigs/heft-node-rig '@rushstack/node-core-library': link:../../libraries/node-core-library '@rushstack/rush-redis-cobuild-plugin': link:../../rush-plugins/rush-redis-cobuild-plugin '@types/http-proxy': 1.17.11 @@ -3230,6 +3232,9 @@ packages: /@aws-cdk/cloud-assembly-schema/2.7.0: resolution: {integrity: sha512-vKTKLMPvzUhsYo3c4/EbMJq+bwIgHkwK0lV9fc5mQlnTUTyHe6nGIvyzmWWMd5BVEkgNzw+QdecxeeYJNu/doA==} engines: {node: '>= 14.15.0'} + dependencies: + jsonschema: 1.4.1 + semver: 7.5.3 dev: true bundledDependencies: - jsonschema @@ -3253,6 +3258,7 @@ packages: engines: {node: '>= 14.15.0'} dependencies: '@aws-cdk/cloud-assembly-schema': 2.7.0 + semver: 7.5.3 dev: true bundledDependencies: - semver @@ -4986,6 +4992,10 @@ packages: '@babel/helper-validator-identifier': 7.22.5 to-fast-properties: 2.0.0 + /@balena/dockerignore/1.0.2: + resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} + dev: true + /@base2/pretty-print-object/1.0.1: resolution: {integrity: sha512-4iri8i1AqYHJE2DstZYkyEprg6Pq6sKx3xn5FpySk9sNhH7qN2LLlHJCfDTZRILNwQNPD7mATWM0TBui7uC1pA==} dev: true @@ -9180,8 +9190,8 @@ packages: expect: 29.5.0 pretty-format: 29.5.0 - /@types/jest/29.5.2: - resolution: {integrity: sha512-mSoZVJF5YzGVCk+FsDxzDuH7s+SCkzrgKZzf0Z0T2WudhBUPoF6ktoTPC4R0ZoCPCV5xUvuU6ias5NvxcBcMMg==} + /@types/jest/29.5.3: + resolution: {integrity: sha512-1Nq7YrO/vJE/FYnqYyw0FS8LdrjExSgIiHyKg7xPpn+yi8Q4huZryKnkJatN1ZRH89Kw2v33/8ZMB7DuZeSLlA==} dependencies: expect: 29.5.0 pretty-format: 29.5.0 @@ -9281,8 +9291,8 @@ packages: resolution: {integrity: sha512-xA6drNNeqb5YyV5fO3OAEsnXLfO7uF0whiOfPTz5AeDo8KeZFmODKnvwPymMNO8qE/an8pVY/O50tig2SQCrGw==} dev: true - /@types/node/20.3.2: - resolution: {integrity: sha512-vOBLVQeCQfIcF/2Y7eKFTqrMnizK5lRNQ7ykML/5RuwVXVWxYkgwS7xbt4B6fKCUPgbSL5FSsjHQpaGQP/dQmw==} + /@types/node/20.4.1: + resolution: {integrity: sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==} dev: true /@types/normalize-package-data/2.4.1: @@ -10630,7 +10640,16 @@ packages: peerDependencies: constructs: ^10.0.0 dependencies: + '@balena/dockerignore': 1.0.2 + case: 1.6.3 constructs: 10.0.130 + fs-extra: 9.1.0 + ignore: 5.2.4 + jsonschema: 1.4.1 + minimatch: 3.1.2 + punycode: 2.3.0 + semver: 7.5.3 + yaml: 1.10.2 dev: true bundledDependencies: - '@balena/dockerignore' @@ -11404,6 +11423,11 @@ packages: engines: {node: '>=4'} dev: true + /case/1.6.3: + resolution: {integrity: sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ==} + engines: {node: '>= 0.8.0'} + dev: true + /ccount/1.1.0: resolution: {integrity: sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==} dev: true @@ -16613,6 +16637,10 @@ packages: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} + /jsonschema/1.4.1: + resolution: {integrity: sha512-S6cATIPVv1z0IlxdN+zUk5EPjkGCdnhN4wVSBlvoUO1tOLJootbo9CquNJmbIh4yikWHiUedhRYrNPn1arpEmQ==} + dev: true + /jsonwebtoken/9.0.0: resolution: {integrity: sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==} engines: {node: '>=12', npm: '>=6'} diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index 179444cad4d..83eb3efa653 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "343a65be71bc82e4e59e741b0c4d29b4eaf5c561", + "pnpmShrinkwrapHash": "abf9008e956d819415ba592c72fb2f5750e33e27", "preferredVersionsHash": "1926a5b12ac8f4ab41e76503a0d1d0dccc9c0e06" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index d70685b733a..9df248dc358 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -102,8 +102,6 @@ export class CobuildConfiguration { get cobuildLockProvider(): ICobuildLockProvider; readonly cobuildRunnerId: string; // (undocumented) - get contextId(): string | undefined; - // (undocumented) createLockProviderAsync(terminal: ITerminal): Promise; // (undocumented) destroyLockProviderAsync(): Promise; @@ -283,8 +281,14 @@ export interface ICobuildCompletedState { // @beta (undocumented) export interface ICobuildContext { cacheId: string; + clusterId: string; + completedStateKey: string; contextId: string; - version: number; + lockExpireTimeInSeconds: number; + lockKey: string; + packageName: string; + phaseName: string; + runnerId: string; } // @beta (undocumented) @@ -297,18 +301,12 @@ export interface ICobuildJson { // @beta (undocumented) export interface ICobuildLockProvider { - // (undocumented) - acquireLockAsync(context: ICobuildContext): Promise; - // (undocumented) + acquireLockAsync(context: Readonly): Promise; connectAsync(): Promise; - // (undocumented) disconnectAsync(): Promise; - // (undocumented) - getCompletedStateAsync(context: ICobuildContext): Promise; - // (undocumented) - renewLockAsync(context: ICobuildContext): Promise; - // (undocumented) - setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; + getCompletedStateAsync(context: Readonly): Promise; + renewLockAsync(context: Readonly): Promise; + setCompletedStateAsync(context: Readonly, state: ICobuildCompletedState): Promise; } // @public @@ -1061,7 +1059,6 @@ export class RushConstants { static readonly bypassPolicyFlagLongName: '--bypass-policy'; static readonly changeFilesFolderName: string; static readonly cobuildFilename: string; - static readonly cobuildLockVersion: number; static readonly commandLineFilename: string; static readonly commonFolderName: string; static readonly commonVersionsFilename: string; diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index 743f0b95b65..c59f483e00a 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -6,13 +6,13 @@ /// -import type { ICobuildCompletedState } from '@rushstack/rush-sdk'; -import type { ICobuildContext } from '@rushstack/rush-sdk'; -import type { ICobuildLockProvider } from '@rushstack/rush-sdk'; +import { ICobuildCompletedState } from '@rushstack/rush-sdk'; +import { ICobuildContext } from '@rushstack/rush-sdk'; +import { ICobuildLockProvider } from '@rushstack/rush-sdk'; import type { IRushPlugin } from '@rushstack/rush-sdk'; import type { RedisClientOptions } from '@redis/client'; import type { RushConfiguration } from '@rushstack/rush-sdk'; -import type { RushSession } from '@rushstack/rush-sdk'; +import { RushSession } from '@rushstack/rush-sdk'; // @beta export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { @@ -22,7 +22,6 @@ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { // @beta (undocumented) export class RedisCobuildLockProvider implements ICobuildLockProvider { constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession); - // (undocumented) acquireLockAsync(context: ICobuildContext): Promise; // (undocumented) connectAsync(): Promise; @@ -32,8 +31,6 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { static expandOptionsWithEnvironmentVariables(options: IRedisCobuildLockProviderOptions, environment?: NodeJS.ProcessEnv): IRedisCobuildLockProviderOptions; // (undocumented) getCompletedStateAsync(context: ICobuildContext): Promise; - getCompletedStateKey(context: ICobuildContext): string; - getLockKey(context: ICobuildContext): string; // (undocumented) renewLockAsync(context: ICobuildContext): Promise; // (undocumented) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 3c014c8462a..255e8db7383 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -133,10 +133,6 @@ export class CobuildConfiguration { }); } - public get contextId(): string | undefined { - return this.cobuildContextId; - } - public async createLockProviderAsync(terminal: ITerminal): Promise { if (this.cobuildEnabled) { terminal.writeLine(`Running cobuild (runner ${this.cobuildContextId}/${this.cobuildRunnerId})`); diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 37fa392ea36..ecb2b03f4b4 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -156,7 +156,10 @@ export const EnvironmentVariableNames = { RUSH_COBUILD_ENABLED: 'RUSH_COBUILD_ENABLED', /** - * Setting this environment variable opt into running with cobuilds. + * Setting this environment variable opt into running with cobuilds. The context id should be the same across + * multiple VMs, but changed when it is a new round of cobuilds. + * + * e.g. `Build.BuildNumber` in Azure DevOps Pipeline. * * @remarks * If there is no cobuild configured, then this environment variable is ignored. diff --git a/libraries/rush-lib/src/logic/RushConstants.ts b/libraries/rush-lib/src/logic/RushConstants.ts index af93ecc4f66..f444e342649 100644 --- a/libraries/rush-lib/src/logic/RushConstants.ts +++ b/libraries/rush-lib/src/logic/RushConstants.ts @@ -187,12 +187,6 @@ export class RushConstants { */ public static readonly cobuildFilename: string = 'cobuild.json'; - /** - * Cobuild version number, incremented when the logic to create cobuild lock changes. - * Changing this ensures that lock generated by an old version will no longer access as a cobuild lock. - */ - public static readonly cobuildLockVersion: number = 1; - /** * Per-project configuration filename. */ diff --git a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts index 2b36c66f777..a8b16502885 100644 --- a/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts +++ b/libraries/rush-lib/src/logic/cobuild/CobuildLock.ts @@ -2,16 +2,36 @@ // See LICENSE in the project root for license information. import { InternalError } from '@rushstack/node-core-library'; -import { RushConstants } from '../RushConstants'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; -import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; import type { OperationStatus } from '../operations/OperationStatus'; import type { ICobuildContext } from './ICobuildLockProvider'; +import type { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; + +const KEY_SEPARATOR: ':' = ':'; export interface ICobuildLockOptions { + /** + * {@inheritdoc CobuildConfiguration} + */ cobuildConfiguration: CobuildConfiguration; + /** + * {@inheritdoc ICobuildContext.clusterId} + */ + cobuildClusterId: string; + /** + * {@inheritdoc ICobuildContext.packageName} + */ + packageName: string; + /** + * {@inheritdoc ICobuildContext.phaseName} + */ + phaseName: string; projectBuildCache: ProjectBuildCache; + /** + * The expire time of the lock in seconds. + */ + lockExpireTimeInSeconds: number; } export interface ICobuildCompletedState { @@ -20,18 +40,24 @@ export interface ICobuildCompletedState { } export class CobuildLock { - public readonly projectBuildCache: ProjectBuildCache; public readonly cobuildConfiguration: CobuildConfiguration; + public readonly projectBuildCache: ProjectBuildCache; private _cobuildContext: ICobuildContext; public constructor(options: ICobuildLockOptions) { - const { cobuildConfiguration, projectBuildCache } = options; - this.projectBuildCache = projectBuildCache; - this.cobuildConfiguration = cobuildConfiguration; - - const { contextId } = cobuildConfiguration; + const { + cobuildConfiguration, + projectBuildCache, + cobuildClusterId: clusterId, + lockExpireTimeInSeconds, + packageName, + phaseName + } = options; + const { cobuildContextId: contextId, cobuildRunnerId: runnerId } = cobuildConfiguration; const { cacheId } = projectBuildCache; + this.cobuildConfiguration = cobuildConfiguration; + this.projectBuildCache = projectBuildCache; if (!cacheId) { // This should never happen @@ -43,10 +69,22 @@ export class CobuildLock { throw new InternalError(`Cobuild context id is require for cobuild lock`); } + // Example: cobuild:lock:: + const lockKey: string = ['cobuild', 'lock', contextId, clusterId].join(KEY_SEPARATOR); + + // Example: cobuild:completed:: + const completedStateKey: string = ['cobuild', 'completed', contextId, cacheId].join(KEY_SEPARATOR); + this._cobuildContext = { contextId, - cacheId, - version: RushConstants.cobuildLockVersion + clusterId, + runnerId, + lockKey, + completedStateKey, + packageName, + phaseName, + lockExpireTimeInSeconds: lockExpireTimeInSeconds, + cacheId }; } @@ -64,6 +102,10 @@ export class CobuildLock { const acquireLockResult: boolean = await this.cobuildConfiguration.cobuildLockProvider.acquireLockAsync( this._cobuildContext ); + if (acquireLockResult) { + // renew the lock in a redundant way in case of losing the lock + await this.renewLockAsync(); + } return acquireLockResult; } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index 817dd719dcd..dfc0c7eaafb 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -7,13 +7,34 @@ import type { OperationStatus } from '../operations/OperationStatus'; * @beta */ export interface ICobuildContext { + /** + * The key for acquiring lock. + */ + lockKey: string; + /** + * The expire time of the lock in seconds. + */ + lockExpireTimeInSeconds: number; + /** + * The key for storing completed state. + */ + completedStateKey: string; /** * The contextId is provided by the monorepo maintainer, it reads from environment variable {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID}. - * It ensure only the builds from the same given contextId cooperated. If user was more permissive, - * and wanted all PR and CI builds building anything with the same contextId to cooperate, then just - * set it to a static value. + * It ensure only the builds from the same given contextId cooperated. */ contextId: string; + /** + * The id of the cluster. The operations in the same cluster shares the same clusterId and + * will be executed in the same machine. + */ + clusterId: string; + /** + * The id of the runner. The identifier for the running machine. + * + * It can be specified via assigning `RUSH_COBUILD_RUNNER_ID` environment variable. + */ + runnerId: string; /** * The id of cache. It should be keep same as the normal cacheId from ProjectBuildCache. * Otherwise, there is a discrepancy in the success case then turning on cobuilds will @@ -21,9 +42,17 @@ export interface ICobuildContext { */ cacheId: string; /** - * {@inheritdoc RushConstants.cobuildLockVersion} + * The name of NPM package + * + * Example: `@scope/MyProject` + */ + packageName: string; + /** + * The name of the phase. + * + * Example: _phase:build */ - version: number; + phaseName: string; } /** @@ -42,10 +71,34 @@ export interface ICobuildCompletedState { * @beta */ export interface ICobuildLockProvider { + /** + * The callback function invoked to connect to the lock provider. + * For example, initializing the connection to the redis server. + */ connectAsync(): Promise; + /** + * The callback function invoked to disconnect the lock provider. + */ disconnectAsync(): Promise; - acquireLockAsync(context: ICobuildContext): Promise; - renewLockAsync(context: ICobuildContext): Promise; - setCompletedStateAsync(context: ICobuildContext, state: ICobuildCompletedState): Promise; - getCompletedStateAsync(context: ICobuildContext): Promise; + /** + * The callback function to acquire a lock with a lock key and specific contexts. + * + * NOTE: This lock implementation must be a ReentrantLock. It says the lock might be acquired + * multiple times, since tasks in the same cluster can be run in the same VM. + */ + acquireLockAsync(context: Readonly): Promise; + /** + * The callback function to renew a lock with a lock key and specific contexts. + * + * NOTE: If the lock key expired + */ + renewLockAsync(context: Readonly): Promise; + /** + * The callback function to set completed state. + */ + setCompletedStateAsync(context: Readonly, state: ICobuildCompletedState): Promise; + /** + * The callback function to get completed state. + */ + getCompletedStateAsync(context: Readonly): Promise; } diff --git a/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts index 2603dbafb93..4f4d842e6a9 100644 --- a/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts +++ b/libraries/rush-lib/src/logic/cobuild/test/CobuildLock.test.ts @@ -1,28 +1,43 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { CobuildConfiguration } from '../../../api/CobuildConfiguration'; -import { ProjectBuildCache } from '../../buildCache/ProjectBuildCache'; -import { CobuildLock } from '../CobuildLock'; +import { CobuildLock, ICobuildLockOptions } from '../CobuildLock'; + +import type { CobuildConfiguration } from '../../../api/CobuildConfiguration'; +import type { ProjectBuildCache } from '../../buildCache/ProjectBuildCache'; +import type { ICobuildContext } from '../ICobuildLockProvider'; describe(CobuildLock.name, () => { function prepareSubject(): CobuildLock { - const subject: CobuildLock = new CobuildLock({ + const cobuildLockOptions: ICobuildLockOptions = { cobuildConfiguration: { - contextId: 'foo' + cobuildContextId: 'context_id', + cobuildRunnerId: 'runner_id' } as unknown as CobuildConfiguration, projectBuildCache: { - cacheId: 'bar' - } as unknown as ProjectBuildCache - }); + cacheId: 'cache_id' + } as unknown as ProjectBuildCache, + cobuildClusterId: 'cluster_id', + lockExpireTimeInSeconds: 30, + packageName: 'package_name', + phaseName: 'phase_name' + }; + const subject: CobuildLock = new CobuildLock(cobuildLockOptions); return subject; } it('returns cobuild context', () => { const subject: CobuildLock = prepareSubject(); - expect(subject.cobuildContext).toEqual({ - contextId: 'foo', - cacheId: 'bar', - version: 1 - }); + const expected: ICobuildContext = { + lockKey: 'cobuild:lock:context_id:cluster_id', + completedStateKey: 'cobuild:completed:context_id:cache_id', + lockExpireTimeInSeconds: 30, + contextId: 'context_id', + cacheId: 'cache_id', + clusterId: 'cluster_id', + runnerId: 'runner_id', + packageName: 'package_name', + phaseName: 'phase_name' + }; + expect(subject.cobuildContext).toEqual(expected); }); }); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 545dc712527..f24dd1934ee 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -1,7 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { ColorValue, InternalError, ITerminal, JsonObject, Terminal } from '@rushstack/node-core-library'; +import * as crypto from 'crypto'; +import { + Async, + ColorValue, + ConsoleTerminalProvider, + InternalError, + ITerminal, + JsonObject, + Terminal +} from '@rushstack/node-core-library'; import { CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; import { DiscardStdoutTransform, PrintUtilities } from '@rushstack/terminal'; import { SplitterTransform, TerminalWritable } from '@rushstack/terminal'; @@ -21,7 +30,7 @@ import type { IOperationRunnerBeforeExecuteContext } from './OperationRunnerHooks'; import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; -import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ICreateOperationsContext, IPhasedCommandPlugin, @@ -31,7 +40,8 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import { DisjointSet } from '../cobuild/DisjointSet'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; @@ -41,6 +51,8 @@ export interface IOperationBuildCacheContext { isSkipAllowed: boolean; projectBuildCache: ProjectBuildCache | undefined; cobuildLock: CobuildLock | undefined; + // The id of the cluster contains the operation, used when acquiring cobuild lock + cobuildClusterId: string | undefined; // Controls the log for the cache subsystem buildCacheTerminal: ITerminal | undefined; buildCacheProjectLogWritable: ProjectLogWritable | undefined; @@ -56,12 +68,18 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { hooks.createOperations.tapPromise( PLUGIN_NAME, async (operations: Set, context: ICreateOperationsContext): Promise> => { - const { buildCacheConfiguration, isIncrementalBuildAllowed } = context; + const { buildCacheConfiguration, isIncrementalBuildAllowed, cobuildConfiguration } = context; if (!buildCacheConfiguration) { return operations; } + let disjointSet: DisjointSet | undefined; + if (cobuildConfiguration?.cobuildEnabled) { + disjointSet = new DisjointSet(); + } + for (const operation of operations) { + disjointSet?.add(operation); const { runner } = operation; if (runner) { const buildCacheContext: IOperationBuildCacheContext = { @@ -71,6 +89,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { isSkipAllowed: isIncrementalBuildAllowed, projectBuildCache: undefined, cobuildLock: undefined, + cobuildClusterId: undefined, buildCacheTerminal: undefined, buildCacheProjectLogWritable: undefined }; @@ -83,6 +102,57 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } + if (disjointSet) { + // If disjoint set exists, connect build cache disabled project with its consumers + await Async.forEachAsync( + operations, + async (operation) => { + const { associatedProject: project, associatedPhase: phase } = operation; + if (project && phase && operation.runner instanceof ShellOperationRunner) { + const buildCacheEnabled: boolean = await this._tryGetProjectBuildEnabledAsync({ + buildCacheConfiguration, + rushProject: project, + commandName: phase.name + }); + if (!buildCacheEnabled) { + for (const consumer of operation.consumers) { + if (consumer.runner instanceof ShellOperationRunner) { + disjointSet?.union(operation, consumer); + } + } + } + } + }, + { + concurrency: 10 + } + ); + + for (const set of disjointSet.getAllSets()) { + if (cobuildConfiguration?.cobuildEnabled && cobuildConfiguration.cobuildContextId) { + const hash: crypto.Hash = crypto.createHash('sha1'); + for (const operation of set) { + const { associatedPhase: phase, associatedProject: project } = operation; + if (project && phase) { + hash.update(project.projectRelativeFolder); + hash.update(RushConstants.hashDelimiter); + hash.update(phase.name); + hash.update(RushConstants.hashDelimiter); + } + } + const cobuildClusterId: string = hash.digest('hex'); + for (const operation of set) { + const { runner } = operation; + if (runner instanceof ShellOperationRunner) { + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByRunnerOrThrow(runner); + buildCacheContext.cobuildClusterId = cobuildClusterId; + } + } + } + } + } + return operations; } ); @@ -125,12 +195,13 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } } - - buildCacheContext?.buildCacheProjectLogWritable?.close(); } ); hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { + for (const { buildCacheProjectLogWritable } of this._buildCacheContextByOperationRunner.values()) { + buildCacheProjectLogWritable?.close(); + } this._buildCacheContextByOperationRunner.clear(); }); } @@ -243,7 +314,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cobuildLock = await this._tryGetCobuildLockAsync({ runner, projectBuildCache, - cobuildConfiguration + cobuildConfiguration, + packageName: rushProject.packageName, + phaseName: phase.name }); } @@ -435,6 +508,34 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext; } + private async _tryGetProjectBuildEnabledAsync({ + buildCacheConfiguration, + rushProject, + commandName + }: { + buildCacheConfiguration: BuildCacheConfiguration; + rushProject: RushConfigurationProject; + commandName: string; + }): Promise { + const consoleTerminalProvider: ConsoleTerminalProvider = new ConsoleTerminalProvider(); + const terminal: ITerminal = new Terminal(consoleTerminalProvider); + // This is a silent terminal + terminal.unregisterProvider(consoleTerminalProvider); + + if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { + const projectConfiguration: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); + if (projectConfiguration && projectConfiguration.disableBuildCacheForProject) { + const operationSettings: IOperationSettings | undefined = + projectConfiguration.operationSettingsByOperationName.get(commandName); + if (operationSettings && !operationSettings.disableBuildCacheForOperation) { + return true; + } + } + } + return false; + } + private async _tryGetProjectBuildCacheAsync({ buildCacheConfiguration, runner, @@ -542,6 +643,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext.projectBuildCache; } + // Get a ProjectBuildCache only cache/restore log files private async _tryGetLogOnlyProjectBuildCacheAsync({ runner, rushProject, @@ -631,20 +733,30 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private async _tryGetCobuildLockAsync({ cobuildConfiguration, runner, - projectBuildCache + projectBuildCache, + packageName, + phaseName }: { cobuildConfiguration: CobuildConfiguration | undefined; runner: IOperationRunner; projectBuildCache: ProjectBuildCache | undefined; + packageName: string; + phaseName: string; }): Promise { const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.cobuildLock) { - buildCacheContext.cobuildLock = undefined; - if (projectBuildCache && cobuildConfiguration && cobuildConfiguration.cobuildEnabled) { + if (!buildCacheContext.cobuildClusterId) { + // This should not happen + throw new InternalError('Cobuild cluster id is not defined'); + } buildCacheContext.cobuildLock = new CobuildLock({ cobuildConfiguration, - projectBuildCache + projectBuildCache, + cobuildClusterId: buildCacheContext.cobuildClusterId, + lockExpireTimeInSeconds: ShellOperationRunner.periodicCallbackIntervalInSeconds * 3, + packageName, + phaseName }); } } @@ -727,7 +839,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (!buildCacheConfiguration?.buildCacheEnabled) { return; } - const buildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.buildCacheProjectLogWritable) { buildCacheContext.buildCacheProjectLogWritable = new ProjectLogWritable( rushProject, diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index e5e88352419..e5b3fd217df 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -73,6 +73,7 @@ export class ShellOperationRunner implements IOperationRunner { public readonly hooks: OperationRunnerHooks; public readonly periodicCallback: PeriodicCallback; public readonly logFilenameIdentifier: string; + public static readonly periodicCallbackIntervalInSeconds: number = 10; private readonly _rushProject: RushConfigurationProject; private readonly _phase: IPhase; @@ -101,7 +102,7 @@ export class ShellOperationRunner implements IOperationRunner { this.hooks = new OperationRunnerHooks(); this.periodicCallback = new PeriodicCallback({ - interval: 10 * 1000 + interval: ShellOperationRunner.periodicCallbackIntervalInSeconds * 1000 }); } diff --git a/rush-plugins/rush-redis-cobuild-plugin/package.json b/rush-plugins/rush-redis-cobuild-plugin/package.json index 19caebf3bcd..c4112a7facb 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/package.json +++ b/rush-plugins/rush-redis-cobuild-plugin/package.json @@ -13,10 +13,10 @@ "license": "MIT", "scripts": { "build": "heft build --clean", - "start": "heft test --clean --watch", - "test": "heft test", - "_phase:build": "heft build --clean", - "_phase:test": "heft test --no-build" + "start": "heft test-watch", + "test": "heft test --clean", + "_phase:build": "heft run --only build -- --clean", + "_phase:test": "heft run --only test -- --clean" }, "dependencies": { "@redis/client": "~1.5.5", diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index f6d6cc297b4..54dee4e2db4 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -3,7 +3,7 @@ import { createClient } from '@redis/client'; -import type { +import { ICobuildLockProvider, ICobuildContext, ICobuildCompletedState, @@ -29,7 +29,6 @@ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { passwordEnvironmentVariable?: string; } -const KEY_SEPARATOR: ':' = ':'; const COMPLETED_STATE_SEPARATOR: ';' = ';'; /** @@ -38,13 +37,16 @@ const COMPLETED_STATE_SEPARATOR: ';' = ';'; export class RedisCobuildLockProvider implements ICobuildLockProvider { private readonly _options: IRedisCobuildLockProviderOptions; private readonly _terminal: ITerminal; - - private readonly _redisClient: RedisClientType; - private readonly _lockKeyMap: WeakMap = new WeakMap(); - private readonly _completedKeyMap: WeakMap = new WeakMap< + private readonly _lockKeyIdentifierMap: WeakMap = new WeakMap< ICobuildContext, string >(); + private readonly _completedStateKeyIdentifierMap: WeakMap = new WeakMap< + ICobuildContext, + string + >(); + + private readonly _redisClient: RedisClientType; public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); @@ -103,32 +105,54 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { } } + /** + * Acquiring the lock based on the specific context. + * + * NOTE: this is a reentrant lock implementation + */ public async acquireLockAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; - const lockKey: string = this.getLockKey(context); + const { lockKey, lockExpireTimeInSeconds, runnerId } = context; let result: boolean = false; + const lockKeyIdentifier: string = this._getLockKeyIdentifier(context); try { - const incrResult: number = await this._redisClient.incr(lockKey); - result = incrResult === 1; - terminal.writeDebugLine(`Acquired lock for ${lockKey}: ${incrResult}, 1 is success`); + // According to the doc, the reply of set command is either "OK" or nil. The reply doesn't matter + await this._redisClient.set(lockKey, runnerId, { + NX: true, + // call EXPIRE in an atomic command + EX: lockExpireTimeInSeconds + // Do not specify GET here since using NX ane GET together requires Redis@7. + }); + // Just read the value by lock key to see wether it equals current runner id + const value: string | null = await this._redisClient.get(lockKey); + if (value === null) { + // This should not happen. + throw new Error(`Get redis key failed: ${lockKey}`); + } + result = value === runnerId; if (result) { - await this.renewLockAsync(context); + terminal.writeDebugLine( + `Successfully acquired ${lockKeyIdentifier} to runner(${runnerId}) and it expires in ${lockExpireTimeInSeconds}s` + ); + } else { + terminal.writeDebugLine(`Failed to acquire ${lockKeyIdentifier}, locked by runner ${value}`); } } catch (e) { - throw new Error(`Failed to acquire lock for ${lockKey}: ${e.message}`); + throw new Error(`Error occurs when acquiring ${lockKeyIdentifier}: ${e.message}`); } return result; } public async renewLockAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; - const lockKey: string = this.getLockKey(context); + const { lockKey, lockExpireTimeInSeconds } = context; + const lockKeyIdentifier: string = this._getLockKeyIdentifier(context); try { - await this._redisClient.expire(lockKey, 30); + await this._redisClient.expire(lockKey, lockExpireTimeInSeconds); } catch (e) { - throw new Error(`Failed to renew lock for ${lockKey}: ${e.message}`); + throw new Error(`Failed to renew ${lockKeyIdentifier}: ${e.message}`); } - terminal.writeDebugLine(`Renewed lock for ${lockKey}`); + terminal.writeDebugLine(`Renewed ${lockKeyIdentifier} expires in ${lockExpireTimeInSeconds} seconds`); } public async setCompletedStateAsync( @@ -136,68 +160,63 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { state: ICobuildCompletedState ): Promise { const { _terminal: terminal } = this; - const key: string = this.getCompletedStateKey(context); + const { completedStateKey: key } = context; const value: string = this._serializeCompletedState(state); + const completedStateKeyIdentifier: string = this._getCompletedStateKeyIdentifier(context); try { await this._redisClient.set(key, value); } catch (e) { - throw new Error(`Failed to set completed state for ${key}: ${e.message}`); + throw new Error(`Failed to set ${completedStateKeyIdentifier}: ${e.message}`); } - terminal.writeDebugLine(`Set completed state for ${key}: ${value}`); + terminal.writeDebugLine(`Set ${completedStateKeyIdentifier}: ${value}`); } public async getCompletedStateAsync(context: ICobuildContext): Promise { const { _terminal: terminal } = this; - const key: string = this.getCompletedStateKey(context); + const { completedStateKey: key } = context; + const completedStateKeyIdentifier: string = this._getCompletedStateKeyIdentifier(context); let state: ICobuildCompletedState | undefined; try { const value: string | null = await this._redisClient.get(key); if (value) { state = this._deserializeCompletedState(value); } - terminal.writeDebugLine(`Get completed state for ${key}: ${value}`); + terminal.writeDebugLine(`Get ${completedStateKeyIdentifier}: ${value}`); } catch (e) { - throw new Error(`Failed to get completed state for ${key}: ${e.message}`); + throw new Error(`Failed to get ${completedStateKeyIdentifier}: ${e.message}`); } return state; } - /** - * Returns the lock key for the given context - * Example: cobuild:v1:::lock - */ - public getLockKey(context: ICobuildContext): string { - const { version, contextId, cacheId } = context; - let lockKey: string | undefined = this._lockKeyMap.get(context); - if (!lockKey) { - lockKey = ['cobuild', `v${version}`, contextId, cacheId, 'lock'].join(KEY_SEPARATOR); - this._lockKeyMap.set(context, lockKey); - } - return lockKey; - } - - /** - * Returns the completed key for the given context - * Example: cobuild:v1:::completed - */ - public getCompletedStateKey(context: ICobuildContext): string { - const { version, contextId, cacheId } = context; - let completedKey: string | undefined = this._completedKeyMap.get(context); - if (!completedKey) { - completedKey = ['cobuild', `v${version}`, contextId, cacheId, 'completed'].join(KEY_SEPARATOR); - this._completedKeyMap.set(context, completedKey); - } - return completedKey; - } - private _serializeCompletedState(state: ICobuildCompletedState): string { // Example: SUCCESS;1234567890 // Example: FAILURE;1234567890 - return `${state.status}${COMPLETED_STATE_SEPARATOR}${state.cacheId}`; + const { status, cacheId } = state; + return [status, cacheId].join(COMPLETED_STATE_SEPARATOR); } private _deserializeCompletedState(state: string): ICobuildCompletedState | undefined { const [status, cacheId] = state.split(COMPLETED_STATE_SEPARATOR); return { status: status as ICobuildCompletedState['status'], cacheId }; } + + private _getLockKeyIdentifier(context: ICobuildContext): string { + let lockKeyIdentifier: string | undefined = this._lockKeyIdentifierMap.get(context); + if (lockKeyIdentifier === undefined) { + const { lockKey, packageName, phaseName } = context; + lockKeyIdentifier = `lock(${lockKey})_package(${packageName})_phase(${phaseName})`; + this._lockKeyIdentifierMap.set(context, lockKeyIdentifier); + } + return lockKeyIdentifier; + } + + private _getCompletedStateKeyIdentifier(context: ICobuildContext): string { + let completedStateKeyIdentifier: string | undefined = this._completedStateKeyIdentifierMap.get(context); + if (completedStateKeyIdentifier === undefined) { + const { completedStateKey, packageName, phaseName } = context; + completedStateKeyIdentifier = `completed_state(${completedStateKey})_package(${packageName})_phase(${phaseName})`; + this._completedStateKeyIdentifierMap.set(context, completedStateKeyIdentifier); + } + return completedStateKeyIdentifier; + } } diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 8b275b9875a..69e0b016b27 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -15,18 +15,42 @@ const rushSession: RushSession = new RushSession({ }); describe(RedisCobuildLockProvider.name, () => { - let storage: Record = {}; + let storage: Record = {}; beforeEach(() => { jest.spyOn(redisAPI, 'createClient').mockImplementation(() => { return { - incr: jest.fn().mockImplementation((key: string) => { - storage[key] = (Number(storage[key]) || 0) + 1; - return storage[key]; - }), expire: jest.fn().mockResolvedValue(undefined), - set: jest.fn().mockImplementation((key: string, value: string) => { - storage[key] = value; - }), + set: jest + .fn() + .mockImplementation((key: string, value: string, options?: { NX?: boolean; GET?: boolean }) => { + // https://redis.io/commands/set/ + const oldValue: string | undefined = storage[key]; + const { NX, GET } = options || {}; + let didSet: boolean = false; + if (NX) { + if (!storage[key]) { + storage[key] = value; + didSet = true; + } + } else { + storage[key] = value; + didSet = true; + } + + if (GET) { + if (oldValue === undefined) { + return null; + } else { + return oldValue; + } + } else { + if (didSet) { + return 'OK'; + } else { + return null; + } + } + }), get: jest.fn().mockImplementation((key: string) => { return storage[key]; }) @@ -44,9 +68,15 @@ describe(RedisCobuildLockProvider.name, () => { } const context: ICobuildContext = { - contextId: '123', - cacheId: 'abc', - version: 1 + contextId: 'context_id', + cacheId: 'cache_id', + lockKey: 'lock_key', + lockExpireTimeInSeconds: 30, + completedStateKey: 'completed_state_key', + clusterId: 'cluster_id', + runnerId: 'runner_id', + packageName: 'package_name', + phaseName: 'phase_name' }; it('expands options with environment variables', () => { @@ -75,32 +105,28 @@ describe(RedisCobuildLockProvider.name, () => { }).toThrowErrorMatchingSnapshot(); }); - it('getLockKey works', () => { - const subject: RedisCobuildLockProvider = prepareSubject(); - const lockKey: string = subject.getLockKey(context); - expect(lockKey).toMatchSnapshot(); - }); - - it('getCompletedStateKey works', () => { - const subject: RedisCobuildLockProvider = prepareSubject(); - const completedStateKey: string = subject.getCompletedStateKey(context); - expect(completedStateKey).toMatchSnapshot(); - }); - it('acquires lock success', async () => { const subject: RedisCobuildLockProvider = prepareSubject(); const result: boolean = await subject.acquireLockAsync(context); expect(result).toBe(true); }); - it('acquires lock fails at the second time', async () => { + it('acquires lock is a reentrant lock', async () => { const subject: RedisCobuildLockProvider = prepareSubject(); + const result1: boolean = await subject.acquireLockAsync(context); + expect(result1).toBe(true); + const result2: boolean = await subject.acquireLockAsync(context); + expect(result2).toBe(true); + }); + + it('acquires lock fails with a different runner', async () => { + const subject: RedisCobuildLockProvider = prepareSubject(); + const result1: boolean = await subject.acquireLockAsync(context); + expect(result1).toBe(true); const cobuildContext: ICobuildContext = { ...context, - contextId: 'abc' + runnerId: 'other_runner_id' }; - const result1: boolean = await subject.acquireLockAsync(cobuildContext); - expect(result1).toBe(true); const result2: boolean = await subject.acquireLockAsync(cobuildContext); expect(result2).toBe(false); }); From b08da172fc0a81610ca467de568e9adf05e3e148 Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 12 Jul 2023 19:52:48 +0800 Subject: [PATCH 056/100] chore: migrate redis cobuild test repo to use phases --- .../README.md | 71 ++++++++++--------- .../repo/common/config/rush/command-line.json | 34 ++++++++- .../repo/common/config/rush/experiments.json | 55 ++++++++++++++ .../repo/projects/a/config/rush-project.json | 4 ++ .../sandbox/repo/projects/a/package.json | 5 +- .../repo/projects/b/config/rush-project.json | 4 ++ .../sandbox/repo/projects/b/package.json | 3 +- .../repo/projects/c/config/rush-project.json | 4 ++ .../sandbox/repo/projects/c/package.json | 3 +- .../repo/projects/d/config/rush-project.json | 4 ++ .../sandbox/repo/projects/d/package.json | 3 +- .../repo/projects/e/config/rush-project.json | 4 ++ .../sandbox/repo/projects/e/package.json | 3 +- .../repo/projects/f/config/rush-project.json | 4 ++ .../sandbox/repo/projects/f/package.json | 4 +- .../sandbox/repo/projects/g/package.json | 4 +- .../sandbox/repo/projects/pre-build.js | 11 +++ .../repo/projects/validate-pre-build.js | 13 ++++ 18 files changed, 187 insertions(+), 46 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/experiments.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/pre-build.js create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/validate-pre-build.js diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index 2cb561c5168..64971720eaf 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -35,34 +35,26 @@ Sandbox repo folder: **build-tests/rush-redis-cobuild-plugin-integration-test/sa ```sh cd sandbox/repo -rush update +node ../../lib/runRush.js update ``` -## Case 1: Disable cobuild by setting `RUSH_COBUILD_ENABLED=0` +## Case 1: Normal build, Cobuild is disabled because of missing RUSH_COBUILD_CONTEXT_ID -```sh -rm -rf common/temp/build-cache && RUSH_COBUILD_ENABLED=0 REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild -``` - -Expected behavior: Cobuild feature is disabled. Run command successfully. +1. Write to build cache ```sh -RUSH_COBUILD_ENABLED=0 REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && node ../../lib/runRush.js --debug cobuild ``` -Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. - -## Case 2: Cobuild enabled without specifying RUSH_COBUILD_CONTEXT_ID - -Run `rush cobuild` command without specifying cobuild context id. +2. Read from build cache ```sh -rm -rf common/temp/build-cache && REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild +node ../../lib/runRush.js --debug cobuild ``` -Expected behavior: Cobuild feature is disabled. Build cache was restored successfully. +Expected behavior: Cobuild feature is disabled. Build cache is saved/restored as normal. -## Case 3: Cobuild enabled, run one cobuild command only +# Case 2: Cobuild enabled by specifying RUSH_COBUILD_CONTEXT_ID and Redis authentication 1. Clear redis server @@ -70,39 +62,47 @@ Expected behavior: Cobuild feature is disabled. Build cache was restored success (cd ../.. && docker compose down && docker compose up -d) ``` -2. Run `rush cobuild` command +2. Run cobuilds ```sh -rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo REDIS_PASS=redis123 node ../../lib/runRush.js --debug cobuild +rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo REDIS_PASS=redis123 RUSH_COBUILD_RUNNER_ID=runner1 node ../../lib/runRush.js --debug cobuild ``` Expected behavior: Cobuild feature is enabled. Run command successfully. You can also see cobuild related logs in the terminal. ```sh -Get completed state for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: null -Acquired lock for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:lock: 1, 1 is success -Set completed state for cobuild:v1:foo:c2df36270ec5faa8ef6497fa7367a476de3e2861:completed: SUCCESS;c2df36270ec5faa8ef6497fa7367a476de3e2861 +Running cobuild (runner foo/runner1) +Analyzing repo state... DONE (0.11 seconds) + +Executing a maximum of 10 simultaneous processes... + +==[ b (build) ]====================================================[ 1 of 9 ]== +Get completed_state(cobuild:completed:foo:2e477baf39a85b28fc40e63b417692fe8afcc023)_package(b)_phase(_phase:build): SUCCESS;2e477baf39a85b28fc40e63b417692fe8afcc023 +Get completed_state(cobuild:completed:foo:cfc620db4e74a6f0db41b1a86d0b5402966b97f3)_package(a)_phase(_phase:build): SUCCESS;cfc620db4e74a6f0db41b1a86d0b5402966b97f3 +Successfully acquired lock(cobuild:lock:foo:4c36160884a7a502f9894e8f0adae05c45c8cc4b)_package(b)_phase(_phase:build) to runner(runner1) and it expires in 30s ``` ## Case 4: Cobuild enabled, run two cobuild commands in parallel > Note: This test requires Visual Studio Code to be installed. -1. Clear redis server +1. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. + +2. Clear redis server ```sh -(cd ../.. && docker compose down && docker compose up -d) +# Under rushstack/build-tests/rush-redis-cobuild-plugin-integration-test +docker compose down && docker compose up -d ``` -2. Clear build cache +3. Clear build cache ```sh +# Under rushstack/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo rm -rf common/temp/build-cache ``` -3. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. - 4. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. > In this step, two dedicated terminal windows will open. Running `rush cobuild` command under sandbox repo respectively. @@ -113,32 +113,33 @@ Expected behavior: Cobuild feature is enabled, cobuild related logs out in both > Note: This test requires Visual Studio Code to be installed. -1. Making the cobuild command of project "A" fails +1. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. + +2. Making the cobuild command of project "A" fails **sandbox/repo/projects/a/package.json** ```diff "scripts": { -- "cobuild": "node ../build.js a", -+ "cobuild": "sleep 5 && exit 1", - "build": "node ../build.js a" +- "_phase:build": "node ../build.js a", ++ "_phase:build": "exit 1", } ``` -2. Clear redis server +3. Clear redis server ```sh -(cd ../.. && docker compose down && docker compose up -d) +# Under rushstack/build-tests/rush-redis-cobuild-plugin-integration-test +docker compose down && docker compose up -d ``` -3. Clear build cache +4. Clear build cache ```sh +# Under rushstack/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo rm -rf common/temp/build-cache ``` -4. Open predefined `.vscode/redis-cobuild.code-workspace` in Visual Studio Code. - 5. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`. Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. These two cobuild commands fail because of the failing build of project "A". And, one of them restored the failing build cache created by the other one. diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json index 1a8f7837fa3..c8c1ccc022d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/command-line.json @@ -12,12 +12,13 @@ */ "commands": [ { - "commandKind": "bulk", + "commandKind": "phased", "summary": "Concurrent version of rush build", "name": "cobuild", "safeForSimultaneousRushProcesses": true, "enableParallelism": true, - "incremental": true + "incremental": true, + "phases": ["_phase:pre-build", "_phase:build"] } // { @@ -177,7 +178,34 @@ // } ], - "phases": [], + "phases": [ + { + /** + * The name of the phase. Note that this value must start with the \"_phase:\" prefix. + */ + "name": "_phase:build", + /** + * The dependencies of this phase. + */ + "dependencies": { + "upstream": ["_phase:build"], + "self": ["_phase:pre-build"] + } + }, + { + /** + * The name of the phase. Note that this value must start with the \"_phase:\" prefix. + */ + "name": "_phase:pre-build", + /** + * The dependencies of this phase. + */ + "dependencies": { + "upstream": ["_phase:build"] + }, + "missingScriptBehavior": "silent" + } + ], /** * Custom "parameters" introduce new parameters for specified Rush command-line commands. diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/experiments.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/experiments.json new file mode 100644 index 00000000000..fef826208c3 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/experiments.json @@ -0,0 +1,55 @@ +/** + * This configuration file allows repo maintainers to enable and disable experimental + * Rush features. More documentation is available on the Rush website: https://rushjs.io + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/experiments.schema.json", + + /** + * By default, 'rush install' passes --no-prefer-frozen-lockfile to 'pnpm install'. + * Set this option to true to pass '--frozen-lockfile' instead for faster installs. + */ + "usePnpmFrozenLockfileForRushInstall": true, + + /** + * By default, 'rush update' passes --no-prefer-frozen-lockfile to 'pnpm install'. + * Set this option to true to pass '--prefer-frozen-lockfile' instead to minimize shrinkwrap changes. + */ + "usePnpmPreferFrozenLockfileForRushUpdate": true, + + /** + * If using the 'preventManualShrinkwrapChanges' option, restricts the hash to only include the layout of external dependencies. + * Used to allow links between workspace projects or the addition/removal of references to existing dependency versions to not + * cause hash changes. + */ + "omitImportersFromPreventManualShrinkwrapChanges": true, + + /** + * If true, the chmod field in temporary project tar headers will not be normalized. + * This normalization can help ensure consistent tarball integrity across platforms. + */ + // "noChmodFieldInTarHeaderNormalization": true, + + /** + * If true, build caching will respect the allowWarningsInSuccessfulBuild flag and cache builds with warnings. + * This will not replay warnings from the cached build. + */ + // "buildCacheWithAllowWarningsInSuccessfulBuild": true, + + /** + * If true, the phased commands feature is enabled. To use this feature, create a "phased" command + * in common/config/rush/command-line.json. + */ + "phasedCommands": true + + /** + * If true, perform a clean install after when running `rush install` or `rush update` if the + * `.npmrc` file has changed since the last install. + */ + // "cleanInstallAfterNpmrcChanges": true, + + /** + * If true, print the outputs of shell commands defined in event hooks to the console. + */ + // "printEventHooksOutputToConsole": true +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json index f0036196559..3206537349d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/config/rush-project.json @@ -1,5 +1,9 @@ { "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, { "operationName": "cobuild", "outputFolderNames": ["dist"] diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json index f8b84111f98..98957112d5e 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/a/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js a", - "_cobuild": "sleep 5 && exit 1", - "build": "node ../build.js a" + "build": "node ../build.js a", + "__phase:build": "exit 1", + "_phase:build": "node ../build.js a" } } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json index f0036196559..3206537349d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/config/rush-project.json @@ -1,5 +1,9 @@ { "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, { "operationName": "cobuild", "outputFolderNames": ["dist"] diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json index a8cd24f8006..8b17917b744 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/b/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js", - "build": "node ../build.js" + "build": "node ../build.js", + "_phase:build": "node ../build.js b" } } diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json index f0036196559..3206537349d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/config/rush-project.json @@ -1,5 +1,9 @@ { "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, { "operationName": "cobuild", "outputFolderNames": ["dist"] diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json index b25880f2c84..738c1444ab0 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/c/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js", - "build": "node ../build.js" + "build": "node ../build.js", + "_phase:build": "node ../build.js" }, "dependencies": { "b": "workspace:*" diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json index f0036196559..3206537349d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/config/rush-project.json @@ -1,5 +1,9 @@ { "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, { "operationName": "cobuild", "outputFolderNames": ["dist"] diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json index 6580cb02700..67707275a1e 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/d/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js", - "build": "node ../build.js" + "build": "node ../build.js", + "_phase:build": "node ../build.js" }, "dependencies": { "b": "workspace:*", diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json index f0036196559..3206537349d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/config/rush-project.json @@ -1,5 +1,9 @@ { "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, { "operationName": "cobuild", "outputFolderNames": ["dist"] diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json index 69ac8b1cc97..0b91c05a805 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/e/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js", - "build": "node ../build.js" + "build": "node ../build.js", + "_phase:build": "node ../build.js" }, "dependencies": { "b": "workspace:*", diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json index 23e6a93085e..4e94028e909 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/config/rush-project.json @@ -1,6 +1,10 @@ { "disableBuildCacheForProject": true, "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, { "operationName": "cobuild", "outputFolderNames": ["dist"] diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json index f703b70f3b2..7bf2634a508 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/f/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js", - "build": "node ../build.js" + "build": "node ../build.js", + "_phase:pre-build": "node ../pre-build.js", + "_phase:build": "node ../validate-pre-build.js && node ../build.js f" }, "dependencies": { "b": "workspace:*" diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json index 14c42344694..29cd2f39532 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/g/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "scripts": { "cobuild": "node ../build.js", - "build": "node ../build.js" + "build": "node ../build.js", + "_phase:pre-build": "node ../pre-build.js", + "_phase:build": "node ../validate-pre-build.js && node ../build.js g" }, "dependencies": { "b": "workspace:*" diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/pre-build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/pre-build.js new file mode 100644 index 00000000000..4d9f43e7afa --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/pre-build.js @@ -0,0 +1,11 @@ +/* eslint-env es6 */ +const path = require('path'); +const { FileSystem } = require('@rushstack/node-core-library'); + +setTimeout(() => { + const outputFolder = path.resolve(process.cwd(), 'dist'); + const outputFile = path.resolve(outputFolder, 'pre-build'); + FileSystem.ensureFolder(outputFolder); + FileSystem.writeFile(outputFile, `Hello world!`); + console.log('done'); +}, 2000); diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/validate-pre-build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/validate-pre-build.js new file mode 100644 index 00000000000..218484000e3 --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/validate-pre-build.js @@ -0,0 +1,13 @@ +/* eslint-env es6 */ +const path = require('path'); +const { FileSystem } = require('@rushstack/node-core-library'); + +const outputFolder = path.resolve(process.cwd(), 'dist'); +const outputFile = path.resolve(outputFolder, 'pre-build'); + +if (!FileSystem.exists(outputFile)) { + console.error(`${outputFile} does not exist.`); + process.exit(1); +} + +console.log(`${outputFile} exists`); From 76e968b34c67fd9ac909a12c391b8dab82181686 Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 12 Jul 2023 20:00:07 +0800 Subject: [PATCH 057/100] :memo: --- .../rush-redis-cobuild-plugin-integration-test/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md index 64971720eaf..330224b8f7e 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/README.md +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/README.md @@ -38,7 +38,7 @@ cd sandbox/repo node ../../lib/runRush.js update ``` -## Case 1: Normal build, Cobuild is disabled because of missing RUSH_COBUILD_CONTEXT_ID +#### Case 1: Normal build, Cobuild is disabled because of missing RUSH_COBUILD_CONTEXT_ID 1. Write to build cache @@ -54,7 +54,7 @@ node ../../lib/runRush.js --debug cobuild Expected behavior: Cobuild feature is disabled. Build cache is saved/restored as normal. -# Case 2: Cobuild enabled by specifying RUSH_COBUILD_CONTEXT_ID and Redis authentication +#### Case 2: Cobuild enabled by specifying RUSH_COBUILD_CONTEXT_ID and Redis authentication 1. Clear redis server @@ -83,7 +83,7 @@ Get completed_state(cobuild:completed:foo:cfc620db4e74a6f0db41b1a86d0b5402966b97 Successfully acquired lock(cobuild:lock:foo:4c36160884a7a502f9894e8f0adae05c45c8cc4b)_package(b)_phase(_phase:build) to runner(runner1) and it expires in 30s ``` -## Case 4: Cobuild enabled, run two cobuild commands in parallel +#### Case 3: Cobuild enabled, run two cobuild commands in parallel > Note: This test requires Visual Studio Code to be installed. @@ -109,7 +109,7 @@ rm -rf common/temp/build-cache Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. -## Case 5: Cobuild enabled, run two cobuild commands in parallel, one of them failed +#### Case 4: Cobuild enabled, run two cobuild commands in parallel, one of them failed > Note: This test requires Visual Studio Code to be installed. From 0b1bb90da2fd170b6c3abe322b461a6a0dcaccdf Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 19 Jul 2023 16:05:52 +0800 Subject: [PATCH 058/100] chore: comment --- .../rush-lib/src/logic/operations/CacheableOperationPlugin.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index f24dd1934ee..07bb624596b 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -130,6 +130,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const set of disjointSet.getAllSets()) { if (cobuildConfiguration?.cobuildEnabled && cobuildConfiguration.cobuildContextId) { + // Generates cluster id, cluster id comes from the project folder and phase name of all operations in the same cluster. const hash: crypto.Hash = crypto.createHash('sha1'); for (const operation of set) { const { associatedPhase: phase, associatedProject: project } = operation; @@ -141,6 +142,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } const cobuildClusterId: string = hash.digest('hex'); + + // Assign same cluster id to all operations in the same cluster. for (const operation of set) { const { runner } = operation; if (runner instanceof ShellOperationRunner) { From e46502c069b3ccf73121a158ef0845f78b53fb1b Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 24 Jul 2023 13:37:51 +0800 Subject: [PATCH 059/100] chore: count on cobuild context id when calculating log only build cache --- .../src/logic/operations/CacheableOperationPlugin.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 07bb624596b..7a7582f5d28 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -293,6 +293,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // a log files only project build cache projectBuildCache = await this._tryGetLogOnlyProjectBuildCacheAsync({ buildCacheConfiguration, + cobuildConfiguration, runner, rushProject, phase, @@ -654,12 +655,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { commandName, commandToRun, buildCacheConfiguration, + cobuildConfiguration, phase, trackedProjectFiles, projectChangeAnalyzer, operationMetadataManager }: { buildCacheConfiguration: BuildCacheConfiguration | undefined; + cobuildConfiguration: CobuildConfiguration; runner: IOperationRunner; rushProject: RushConfigurationProject; phase: IPhase; @@ -685,6 +688,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Force the cache to be a log files only cache logFilesOnly: '1' }; + if (cobuildConfiguration.cobuildContextId) { + additionalContext.cobuildContextId = cobuildConfiguration.cobuildContextId; + } if (projectConfiguration) { const operationSettings: IOperationSettings | undefined = projectConfiguration.operationSettingsByOperationName.get(commandName); From adac6eb7c3509953409c495c3dc895deb6c388e7 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 14:03:50 +0800 Subject: [PATCH 060/100] Apply suggestions from code review Co-authored-by: Ian Clanton-Thuon Co-authored-by: David Michon --- .../src/runRush.ts | 3 ++ .../src/testLockProvider.ts | 3 ++ .../rush-lib/src/api/CobuildConfiguration.ts | 38 ++++++++++++------- .../src/api/EnvironmentConfiguration.ts | 4 +- .../rush-lib/src/cli/RushCommandLineParser.ts | 2 +- .../rush-lib/src/logic/cobuild/DisjointSet.ts | 15 +++++--- .../src/logic/cobuild/ICobuildLockProvider.ts | 8 ++-- .../logic/cobuild/test/DisjointSet.test.ts | 3 ++ .../operations/CacheableOperationPlugin.ts | 6 +-- rush.json | 2 +- 10 files changed, 55 insertions(+), 29 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts index 2547cf233fc..5dd95e8871d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/runRush.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + // Import from lib-commonjs for easy debugging import { RushCommandLineParser } from '@microsoft/rush-lib/lib-commonjs/cli/RushCommandLineParser'; import * as rushLib from '@microsoft/rush-lib/lib-commonjs'; diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 9eaa09bef34..276c07073f2 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + import { RedisCobuildLockProvider, IRedisCobuildLockProviderOptions diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 255e8db7383..16212abdbf9 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -1,16 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as path from 'path'; +import path from 'path'; import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; -import schemaJson from '../schemas/cobuild.schema.json'; +import { v4 as uuidv4 } from 'uuid'; + import { EnvironmentConfiguration } from './EnvironmentConfiguration'; import { CobuildLockProviderFactory, RushSession } from '../pluginFramework/RushSession'; import { RushConstants } from '../logic/RushConstants'; -import { v4 as uuidv4 } from 'uuid'; - import type { ICobuildLockProvider } from '../logic/cobuild/ICobuildLockProvider'; import type { RushConfiguration } from './RushConfiguration'; +import schemaJson from '../schemas/cobuild.schema.json'; /** * @beta @@ -98,14 +98,17 @@ export class CobuildConfiguration { rushSession: RushSession ): Promise { const jsonFilePath: string = CobuildConfiguration.getCobuildConfigFilePath(rushConfiguration); - if (!FileSystem.exists(jsonFilePath)) { - return undefined; + try { + return await CobuildConfiguration._loadAsync(jsonFilePath, terminal, rushConfiguration, rushSession); + } catch (err) { + if (!FileSystem.isNotExistError(err)) { + throw err; + } } - return await CobuildConfiguration._loadAsync(jsonFilePath, terminal, rushConfiguration, rushSession); } public static getCobuildConfigFilePath(rushConfiguration: RushConfiguration): string { - return path.resolve(rushConfiguration.commonRushConfigFolder, RushConstants.cobuildFilename); + return `${rushConfiguration.commonRushConfigFolder}/${RushConstants.cobuildFilename}`; } private static async _loadAsync( @@ -113,11 +116,20 @@ export class CobuildConfiguration { terminal: ITerminal, rushConfiguration: RushConfiguration, rushSession: RushSession - ): Promise { - const cobuildJson: ICobuildJson = await JsonFile.loadAndValidateAsync( - jsonFilePath, - CobuildConfiguration._jsonSchema - ); + ): Promise { + let cobuildJson: ICobuildJson | undefined; + try { + cobuildJson = await JsonFile.loadAndValidateAsync( + jsonFilePath, + CobuildConfiguration._jsonSchema + ); + } catch (e) { + if (FileSystem.isNotExistError(e) { + return undefined; + } else { + throw e; + } + } const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = rushSession.getCobuildLockProviderFactory(cobuildJson.cobuildLockProvider); diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index ecb2b03f4b4..4f27f9a676d 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -156,7 +156,7 @@ export const EnvironmentVariableNames = { RUSH_COBUILD_ENABLED: 'RUSH_COBUILD_ENABLED', /** - * Setting this environment variable opt into running with cobuilds. The context id should be the same across + * Setting this environment variable opts into running with cobuilds. The context id should be the same across * multiple VMs, but changed when it is a new round of cobuilds. * * e.g. `Build.BuildNumber` in Azure DevOps Pipeline. @@ -169,7 +169,7 @@ export const EnvironmentVariableNames = { /** * Explicitly specifies a name for each participating cobuild runner. * - * Setting this environment variable opt into running with cobuilds. + * Setting this environment variable opts into running with cobuilds. * * @remarks * This environment variable is optional, if it is not provided, a random id is used. diff --git a/libraries/rush-lib/src/cli/RushCommandLineParser.ts b/libraries/rush-lib/src/cli/RushCommandLineParser.ts index e628e4a18e9..5dc44071fd9 100644 --- a/libraries/rush-lib/src/cli/RushCommandLineParser.ts +++ b/libraries/rush-lib/src/cli/RushCommandLineParser.ts @@ -191,7 +191,7 @@ export class RushCommandLineParser extends CommandLineParser { public async execute(args?: string[]): Promise { // debugParameter will be correctly parsed during super.execute(), so manually parse here. - this._terminalProvider.debugEnabled = process.argv.indexOf('--debug') >= 0; + this._terminalProvider.verboseEnabled = this._terminalProvider.debugEnabled = process.argv.indexOf('--debug') >= 0; await this.pluginManager.tryInitializeUnassociatedPluginsAsync(); diff --git a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts index cf704865cd0..07940014978 100644 --- a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts +++ b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts @@ -52,13 +52,15 @@ export class DisjointSet { return; } - if (this._getSize(x) < this._getSize(y)) { + const xSize: number = this._getSize(x); + const ySize: number = this._getSize(y); + if (xSize < ySize) { const t: T = x; x = y; y = t; } this._parentMap.set(y, x); - this._sizeMap.set(x, this._getSize(x) + this._getSize(y)); + this._sizeMap.set(x, xSize + ySize); this._setByElement = undefined; } @@ -88,9 +90,12 @@ export class DisjointSet { private _find(a: T): T { let x: T = a; - while (this._getParent(x) !== x) { - this._parentMap.set(x, this._getParent(this._getParent(x))); - x = this._getParent(x); + let parent: T = this.getParent(x); + while (parent !== x) { + parent = this._getParent(parent); + this._parentMap.set(x, parent); + x = parent; + parent = this._getParent(x); } return x; } diff --git a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts index dfc0c7eaafb..027d8556a0c 100644 --- a/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts +++ b/libraries/rush-lib/src/logic/cobuild/ICobuildLockProvider.ts @@ -25,8 +25,8 @@ export interface ICobuildContext { */ contextId: string; /** - * The id of the cluster. The operations in the same cluster shares the same clusterId and - * will be executed in the same machine. + * The id of the cluster. The operations in the same cluster share the same clusterId and + * will be executed on the same machine. */ clusterId: string; /** @@ -36,8 +36,8 @@ export interface ICobuildContext { */ runnerId: string; /** - * The id of cache. It should be keep same as the normal cacheId from ProjectBuildCache. - * Otherwise, there is a discrepancy in the success case then turning on cobuilds will + * The id of the cache entry. It should be kept the same as the normal cacheId from ProjectBuildCache. + * Otherwise, there is a discrepancy in the success case wherein turning on cobuilds will * fail to populate the normal build cache. */ cacheId: string; diff --git a/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts b/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts index 255f0a426e7..56bb80695d5 100644 --- a/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts +++ b/libraries/rush-lib/src/logic/cobuild/test/DisjointSet.test.ts @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + import { DisjointSet } from '../DisjointSet'; describe(DisjointSet.name, () => { diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 7a7582f5d28..33b3296eb30 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -286,7 +286,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (cobuildConfiguration?.cobuildEnabled) { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && - rushProject.consumingProjects.size === 0 && + runner.consumers.size === 0 && !projectBuildCache ) { // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get @@ -447,7 +447,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { let setCacheEntryPromise: Promise | undefined; if (cobuildLock && isCacheWriteAllowed) { if (context.error) { - // In order to preventing the worst case that all cobuild tasks go through the same failure, + // In order to prevent the worst case that all cobuild tasks go through the same failure, // allowing a failing build to be cached and retrieved, print the error message to the terminal // and clear the error in context. const message: string | undefined = context.error?.message; @@ -512,7 +512,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext; } - private async _tryGetProjectBuildEnabledAsync({ + private async _tryGetProjectBuildCacheEnabledAsync({ buildCacheConfiguration, rushProject, commandName diff --git a/rush.json b/rush.json index 0fcae50a6c2..2fdb0b00027 100644 --- a/rush.json +++ b/rush.json @@ -1107,7 +1107,7 @@ "packageName": "@rushstack/rush-redis-cobuild-plugin", "projectFolder": "rush-plugins/rush-redis-cobuild-plugin", "reviewCategory": "libraries", - "shouldPublish": true + "versionPolicyName": "rush" }, { "packageName": "@rushstack/rush-serve-plugin", From ab678c8cf9df24f9e1be32069760c17543dd36f2 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:28:42 +0800 Subject: [PATCH 061/100] chore: lower the build time of build in the cobuild sandbox repo --- .../sandbox/repo/projects/build.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js index 20f983ed146..dc0589b7d62 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js @@ -11,4 +11,4 @@ setTimeout(() => { FileSystem.ensureFolder(outputFolder); FileSystem.writeFile(outputFile, `Hello world! ${args.join(' ')}`); console.log('done'); -}, 5000); +}, 1000); From 7c152ee4f2599ededf93e6ac3ae2c36caae16de8 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:29:28 +0800 Subject: [PATCH 062/100] chore: housekeeping after suggestion commit --- .../rush-lib/src/api/CobuildConfiguration.ts | 15 +++++++-------- .../rush-lib/src/logic/cobuild/DisjointSet.ts | 2 +- .../logic/operations/CacheableOperationPlugin.ts | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 16212abdbf9..e92c5cc88a9 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import path from 'path'; import { FileSystem, ITerminal, JsonFile, JsonSchema } from '@rushstack/node-core-library'; import { v4 as uuidv4 } from 'uuid'; @@ -119,16 +118,16 @@ export class CobuildConfiguration { ): Promise { let cobuildJson: ICobuildJson | undefined; try { - cobuildJson = await JsonFile.loadAndValidateAsync( - jsonFilePath, - CobuildConfiguration._jsonSchema - ); + cobuildJson = await JsonFile.loadAndValidateAsync(jsonFilePath, CobuildConfiguration._jsonSchema); } catch (e) { - if (FileSystem.isNotExistError(e) { + if (FileSystem.isNotExistError(e)) { return undefined; - } else { - throw e; } + throw e; + } + + if (!cobuildJson) { + return undefined; } const cobuildLockProviderFactory: CobuildLockProviderFactory | undefined = diff --git a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts index 07940014978..3a33aef59ae 100644 --- a/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts +++ b/libraries/rush-lib/src/logic/cobuild/DisjointSet.ts @@ -90,7 +90,7 @@ export class DisjointSet { private _find(a: T): T { let x: T = a; - let parent: T = this.getParent(x); + let parent: T = this._getParent(x); while (parent !== x) { parent = this._getParent(parent); this._parentMap.set(x, parent); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 33b3296eb30..68a20476006 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -109,7 +109,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { async (operation) => { const { associatedProject: project, associatedPhase: phase } = operation; if (project && phase && operation.runner instanceof ShellOperationRunner) { - const buildCacheEnabled: boolean = await this._tryGetProjectBuildEnabledAsync({ + const buildCacheEnabled: boolean = await this._tryGetProjectBuildCacheEnabledAsync({ buildCacheConfiguration, rushProject: project, commandName: phase.name @@ -286,7 +286,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (cobuildConfiguration?.cobuildEnabled) { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && - runner.consumers.size === 0 && + context.consumers.size === 0 && !projectBuildCache ) { // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get From 017d01a84b53f592a12ca344a1abe60107b35832 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:29:57 +0800 Subject: [PATCH 063/100] chore: remove change json for rush-redis-cobuild-plugin --- .../feat-cobuild_2023-02-17-07-02.json | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json b/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json deleted file mode 100644 index 77f64cbf75a..00000000000 --- a/common/changes/@rushstack/rush-redis-cobuild-plugin/feat-cobuild_2023-02-17-07-02.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "changes": [ - { - "packageName": "@rushstack/rush-redis-cobuild-plugin", - "comment": "Implement a redis lock provider for cobuild feature", - "type": "minor" - } - ], - "packageName": "@rushstack/rush-redis-cobuild-plugin" -} \ No newline at end of file From 9b057d0f49325db8b7c69e331372f54983e39897 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:30:22 +0800 Subject: [PATCH 064/100] chore: add a feature note link to change json --- .../changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json index c569757d07d..ac388318c6a 100644 --- a/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json +++ b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "(EXPERIMENTAL) Add a cheap way to get distributed builds called \"cobuild\"", + "comment": "(EXPERIMENTAL) Add a cheap way to get distributed builds called \"cobuild\". See [Rush.js Cobuild Feature Note](https://docs.google.com/document/d/1ydh4dVMpqSk_3mi-NgtWhI_g3TTmvkKsFQuddp8-4dI/edit)", "type": "none" } ], From a1f232ab0be858ebc24c5a736bd10e58e5e3548f Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:30:38 +0800 Subject: [PATCH 065/100] chore: remove useless eslint-disable comment --- .../src/test/RedisCobuildLockProvider.test.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts index 69e0b016b27..c90fbb16075 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/test/RedisCobuildLockProvider.test.ts @@ -1,14 +1,13 @@ -/* eslint-disable @typescript-eslint/no-floating-promises */ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. import { ConsoleTerminalProvider } from '@rushstack/node-core-library'; -import { ICobuildCompletedState, ICobuildContext, OperationStatus, RushSession } from '@rushstack/rush-sdk'; -import { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from '../RedisCobuildLockProvider'; - import * as redisAPI from '@redis/client'; import type { RedisClientType } from '@redis/client'; +import { ICobuildCompletedState, ICobuildContext, OperationStatus, RushSession } from '@rushstack/rush-sdk'; +import { IRedisCobuildLockProviderOptions, RedisCobuildLockProvider } from '../RedisCobuildLockProvider'; + const rushSession: RushSession = new RushSession({ terminalProvider: new ConsoleTerminalProvider(), getIsDebugMode: () => false From da224ffff83e309d36425515b902f3824c098a3c Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:34:09 +0800 Subject: [PATCH 066/100] chore: set exitCode to 1 before running the logic --- .../src/testLockProvider.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts index 276c07073f2..927d07b810e 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/src/testLockProvider.ts @@ -43,7 +43,18 @@ async function main(): Promise { console.log('Completed state: ', completedState); await lockProvider.disconnectAsync(); } -main().catch((err) => { - console.error(err); - process.exit(1); -}); + +process.exitCode = 1; + +main() + .then(() => { + process.exitCode = 0; + }) + .catch((err) => { + console.error(err); + }) + .finally(() => { + if (process.exitCode !== undefined) { + process.exit(process.exitCode); + } + }); From c8537a0ab4a5302fdddce4439e617768374b568d Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 15:47:58 +0800 Subject: [PATCH 067/100] chore: merge the assignment --- libraries/rush-lib/src/api/CobuildConfiguration.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index e92c5cc88a9..11b330597b5 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -74,14 +74,13 @@ export class CobuildConfiguration { private constructor(options: ICobuildConfigurationOptions) { const { cobuildJson, cobuildLockProviderFactory } = options; - this.cobuildEnabled = EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; + this.cobuildEnabled = this.cobuildContextId + ? EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled + : false; this.cobuildRunnerId = EnvironmentConfiguration.cobuildRunnerId || uuidv4(); this.cobuildLeafProjectLogOnlyAllowed = EnvironmentConfiguration.cobuildLeafProjectLogOnlyAllowed ?? false; - if (!this.cobuildContextId) { - this.cobuildEnabled = false; - } this._cobuildLockProviderFactory = cobuildLockProviderFactory; this._cobuildJson = cobuildJson; From 80e39f5054ac75550d0e540b3ce4ec647a4843b8 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 16:01:17 +0800 Subject: [PATCH 068/100] chore: move destroyLockProviderAsync in finally block --- .../cli/scriptActions/PhasedScriptAction.ts | 136 +++++++++--------- 1 file changed, 70 insertions(+), 66 deletions(-) diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index bcffbeeaf4a..67c261da736 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -323,83 +323,87 @@ export class PhasedScriptAction extends BaseScriptAction { await cobuildConfiguration?.createLockProviderAsync(terminal); } - const projectSelection: Set = - await this._selectionParameters.getSelectedProjectsAsync(terminal); + try { + const projectSelection: Set = + await this._selectionParameters.getSelectedProjectsAsync(terminal); + + if (!projectSelection.size) { + terminal.writeLine( + colors.yellow(`The command line selection parameters did not match any projects.`) + ); + return; + } - if (!projectSelection.size) { - terminal.writeLine(colors.yellow(`The command line selection parameters did not match any projects.`)); - return; - } + const isWatch: boolean = this._watchParameter?.value || this._alwaysWatch; - const isWatch: boolean = this._watchParameter?.value || this._alwaysWatch; + const customParametersByName: Map = new Map(); + for (const [configParameter, parserParameter] of this.customParameters) { + customParametersByName.set(configParameter.longName, parserParameter); + } - const customParametersByName: Map = new Map(); - for (const [configParameter, parserParameter] of this.customParameters) { - customParametersByName.set(configParameter.longName, parserParameter); - } + const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); + const initialCreateOperationsContext: ICreateOperationsContext = { + buildCacheConfiguration, + cobuildConfiguration, + customParameters: customParametersByName, + isIncrementalBuildAllowed: this._isIncrementalBuildAllowed, + isInitial: true, + isWatch, + rushConfiguration: this.rushConfiguration, + phaseOriginal: new Set(this._originalPhases), + phaseSelection: new Set(this._initialPhases), + projectChangeAnalyzer, + projectSelection, + projectsInUnknownState: projectSelection + }; - const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); - const initialCreateOperationsContext: ICreateOperationsContext = { - buildCacheConfiguration, - cobuildConfiguration, - customParameters: customParametersByName, - isIncrementalBuildAllowed: this._isIncrementalBuildAllowed, - isInitial: true, - isWatch, - rushConfiguration: this.rushConfiguration, - phaseOriginal: new Set(this._originalPhases), - phaseSelection: new Set(this._initialPhases), - projectChangeAnalyzer, - projectSelection, - projectsInUnknownState: projectSelection - }; + const executionManagerOptions: IOperationExecutionManagerOptions = { + quietMode: isQuietMode, + debugMode: this.parser.isDebug, + parallelism, + changedProjectsOnly, + beforeExecuteOperation: async (record: IOperationRunnerContext) => { + await this.hooks.beforeExecuteOperation.promise(record); + }, + afterExecuteOperation: async (record: IOperationRunnerContext) => { + await this.hooks.afterExecuteOperation.promise(record); + }, + beforeExecuteOperations: async (records: Map) => { + await this.hooks.beforeExecuteOperations.promise(records); + }, + onOperationStatusChanged: (record: OperationExecutionRecord) => { + this.hooks.onOperationStatusChanged.call(record); + } + }; - const executionManagerOptions: IOperationExecutionManagerOptions = { - quietMode: isQuietMode, - debugMode: this.parser.isDebug, - parallelism, - changedProjectsOnly, - beforeExecuteOperation: async (record: IOperationRunnerContext) => { - await this.hooks.beforeExecuteOperation.promise(record); - }, - afterExecuteOperation: async (record: IOperationRunnerContext) => { - await this.hooks.afterExecuteOperation.promise(record); - }, - beforeExecuteOperations: async (records: Map) => { - await this.hooks.beforeExecuteOperations.promise(records); - }, - onOperationStatusChanged: (record: OperationExecutionRecord) => { - this.hooks.onOperationStatusChanged.call(record); - } - }; + const internalOptions: IRunPhasesOptions = { + initialCreateOperationsContext, + executionManagerOptions, + stopwatch, + terminal + }; - const internalOptions: IRunPhasesOptions = { - initialCreateOperationsContext, - executionManagerOptions, - stopwatch, - terminal - }; + terminal.write('Analyzing repo state... '); + const repoStateStopwatch: Stopwatch = new Stopwatch(); + repoStateStopwatch.start(); + await projectChangeAnalyzer._ensureInitializedAsync(terminal); + repoStateStopwatch.stop(); + terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); + terminal.writeLine(); - terminal.write('Analyzing repo state... '); - const repoStateStopwatch: Stopwatch = new Stopwatch(); - repoStateStopwatch.start(); - await projectChangeAnalyzer._ensureInitializedAsync(terminal); - repoStateStopwatch.stop(); - terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); - terminal.writeLine(); + await this._runInitialPhases(internalOptions); - await this._runInitialPhases(internalOptions); + if (isWatch) { + if (buildCacheConfiguration) { + // Cache writes are not supported during watch mode, only reads. + buildCacheConfiguration.cacheWriteEnabled = false; + } - if (isWatch) { - if (buildCacheConfiguration) { - // Cache writes are not supported during watch mode, only reads. - buildCacheConfiguration.cacheWriteEnabled = false; + await this._runWatchPhases(internalOptions); } - - await this._runWatchPhases(internalOptions); + } finally { + await cobuildConfiguration?.destroyLockProviderAsync(); } - - await cobuildConfiguration?.destroyLockProviderAsync(); } private async _runInitialPhases(options: IRunPhasesOptions): Promise { From 70c1add46920b2461f2e64dc03b6d5a6aa7c799f Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 17:01:44 +0800 Subject: [PATCH 069/100] chore: wrap the lifetime of project writable object between beforeExecute and afterExecute --- .../repo/common/scripts/install-run.js | 59 ++++++-- .../operations/CacheableOperationPlugin.ts | 131 ++++++++++++------ .../logic/operations/OperationRunnerHooks.ts | 1 + .../logic/operations/ShellOperationRunner.ts | 8 +- 4 files changed, 141 insertions(+), 58 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js index 68b1b56fc58..c04c587be19 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/scripts/install-run.js @@ -359,6 +359,23 @@ function _getRushTempFolder(rushCommonFolder) { return _ensureAndJoinPath(rushCommonFolder, 'temp'); } } +/** + * Compare version strings according to semantic versioning. + * Returns a positive integer if "a" is a later version than "b", + * a negative integer if "b" is later than "a", + * and 0 otherwise. + */ +function _compareVersionStrings(a, b) { + const aParts = a.split(/[.-]/); + const bParts = b.split(/[.-]/); + const numberOfParts = Math.max(aParts.length, bParts.length); + for (let i = 0; i < numberOfParts; i++) { + if (aParts[i] !== bParts[i]) { + return (Number(aParts[i]) || 0) - (Number(bParts[i]) || 0); + } + } + return 0; +} /** * Resolve a package specifier to a static version */ @@ -379,12 +396,23 @@ function _resolvePackageVersion(logger, rushCommonFolder, { name, version }) { (0,_utilities_npmrcUtilities__WEBPACK_IMPORTED_MODULE_4__.syncNpmrc)(sourceNpmrcFolder, rushTempFolder, undefined, logger); const npmPath = getNpmPath(); // This returns something that looks like: - // @microsoft/rush@3.0.0 '3.0.0' - // @microsoft/rush@3.0.1 '3.0.1' - // ... - // @microsoft/rush@3.0.20 '3.0.20' - // - const npmVersionSpawnResult = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier'], { + // ``` + // [ + // "3.0.0", + // "3.0.1", + // ... + // "3.0.20" + // ] + // ``` + // + // if multiple versions match the selector, or + // + // ``` + // "3.0.0" + // ``` + // + // if only a single version matches. + const npmVersionSpawnResult = child_process__WEBPACK_IMPORTED_MODULE_0__.spawnSync(npmPath, ['view', `${name}@${version}`, 'version', '--no-update-notifier', '--json'], { cwd: rushTempFolder, stdio: [] }); @@ -392,16 +420,21 @@ function _resolvePackageVersion(logger, rushCommonFolder, { name, version }) { throw new Error(`"npm view" returned error code ${npmVersionSpawnResult.status}`); } const npmViewVersionOutput = npmVersionSpawnResult.stdout.toString(); - const versionLines = npmViewVersionOutput.split('\n').filter((line) => !!line); - const latestVersion = versionLines[versionLines.length - 1]; + const parsedVersionOutput = JSON.parse(npmViewVersionOutput); + const versions = Array.isArray(parsedVersionOutput) + ? parsedVersionOutput + : [parsedVersionOutput]; + let latestVersion = versions[0]; + for (let i = 1; i < versions.length; i++) { + const version = versions[i]; + if (_compareVersionStrings(version, latestVersion) > 0) { + latestVersion = version; + } + } if (!latestVersion) { throw new Error('No versions found for the specified version range.'); } - const versionMatches = latestVersion.match(/^.+\s\'(.+)\'$/); - if (!versionMatches) { - throw new Error(`Invalid npm output ${latestVersion}`); - } - return versionMatches[1]; + return latestVersion; } catch (e) { throw new Error(`Unable to resolve version ${version} of package ${name}: ${e}`); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 68a20476006..f6bb33fcd27 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -202,9 +202,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { ); hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { - for (const { buildCacheProjectLogWritable } of this._buildCacheContextByOperationRunner.values()) { - buildCacheProjectLogWritable?.close(); - } this._buildCacheContextByOperationRunner.clear(); }); } @@ -237,7 +234,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { phase, commandName, commandToRun, - earlyReturnStatus + earlyReturnStatus, + finallyCallbacks } = beforeExecuteContext; if (earlyReturnStatus) { // If there is existing early return status, we don't need to do anything @@ -267,6 +265,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); buildCacheContext.buildCacheTerminal = buildCacheTerminal; + finallyCallbacks.push(() => { + /** + * A known issue is that on some operating system configurations, attempting to read the file while this + * process has it open for writing throws an error, so the lifetime of the buildCacheProjectLogWritable + * needs to be wrapped between the beforeExecute and afterExecute hooks. + */ + buildCacheContext.buildCacheProjectLogWritable?.close(); + }); + let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ buildCacheConfiguration, runner, @@ -791,44 +798,82 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }): ITerminal { const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); if (!buildCacheContext.buildCacheTerminal) { - let cacheConsoleWritable: TerminalWritable; - const cacheProjectLogWritable: ProjectLogWritable | undefined = - this._tryGetBuildCacheProjectLogWritable({ - runner, - buildCacheConfiguration, - rushProject, - collatedTerminal: collatedWriter.terminal, - logFilenameIdentifier - }); + buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ + runner, + buildCacheConfiguration, + rushProject, + collatedWriter, + logFilenameIdentifier, + quietMode, + debugMode + }); + } else if (!buildCacheContext.buildCacheProjectLogWritable?.isOpen) { + // The ProjectLogWritable is closed, re-create one + buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ + runner, + buildCacheConfiguration, + rushProject, + collatedWriter, + logFilenameIdentifier, + quietMode, + debugMode + }); + } - if (quietMode) { - cacheConsoleWritable = new DiscardStdoutTransform({ - destination: collatedWriter - }); - } else { - cacheConsoleWritable = collatedWriter; - } + return buildCacheContext.buildCacheTerminal; + } - let cacheCollatedTerminal: CollatedTerminal; - if (cacheProjectLogWritable) { - const cacheSplitterTransform: SplitterTransform = new SplitterTransform({ - destinations: [cacheConsoleWritable, cacheProjectLogWritable] - }); - cacheCollatedTerminal = new CollatedTerminal(cacheSplitterTransform); - } else { - cacheCollatedTerminal = new CollatedTerminal(cacheConsoleWritable); - } + private _createBuildCacheTerminal({ + runner, + buildCacheConfiguration, + rushProject, + collatedWriter, + logFilenameIdentifier, + quietMode, + debugMode + }: { + runner: ShellOperationRunner; + buildCacheConfiguration: BuildCacheConfiguration | undefined; + rushProject: RushConfigurationProject; + collatedWriter: CollatedWriter; + logFilenameIdentifier: string; + quietMode: boolean; + debugMode: boolean; + }): ITerminal { + let cacheConsoleWritable: TerminalWritable; + const cacheProjectLogWritable: ProjectLogWritable | undefined = this._tryGetBuildCacheProjectLogWritable({ + runner, + buildCacheConfiguration, + rushProject, + collatedTerminal: collatedWriter.terminal, + logFilenameIdentifier + }); - const buildCacheTerminalProvider: CollatedTerminalProvider = new CollatedTerminalProvider( - cacheCollatedTerminal, - { - debugEnabled: debugMode - } - ); - buildCacheContext.buildCacheTerminal = new Terminal(buildCacheTerminalProvider); + if (quietMode) { + cacheConsoleWritable = new DiscardStdoutTransform({ + destination: collatedWriter + }); + } else { + cacheConsoleWritable = collatedWriter; } - return buildCacheContext.buildCacheTerminal; + let cacheCollatedTerminal: CollatedTerminal; + if (cacheProjectLogWritable) { + const cacheSplitterTransform: SplitterTransform = new SplitterTransform({ + destinations: [cacheConsoleWritable, cacheProjectLogWritable] + }); + cacheCollatedTerminal = new CollatedTerminal(cacheSplitterTransform); + } else { + cacheCollatedTerminal = new CollatedTerminal(cacheConsoleWritable); + } + + const buildCacheTerminalProvider: CollatedTerminalProvider = new CollatedTerminalProvider( + cacheCollatedTerminal, + { + debugEnabled: debugMode + } + ); + return new Terminal(buildCacheTerminalProvider); } private _tryGetBuildCacheProjectLogWritable({ @@ -849,13 +894,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); - if (!buildCacheContext.buildCacheProjectLogWritable) { - buildCacheContext.buildCacheProjectLogWritable = new ProjectLogWritable( - rushProject, - collatedTerminal, - `${logFilenameIdentifier}.cache` - ); - } + buildCacheContext.buildCacheProjectLogWritable = new ProjectLogWritable( + rushProject, + collatedTerminal, + `${logFilenameIdentifier}.cache` + ); return buildCacheContext.buildCacheProjectLogWritable; } } diff --git a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index 66415c697af..623338af2f4 100644 --- a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -34,6 +34,7 @@ export interface IOperationRunnerBeforeExecuteContext { phase: IPhase; commandName: string; commandToRun: string; + finallyCallbacks: (() => void)[]; } export interface IOperationRunnerAfterExecuteContext { diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index e5b3fd217df..a55cee7da79 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -121,6 +121,8 @@ export class ShellOperationRunner implements IOperationRunner { this.logFilenameIdentifier ); + const finallyCallbacks: (() => {})[] = []; + try { //#region OPERATION LOGGING // TERMINAL PIPELINE: @@ -229,7 +231,8 @@ export class ShellOperationRunner implements IOperationRunner { phase: this._phase, commandName: this._commandName, commandToRun: this._commandToRun, - earlyReturnStatus: undefined + earlyReturnStatus: undefined, + finallyCallbacks }; await this.hooks.beforeExecute.promise(beforeExecuteContext); @@ -361,6 +364,9 @@ export class ShellOperationRunner implements IOperationRunner { } finally { projectLogWritable.close(); this.periodicCallback.stop(); + for (const callback of finallyCallbacks) { + callback(); + } } } } From 32b397bd244aa364668a8abf60eb1ff06f28074e Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 17:17:35 +0800 Subject: [PATCH 070/100] chore: cacheable logic taps to beforeExecuteOperations instead of createOperations --- .../cli/scriptActions/PhasedScriptAction.ts | 2 +- .../operations/CacheableOperationPlugin.ts | 19 ++++++++++++------- .../src/pluginFramework/PhasedCommandHooks.ts | 5 +++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 67c261da736..37b856a13eb 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -369,7 +369,7 @@ export class PhasedScriptAction extends BaseScriptAction { await this.hooks.afterExecuteOperation.promise(record); }, beforeExecuteOperations: async (records: Map) => { - await this.hooks.beforeExecuteOperations.promise(records); + await this.hooks.beforeExecuteOperations.promise(records, initialCreateOperationsContext); }, onOperationStatusChanged: (record: OperationExecutionRecord) => { this.hooks.onOperationStatusChanged.call(record); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index f6bb33fcd27..6dd882b6afd 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -24,6 +24,9 @@ import { RushConstants } from '../RushConstants'; import { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; import { ProjectLogWritable } from './ProjectLogWritable'; +import { CobuildConfiguration } from '../../api/CobuildConfiguration'; +import { DisjointSet } from '../cobuild/DisjointSet'; + import type { Operation } from './Operation'; import type { IOperationRunnerAfterExecuteContext, @@ -40,8 +43,7 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; -import { CobuildConfiguration } from '../../api/CobuildConfiguration'; -import { DisjointSet } from '../cobuild/DisjointSet'; +import type { IOperationExecutionResult } from './IOperationExecutionResult'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; @@ -65,12 +67,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { >(); public apply(hooks: PhasedCommandHooks): void { - hooks.createOperations.tapPromise( + hooks.beforeExecuteOperations.tapPromise( PLUGIN_NAME, - async (operations: Set, context: ICreateOperationsContext): Promise> => { + async ( + records: Map, + context: ICreateOperationsContext + ): Promise => { const { buildCacheConfiguration, isIncrementalBuildAllowed, cobuildConfiguration } = context; if (!buildCacheConfiguration) { - return operations; + return; } let disjointSet: DisjointSet | undefined; @@ -78,6 +83,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { disjointSet = new DisjointSet(); } + const operations: IterableIterator = records.keys(); + for (const operation of operations) { disjointSet?.add(operation); const { runner } = operation; @@ -155,8 +162,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } } - - return operations; } ); diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index aed20e8389f..2c151623b8b 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -104,8 +104,9 @@ export class PhasedCommandHooks { * Hook invoked before operation start * Hook is series for stable output. */ - public readonly beforeExecuteOperations: AsyncSeriesHook<[Map]> = - new AsyncSeriesHook(['records']); + public readonly beforeExecuteOperations: AsyncSeriesHook< + [Map, ICreateOperationsContext] + > = new AsyncSeriesHook(['records', 'context']); /** * Hook invoked when operation status changed From 4ea76ae81fa6a47bc9f65d4d2fc2d317e9d65dd8 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 19:11:41 +0800 Subject: [PATCH 071/100] chore: sort grouped operations of each disjoin set --- .../src/logic/operations/CacheableOperationPlugin.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 6dd882b6afd..1dfb0cb0f32 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -9,6 +9,7 @@ import { InternalError, ITerminal, JsonObject, + Sort, Terminal } from '@rushstack/node-core-library'; import { CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; @@ -137,9 +138,16 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const set of disjointSet.getAllSets()) { if (cobuildConfiguration?.cobuildEnabled && cobuildConfiguration.cobuildContextId) { + // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. + const groupedOperations: Operation[] = Array.from(set); + Sort.sortBy(groupedOperations, (operation: Operation) => { + const { associatedProject, associatedPhase } = operation; + return `${associatedProject?.packageName}${RushConstants.hashDelimiter}${associatedPhase?.name}`; + }); + // Generates cluster id, cluster id comes from the project folder and phase name of all operations in the same cluster. const hash: crypto.Hash = crypto.createHash('sha1'); - for (const operation of set) { + for (const operation of groupedOperations) { const { associatedPhase: phase, associatedProject: project } = operation; if (project && phase) { hash.update(project.projectRelativeFolder); @@ -151,7 +159,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const cobuildClusterId: string = hash.digest('hex'); // Assign same cluster id to all operations in the same cluster. - for (const operation of set) { + for (const operation of groupedOperations) { const { runner } = operation; if (runner instanceof ShellOperationRunner) { const buildCacheContext: IOperationBuildCacheContext = From 629d8644a1014b3b99587ab7ab62f01d96a10d5c Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 19:13:08 +0800 Subject: [PATCH 072/100] chore: update comment for isSkipAllowed --- .../rush-lib/src/logic/operations/CacheableOperationPlugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 1dfb0cb0f32..031439ea48d 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -360,7 +360,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // has changed happens inside the hashing logic. // // - For skipping, "isSkipAllowed" is set to true initially, and during - // the process of running dependents, it will be changed by OperationExecutionManager to + // the process of running dependents, it will be changed by this plugin to // false if a dependency wasn't able to be skipped. // let buildCacheReadAttempted: boolean = false; From c1e8adb1b86ef87dbd60f47d21802e96630d47c6 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 19:25:49 +0800 Subject: [PATCH 073/100] chore: refactor beforeExecute hook of runner to bail hook --- common/reviews/api/rush-lib.api.md | 5 ++++- .../logic/operations/CacheableOperationPlugin.ts | 10 +--------- .../src/logic/operations/OperationRunnerHooks.ts | 15 ++++++++------- .../src/logic/operations/ShellOperationRunner.ts | 10 +++++----- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9df248dc358..0ba132aa998 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -841,7 +841,10 @@ export class PhasedCommandHooks { readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; - readonly beforeExecuteOperations: AsyncSeriesHook<[Map]>; + readonly beforeExecuteOperations: AsyncSeriesHook<[ + Map, + ICreateOperationsContext + ]>; readonly beforeLog: SyncHook; readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]>; readonly onOperationStatusChanged: SyncHook<[IOperationExecutionResult]>; diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 031439ea48d..bf0842b04a5 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -247,13 +247,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { phase, commandName, commandToRun, - earlyReturnStatus, finallyCallbacks } = beforeExecuteContext; - if (earlyReturnStatus) { - // If there is existing early return status, we don't need to do anything - return earlyReturnStatus; - } if (!projectDeps && buildCacheContext.isSkipAllowed) { // To test this code path: @@ -434,10 +429,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } })(); - if (earlyReturnStatus) { - beforeExecuteContext.earlyReturnStatus = earlyReturnStatus; - } - return beforeExecuteContext; + return earlyReturnStatus; } ); diff --git a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts index 623338af2f4..7a9e526c327 100644 --- a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts +++ b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { AsyncSeriesWaterfallHook } from 'tapable'; +import { AsyncSeriesBailHook, AsyncSeriesWaterfallHook } from 'tapable'; import type { ITerminal } from '@rushstack/node-core-library'; import type { IOperationRunnerContext } from './IOperationRunner'; @@ -23,7 +23,6 @@ export interface IOperationRunnerPlugin { export interface IOperationRunnerBeforeExecuteContext { context: IOperationRunnerContext; runner: ShellOperationRunner; - earlyReturnStatus: OperationStatus | undefined; terminal: ITerminal; projectDeps: IProjectDeps | undefined; lastProjectDeps: IProjectDeps | undefined; @@ -55,11 +54,13 @@ export interface IOperationRunnerAfterExecuteContext { * */ export class OperationRunnerHooks { - public beforeExecute: AsyncSeriesWaterfallHook = - new AsyncSeriesWaterfallHook( - ['beforeExecuteContext'], - 'beforeExecute' - ); + public beforeExecute: AsyncSeriesBailHook< + IOperationRunnerBeforeExecuteContext, + OperationStatus | undefined + > = new AsyncSeriesBailHook( + ['beforeExecuteContext'], + 'beforeExecute' + ); public afterExecute: AsyncSeriesWaterfallHook = new AsyncSeriesWaterfallHook( diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index a55cee7da79..06e72e1ea60 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -231,14 +231,14 @@ export class ShellOperationRunner implements IOperationRunner { phase: this._phase, commandName: this._commandName, commandToRun: this._commandToRun, - earlyReturnStatus: undefined, finallyCallbacks }; - await this.hooks.beforeExecute.promise(beforeExecuteContext); - - if (beforeExecuteContext.earlyReturnStatus) { - return beforeExecuteContext.earlyReturnStatus; + const earlyReturnStatus: OperationStatus | undefined = await this.hooks.beforeExecute.promise( + beforeExecuteContext + ); + if (earlyReturnStatus) { + return earlyReturnStatus; } // If the deps file exists, remove it before starting execution. From 80a7990667b28f92b370f193dd376d5146b23bc0 Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 19:35:18 +0800 Subject: [PATCH 074/100] chore: revert back context.stopWatch to IStopwatchResult --- common/reviews/api/rush-lib.api.md | 3 +-- .../rush-lib/src/logic/operations/CacheableOperationPlugin.ts | 3 ++- libraries/rush-lib/src/logic/operations/IOperationRunner.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 0ba132aa998..274c4c22dea 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -510,8 +510,7 @@ export interface IOperationRunnerContext { readonly runner: IOperationRunner; status: OperationStatus; stdioSummarizer: StdioSummarizer; - // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts - stopwatch: Stopwatch; + stopwatch: IStopwatchResult; } // @internal (undocumented) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index bf0842b04a5..ef5ca3d24fc 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -45,6 +45,7 @@ import { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { Stopwatch } from '../../utilities/Stopwatch'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; @@ -424,7 +425,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } else { // failed to acquire the lock, mark current operation to remote executing - context.stopwatch.reset(); + (context.stopwatch as Stopwatch).reset(); return OperationStatus.RemoteExecuting; } } diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 49ac8f109a8..17f96946184 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -6,7 +6,7 @@ import type { CollatedWriter } from '@rushstack/stream-collator'; import type { OperationStatus } from './OperationStatus'; import type { OperationMetadataManager } from './OperationMetadataManager'; -import type { Stopwatch } from '../../utilities/Stopwatch'; +import type { IStopwatchResult } from '../../utilities/Stopwatch'; /** * Information passed to the executing `IOperationRunner` @@ -39,7 +39,7 @@ export interface IOperationRunnerContext { /** * Object used to track elapsed time. */ - stopwatch: Stopwatch; + stopwatch: IStopwatchResult; /** * The current execution status of an operation. Operations start in the 'ready' state, * but can be 'blocked' if an upstream operation failed. It is 'executing' when From 9a5b35ef13c28825561e3e6a02eea82d4a278eca Mon Sep 17 00:00:00 2001 From: Cheng Date: Tue, 8 Aug 2023 19:46:41 +0800 Subject: [PATCH 075/100] chore: hoist function definition of beforeExecuteCallback function --- .../operations/CacheableOperationPlugin.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index ef5ca3d24fc..7ceb7208ee2 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -234,7 +234,19 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { hooks.beforeExecute.tapPromise( PLUGIN_NAME, async (beforeExecuteContext: IOperationRunnerBeforeExecuteContext) => { - const earlyReturnStatus: OperationStatus | undefined = await (async () => { + const beforeExecuteCallbackAsync = async ({ + buildCacheContext, + buildCacheConfiguration, + cobuildConfiguration, + selectedPhases, + projectChangeAnalyzer + }: { + buildCacheContext: IOperationBuildCacheContext; + buildCacheConfiguration: BuildCacheConfiguration | undefined; + cobuildConfiguration: CobuildConfiguration | undefined; + selectedPhases: ReadonlySet; + projectChangeAnalyzer: ProjectChangeAnalyzer; + }): Promise => { const { context, runner, @@ -429,7 +441,14 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return OperationStatus.RemoteExecuting; } } - })(); + }; + const earlyReturnStatus: OperationStatus | undefined = await beforeExecuteCallbackAsync({ + buildCacheContext, + buildCacheConfiguration, + cobuildConfiguration, + selectedPhases, + projectChangeAnalyzer + }); return earlyReturnStatus; } ); From eda3e8e37890d3994b9dd70ecc6b15aebbc7121f Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:55:08 -0700 Subject: [PATCH 076/100] rush update --- common/config/rush/repo-state.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/config/rush/repo-state.json b/common/config/rush/repo-state.json index a1f786ca80c..42ed1ee3ebd 100644 --- a/common/config/rush/repo-state.json +++ b/common/config/rush/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "4cfef707a58cfebf1748defd16d0ba994df96319", + "pnpmShrinkwrapHash": "264f8737e2fb8d41bdd942b0689003bd68e0515e", "preferredVersionsHash": "1926a5b12ac8f4ab41e76503a0d1d0dccc9c0e06" } From d7ac5f12dc5e81ee5d883025f54c94e4d27e1592 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 10 Aug 2023 20:14:58 +0800 Subject: [PATCH 077/100] refactor: use the operation-level to handle cache interaction --- .../sandbox/repo/projects/build.js | 2 +- common/reviews/api/rush-lib.api.md | 6 +- .../cli/scriptActions/PhasedScriptAction.ts | 2 +- .../src/logic/buildCache/ProjectBuildCache.ts | 15 +- .../operations/CacheableOperationPlugin.ts | 717 +++++++++++------- .../src/logic/operations/IOperationRunner.ts | 15 +- .../operations/OperationExecutionManager.ts | 22 +- .../operations/OperationExecutionRecord.ts | 29 +- .../logic/operations/OperationRunnerHooks.ts | 70 -- .../logic/operations/ProjectLogWritable.ts | 100 +-- .../logic/operations/ShellOperationRunner.ts | 187 +---- .../test/ShellOperationRunnerPlugin.test.ts | 2 +- .../src/pluginFramework/PhasedCommandHooks.ts | 13 +- .../src/utilities/NullTerminalProvider.ts | 10 + 14 files changed, 578 insertions(+), 612 deletions(-) delete mode 100644 libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts create mode 100644 libraries/rush-lib/src/utilities/NullTerminalProvider.ts diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js index dc0589b7d62..15441feef29 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/build.js @@ -11,4 +11,4 @@ setTimeout(() => { FileSystem.ensureFolder(outputFolder); FileSystem.writeFile(outputFile, `Hello world! ${args.join(' ')}`); console.log('done'); -}, 1000); +}, 2000); diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 274c4c22dea..90ebc448947 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -7,6 +7,7 @@ /// import { AsyncParallelHook } from 'tapable'; +import { AsyncSeriesBailHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; import type { CollatedWriter } from '@rushstack/stream-collator'; @@ -490,6 +491,7 @@ export interface IOperationOptions { // @beta export interface IOperationRunner { + commandToRun?: string; executeAsync(context: IOperationRunnerContext): Promise; readonly name: string; reportTiming: boolean; @@ -501,13 +503,11 @@ export interface IOperationRunner { export interface IOperationRunnerContext { readonly changedProjectsOnly: boolean; collatedWriter: CollatedWriter; - readonly consumers: Set; debugMode: boolean; error?: Error; // @internal _operationMetadataManager?: _OperationMetadataManager; quietMode: boolean; - readonly runner: IOperationRunner; status: OperationStatus; stdioSummarizer: StdioSummarizer; stopwatch: IStopwatchResult; @@ -839,7 +839,7 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage export class PhasedCommandHooks { readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; - readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; + readonly beforeExecuteOperation: AsyncSeriesBailHook<[IOperationRunnerContext], OperationStatus | undefined>; readonly beforeExecuteOperations: AsyncSeriesHook<[ Map, ICreateOperationsContext diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 37b856a13eb..07950a20132 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -363,7 +363,7 @@ export class PhasedScriptAction extends BaseScriptAction { parallelism, changedProjectsOnly, beforeExecuteOperation: async (record: IOperationRunnerContext) => { - await this.hooks.beforeExecuteOperation.promise(record); + return await this.hooks.beforeExecuteOperation.promise(record); }, afterExecuteOperation: async (record: IOperationRunnerContext) => { await this.hooks.afterExecuteOperation.promise(record); diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index b8c224e53df..d9c778c64f8 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -184,7 +184,7 @@ export class ProjectBuildCache { const tarUtility: TarExecutable | undefined = await ProjectBuildCache._tryGetTarUtility(terminal); let restoreSuccess: boolean = false; if (tarUtility && localCacheEntryPath) { - const logFilePath: string = this._getTarLogFilePath(); + const logFilePath: string = this._getTarLogFilePath('untar'); const tarExitCode: number = await tarUtility.tryUntarAsync({ archivePath: localCacheEntryPath, outputFolderPath: projectFolderPath, @@ -194,7 +194,10 @@ export class ProjectBuildCache { restoreSuccess = true; terminal.writeLine('Successfully restored output from the build cache.'); } else { - terminal.writeWarningLine('Unable to restore output from the build cache.'); + terminal.writeWarningLine( + 'Unable to restore output from the build cache. ' + + `See "${logFilePath}" for logs from the tar process.` + ); } } @@ -238,7 +241,7 @@ export class ProjectBuildCache { const randomSuffix: string = crypto.randomBytes(8).toString('hex'); const tempLocalCacheEntryPath: string = `${finalLocalCacheEntryPath}-${randomSuffix}.temp`; - const logFilePath: string = this._getTarLogFilePath(); + const logFilePath: string = this._getTarLogFilePath('tar'); const tarExitCode: number = await tarUtility.tryCreateArchiveFromProjectPathsAsync({ archivePath: tempLocalCacheEntryPath, paths: filesToCache.outputFilePaths, @@ -382,7 +385,7 @@ export class ProjectBuildCache { // Add additional output file paths await Async.forEachAsync( this._additionalProjectOutputFilePaths, - async (additionalProjectOutputFilePath) => { + async (additionalProjectOutputFilePath: string) => { const fullPath: string = `${projectFolderPath}/${additionalProjectOutputFilePath}`; const pathExists: boolean = await FileSystem.existsAsync(fullPath); if (pathExists) { @@ -401,8 +404,8 @@ export class ProjectBuildCache { }; } - private _getTarLogFilePath(): string { - return path.join(this._project.projectRushTempFolder, `${this._cacheId}.log`); + private _getTarLogFilePath(mode: 'tar' | 'untar'): string { + return path.join(this._project.projectRushTempFolder, `${this._cacheId}.${mode}.log`); } private static async _getCacheId(options: IProjectBuildCacheOptions): Promise { diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 7ceb7208ee2..96ef0cd3b58 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -1,13 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import * as path from 'path'; import * as crypto from 'crypto'; import { Async, ColorValue, - ConsoleTerminalProvider, + FileSystem, InternalError, ITerminal, + JsonFile, JsonObject, Sort, Terminal @@ -17,7 +19,6 @@ import { DiscardStdoutTransform, PrintUtilities } from '@rushstack/terminal'; import { SplitterTransform, TerminalWritable } from '@rushstack/terminal'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import { ShellOperationRunner } from './ShellOperationRunner'; import { OperationStatus } from './OperationStatus'; import { CobuildLock, ICobuildCompletedState } from '../cobuild/CobuildLock'; import { ProjectBuildCache } from '../buildCache/ProjectBuildCache'; @@ -27,27 +28,31 @@ import { getHashesForGlobsAsync } from '../buildCache/getHashesForGlobsAsync'; import { ProjectLogWritable } from './ProjectLogWritable'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { DisjointSet } from '../cobuild/DisjointSet'; +import { PeriodicCallback } from './PeriodicCallback'; +import { NullTerminalProvider } from '../../utilities/NullTerminalProvider'; import type { Operation } from './Operation'; -import type { - IOperationRunnerAfterExecuteContext, - IOperationRunnerBeforeExecuteContext -} from './OperationRunnerHooks'; -import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; -import { RushConfigurationProject } from '../../api/RushConfigurationProject'; +import type { IOperationRunnerContext } from './IOperationRunner'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ICreateOperationsContext, IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; +import type { IRawRepoState, ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { OperationMetadataManager } from './OperationMetadataManager'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; -import type { Stopwatch } from '../../utilities/Stopwatch'; +import type { OperationExecutionRecord } from './OperationExecutionRecord'; const PLUGIN_NAME: 'CacheablePhasedOperationPlugin' = 'CacheablePhasedOperationPlugin'; +const PERIODIC_CALLBACK_INTERVAL_IN_SECONDS: number = 10; + +export interface IProjectDeps { + files: { [filePath: string]: string }; + arguments: string; +} export interface IOperationBuildCacheContext { isCacheWriteAllowed: boolean; @@ -60,19 +65,26 @@ export interface IOperationBuildCacheContext { // Controls the log for the cache subsystem buildCacheTerminal: ITerminal | undefined; buildCacheProjectLogWritable: ProjectLogWritable | undefined; + periodicCallback: PeriodicCallback; + projectDeps: IProjectDeps | undefined; + currentDepsPath: string | undefined; + // True if a operation status has been return by the beforeExecutionOperation callback function + hasReturnedStatus: boolean; } export class CacheableOperationPlugin implements IPhasedCommandPlugin { - private _buildCacheContextByOperationRunner: Map = new Map< - IOperationRunner, + private _buildCacheContextByOperationExecutionRecord: Map< + OperationExecutionRecord, IOperationBuildCacheContext - >(); + > = new Map(); + + private _createContext: ICreateOperationsContext | undefined; public apply(hooks: PhasedCommandHooks): void { hooks.beforeExecuteOperations.tapPromise( PLUGIN_NAME, async ( - records: Map, + recordByOperation: Map, context: ICreateOperationsContext ): Promise => { const { buildCacheConfiguration, isIncrementalBuildAllowed, cobuildConfiguration } = context; @@ -80,54 +92,63 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } - let disjointSet: DisjointSet | undefined; + this._createContext = context; + + let disjointSet: DisjointSet | undefined; if (cobuildConfiguration?.cobuildEnabled) { - disjointSet = new DisjointSet(); + disjointSet = new DisjointSet(); } - const operations: IterableIterator = records.keys(); - - for (const operation of operations) { - disjointSet?.add(operation); - const { runner } = operation; - if (runner) { - const buildCacheContext: IOperationBuildCacheContext = { - // ShellOperationRunner supports cache writes by default. - isCacheWriteAllowed: true, - isCacheReadAllowed: isIncrementalBuildAllowed, - isSkipAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - cobuildLock: undefined, - cobuildClusterId: undefined, - buildCacheTerminal: undefined, - buildCacheProjectLogWritable: undefined - }; - // Upstream runners may mutate the property of build cache context for downstream runners - this._buildCacheContextByOperationRunner.set(runner, buildCacheContext); - - if (runner instanceof ShellOperationRunner) { - this._applyShellOperationRunner(runner, context); - } - } + const records: IterableIterator = + recordByOperation.values() as IterableIterator; + + for (const record of records) { + disjointSet?.add(record); + const buildCacheContext: IOperationBuildCacheContext = { + // Supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + isSkipAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + cobuildLock: undefined, + cobuildClusterId: undefined, + buildCacheTerminal: undefined, + buildCacheProjectLogWritable: undefined, + periodicCallback: new PeriodicCallback({ + interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 + }), + projectDeps: undefined, + currentDepsPath: undefined, + hasReturnedStatus: false + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperationExecutionRecord.set(record, buildCacheContext); } if (disjointSet) { // If disjoint set exists, connect build cache disabled project with its consumers await Async.forEachAsync( - operations, - async (operation) => { - const { associatedProject: project, associatedPhase: phase } = operation; - if (project && phase && operation.runner instanceof ShellOperationRunner) { + records, + async (record: OperationExecutionRecord) => { + const { associatedProject: project, associatedPhase: phase } = record; + if (project && phase) { const buildCacheEnabled: boolean = await this._tryGetProjectBuildCacheEnabledAsync({ buildCacheConfiguration, rushProject: project, commandName: phase.name }); if (!buildCacheEnabled) { - for (const consumer of operation.consumers) { - if (consumer.runner instanceof ShellOperationRunner) { - disjointSet?.union(operation, consumer); - } + /** + * Group the project build cache disabled with its consumers. This won't affect too much in + * a monorepo with high build cache coverage. + * + * The mental model is that if X disables the cache, and Y depends on X, then: + * 1. Y must be built by the same VM that build X; + * 2. OR, Y must be rebuilt on each VM that needs it. + * Approach 1 is probably the better choice. + */ + for (const consumer of record.consumers) { + disjointSet?.union(record, consumer); } } } @@ -140,16 +161,16 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { for (const set of disjointSet.getAllSets()) { if (cobuildConfiguration?.cobuildEnabled && cobuildConfiguration.cobuildContextId) { // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. - const groupedOperations: Operation[] = Array.from(set); - Sort.sortBy(groupedOperations, (operation: Operation) => { - const { associatedProject, associatedPhase } = operation; + const groupedRecords: OperationExecutionRecord[] = Array.from(set); + Sort.sortBy(groupedRecords, (record: OperationExecutionRecord) => { + const { associatedProject, associatedPhase } = record; return `${associatedProject?.packageName}${RushConstants.hashDelimiter}${associatedPhase?.name}`; }); // Generates cluster id, cluster id comes from the project folder and phase name of all operations in the same cluster. const hash: crypto.Hash = crypto.createHash('sha1'); - for (const operation of groupedOperations) { - const { associatedPhase: phase, associatedProject: project } = operation; + for (const record of groupedRecords) { + const { associatedPhase: phase, associatedProject: project } = record; if (project && phase) { hash.update(project.projectRelativeFolder); hash.update(RushConstants.hashDelimiter); @@ -160,13 +181,10 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const cobuildClusterId: string = hash.digest('hex'); // Assign same cluster id to all operations in the same cluster. - for (const operation of groupedOperations) { - const { runner } = operation; - if (runner instanceof ShellOperationRunner) { - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByRunnerOrThrow(runner); - buildCacheContext.cobuildClusterId = cobuildClusterId; - } + for (const record of groupedRecords) { + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); + buildCacheContext.cobuildClusterId = cobuildClusterId; } } } @@ -174,99 +192,114 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } ); - hooks.afterExecuteOperation.tapPromise( + hooks.beforeExecuteOperation.tapPromise( PLUGIN_NAME, - async (runnerContext: IOperationRunnerContext): Promise => { - const { runner, status, consumers, changedProjectsOnly } = runnerContext; - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(runner); - - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; - - switch (status) { - case OperationStatus.Skipped: { - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; - break; - } - - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: { - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !changedProjectsOnly; - break; - } + async (runnerContext: IOperationRunnerContext): Promise => { + const { _createContext: createContext } = this; + if (!createContext) { + return; } + const { + projectChangeAnalyzer, + buildCacheConfiguration, + cobuildConfiguration, + phaseSelection: selectedPhases + } = createContext; - // Apply status changes to direct dependents - for (const item of consumers) { - const itemRunnerBuildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(item.runner); - if (itemRunnerBuildCacheContext) { - if (blockCacheWrite) { - itemRunnerBuildCacheContext.isCacheWriteAllowed = false; - } - if (blockSkip) { - itemRunnerBuildCacheContext.isSkipAllowed = false; - } - } - } - } - ); + const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; + const { + associatedProject: project, + associatedPhase: phase, + _operationMetadataManager: operationMetadataManager + } = record; - hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { - this._buildCacheContextByOperationRunner.clear(); - }); - } + if (!project || !phase) { + return; + } - private _applyShellOperationRunner(runner: ShellOperationRunner, context: ICreateOperationsContext): void { - const { - buildCacheConfiguration, - cobuildConfiguration, - phaseSelection: selectedPhases, - projectChangeAnalyzer - } = context; - const { hooks } = runner; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperationExecutionRecord(record); - const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + if (!buildCacheContext) { + return; + } - hooks.beforeExecute.tapPromise( - PLUGIN_NAME, - async (beforeExecuteContext: IOperationRunnerBeforeExecuteContext) => { - const beforeExecuteCallbackAsync = async ({ - buildCacheContext, + const runBeforeExecute = async ({ + projectChangeAnalyzer, buildCacheConfiguration, cobuildConfiguration, selectedPhases, - projectChangeAnalyzer + project, + phase, + operationMetadataManager, + buildCacheContext, + record }: { - buildCacheContext: IOperationBuildCacheContext; + projectChangeAnalyzer: ProjectChangeAnalyzer; buildCacheConfiguration: BuildCacheConfiguration | undefined; cobuildConfiguration: CobuildConfiguration | undefined; selectedPhases: ReadonlySet; - projectChangeAnalyzer: ProjectChangeAnalyzer; + project: RushConfigurationProject; + phase: IPhase; + operationMetadataManager: OperationMetadataManager | undefined; + buildCacheContext: IOperationBuildCacheContext; + record: OperationExecutionRecord; }): Promise => { - const { - context, - runner, - terminal, - lastProjectDeps, - projectDeps, - trackedProjectFiles, - logPath, - errorLogPath, - rushProject, - phase, - commandName, - commandToRun, - finallyCallbacks - } = beforeExecuteContext; + const buildCacheTerminal: ITerminal = this._getBuildCacheTerminal({ + record, + buildCacheConfiguration, + rushProject: project, + logFilenameIdentifier: phase.logFilenameIdentifier, + quietMode: record.quietMode, + debugMode: record.debugMode + }); + buildCacheContext.buildCacheTerminal = buildCacheTerminal; + + const commandToRun: string = record.runner.commandToRun || ''; + + const packageDepsFilename: string = `package-deps_${phase.logFilenameIdentifier}.json`; + const currentDepsPath: string = path.join(project.projectRushTempFolder, packageDepsFilename); + buildCacheContext.currentDepsPath = currentDepsPath; + + let projectDeps: IProjectDeps | undefined; + let trackedProjectFiles: string[] | undefined; + try { + const fileHashes: Map | undefined = + await createContext.projectChangeAnalyzer._tryGetProjectDependenciesAsync( + project, + buildCacheTerminal + ); + + if (fileHashes) { + const files: { [filePath: string]: string } = {}; + trackedProjectFiles = []; + for (const [filePath, fileHash] of fileHashes) { + files[filePath] = fileHash; + trackedProjectFiles.push(filePath); + } + + projectDeps = { + files, + arguments: commandToRun + }; + buildCacheContext.projectDeps = projectDeps; + } + } catch (error) { + // To test this code path: + // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" + buildCacheTerminal.writeLine( + 'Unable to calculate incremental state: ' + (error as Error).toString() + ); + buildCacheTerminal.writeLine({ + text: 'Rush will proceed without incremental execution, caching, and change detection.', + foregroundColor: ColorValue.Cyan + }); + } if (!projectDeps && buildCacheContext.isSkipAllowed) { // To test this code path: // Remove the `.git` folder then run "rush build --verbose" - terminal.writeLine({ + buildCacheTerminal.writeLine({ text: PrintUtilities.wrapWords( 'This workspace does not appear to be tracked by Git. ' + 'Rush will proceed without incremental execution, caching, and change detection.' @@ -275,38 +308,17 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } - const buildCacheTerminal: ITerminal = this._getBuildCacheTerminal({ - runner, - buildCacheConfiguration, - rushProject, - collatedWriter: context.collatedWriter, - logFilenameIdentifier: runner.logFilenameIdentifier, - quietMode: context.quietMode, - debugMode: context.debugMode - }); - buildCacheContext.buildCacheTerminal = buildCacheTerminal; - - finallyCallbacks.push(() => { - /** - * A known issue is that on some operating system configurations, attempting to read the file while this - * process has it open for writing throws an error, so the lifetime of the buildCacheProjectLogWritable - * needs to be wrapped between the beforeExecute and afterExecute hooks. - */ - buildCacheContext.buildCacheProjectLogWritable?.close(); - }); - let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ + record, buildCacheConfiguration, - runner, - rushProject, + rushProject: project, phase, selectedPhases, projectChangeAnalyzer, - commandName, commandToRun, terminal: buildCacheTerminal, trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager + operationMetadataManager }); // Try to acquire the cobuild lock @@ -314,7 +326,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (cobuildConfiguration?.cobuildEnabled) { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && - context.consumers.size === 0 && + record.consumers.size === 0 && !projectBuildCache ) { // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get @@ -322,32 +334,31 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { projectBuildCache = await this._tryGetLogOnlyProjectBuildCacheAsync({ buildCacheConfiguration, cobuildConfiguration, - runner, - rushProject, + record, + rushProject: project, phase, projectChangeAnalyzer, - commandName, commandToRun, terminal: buildCacheTerminal, trackedProjectFiles, - operationMetadataManager: context._operationMetadataManager + operationMetadataManager }); if (projectBuildCache) { buildCacheTerminal.writeVerboseLine( - `Log files only build cache is enabled for the project "${rushProject.packageName}" because the cobuild leaf project log only is allowed` + `Log files only build cache is enabled for the project "${project.packageName}" because the cobuild leaf project log only is allowed` ); } else { buildCacheTerminal.writeWarningLine( - `Failed to get log files only build cache for the project "${rushProject.packageName}"` + `Failed to get log files only build cache for the project "${project.packageName}"` ); } } cobuildLock = await this._tryGetCobuildLockAsync({ - runner, + record, projectBuildCache, cobuildConfiguration, - packageName: rushProject.packageName, + packageName: project.packageName, phaseName: phase.name }); } @@ -373,6 +384,10 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // let buildCacheReadAttempted: boolean = false; + const { logPath, errorLogPath } = ProjectLogWritable.getLogFilePaths({ + project, + logFilenameIdentifier: phase.logFilenameIdentifier + }); if (cobuildLock) { // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to // the build cache once completed, but does not retrieve them (since the "incremental" @@ -388,8 +403,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (restoreFromCacheSuccess) { // Restore the original state of the operation without cache - await context._operationMetadataManager?.tryRestoreAsync({ - terminal, + await operationMetadataManager?.tryRestoreAsync({ + terminal: buildCacheTerminal, logPath, errorLogPath }); @@ -406,8 +421,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (restoreFromCacheSuccess) { // Restore the original state of the operation without cache - await context._operationMetadataManager?.tryRestoreAsync({ - terminal, + await operationMetadataManager?.tryRestoreAsync({ + terminal: buildCacheTerminal, logPath, errorLogPath }); @@ -415,6 +430,17 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } if (buildCacheContext.isSkipAllowed && !buildCacheReadAttempted) { + let lastProjectDeps: IProjectDeps | undefined = undefined; + try { + lastProjectDeps = JsonFile.load(currentDepsPath); + } catch (e) { + // Warn and ignore - treat failing to load the file as the project being not built. + buildCacheTerminal.writeWarningLine( + `Warning: error parsing ${packageDepsFilename}: ${e}. Ignoring and ` + + `treating the command "${commandToRun}" as not run.` + ); + } + const isPackageUnchanged: boolean = !!( lastProjectDeps && projectDeps && @@ -430,116 +456,241 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); if (acquireSuccess) { - // The operation may be used to marked remote executing, now change it to executing - context.status = OperationStatus.Executing; - runner.periodicCallback.addCallback(async () => { + const { periodicCallback } = buildCacheContext; + periodicCallback.addCallback(async () => { await cobuildLock?.renewLockAsync(); }); + periodicCallback.start(); } else { // failed to acquire the lock, mark current operation to remote executing - (context.stopwatch as Stopwatch).reset(); return OperationStatus.RemoteExecuting; } } + + // If the deps file exists, remove it before starting execution. + FileSystem.deleteFile(currentDepsPath); + + // TODO: Remove legacyDepsPath with the next major release of Rush + const legacyDepsPath: string = path.join(project.projectFolder, 'package-deps.json'); + // Delete the legacy package-deps.json + FileSystem.deleteFile(legacyDepsPath); + + if (!commandToRun) { + // Write deps on success. + if (projectDeps) { + JsonFile.save(projectDeps, currentDepsPath, { + ensureFolderExists: true + }); + } + } }; - const earlyReturnStatus: OperationStatus | undefined = await beforeExecuteCallbackAsync({ - buildCacheContext, - buildCacheConfiguration, - cobuildConfiguration, - selectedPhases, - projectChangeAnalyzer - }); - return earlyReturnStatus; + + try { + const earlyReturnStatus: OperationStatus | undefined = await runBeforeExecute({ + projectChangeAnalyzer, + buildCacheConfiguration, + cobuildConfiguration, + selectedPhases, + project, + phase, + operationMetadataManager, + buildCacheContext, + record + }); + if (earlyReturnStatus) { + buildCacheContext.hasReturnedStatus = true; + } + return earlyReturnStatus; + } catch (e) { + buildCacheContext.buildCacheProjectLogWritable?.close(); + throw e; + } } ); - runner.hooks.afterExecute.tapPromise( + hooks.afterExecuteOperation.tapPromise( PLUGIN_NAME, - async (afterExecuteContext: IOperationRunnerAfterExecuteContext) => { - const { context, status, taskIsSuccessful, logPath, errorLogPath } = afterExecuteContext; - - const { cobuildLock, projectBuildCache, isCacheWriteAllowed, buildCacheTerminal } = buildCacheContext; - - // Save the metadata to disk - const { duration: durationInSeconds } = context.stopwatch; - await context._operationMetadataManager?.saveAsync({ - durationInSeconds, - cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, - cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, - logPath, - errorLogPath - }); + async (runnerContext: IOperationRunnerContext): Promise => { + const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; + const { + status, + consumers, + changedProjectsOnly, + stopwatch, + _operationMetadataManager: operationMetadataManager, + associatedProject: project, + associatedPhase: phase + } = record; + + if (!project || !phase) { + return; + } - if (!buildCacheTerminal) { - // This should not happen - throw new InternalError(`Build Cache Terminal is not created`); + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperationExecutionRecord(record); + + if (!buildCacheContext) { + return; } + const { + cobuildLock, + projectBuildCache, + isCacheWriteAllowed, + buildCacheTerminal, + projectDeps, + currentDepsPath, + hasReturnedStatus: hasStatusReturned + } = buildCacheContext; + + if (hasStatusReturned) { + // Status has been returned in beforeExecutionOperation hook, skip the following logic. + return; + } + + try { + const { logFilenameIdentifier } = phase; - let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; - let setCacheEntryPromise: Promise | undefined; - if (cobuildLock && isCacheWriteAllowed) { - if (context.error) { - // In order to prevent the worst case that all cobuild tasks go through the same failure, - // allowing a failing build to be cached and retrieved, print the error message to the terminal - // and clear the error in context. - const message: string | undefined = context.error?.message; - if (message) { - context.collatedWriter.terminal.writeStderrLine(message); + // Save the metadata to disk + const { duration: durationInSeconds } = stopwatch; + const { logPath, errorLogPath } = ProjectLogWritable.getLogFilePaths({ + project, + logFilenameIdentifier + }); + await operationMetadataManager?.saveAsync({ + durationInSeconds, + cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, + cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, + logPath, + errorLogPath + }); + + if (!buildCacheTerminal) { + // This should not happen + throw new InternalError(`Build Cache Terminal is not created`); + } + + let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; + let setCacheEntryPromise: Promise | undefined; + if (cobuildLock && isCacheWriteAllowed) { + if (record.error) { + // In order to prevent the worst case that all cobuild tasks go through the same failure, + // allowing a failing build to be cached and retrieved, print the error message to the terminal + // and clear the error in context. + const message: string | undefined = record.error?.message; + if (message) { + record.collatedWriter.terminal.writeStderrLine(message); + } + record.error = undefined; } - context.error = undefined; + const { cacheId, contextId } = cobuildLock.cobuildContext; + + const finalCacheId: string = + status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, + cacheId: finalCacheId + }); + }; + setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( + buildCacheTerminal, + finalCacheId + ); + } + } + } + + const taskIsSuccessful: boolean = + status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + record.runner.warningsAreAllowed && + !!project.rushConfiguration.experimentsConfiguration.configuration + .buildCacheWithAllowWarningsInSuccessfulBuild); + + if (taskIsSuccessful && projectDeps && currentDepsPath) { + // Write deps on success. + await JsonFile.saveAsync(projectDeps, currentDepsPath, { + ensureFolderExists: true + }); } - const { cacheId, contextId } = cobuildLock.cobuildContext; - const finalCacheId: string = - status === OperationStatus.Failure ? `${cacheId}-${contextId}-failed` : cacheId; - switch (status) { + // If the command is successful, we can calculate project hash, and no dependencies were skipped, + // write a new cache entry. + if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && projectBuildCache) { + setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal); + } + const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise; + await setCompletedStatePromiseFunction?.(); + + if (cacheWriteSuccess === false && status === OperationStatus.Success) { + record.status = OperationStatus.SuccessWithWarning; + } + + // Status changes to direct dependents + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; + + switch (record.status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; + } + case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: - case OperationStatus.Failure: { - const currentStatus: ICobuildCompletedState['status'] = status; - setCompletedStatePromiseFunction = () => { - return cobuildLock?.setCompletedStateAsync({ - status: currentStatus, - cacheId: finalCacheId - }); - }; - setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( - buildCacheTerminal, - finalCacheId - ); + case OperationStatus.Success: { + // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. + blockSkip ||= !changedProjectsOnly; + break; } } - } - // If the command is successful, we can calculate project hash, and no dependencies were skipped, - // write a new cache entry. - if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && projectBuildCache) { - setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal); - } - const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise; - await setCompletedStatePromiseFunction?.(); - - if (cacheWriteSuccess === false && afterExecuteContext.status === OperationStatus.Success) { - afterExecuteContext.status = OperationStatus.SuccessWithWarning; + // Apply status changes to direct dependents + for (const consumer of consumers) { + const consumerBuildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperationExecutionRecord(consumer); + if (consumerBuildCacheContext) { + if (blockCacheWrite) { + consumerBuildCacheContext.isCacheWriteAllowed = false; + } + if (blockSkip) { + consumerBuildCacheContext.isSkipAllowed = false; + } + } + } + } finally { + buildCacheContext.buildCacheProjectLogWritable?.close(); + buildCacheContext.periodicCallback.stop(); } - - return afterExecuteContext; } ); + + hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { + this._buildCacheContextByOperationExecutionRecord.clear(); + }); } - private _getBuildCacheContextByRunner(runner: IOperationRunner): IOperationBuildCacheContext | undefined { + private _getBuildCacheContextByOperationExecutionRecord( + record: OperationExecutionRecord + ): IOperationBuildCacheContext | undefined { const buildCacheContext: IOperationBuildCacheContext | undefined = - this._buildCacheContextByOperationRunner.get(runner); + this._buildCacheContextByOperationExecutionRecord.get(record); return buildCacheContext; } - private _getBuildCacheContextByRunnerOrThrow(runner: IOperationRunner): IOperationBuildCacheContext { + private _getBuildCacheContextByOperationExecutionRecordOrThrow( + record: OperationExecutionRecord + ): IOperationBuildCacheContext { const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByRunner(runner); + this._getBuildCacheContextByOperationExecutionRecord(record); if (!buildCacheContext) { // This should not happen - throw new InternalError(`Build cache context for runner ${runner.name} should be defined`); + throw new InternalError(`Build cache context for runner ${record.name} should be defined`); } return buildCacheContext; } @@ -553,10 +704,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { rushProject: RushConfigurationProject; commandName: string; }): Promise { - const consoleTerminalProvider: ConsoleTerminalProvider = new ConsoleTerminalProvider(); - const terminal: ITerminal = new Terminal(consoleTerminalProvider); + const nullTerminalProvider: NullTerminalProvider = new NullTerminalProvider(); // This is a silent terminal - terminal.unregisterProvider(consoleTerminalProvider); + const terminal: ITerminal = new Terminal(nullTerminalProvider); if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { const projectConfiguration: RushProjectConfiguration | undefined = @@ -574,30 +724,29 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private async _tryGetProjectBuildCacheAsync({ buildCacheConfiguration, - runner, + record, rushProject, phase, selectedPhases, projectChangeAnalyzer, - commandName, commandToRun, terminal, trackedProjectFiles, operationMetadataManager }: { + record: OperationExecutionRecord; buildCacheConfiguration: BuildCacheConfiguration | undefined; - runner: IOperationRunner; rushProject: RushConfigurationProject; phase: IPhase; selectedPhases: Iterable; projectChangeAnalyzer: ProjectChangeAnalyzer; - commandName: string; commandToRun: string; terminal: ITerminal; trackedProjectFiles: string[] | undefined; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); if (!buildCacheContext.projectBuildCache) { if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { // Disable legacy skip logic if the build cache is in play @@ -606,6 +755,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const projectConfiguration: RushProjectConfiguration | undefined = await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); if (projectConfiguration) { + const commandName: string = phase.name; projectConfiguration.validatePhaseConfiguration(selectedPhases, terminal); if (projectConfiguration.disableBuildCacheForProject) { terminal.writeVerboseLine('Caching has been disabled for this project.'); @@ -681,10 +831,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Get a ProjectBuildCache only cache/restore log files private async _tryGetLogOnlyProjectBuildCacheAsync({ - runner, + record, rushProject, terminal, - commandName, commandToRun, buildCacheConfiguration, cobuildConfiguration, @@ -693,19 +842,19 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { projectChangeAnalyzer, operationMetadataManager }: { + record: OperationExecutionRecord; buildCacheConfiguration: BuildCacheConfiguration | undefined; cobuildConfiguration: CobuildConfiguration; - runner: IOperationRunner; rushProject: RushConfigurationProject; phase: IPhase; commandToRun: string; - commandName: string; terminal: ITerminal; trackedProjectFiles: string[] | undefined; projectChangeAnalyzer: ProjectChangeAnalyzer; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { // Disable legacy skip logic if the build cache is in play buildCacheContext.isSkipAllowed = false; @@ -724,6 +873,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { additionalContext.cobuildContextId = cobuildConfiguration.cobuildContextId; } if (projectConfiguration) { + const commandName: string = phase.name; const operationSettings: IOperationSettings | undefined = projectConfiguration.operationSettingsByOperationName.get(commandName); if (operationSettings) { @@ -773,18 +923,19 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private async _tryGetCobuildLockAsync({ cobuildConfiguration, - runner, + record, projectBuildCache, packageName, phaseName }: { cobuildConfiguration: CobuildConfiguration | undefined; - runner: IOperationRunner; + record: OperationExecutionRecord; projectBuildCache: ProjectBuildCache | undefined; packageName: string; phaseName: string; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); if (!buildCacheContext.cobuildLock) { if (projectBuildCache && cobuildConfiguration && cobuildConfiguration.cobuildEnabled) { if (!buildCacheContext.cobuildClusterId) { @@ -795,7 +946,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cobuildConfiguration, projectBuildCache, cobuildClusterId: buildCacheContext.cobuildClusterId, - lockExpireTimeInSeconds: ShellOperationRunner.periodicCallbackIntervalInSeconds * 3, + lockExpireTimeInSeconds: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 3, packageName, phaseName }); @@ -805,29 +956,27 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } private _getBuildCacheTerminal({ - runner, + record, buildCacheConfiguration, rushProject, - collatedWriter, logFilenameIdentifier, quietMode, debugMode }: { - runner: ShellOperationRunner; + record: OperationExecutionRecord; buildCacheConfiguration: BuildCacheConfiguration | undefined; rushProject: RushConfigurationProject; - collatedWriter: CollatedWriter; logFilenameIdentifier: string; quietMode: boolean; debugMode: boolean; }): ITerminal { - const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); if (!buildCacheContext.buildCacheTerminal) { buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ - runner, + record, buildCacheConfiguration, rushProject, - collatedWriter, logFilenameIdentifier, quietMode, debugMode @@ -835,10 +984,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } else if (!buildCacheContext.buildCacheProjectLogWritable?.isOpen) { // The ProjectLogWritable is closed, re-create one buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ - runner, + record, buildCacheConfiguration, rushProject, - collatedWriter, logFilenameIdentifier, quietMode, debugMode @@ -849,25 +997,31 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } private _createBuildCacheTerminal({ - runner, + record, buildCacheConfiguration, rushProject, - collatedWriter, logFilenameIdentifier, quietMode, debugMode }: { - runner: ShellOperationRunner; + record: OperationExecutionRecord; buildCacheConfiguration: BuildCacheConfiguration | undefined; rushProject: RushConfigurationProject; - collatedWriter: CollatedWriter; logFilenameIdentifier: string; quietMode: boolean; debugMode: boolean; }): ITerminal { + const silent: boolean = record.runner.silent; + if (silent) { + const nullTerminalProvider: NullTerminalProvider = new NullTerminalProvider(); + return new Terminal(nullTerminalProvider); + } + let cacheConsoleWritable: TerminalWritable; + // This creates the writer, only do this if necessary. + const collatedWriter: CollatedWriter = record.collatedWriter; const cacheProjectLogWritable: ProjectLogWritable | undefined = this._tryGetBuildCacheProjectLogWritable({ - runner, + record, buildCacheConfiguration, rushProject, collatedTerminal: collatedWriter.terminal, @@ -904,13 +1058,13 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private _tryGetBuildCacheProjectLogWritable({ buildCacheConfiguration, rushProject, - runner, + record, collatedTerminal, logFilenameIdentifier }: { buildCacheConfiguration: BuildCacheConfiguration | undefined; rushProject: RushConfigurationProject; - runner: ShellOperationRunner; + record: OperationExecutionRecord; collatedTerminal: CollatedTerminal; logFilenameIdentifier: string; }): ProjectLogWritable | undefined { @@ -918,7 +1072,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (!buildCacheConfiguration?.buildCacheEnabled) { return; } - const buildCacheContext: IOperationBuildCacheContext = this._getBuildCacheContextByRunnerOrThrow(runner); + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); buildCacheContext.buildCacheProjectLogWritable = new ProjectLogWritable( rushProject, collatedTerminal, diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 17f96946184..13871c68c19 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -54,16 +54,6 @@ export interface IOperationRunnerContext { */ error?: Error; - /** - * The set of operations that depend on this operation. - */ - readonly consumers: Set; - - /** - * The operation runner that is executing this operation. - */ - readonly runner: IOperationRunner; - /** * Normally the incremental build logic will rebuild changed projects as well as * any projects that directly or indirectly depend on a changed project. @@ -102,6 +92,11 @@ export interface IOperationRunner { */ warningsAreAllowed: boolean; + /** + * Full shell command string to run by this runner. + */ + commandToRun?: string; + /** * Method to be executed for the operation. */ diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 85450514ac2..1a4b348d51b 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -24,7 +24,7 @@ export interface IOperationExecutionManagerOptions { changedProjectsOnly: boolean; destination?: TerminalWritable; - beforeExecuteOperation?: (operation: OperationExecutionRecord) => Promise; + beforeExecuteOperation?: (operation: OperationExecutionRecord) => Promise; afterExecuteOperation?: (operation: OperationExecutionRecord) => Promise; onOperationStatusChanged?: (record: OperationExecutionRecord) => void; beforeExecuteOperations?: (records: Map) => Promise; @@ -61,7 +61,9 @@ export class OperationExecutionManager { private readonly _terminal: CollatedTerminal; - private readonly _beforeExecuteOperation?: (operation: OperationExecutionRecord) => Promise; + private readonly _beforeExecuteOperation?: ( + operation: OperationExecutionRecord + ) => Promise; private readonly _afterExecuteOperation?: (operation: OperationExecutionRecord) => Promise; private readonly _onOperationStatusChanged?: (record: OperationExecutionRecord) => void; private readonly _beforeExecuteOperations?: ( @@ -224,11 +226,17 @@ export class OperationExecutionManager { // This function is a callback because it may write to the collatedWriter before // operation.executeAsync returns (and cleans up the writer) - const onOperationComplete: (record: OperationExecutionRecord) => Promise = async ( + const onOperationCompleteAsync: (record: OperationExecutionRecord) => Promise = async ( record: OperationExecutionRecord ) => { - this._onOperationComplete(record); await this._afterExecuteOperation?.(record); + this._onOperationComplete(record); + }; + + const onOperationStartAsync: ( + record: OperationExecutionRecord + ) => Promise = async (record: OperationExecutionRecord) => { + return await this._beforeExecuteOperation?.(record); }; await Async.forEachAsync( @@ -252,8 +260,10 @@ export class OperationExecutionManager { // Fail to assign a operation, start over again return; } else { - await this._beforeExecuteOperation?.(record); - await record.executeAsync(onOperationComplete); + await record.executeAsync({ + onStart: onOperationStartAsync, + onResult: onOperationCompleteAsync + }); } }, { diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index d75ed3f968e..29e242e01dd 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -10,6 +10,8 @@ import { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import { Operation } from './Operation'; import { Stopwatch } from '../../utilities/Stopwatch'; import { OperationMetadataManager } from './OperationMetadataManager'; +import type { IPhase } from '../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; @@ -87,6 +89,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext { public readonly runner: IOperationRunner; public readonly weight: number; + public readonly associatedPhase: IPhase | undefined; + public readonly associatedProject: RushConfigurationProject | undefined; public readonly _operationMetadataManager: OperationMetadataManager | undefined; private readonly _context: IOperationExecutionRecordContext; @@ -94,16 +98,18 @@ export class OperationExecutionRecord implements IOperationRunnerContext { private _collatedWriter: CollatedWriter | undefined = undefined; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { - const { runner } = operation; + const { runner, associatedPhase, associatedProject } = operation; if (!runner) { throw new InternalError( - `Operation for phase '${operation.associatedPhase?.name}' and project '${operation.associatedProject?.packageName}' has no runner.` + `Operation for phase '${associatedPhase?.name}' and project '${associatedProject?.packageName}' has no runner.` ); } this.runner = runner; this.weight = operation.weight; + this.associatedPhase = associatedPhase; + this.associatedProject = associatedProject; if (operation.associatedPhase && operation.associatedProject) { this._operationMetadataManager = new OperationMetadataManager({ phase: operation.associatedPhase, @@ -147,13 +153,28 @@ export class OperationExecutionRecord implements IOperationRunnerContext { return this._operationMetadataManager?.stateFile.state?.cobuildRunnerId; } - public async executeAsync(onResult: (record: OperationExecutionRecord) => Promise): Promise { + public async executeAsync({ + onStart, + onResult + }: { + onStart: (record: OperationExecutionRecord) => Promise; + onResult: (record: OperationExecutionRecord) => Promise; + }): Promise { + if (this.status === OperationStatus.RemoteExecuting) { + this.stopwatch.reset(); + } this.status = OperationStatus.Executing; this.stopwatch.start(); this._context.onOperationStatusChanged?.(this); try { - this.status = await this.runner.executeAsync(this); + const earlyReturnStatus: OperationStatus | undefined = await onStart(this); + // When the operation status returns by the hook, bypass the runner execution. + if (earlyReturnStatus) { + this.status = earlyReturnStatus; + } else { + this.status = await this.runner.executeAsync(this); + } // Delegate global state reporting await onResult(this); } catch (error) { diff --git a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts b/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts deleted file mode 100644 index 7a9e526c327..00000000000 --- a/libraries/rush-lib/src/logic/operations/OperationRunnerHooks.ts +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { AsyncSeriesBailHook, AsyncSeriesWaterfallHook } from 'tapable'; - -import type { ITerminal } from '@rushstack/node-core-library'; -import type { IOperationRunnerContext } from './IOperationRunner'; -import type { OperationStatus } from './OperationStatus'; -import type { IProjectDeps, ShellOperationRunner } from './ShellOperationRunner'; -import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; -import type { IPhase } from '../../api/CommandLineConfiguration'; - -/** - * A plugin tht interacts with a operation runner - */ -export interface IOperationRunnerPlugin { - /** - * Applies this plugin. - */ - apply(hooks: OperationRunnerHooks): void; -} - -export interface IOperationRunnerBeforeExecuteContext { - context: IOperationRunnerContext; - runner: ShellOperationRunner; - terminal: ITerminal; - projectDeps: IProjectDeps | undefined; - lastProjectDeps: IProjectDeps | undefined; - trackedProjectFiles: string[] | undefined; - logPath: string; - errorLogPath: string; - rushProject: RushConfigurationProject; - phase: IPhase; - commandName: string; - commandToRun: string; - finallyCallbacks: (() => void)[]; -} - -export interface IOperationRunnerAfterExecuteContext { - context: IOperationRunnerContext; - terminal: ITerminal; - /** - * Exit code of the operation command - */ - exitCode: number; - status: OperationStatus; - taskIsSuccessful: boolean; - logPath: string; - errorLogPath: string; -} - -/** - * Hooks into the lifecycle of the operation runner - * - */ -export class OperationRunnerHooks { - public beforeExecute: AsyncSeriesBailHook< - IOperationRunnerBeforeExecuteContext, - OperationStatus | undefined - > = new AsyncSeriesBailHook( - ['beforeExecuteContext'], - 'beforeExecute' - ); - - public afterExecute: AsyncSeriesWaterfallHook = - new AsyncSeriesWaterfallHook( - ['afterExecuteContext'], - 'afterExecute' - ); -} diff --git a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts index adf338a1fbd..7c1ee1324a1 100644 --- a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts +++ b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts @@ -30,57 +30,20 @@ export class ProjectLogWritable extends TerminalWritable { this._project = project; this._terminal = terminal; - function getLogFilePaths( - projectFolder: string, - logFilenameIdentifier: string, - logFolder?: string - ): { - logPath: string; - errorLogPath: string; - relativeLogPath: string; - relativeErrorLogPath: string; - } { - const unscopedProjectName: string = PackageNameParsers.permissive.getUnscopedName(project.packageName); - const logFileBaseName: string = `${unscopedProjectName}.${logFilenameIdentifier}`; - const logFilename: string = `${logFileBaseName}.log`; - const errorLogFilename: string = `${logFileBaseName}.error.log`; - - const relativeLogPath: string = logFolder ? `${logFolder}/${logFilename}` : logFilename; - const relativeErrorLogPath: string = logFolder ? `${logFolder}/${errorLogFilename}` : errorLogFilename; - - const logPath: string = `${projectFolder}/${relativeLogPath}`; - const errorLogPath: string = `${projectFolder}/${relativeErrorLogPath}`; - - return { - logPath, - errorLogPath, - relativeLogPath, - relativeErrorLogPath - }; - } - - const projectFolder: string = this._project.projectFolder; // Delete the legacy logs - const { logPath: legacyLogPath, errorLogPath: legacyErrorLogPath } = getLogFilePaths( - projectFolder, - 'build' - ); + const { logPath: legacyLogPath, errorLogPath: legacyErrorLogPath } = ProjectLogWritable.getLogFilePaths({ + project, + logFilenameIdentifier: 'build', + isLegacyLog: true + }); FileSystem.deleteFile(legacyLogPath); FileSystem.deleteFile(legacyErrorLogPath); - // If the phased commands experiment is enabled, put logs under `rush-logs` - let logFolder: string | undefined; - if (project.rushConfiguration.experimentsConfiguration.configuration.phasedCommands) { - const logPathPrefix: string = `${projectFolder}/${RushConstants.rushLogsFolderName}`; - FileSystem.ensureFolder(logPathPrefix); - logFolder = RushConstants.rushLogsFolderName; - } - - const { logPath, errorLogPath, relativeLogPath, relativeErrorLogPath } = getLogFilePaths( - projectFolder, - logFilenameIdentifier, - logFolder - ); + const { logPath, errorLogPath, relativeLogPath, relativeErrorLogPath } = + ProjectLogWritable.getLogFilePaths({ + project, + logFilenameIdentifier + }); this.logPath = logPath; this.errorLogPath = errorLogPath; this.relativeLogPath = relativeLogPath; @@ -97,6 +60,49 @@ export class ProjectLogWritable extends TerminalWritable { this._logWriter = FileWriter.open(this.logPath); } + public static getLogFilePaths({ + project, + logFilenameIdentifier, + isLegacyLog = false + }: { + project: RushConfigurationProject; + logFilenameIdentifier: string; + isLegacyLog?: boolean; + }): { + logPath: string; + errorLogPath: string; + relativeLogPath: string; + relativeErrorLogPath: string; + } { + const unscopedProjectName: string = PackageNameParsers.permissive.getUnscopedName(project.packageName); + const logFileBaseName: string = `${unscopedProjectName}.${logFilenameIdentifier}`; + const logFilename: string = `${logFileBaseName}.log`; + const errorLogFilename: string = `${logFileBaseName}.error.log`; + + const { projectFolder } = project; + + // If the phased commands experiment is enabled, put logs under `rush-logs` + let logFolder: string | undefined; + if (!isLegacyLog && project.rushConfiguration.experimentsConfiguration.configuration.phasedCommands) { + const logPathPrefix: string = `${projectFolder}/${RushConstants.rushLogsFolderName}`; + FileSystem.ensureFolder(logPathPrefix); + logFolder = RushConstants.rushLogsFolderName; + } + + const relativeLogPath: string = logFolder ? `${logFolder}/${logFilename}` : logFilename; + const relativeErrorLogPath: string = logFolder ? `${logFolder}/${errorLogFilename}` : errorLogFilename; + + const logPath: string = `${projectFolder}/${relativeLogPath}`; + const errorLogPath: string = `${projectFolder}/${relativeErrorLogPath}`; + + return { + logPath, + errorLogPath, + relativeLogPath, + relativeErrorLogPath + }; + } + protected onWriteChunk(chunk: ITerminalChunk): void { if (!this._logWriter) { throw new InternalError('Output file was closed'); diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 06e72e1ea60..4ada66c5b45 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -2,16 +2,7 @@ // See LICENSE in the project root for license information. import * as child_process from 'child_process'; -import * as path from 'path'; -import { - JsonFile, - Text, - FileSystem, - NewlineKind, - InternalError, - Terminal, - ColorValue -} from '@rushstack/node-core-library'; +import { Text, NewlineKind, InternalError, Terminal } from '@rushstack/node-core-library'; import { TerminalChunkKind, TextRewriterTransform, @@ -28,23 +19,12 @@ import { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import { ProjectLogWritable } from './ProjectLogWritable'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; -import { PeriodicCallback } from './PeriodicCallback'; -import { - IOperationRunnerAfterExecuteContext, - IOperationRunnerBeforeExecuteContext, - OperationRunnerHooks -} from './OperationRunnerHooks'; import type { RushConfiguration } from '../../api/RushConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; import type { IPhase } from '../../api/CommandLineConfiguration'; -export interface IProjectDeps { - files: { [filePath: string]: string }; - arguments: string; -} - export interface IOperationRunnerOptions { rushProject: RushConfigurationProject; rushConfiguration: RushConfiguration; @@ -70,40 +50,22 @@ export class ShellOperationRunner implements IOperationRunner { public readonly silent: boolean = false; public readonly warningsAreAllowed: boolean; - public readonly hooks: OperationRunnerHooks; - public readonly periodicCallback: PeriodicCallback; - public readonly logFilenameIdentifier: string; - public static readonly periodicCallbackIntervalInSeconds: number = 10; + public readonly commandToRun: string; + private readonly _logFilenameIdentifier: string; private readonly _rushProject: RushConfigurationProject; - private readonly _phase: IPhase; private readonly _rushConfiguration: RushConfiguration; - private readonly _commandName: string; - private readonly _commandToRun: string; - private readonly _projectChangeAnalyzer: ProjectChangeAnalyzer; - private readonly _packageDepsFilename: string; - private readonly _selectedPhases: Iterable; public constructor(options: IOperationRunnerOptions) { const { phase } = options; this.name = options.displayName; this._rushProject = options.rushProject; - this._phase = phase; this._rushConfiguration = options.rushConfiguration; - this._commandName = phase.name; - this._commandToRun = options.commandToRun; - this._projectChangeAnalyzer = options.projectChangeAnalyzer; - this._packageDepsFilename = `package-deps_${phase.logFilenameIdentifier}.json`; + this.commandToRun = options.commandToRun; this.warningsAreAllowed = EnvironmentConfiguration.allowWarningsInSuccessfulBuild || phase.allowWarningsOnSuccess || false; - this.logFilenameIdentifier = phase.logFilenameIdentifier; - this._selectedPhases = options.selectedPhases; - - this.hooks = new OperationRunnerHooks(); - this.periodicCallback = new PeriodicCallback({ - interval: ShellOperationRunner.periodicCallbackIntervalInSeconds * 1000 - }); + this._logFilenameIdentifier = phase.logFilenameIdentifier; } public async executeAsync(context: IOperationRunnerContext): Promise { @@ -118,11 +80,9 @@ export class ShellOperationRunner implements IOperationRunner { const projectLogWritable: ProjectLogWritable = new ProjectLogWritable( this._rushProject, context.collatedWriter.terminal, - this.logFilenameIdentifier + this._logFilenameIdentifier ); - const finallyCallbacks: (() => {})[] = []; - try { //#region OPERATION LOGGING // TERMINAL PIPELINE: @@ -170,102 +130,12 @@ export class ShellOperationRunner implements IOperationRunner { let hasWarningOrError: boolean = false; const projectFolder: string = this._rushProject.projectFolder; - let lastProjectDeps: IProjectDeps | undefined = undefined; - - const currentDepsPath: string = path.join( - this._rushProject.projectRushTempFolder, - this._packageDepsFilename - ); - - if (FileSystem.exists(currentDepsPath)) { - try { - lastProjectDeps = JsonFile.load(currentDepsPath); - } catch (e) { - // Warn and ignore - treat failing to load the file as the project being not built. - terminal.writeWarningLine( - `Warning: error parsing ${this._packageDepsFilename}: ${e}. Ignoring and ` + - `treating the command "${this._commandToRun}" as not run.` - ); - } - } - - let projectDeps: IProjectDeps | undefined; - let trackedProjectFiles: string[] | undefined; - try { - const fileHashes: Map | undefined = - await this._projectChangeAnalyzer._tryGetProjectDependenciesAsync(this._rushProject, terminal); - - if (fileHashes) { - const files: { [filePath: string]: string } = {}; - trackedProjectFiles = []; - for (const [filePath, fileHash] of fileHashes) { - files[filePath] = fileHash; - trackedProjectFiles.push(filePath); - } - - projectDeps = { - files, - arguments: this._commandToRun - }; - } - } catch (error) { - // To test this code path: - // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" - terminal.writeLine('Unable to calculate incremental state: ' + (error as Error).toString()); - terminal.writeLine({ - text: 'Rush will proceed without incremental execution, caching, and change detection.', - foregroundColor: ColorValue.Cyan - }); - } - - const beforeExecuteContext: IOperationRunnerBeforeExecuteContext = { - context, - runner: this, - terminal, - projectDeps, - lastProjectDeps, - trackedProjectFiles, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath, - rushProject: this._rushProject, - phase: this._phase, - commandName: this._commandName, - commandToRun: this._commandToRun, - finallyCallbacks - }; - - const earlyReturnStatus: OperationStatus | undefined = await this.hooks.beforeExecute.promise( - beforeExecuteContext - ); - if (earlyReturnStatus) { - return earlyReturnStatus; - } - - // If the deps file exists, remove it before starting execution. - FileSystem.deleteFile(currentDepsPath); - - // TODO: Remove legacyDepsPath with the next major release of Rush - const legacyDepsPath: string = path.join(this._rushProject.projectFolder, 'package-deps.json'); - // Delete the legacy package-deps.json - FileSystem.deleteFile(legacyDepsPath); - - if (!this._commandToRun) { - // Write deps on success. - if (projectDeps) { - JsonFile.save(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - } - - return OperationStatus.Success; - } // Run the operation - terminal.writeLine('Invoking: ' + this._commandToRun); - this.periodicCallback.start(); + terminal.writeLine('Invoking: ' + this.commandToRun); const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( - this._commandToRun, + this.commandToRun, { rushConfiguration: this._rushConfiguration, workingDirectory: projectFolder, @@ -292,14 +162,12 @@ export class ShellOperationRunner implements IOperationRunner { }); } - let exitCode: number = 1; let status: OperationStatus = await new Promise( (resolve: (status: OperationStatus) => void, reject: (error: OperationError) => void) => { subProcess.on('close', (code: number) => { - exitCode = code; try { if (code !== 0) { - // Do NOT reject here immediately, give a chance for hooks to suppress the error + // Do NOT reject here immediately, give a chance for other logic to suppress the error context.error = new OperationError('error', `Returned error code: ${code}`); resolve(OperationStatus.Failure); } else if (hasWarningOrError) { @@ -323,39 +191,6 @@ export class ShellOperationRunner implements IOperationRunner { throw new InternalError('The output file handle was not closed'); } - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - this.warningsAreAllowed && - !!this._rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - - if (taskIsSuccessful && projectDeps) { - // Write deps on success. - await JsonFile.saveAsync(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - } - - const afterExecuteContext: IOperationRunnerAfterExecuteContext = { - context, - terminal, - exitCode, - status, - taskIsSuccessful, - logPath: projectLogWritable.logPath, - errorLogPath: projectLogWritable.errorLogPath - }; - - await this.hooks.afterExecute.promise(afterExecuteContext); - - if (context.error) { - throw context.error; - } - - // Sync the status in case it was changed by the hook - status = afterExecuteContext.status; - if (terminalProvider.hasErrors) { status = OperationStatus.Failure; } @@ -363,10 +198,6 @@ export class ShellOperationRunner implements IOperationRunner { return status; } finally { projectLogWritable.close(); - this.periodicCallback.stop(); - for (const callback of finallyCallbacks) { - callback(); - } } } } diff --git a/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts index 96fe059ee21..8518e53eb86 100644 --- a/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts @@ -21,7 +21,7 @@ interface ISerializedOperation { function serializeOperation(operation: Operation): ISerializedOperation { return { name: operation.name!, - commandToRun: (operation.runner as ShellOperationRunner)['_commandToRun'] + commandToRun: (operation.runner as ShellOperationRunner).commandToRun }; } diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 2c151623b8b..49c5372b530 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { AsyncSeriesHook, AsyncSeriesWaterfallHook, SyncHook } from 'tapable'; +import { AsyncSeriesBailHook, AsyncSeriesHook, AsyncSeriesWaterfallHook, SyncHook } from 'tapable'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import type { BuildCacheConfiguration } from '../api/BuildCacheConfiguration'; @@ -17,6 +17,7 @@ import type { import type { CobuildConfiguration } from '../api/CobuildConfiguration'; import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; import type { ITelemetryData } from '../logic/Telemetry'; +import type { OperationStatus } from '../logic/operations/OperationStatus'; /** * A plugin that interacts with a phased commands. @@ -125,9 +126,13 @@ export class PhasedCommandHooks { /** * Hook invoked before executing a operation. */ - public readonly beforeExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]> = new AsyncSeriesHook< - [IOperationRunnerContext] - >(['runnerContext'], 'beforeExecuteOperation'); + public readonly beforeExecuteOperation: AsyncSeriesBailHook< + [IOperationRunnerContext], + OperationStatus | undefined + > = new AsyncSeriesBailHook<[IOperationRunnerContext], OperationStatus | undefined>( + ['runnerContext'], + 'beforeExecuteOperation' + ); /** * Hook invoked after executing a operation. diff --git a/libraries/rush-lib/src/utilities/NullTerminalProvider.ts b/libraries/rush-lib/src/utilities/NullTerminalProvider.ts new file mode 100644 index 00000000000..5850747a0cd --- /dev/null +++ b/libraries/rush-lib/src/utilities/NullTerminalProvider.ts @@ -0,0 +1,10 @@ +import type { ITerminalProvider } from '@rushstack/node-core-library'; + +/** + * A terminal provider like /dev/null + */ +export class NullTerminalProvider implements ITerminalProvider { + public supportsColor: boolean = false; + public eolCharacter: string = '\n'; + public write(): void {} +} From 18901443c510568cb7f863067292591d4614d1cc Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 10 Aug 2023 20:16:14 +0800 Subject: [PATCH 078/100] fix: missing export --- common/reviews/api/rush-lib.api.md | 4 +++- common/reviews/api/rush-redis-cobuild-plugin.api.md | 6 +++++- rush-plugins/rush-redis-cobuild-plugin/src/index.ts | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 90ebc448947..646dff183b2 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -839,7 +839,9 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage export class PhasedCommandHooks { readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; - readonly beforeExecuteOperation: AsyncSeriesBailHook<[IOperationRunnerContext], OperationStatus | undefined>; + readonly beforeExecuteOperation: AsyncSeriesBailHook<[ + IOperationRunnerContext + ], OperationStatus | undefined>; readonly beforeExecuteOperations: AsyncSeriesHook<[ Map, ICreateOperationsContext diff --git a/common/reviews/api/rush-redis-cobuild-plugin.api.md b/common/reviews/api/rush-redis-cobuild-plugin.api.md index c59f483e00a..110494810c5 100644 --- a/common/reviews/api/rush-redis-cobuild-plugin.api.md +++ b/common/reviews/api/rush-redis-cobuild-plugin.api.md @@ -19,6 +19,11 @@ export interface IRedisCobuildLockProviderOptions extends RedisClientOptions { passwordEnvironmentVariable?: string; } +// Warning: (ae-incompatible-release-tags) The symbol "IRushRedisCobuildPluginOptions" is marked as @public, but its signature references "IRedisCobuildLockProviderOptions" which is marked as @beta +// +// @public (undocumented) +export type IRushRedisCobuildPluginOptions = IRedisCobuildLockProviderOptions; + // @beta (undocumented) export class RedisCobuildLockProvider implements ICobuildLockProvider { constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession); @@ -39,7 +44,6 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { // @public (undocumented) class RushRedisCobuildPlugin implements IRushPlugin { - // Warning: (ae-forgotten-export) The symbol "IRushRedisCobuildPluginOptions" needs to be exported by the entry point index.d.ts constructor(options: IRushRedisCobuildPluginOptions); // (undocumented) apply(rushSession: RushSession, rushConfiguration: RushConfiguration): void; diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts index b07442ac86d..77bf9156355 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/index.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/index.ts @@ -1,8 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { RushRedisCobuildPlugin } from './RushRedisCobuildPlugin'; +import { RushRedisCobuildPlugin, IRushRedisCobuildPluginOptions } from './RushRedisCobuildPlugin'; export default RushRedisCobuildPlugin; export { RedisCobuildLockProvider } from './RedisCobuildLockProvider'; export { IRedisCobuildLockProviderOptions } from './RedisCobuildLockProvider'; +export { IRushRedisCobuildPluginOptions }; From fceb88badf01ae1dd2d9ac8d757a4df6b98921e7 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 10 Aug 2023 20:46:58 +0800 Subject: [PATCH 079/100] fix: Noop command --- .../rush-lib/src/logic/operations/CacheableOperationPlugin.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 96ef0cd3b58..a598548b4a9 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -475,6 +475,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Delete the legacy package-deps.json FileSystem.deleteFile(legacyDepsPath); + // No-op command if (!commandToRun) { // Write deps on success. if (projectDeps) { @@ -482,6 +483,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { ensureFolderExists: true }); } + return OperationStatus.Success; } }; From 3c1978bda090121bc245dba80df4be326a3c2c74 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 10 Aug 2023 23:41:26 +0800 Subject: [PATCH 080/100] fix: Noop command --- .../repo/common/config/rush/pnpm-lock.yaml | 6 + .../repo/projects/h/config/rush-project.json | 16 ++ .../sandbox/repo/projects/h/package.json | 12 ++ .../sandbox/repo/rush.json | 4 + .../operations/CacheableOperationPlugin.ts | 144 +++++++++--------- 5 files changed, 109 insertions(+), 73 deletions(-) create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/config/rush-project.json create mode 100644 build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/package.json diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml index 36f86a80a56..98b22e9d424 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/pnpm-lock.yaml @@ -44,3 +44,9 @@ importers: b: workspace:* dependencies: b: link:../b + + ../../projects/h: + specifiers: + a: workspace:* + dependencies: + a: link:../a diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/config/rush-project.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/config/rush-project.json new file mode 100644 index 00000000000..3206537349d --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/config/rush-project.json @@ -0,0 +1,16 @@ +{ + "operationSettings": [ + { + "operationName": "_phase:build", + "outputFolderNames": ["dist"] + }, + { + "operationName": "cobuild", + "outputFolderNames": ["dist"] + }, + { + "operationName": "build", + "outputFolderNames": ["dist"] + } + ] +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/package.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/package.json new file mode 100644 index 00000000000..cb74fb60a7d --- /dev/null +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/projects/h/package.json @@ -0,0 +1,12 @@ +{ + "name": "h", + "version": "1.0.0", + "scripts": { + "cobuild": "", + "build": "", + "_phase:build": "" + }, + "dependencies": { + "a": "workspace:*" + } +} diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json index 70c6c4890de..012281bea0d 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/rush.json @@ -32,6 +32,10 @@ { "packageName": "g", "projectFolder": "projects/g" + }, + { + "packageName": "h", + "projectFolder": "projects/h" } ] } diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index a598548b4a9..323f7180d28 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -68,8 +68,7 @@ export interface IOperationBuildCacheContext { periodicCallback: PeriodicCallback; projectDeps: IProjectDeps | undefined; currentDepsPath: string | undefined; - // True if a operation status has been return by the beforeExecutionOperation callback function - hasReturnedStatus: boolean; + cacheRestored: boolean; } export class CacheableOperationPlugin implements IPhasedCommandPlugin { @@ -119,7 +118,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }), projectDeps: undefined, currentDepsPath: undefined, - hasReturnedStatus: false + cacheRestored: false }; // Upstream runners may mutate the property of build cache context for downstream runners this._buildCacheContextByOperationExecutionRecord.set(record, buildCacheContext); @@ -308,6 +307,25 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); } + // If the deps file exists, remove it before starting execution. + FileSystem.deleteFile(currentDepsPath); + + // TODO: Remove legacyDepsPath with the next major release of Rush + const legacyDepsPath: string = path.join(project.projectFolder, 'package-deps.json'); + // Delete the legacy package-deps.json + FileSystem.deleteFile(legacyDepsPath); + + // No-op command + if (!commandToRun) { + // Write deps on success. + if (projectDeps) { + JsonFile.save(projectDeps, currentDepsPath, { + ensureFolderExists: true + }); + } + return OperationStatus.NoOp; + } + let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ record, buildCacheConfiguration, @@ -388,6 +406,23 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { project, logFilenameIdentifier: phase.logFilenameIdentifier }); + const restoreCacheAsync = async ( + projectBuildCache: ProjectBuildCache | undefined, + specifiedCacheId?: string + ): Promise => { + const restoreFromCacheSuccess: boolean | undefined = + await projectBuildCache?.tryRestoreFromCacheAsync(buildCacheTerminal, specifiedCacheId); + if (restoreFromCacheSuccess) { + // Restore the original state of the operation without cache + await operationMetadataManager?.tryRestoreAsync({ + terminal: buildCacheTerminal, + logPath, + errorLogPath + }); + buildCacheContext.cacheRestored = true; + } + return Boolean(restoreFromCacheSuccess); + }; if (cobuildLock) { // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to // the build cache once completed, but does not retrieve them (since the "incremental" @@ -398,16 +433,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (cobuildCompletedState) { const { status, cacheId } = cobuildCompletedState; - const restoreFromCacheSuccess: boolean | undefined = - await cobuildLock.projectBuildCache.tryRestoreFromCacheAsync(buildCacheTerminal, cacheId); + const restoreFromCacheSuccess: boolean = await restoreCacheAsync( + cobuildLock.projectBuildCache, + cacheId + ); if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await operationMetadataManager?.tryRestoreAsync({ - terminal: buildCacheTerminal, - logPath, - errorLogPath - }); if (cobuildCompletedState) { return cobuildCompletedState.status; } @@ -416,16 +447,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } else if (buildCacheContext.isCacheReadAllowed) { buildCacheReadAttempted = !!projectBuildCache; - const restoreFromCacheSuccess: boolean | undefined = - await projectBuildCache?.tryRestoreFromCacheAsync(buildCacheTerminal); + const restoreFromCacheSuccess: boolean = await restoreCacheAsync(projectBuildCache); if (restoreFromCacheSuccess) { - // Restore the original state of the operation without cache - await operationMetadataManager?.tryRestoreAsync({ - terminal: buildCacheTerminal, - logPath, - errorLogPath - }); return OperationStatus.FromCache; } } @@ -466,25 +490,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return OperationStatus.RemoteExecuting; } } - - // If the deps file exists, remove it before starting execution. - FileSystem.deleteFile(currentDepsPath); - - // TODO: Remove legacyDepsPath with the next major release of Rush - const legacyDepsPath: string = path.join(project.projectFolder, 'package-deps.json'); - // Delete the legacy package-deps.json - FileSystem.deleteFile(legacyDepsPath); - - // No-op command - if (!commandToRun) { - // Write deps on success. - if (projectDeps) { - JsonFile.save(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - } - return OperationStatus.Success; - } }; try { @@ -499,9 +504,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheContext, record }); - if (earlyReturnStatus) { - buildCacheContext.hasReturnedStatus = true; - } return earlyReturnStatus; } catch (e) { buildCacheContext.buildCacheProjectLogWritable?.close(); @@ -524,7 +526,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { associatedPhase: phase } = record; - if (!project || !phase) { + if (!project || !phase || record.status === OperationStatus.NoOp) { return; } @@ -541,30 +543,26 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { buildCacheTerminal, projectDeps, currentDepsPath, - hasReturnedStatus: hasStatusReturned + cacheRestored } = buildCacheContext; - if (hasStatusReturned) { - // Status has been returned in beforeExecutionOperation hook, skip the following logic. - return; - } - try { - const { logFilenameIdentifier } = phase; - - // Save the metadata to disk - const { duration: durationInSeconds } = stopwatch; - const { logPath, errorLogPath } = ProjectLogWritable.getLogFilePaths({ - project, - logFilenameIdentifier - }); - await operationMetadataManager?.saveAsync({ - durationInSeconds, - cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, - cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, - logPath, - errorLogPath - }); + if (!cacheRestored) { + // Save the metadata to disk + const { logFilenameIdentifier } = phase; + const { duration: durationInSeconds } = stopwatch; + const { logPath, errorLogPath } = ProjectLogWritable.getLogFilePaths({ + project, + logFilenameIdentifier + }); + await operationMetadataManager?.saveAsync({ + durationInSeconds, + cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, + cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, + logPath, + errorLogPath + }); + } if (!buildCacheTerminal) { // This should not happen @@ -572,7 +570,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; - let setCacheEntryPromise: Promise | undefined; + let setCacheEntryPromise: (() => Promise | undefined) | undefined; if (cobuildLock && isCacheWriteAllowed) { if (record.error) { // In order to prevent the worst case that all cobuild tasks go through the same failure, @@ -599,10 +597,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { cacheId: finalCacheId }); }; - setCacheEntryPromise = cobuildLock.projectBuildCache.trySetCacheEntryAsync( - buildCacheTerminal, - finalCacheId - ); + setCacheEntryPromise = () => + cobuildLock.projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal, finalCacheId); } } } @@ -624,13 +620,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && projectBuildCache) { - setCacheEntryPromise = projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal); + setCacheEntryPromise = () => projectBuildCache.trySetCacheEntryAsync(buildCacheTerminal); } - const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise; - await setCompletedStatePromiseFunction?.(); + if (!cacheRestored) { + const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise?.(); + await setCompletedStatePromiseFunction?.(); - if (cacheWriteSuccess === false && status === OperationStatus.Success) { - record.status = OperationStatus.SuccessWithWarning; + if (cacheWriteSuccess === false && status === OperationStatus.Success) { + record.status = OperationStatus.SuccessWithWarning; + } } // Status changes to direct dependents From b61235b8293789b099e00e40c649772bec3b082c Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 11 Aug 2023 21:25:00 +0800 Subject: [PATCH 081/100] feat: throw clear error when writing in the closed project log writer --- .../rush-lib/src/logic/operations/ProjectLogWritable.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts index 7c1ee1324a1..0f0591167bb 100644 --- a/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts +++ b/libraries/rush-lib/src/logic/operations/ProjectLogWritable.ts @@ -103,6 +103,15 @@ export class ProjectLogWritable extends TerminalWritable { }; } + // Override writeChunk function to throw custom error + public writeChunk(chunk: ITerminalChunk): void { + if (!this._logWriter) { + throw new InternalError(`Log writer was closed for ${this.logPath}`); + } + // Stderr can always get written to a error log writer + super.writeChunk(chunk); + } + protected onWriteChunk(chunk: ITerminalChunk): void { if (!this._logWriter) { throw new InternalError('Output file was closed'); From ba583071e2ee3d8b39f0fe8dc6e705126aff2ba3 Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 11 Aug 2023 21:47:05 +0800 Subject: [PATCH 082/100] chore: use runner.name to sort operation execution records --- .../rush-lib/src/logic/operations/CacheableOperationPlugin.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 323f7180d28..acdcffc2161 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -162,8 +162,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. const groupedRecords: OperationExecutionRecord[] = Array.from(set); Sort.sortBy(groupedRecords, (record: OperationExecutionRecord) => { - const { associatedProject, associatedPhase } = record; - return `${associatedProject?.packageName}${RushConstants.hashDelimiter}${associatedPhase?.name}`; + return record.runner.name; }); // Generates cluster id, cluster id comes from the project folder and phase name of all operations in the same cluster. From 19206d20d7126dadc6c9e3647f6fb9602f11b920 Mon Sep 17 00:00:00 2001 From: Cheng Date: Sat, 12 Aug 2023 00:50:42 +0800 Subject: [PATCH 083/100] fix: cache execution logic --- .../operations/CacheableOperationPlugin.ts | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index acdcffc2161..e35b63e4baa 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -472,6 +472,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { ); if (isPackageUnchanged) { + // Pretend the cache restored when skip + buildCacheContext.cacheRestored = true; return OperationStatus.Skipped; } } @@ -525,10 +527,21 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { associatedPhase: phase } = record; - if (!project || !phase || record.status === OperationStatus.NoOp) { + if (!project || !phase) { return; } + // No need to run for the following operation status + switch (record.status) { + case OperationStatus.NoOp: + case OperationStatus.RemoteExecuting: { + return; + } + default: { + break; + } + } + const buildCacheContext: IOperationBuildCacheContext | undefined = this._getBuildCacheContextByOperationExecutionRecord(record); @@ -571,16 +584,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; let setCacheEntryPromise: (() => Promise | undefined) | undefined; if (cobuildLock && isCacheWriteAllowed) { - if (record.error) { - // In order to prevent the worst case that all cobuild tasks go through the same failure, - // allowing a failing build to be cached and retrieved, print the error message to the terminal - // and clear the error in context. - const message: string | undefined = record.error?.message; - if (message) { - record.collatedWriter.terminal.writeStderrLine(message); - } - record.error = undefined; - } const { cacheId, contextId } = cobuildLock.cobuildContext; const finalCacheId: string = @@ -980,7 +983,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { quietMode, debugMode }); - } else if (!buildCacheContext.buildCacheProjectLogWritable?.isOpen) { + } else if (buildCacheContext.buildCacheProjectLogWritable?.isOpen === false) { // The ProjectLogWritable is closed, re-create one buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ record, From 9b6b27597b790b5de8bef525cb991e2ae7a24682 Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:12:02 -0700 Subject: [PATCH 084/100] Fix build failure --- libraries/load-themed-styles/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/load-themed-styles/src/index.ts b/libraries/load-themed-styles/src/index.ts index 3bd083f2de6..e0c361f7f3e 100644 --- a/libraries/load-themed-styles/src/index.ts +++ b/libraries/load-themed-styles/src/index.ts @@ -221,7 +221,9 @@ export function flush(): void { * register async loadStyles */ function asyncLoadStyles(): number { - return setTimeout(() => { + // Use "self" to distinguish conflicting global typings for setTimeout() from lib.dom.d.ts vs Jest's @types/node + // https://github.com/jestjs/jest/issues/14418 + return self.setTimeout(() => { _themeState.runState.flushTimer = 0; flush(); }, 0); From c3f956505cae244949db939ea6475d773ff39cdc Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:18:03 -0700 Subject: [PATCH 085/100] rush change --- .../feat-cobuild_2023-08-16-03-17.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@microsoft/load-themed-styles/feat-cobuild_2023-08-16-03-17.json diff --git a/common/changes/@microsoft/load-themed-styles/feat-cobuild_2023-08-16-03-17.json b/common/changes/@microsoft/load-themed-styles/feat-cobuild_2023-08-16-03-17.json new file mode 100644 index 00000000000..0a43a4d60f5 --- /dev/null +++ b/common/changes/@microsoft/load-themed-styles/feat-cobuild_2023-08-16-03-17.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/load-themed-styles", + "comment": "Use self.setTimeout() instead of setTimeout() to work around a Jest regression", + "type": "patch" + } + ], + "packageName": "@microsoft/load-themed-styles" +} \ No newline at end of file From 32b01df39f58b3f306d1b0f7141baff4fdea130d Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:38:02 -0700 Subject: [PATCH 086/100] Regenerate README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 231781b1c82..9d1365fa499 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ These GitHub repositories provide supplementary resources for Rush Stack: | [/rush-plugins/rush-amazon-s3-build-cache-plugin](./rush-plugins/rush-amazon-s3-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-amazon-s3-build-cache-plugin) | | [@rushstack/rush-amazon-s3-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-amazon-s3-build-cache-plugin) | | [/rush-plugins/rush-azure-storage-build-cache-plugin](./rush-plugins/rush-azure-storage-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-azure-storage-build-cache-plugin) | | [@rushstack/rush-azure-storage-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-azure-storage-build-cache-plugin) | | [/rush-plugins/rush-http-build-cache-plugin](./rush-plugins/rush-http-build-cache-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-http-build-cache-plugin) | | [@rushstack/rush-http-build-cache-plugin](https://www.npmjs.com/package/@rushstack/rush-http-build-cache-plugin) | -| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | [changelog](./rush-plugins/rush-redis-cobuild-plugin/CHANGELOG.md) | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | +| [/rush-plugins/rush-redis-cobuild-plugin](./rush-plugins/rush-redis-cobuild-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-redis-cobuild-plugin) | | [@rushstack/rush-redis-cobuild-plugin](https://www.npmjs.com/package/@rushstack/rush-redis-cobuild-plugin) | | [/rush-plugins/rush-serve-plugin](./rush-plugins/rush-serve-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Frush-serve-plugin) | | [@rushstack/rush-serve-plugin](https://www.npmjs.com/package/@rushstack/rush-serve-plugin) | | [/webpack/hashed-folder-copy-plugin](./webpack/hashed-folder-copy-plugin/) | [![npm version](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin.svg)](https://badge.fury.io/js/%40rushstack%2Fhashed-folder-copy-plugin) | [changelog](./webpack/hashed-folder-copy-plugin/CHANGELOG.md) | [@rushstack/hashed-folder-copy-plugin](https://www.npmjs.com/package/@rushstack/hashed-folder-copy-plugin) | | [/webpack/loader-load-themed-styles](./webpack/loader-load-themed-styles/) | [![npm version](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles.svg)](https://badge.fury.io/js/%40microsoft%2Floader-load-themed-styles) | [changelog](./webpack/loader-load-themed-styles/CHANGELOG.md) | [@microsoft/loader-load-themed-styles](https://www.npmjs.com/package/@microsoft/loader-load-themed-styles) | From 42345f1a7658f543862cac4ed38dc7a30e28f010 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 17 Aug 2023 01:54:07 -0700 Subject: [PATCH 087/100] TEMPORARY - attempt to highlight a regression. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4715a50f4d..e4945ecdf3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: # run: /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & - name: Rush retest (install-run-rush) - run: node common/scripts/install-run-rush.js retest --verbose --production + run: node common/scripts/install-run-rush.js retest --verbose --production --to rush --to repo-toolbox env: # Prevent time-based browserslist update warning # See https://github.com/microsoft/rushstack/issues/2981 @@ -61,6 +61,10 @@ jobs: - name: Ensure repo README is up-to-date run: node repo-scripts/repo-toolbox/lib/start.js readme --verify + - if: runner.os == 'Windows' + name: Delete the build cache + run: Remove-Item -LiteralPath "common/temp/build-cache" -Force -Recurse + - name: Rush test (rush-lib) run: node apps/rush/lib/start-dev.js test --verbose --production --timeline env: From 616dee25ef7cd5e7eb55706d24bf10806eb05026 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 24 Aug 2023 13:11:16 +0800 Subject: [PATCH 088/100] fix: log cache/restore in Windows OS --- .../logic/operations/CacheableOperationPlugin.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index e35b63e4baa..50362ee168c 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -11,11 +11,12 @@ import { ITerminal, JsonFile, JsonObject, + NewlineKind, Sort, Terminal } from '@rushstack/node-core-library'; import { CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; -import { DiscardStdoutTransform, PrintUtilities } from '@rushstack/terminal'; +import { DiscardStdoutTransform, PrintUtilities, TextRewriterTransform } from '@rushstack/terminal'; import { SplitterTransform, TerminalWritable } from '@rushstack/terminal'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; @@ -412,13 +413,13 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const restoreFromCacheSuccess: boolean | undefined = await projectBuildCache?.tryRestoreFromCacheAsync(buildCacheTerminal, specifiedCacheId); if (restoreFromCacheSuccess) { + buildCacheContext.cacheRestored = true; // Restore the original state of the operation without cache await operationMetadataManager?.tryRestoreAsync({ terminal: buildCacheTerminal, logPath, errorLogPath }); - buildCacheContext.cacheRestored = true; } return Boolean(restoreFromCacheSuccess); }; @@ -1031,9 +1032,15 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); if (quietMode) { - cacheConsoleWritable = new DiscardStdoutTransform({ + const discardTransform: DiscardStdoutTransform = new DiscardStdoutTransform({ destination: collatedWriter }); + const normalizeNewlineTransform: TextRewriterTransform = new TextRewriterTransform({ + destination: discardTransform, + normalizeNewlines: NewlineKind.Lf, + ensureNewlineAtEnd: true + }); + cacheConsoleWritable = normalizeNewlineTransform; } else { cacheConsoleWritable = collatedWriter; } From 219bc4d667a4bb609f5bd1527d4d4ae4d8785101 Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 24 Aug 2023 16:33:58 +0800 Subject: [PATCH 089/100] feat: report unhandled exception from afterExecutionOperation --- .../logic/operations/OperationExecutionManager.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts index 1a4b348d51b..b46dc315caf 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts @@ -229,7 +229,17 @@ export class OperationExecutionManager { const onOperationCompleteAsync: (record: OperationExecutionRecord) => Promise = async ( record: OperationExecutionRecord ) => { - await this._afterExecuteOperation?.(record); + try { + await this._afterExecuteOperation?.(record); + } catch (e) { + // Failed operations get reported here + const message: string | undefined = record.error?.message; + if (message) { + this._terminal.writeStderrLine('Unhandled exception: '); + this._terminal.writeStderrLine(message); + } + throw e; + } this._onOperationComplete(record); }; From f2539ac2bafa4f9a64ea9a77b2d2772694ea202a Mon Sep 17 00:00:00 2001 From: Cheng Liu Date: Thu, 24 Aug 2023 16:52:24 +0800 Subject: [PATCH 090/100] feat: clean up the logReadStream --- .../src/logic/operations/CacheableOperationPlugin.ts | 2 +- .../src/logic/operations/OperationMetadataManager.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 50362ee168c..c7fc0b95e99 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -421,7 +421,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { errorLogPath }); } - return Boolean(restoreFromCacheSuccess); + return !!restoreFromCacheSuccess; }; if (cobuildLock) { // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to diff --git a/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts b/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts index 2c2008f084b..1df3df4ee1e 100644 --- a/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts +++ b/libraries/rush-lib/src/logic/operations/OperationMetadataManager.ts @@ -122,8 +122,9 @@ export class OperationMetadataManager { // Append cached log into current log file terminal.writeLine(`Restoring cached log file at ${this._logPath}`); + let logReadStream: fs.ReadStream | undefined; try { - const logReadStream: fs.ReadStream = fs.createReadStream(this._logPath, { + logReadStream = fs.createReadStream(this._logPath, { encoding: 'utf-8' }); for await (const data of logReadStream) { @@ -133,6 +134,9 @@ export class OperationMetadataManager { if (!FileSystem.isNotExistError(e)) { throw e; } + } finally { + // Clean up the read steam + logReadStream?.close(); } // Try to restore cached error log as error log file From 1df42ddcf9b67a9d9f373b84f63efe9a62696361 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 24 Aug 2023 16:59:51 +0800 Subject: [PATCH 091/100] chore: rush change --- .../feat-cobuild_2023-08-24-08-58.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@rushstack/package-extractor/feat-cobuild_2023-08-24-08-58.json diff --git a/common/changes/@rushstack/package-extractor/feat-cobuild_2023-08-24-08-58.json b/common/changes/@rushstack/package-extractor/feat-cobuild_2023-08-24-08-58.json new file mode 100644 index 00000000000..672fa0fcb35 --- /dev/null +++ b/common/changes/@rushstack/package-extractor/feat-cobuild_2023-08-24-08-58.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/package-extractor", + "comment": "", + "type": "none" + } + ], + "packageName": "@rushstack/package-extractor" +} \ No newline at end of file From 5cd047ca09d113f8519c071d8ad9953b684566b9 Mon Sep 17 00:00:00 2001 From: Pete Gonzalez <4673363+octogonz@users.noreply.github.com> Date: Thu, 24 Aug 2023 12:25:05 -0700 Subject: [PATCH 092/100] Revert "TEMPORARY - attempt to highlight a regression." This reverts commit 42345f1a7658f543862cac4ed38dc7a30e28f010. --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4945ecdf3d..b4715a50f4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: # run: /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & - name: Rush retest (install-run-rush) - run: node common/scripts/install-run-rush.js retest --verbose --production --to rush --to repo-toolbox + run: node common/scripts/install-run-rush.js retest --verbose --production env: # Prevent time-based browserslist update warning # See https://github.com/microsoft/rushstack/issues/2981 @@ -61,10 +61,6 @@ jobs: - name: Ensure repo README is up-to-date run: node repo-scripts/repo-toolbox/lib/start.js readme --verify - - if: runner.os == 'Windows' - name: Delete the build cache - run: Remove-Item -LiteralPath "common/temp/build-cache" -Force -Recurse - - name: Rush test (rush-lib) run: node apps/rush/lib/start-dev.js test --verbose --production --timeline env: From f5573d2734879d7a1703c5874e5a382387dc146b Mon Sep 17 00:00:00 2001 From: David Michon Date: Tue, 29 Aug 2023 20:27:40 -0700 Subject: [PATCH 093/100] [rush] Separate Skip and Build Cache, add flag --- common/reviews/api/rush-lib.api.md | 57 +- .../src/api/RushProjectConfiguration.ts | 84 +- .../api/test/RushProjectConfiguration.test.ts | 73 +- .../RushProjectConfiguration.test.ts.snap | 9 + .../test-project-a/config/rush-project.json | 2 + .../test-project-c/config/rush-project.json | 3 +- .../cli/scriptActions/PhasedScriptAction.ts | 34 +- libraries/rush-lib/src/index.ts | 12 +- .../src/logic/buildCache/ProjectBuildCache.ts | 93 +- .../buildCache/test/ProjectBuildCache.test.ts | 11 +- .../operations/CacheableOperationPlugin.ts | 829 +++++++----------- .../operations/IOperationExecutionResult.ts | 6 +- .../src/logic/operations/IOperationRunner.ts | 13 +- .../src/logic/operations/LegacySkipPlugin.ts | 269 ++++++ .../logic/operations/NullOperationRunner.ts | 10 +- .../operations/OperationExecutionRecord.ts | 6 + .../logic/operations/ShellOperationRunner.ts | 13 +- .../operations/test/MockOperationRunner.ts | 7 +- .../test/ShellOperationRunnerPlugin.test.ts | 3 +- .../src/pluginFramework/PhasedCommandHooks.ts | 18 +- .../test/__snapshots__/script.test.ts.snap | 1 + 21 files changed, 917 insertions(+), 636 deletions(-) create mode 100644 libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index ed5132152a4..9c0809314af 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -345,6 +345,7 @@ export interface ICreateOperationsContext { readonly phaseOriginal: ReadonlySet; readonly phaseSelection: ReadonlySet; readonly projectChangeAnalyzer: ProjectChangeAnalyzer; + readonly projectConfigurations: ReadonlyMap; readonly projectSelection: ReadonlySet; readonly projectsInUnknownState: ReadonlySet; readonly rushConfiguration: RushConfiguration; @@ -482,6 +483,7 @@ export interface IOperationExecutionResult { readonly cobuildRunnerId: string | undefined; readonly error: Error | undefined; readonly nonCachedDurationMs: number | undefined; + readonly operation: Operation; readonly status: OperationStatus; readonly stdioSummarizer: StdioSummarizer; readonly stopwatch: IStopwatchResult; @@ -518,8 +520,9 @@ export interface IOperationOptions { // @beta export interface IOperationRunner { - commandToRun?: string; + cacheable: boolean; executeAsync(context: IOperationRunnerContext): Promise; + getConfigHash(): string; readonly name: string; reportTiming: boolean; silent: boolean; @@ -540,6 +543,15 @@ export interface IOperationRunnerContext { stopwatch: IStopwatchResult; } +// @alpha (undocumented) +export interface IOperationSettings { + dependsOnAdditionalFiles?: string[]; + dependsOnEnvVars?: string[]; + disableBuildCacheForOperation?: boolean; + operationName: string; + outputFolderNames?: string[]; +} + // @internal (undocumented) export interface _IOperationStateFileOptions { // (undocumented) @@ -612,6 +624,16 @@ export interface IPrefixMatch { value: TItem; } +// @internal (undocumented) +export interface _IRawRepoState { + // (undocumented) + projectState: Map> | undefined; + // (undocumented) + rawHashes: Map; + // (undocumented) + rootDir: string; +} + // @beta export interface IRushCommand { readonly actionName: string; @@ -631,6 +653,14 @@ export interface _IRushPluginConfigurationBase { pluginName: string; } +// @internal +export interface _IRushProjectJson { + disableBuildCacheForProject?: boolean; + incrementalBuildIgnoredGlobs?: string[]; + // (undocumented) + operationSettings?: IOperationSettings[]; +} + // @beta (undocumented) export interface IRushSessionOptions { // (undocumented) @@ -864,10 +894,12 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage // @alpha export class PhasedCommandHooks { - readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]>; + readonly afterExecuteOperation: AsyncSeriesHook<[ + IOperationRunnerContext & IOperationExecutionResult + ]>; readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, ICreateOperationsContext]>; readonly beforeExecuteOperation: AsyncSeriesBailHook<[ - IOperationRunnerContext + IOperationRunnerContext & IOperationExecutionResult ], OperationStatus | undefined>; readonly beforeExecuteOperations: AsyncSeriesHook<[ Map, @@ -908,10 +940,8 @@ export type PnpmStoreOptions = 'local' | 'global'; // @beta (undocumented) export class ProjectChangeAnalyzer { constructor(rushConfiguration: RushConfiguration); - // Warning: (ae-forgotten-export) The symbol "IRawRepoState" needs to be exported by the entry point index.d.ts - // // @internal (undocumented) - _ensureInitializedAsync(terminal: ITerminal): Promise; + _ensureInitializedAsync(terminal: ITerminal): Promise<_IRawRepoState | undefined>; // (undocumented) _filterProjectDataAsync(project: RushConfigurationProject, unfilteredProjectData: Map, rootDir: string, terminal: ITerminal): Promise>; getChangedProjectsAsync(options: IGetChangedProjectsOptions): Promise>; @@ -1161,6 +1191,21 @@ export class RushLifecycleHooks { runPhasedCommand: HookMap>; } +// @alpha +export class RushProjectConfiguration { + readonly disableBuildCacheForProject: boolean; + getCacheDisabledReason(trackedFileNames: Iterable, phaseName: string): string | undefined; + readonly incrementalBuildIgnoredGlobs: ReadonlyArray; + // (undocumented) + readonly operationSettingsByOperationName: ReadonlyMap>; + // (undocumented) + readonly project: RushConfigurationProject; + static tryLoadAndValidateForProjectsAsync(projects: Iterable, phases: ReadonlySet, terminal: ITerminal): Promise>; + static tryLoadForProjectAsync(project: RushConfigurationProject, terminal: ITerminal): Promise; + static tryLoadIgnoreGlobsForProjectAsync(project: RushConfigurationProject, terminal: ITerminal): Promise | undefined>; + validatePhaseConfiguration(phases: Iterable, terminal: ITerminal): void; +} + // @beta (undocumented) export class RushSession { constructor(options: IRushSessionOptions); diff --git a/libraries/rush-lib/src/api/RushProjectConfiguration.ts b/libraries/rush-lib/src/api/RushProjectConfiguration.ts index 0245b8c53a9..9f0753c45ae 100644 --- a/libraries/rush-lib/src/api/RushProjectConfiguration.ts +++ b/libraries/rush-lib/src/api/RushProjectConfiguration.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { AlreadyReportedError, ITerminal, Path } from '@rushstack/node-core-library'; +import { AlreadyReportedError, Async, type ITerminal, Path } from '@rushstack/node-core-library'; import { ConfigurationFile, InheritanceType } from '@rushstack/heft-config-file'; import { RigConfig } from '@rushstack/rig-package'; @@ -13,7 +13,8 @@ import schemaJson from '../schemas/rush-project.schema.json'; import anythingSchemaJson from '../schemas/rush-project.schema.json'; /** - * Describes the file structure for the "/config/rush-project.json" config file. + * Describes the file structure for the `/config/rush-project.json` config file. + * @internal */ export interface IRushProjectJson { /** @@ -40,6 +41,9 @@ export interface IRushProjectJson { operationSettings?: IOperationSettings[]; } +/** + * @alpha + */ export interface IOperationSettings { /** * The name of the operation. This should be a key in the `package.json`'s `scripts` object. @@ -188,7 +192,7 @@ const OLD_RUSH_PROJECT_CONFIGURATION_FILE: ConfigurationFile = @@ -197,12 +201,12 @@ export class RushProjectConfiguration { public readonly project: RushConfigurationProject; /** - * {@inheritdoc IRushProjectJson.incrementalBuildIgnoredGlobs} + * {@inheritdoc _IRushProjectJson.incrementalBuildIgnoredGlobs} */ public readonly incrementalBuildIgnoredGlobs: ReadonlyArray; /** - * {@inheritdoc IRushProjectJson.disableBuildCacheForProject} + * {@inheritdoc _IRushProjectJson.disableBuildCacheForProject} */ public readonly disableBuildCacheForProject: boolean; @@ -284,6 +288,53 @@ export class RushProjectConfiguration { } } + /** + * Examines the list of source files for the project and the target phase and returns a reason + * why the project cannot enable the build cache for that phase, or undefined if it is safe to so do. + */ + public getCacheDisabledReason(trackedFileNames: Iterable, phaseName: string): string | undefined { + if (this.disableBuildCacheForProject) { + return 'Caching has been disabled for this project.'; + } + + const normalizedProjectRelativeFolder: string = Path.convertToSlashes(this.project.projectRelativeFolder); + + const operationSettings: IOperationSettings | undefined = + this.operationSettingsByOperationName.get(phaseName); + if (!operationSettings) { + return `This project does not define the caching behavior of the "${phaseName}" command, so caching has been disabled.`; + } + + if (operationSettings.disableBuildCacheForOperation) { + return `Caching has been disabled for this project's "${phaseName}" command.`; + } + + const { outputFolderNames } = operationSettings; + if (!outputFolderNames) { + return; + } + + const normalizedOutputFolders: string[] = outputFolderNames.map( + (outputFolderName) => `${normalizedProjectRelativeFolder}/${outputFolderName}/` + ); + + const inputOutputFiles: string[] = []; + for (const file of trackedFileNames) { + for (const outputFolder of normalizedOutputFolders) { + if (file.startsWith(outputFolder)) { + inputOutputFiles.push(file); + } + } + } + + if (inputOutputFiles.length > 0) { + return ( + 'The following files are used to calculate project state ' + + `and are considered project output: ${inputOutputFiles.join(', ')}` + ); + } + } + /** * Loads the rush-project.json data for the specified project. */ @@ -337,6 +388,29 @@ export class RushProjectConfiguration { return rushProjectJson?.incrementalBuildIgnoredGlobs; } + /** + * Load the rush-project.json data for all selected projects. + * Validate compatibility of output folders across all selected phases. + */ + public static async tryLoadAndValidateForProjectsAsync( + projects: Iterable, + phases: ReadonlySet, + terminal: ITerminal + ): Promise> { + const result: Map = new Map(); + + await Async.forEachAsync(projects, async (project: RushConfigurationProject) => { + const projectConfig: RushProjectConfiguration | undefined = + await RushProjectConfiguration.tryLoadForProjectAsync(project, terminal); + if (projectConfig) { + projectConfig.validatePhaseConfiguration(phases, terminal); + result.set(project, projectConfig); + } + }); + + return result; + } + private static async _tryLoadJsonForProjectAsync( project: RushConfigurationProject, terminal: ITerminal diff --git a/libraries/rush-lib/src/api/test/RushProjectConfiguration.test.ts b/libraries/rush-lib/src/api/test/RushProjectConfiguration.test.ts index 9d737bd9ac5..3957761180f 100644 --- a/libraries/rush-lib/src/api/test/RushProjectConfiguration.test.ts +++ b/libraries/rush-lib/src/api/test/RushProjectConfiguration.test.ts @@ -23,7 +23,8 @@ async function loadProjectConfigurationAsync( const testFolder: string = `${__dirname}/jsonFiles/${testProjectName}`; const rushProject: RushConfigurationProject = { packageName: testProjectName, - projectFolder: testFolder + projectFolder: testFolder, + projectRelativeFolder: testProjectName } as RushConfigurationProject; const terminal: Terminal = new Terminal(new StringBufferTerminalProvider()); try { @@ -108,4 +109,74 @@ describe(RushProjectConfiguration.name, () => { expect(errorWasThrown).toBe(true); }); }); + + describe(RushProjectConfiguration.prototype.getCacheDisabledReason.name, () => { + it('Indicates if the build cache is completely disabled', async () => { + const config: RushProjectConfiguration | undefined = await loadProjectConfigurationAsync( + 'test-project-a' + ); + + if (!config) { + throw new Error('Failed to load config'); + } + + const reason: string | undefined = config.getCacheDisabledReason([], 'z'); + expect(reason).toMatchSnapshot(); + }); + + it('Indicates if the phase behavior is not defined', async () => { + const config: RushProjectConfiguration | undefined = await loadProjectConfigurationAsync( + 'test-project-c' + ); + + if (!config) { + throw new Error('Failed to load config'); + } + + const reason: string | undefined = config.getCacheDisabledReason([], 'z'); + expect(reason).toMatchSnapshot(); + }); + + it('Indicates if the phase has disabled the cache', async () => { + const config: RushProjectConfiguration | undefined = await loadProjectConfigurationAsync( + 'test-project-c' + ); + + if (!config) { + throw new Error('Failed to load config'); + } + + const reason: string | undefined = config.getCacheDisabledReason([], '_phase:a'); + expect(reason).toMatchSnapshot(); + }); + + it('Indicates if tracked files are outputs of the phase', async () => { + const config: RushProjectConfiguration | undefined = await loadProjectConfigurationAsync( + 'test-project-c' + ); + + if (!config) { + throw new Error('Failed to load config'); + } + + const reason: string | undefined = config.getCacheDisabledReason( + ['test-project-c/.cache/b/foo'], + '_phase:b' + ); + expect(reason).toMatchSnapshot(); + }); + + it('returns undfined if the config is safe', async () => { + const config: RushProjectConfiguration | undefined = await loadProjectConfigurationAsync( + 'test-project-c' + ); + + if (!config) { + throw new Error('Failed to load config'); + } + + const reason: string | undefined = config.getCacheDisabledReason([''], '_phase:b'); + expect(reason).toBeUndefined(); + }); + }); }); diff --git a/libraries/rush-lib/src/api/test/__snapshots__/RushProjectConfiguration.test.ts.snap b/libraries/rush-lib/src/api/test/__snapshots__/RushProjectConfiguration.test.ts.snap index c087d0f6ad5..9250e9b1aa0 100644 --- a/libraries/rush-lib/src/api/test/__snapshots__/RushProjectConfiguration.test.ts.snap +++ b/libraries/rush-lib/src/api/test/__snapshots__/RushProjectConfiguration.test.ts.snap @@ -1,8 +1,17 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`RushProjectConfiguration getCacheDisabledReason Indicates if the build cache is completely disabled 1`] = `"Caching has been disabled for this project."`; + +exports[`RushProjectConfiguration getCacheDisabledReason Indicates if the phase behavior is not defined 1`] = `"This project does not define the caching behavior of the \\"z\\" command, so caching has been disabled."`; + +exports[`RushProjectConfiguration getCacheDisabledReason Indicates if the phase has disabled the cache 1`] = `"Caching has been disabled for this project's \\"_phase:a\\" command."`; + +exports[`RushProjectConfiguration getCacheDisabledReason Indicates if tracked files are outputs of the phase 1`] = `"The following files are used to calculate project state and are considered project output: test-project-c/.cache/b/foo"`; + exports[`RushProjectConfiguration operationSettingsByOperationName allows outputFolderNames to be inside subfolders 1`] = ` Map { "_phase:a" => Object { + "disableBuildCacheForOperation": true, "operationName": "_phase:a", "outputFolderNames": Array [ ".cache/a", diff --git a/libraries/rush-lib/src/api/test/jsonFiles/test-project-a/config/rush-project.json b/libraries/rush-lib/src/api/test/jsonFiles/test-project-a/config/rush-project.json index c8743b4eb22..fe852bfeda4 100644 --- a/libraries/rush-lib/src/api/test/jsonFiles/test-project-a/config/rush-project.json +++ b/libraries/rush-lib/src/api/test/jsonFiles/test-project-a/config/rush-project.json @@ -1,6 +1,8 @@ { "extends": "../../rush-project-base.json", + "disableBuildCacheForProject": true, + "operationSettings": [ { "operationName": "_phase:a", diff --git a/libraries/rush-lib/src/api/test/jsonFiles/test-project-c/config/rush-project.json b/libraries/rush-lib/src/api/test/jsonFiles/test-project-c/config/rush-project.json index 3609b5a63b9..6b2625686a3 100644 --- a/libraries/rush-lib/src/api/test/jsonFiles/test-project-c/config/rush-project.json +++ b/libraries/rush-lib/src/api/test/jsonFiles/test-project-c/config/rush-project.json @@ -2,7 +2,8 @@ "operationSettings": [ { "operationName": "_phase:a", - "outputFolderNames": [".cache/a"] + "outputFolderNames": [".cache/a"], + "disableBuildCacheForOperation": true }, { "operationName": "_phase:b", diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index 3656ecddbaf..a69b64d4be1 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -40,7 +40,8 @@ import type { ITelemetryData, ITelemetryOperationResult } from '../../logic/Tele import { parseParallelism } from '../parsing/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; -import type { IOperationRunnerContext } from '../../logic/operations/IOperationRunner'; +import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; +import { LegacySkipPlugin } from '../../logic/operations/LegacySkipPlugin'; /** * Constructor parameters for PhasedScriptAction. @@ -147,8 +148,6 @@ export class PhasedScriptAction extends BaseScriptAction { new PhasedOperationPlugin().apply(this.hooks); // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(this.hooks); - // Applies the build cache related logic to the selected operations - new CacheableOperationPlugin().apply(this.hooks); if (this._enableParallelism) { this._parallelismParameter = this.defineStringParameter({ @@ -342,6 +341,30 @@ export class PhasedScriptAction extends BaseScriptAction { customParametersByName.set(configParameter.longName, parserParameter); } + if (buildCacheConfiguration) { + new CacheableOperationPlugin({ + allowWarningsInSuccessfulBuild: + !!this.rushConfiguration.experimentsConfiguration.configuration + .buildCacheWithAllowWarningsInSuccessfulBuild, + buildCacheConfiguration, + cobuildConfiguration, + terminal + }).apply(this.hooks); + } else { + new LegacySkipPlugin({ + terminal, + changedProjectsOnly, + isIncrementalBuildAllowed: this._isIncrementalBuildAllowed + }).apply(this.hooks); + } + + const projectConfigurations: ReadonlyMap = + await RushProjectConfiguration.tryLoadAndValidateForProjectsAsync( + projectSelection, + this._initialPhases, + terminal + ); + const projectChangeAnalyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); const initialCreateOperationsContext: ICreateOperationsContext = { buildCacheConfiguration, @@ -355,6 +378,7 @@ export class PhasedScriptAction extends BaseScriptAction { phaseSelection: new Set(this._initialPhases), projectChangeAnalyzer, projectSelection, + projectConfigurations, projectsInUnknownState: projectSelection }; @@ -363,10 +387,10 @@ export class PhasedScriptAction extends BaseScriptAction { debugMode: this.parser.isDebug, parallelism, changedProjectsOnly, - beforeExecuteOperation: async (record: IOperationRunnerContext) => { + beforeExecuteOperation: async (record: OperationExecutionRecord) => { return await this.hooks.beforeExecuteOperation.promise(record); }, - afterExecuteOperation: async (record: IOperationRunnerContext) => { + afterExecuteOperation: async (record: OperationExecutionRecord) => { await this.hooks.afterExecuteOperation.promise(record); }, beforeExecuteOperations: async (records: Map) => { diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 350549bfc7f..4e9a3bfd5ad 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -55,6 +55,12 @@ export { PackageManagerName, PackageManager } from './api/packageManager/Package export { RushConfigurationProject } from './api/RushConfigurationProject'; +export { + IRushProjectJson as _IRushProjectJson, + IOperationSettings, + RushProjectConfiguration +} from './api/RushProjectConfiguration'; + export { RushUserConfiguration } from './api/RushUserConfiguration'; export { RushGlobalFolder as _RushGlobalFolder } from './api/RushGlobalFolder'; @@ -98,7 +104,11 @@ export { ICustomTipItemJson } from './api/CustomTipsConfiguration'; -export { ProjectChangeAnalyzer, IGetChangedProjectsOptions } from './logic/ProjectChangeAnalyzer'; +export { + ProjectChangeAnalyzer, + IGetChangedProjectsOptions, + IRawRepoState as _IRawRepoState +} from './logic/ProjectChangeAnalyzer'; export { IOperationRunner, IOperationRunnerContext } from './logic/operations/IOperationRunner'; export { IExecutionResult, IOperationExecutionResult } from './logic/operations/IOperationExecutionResult'; diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index d9c778c64f8..c5196093428 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -3,7 +3,8 @@ import * as path from 'path'; import * as crypto from 'crypto'; -import { FileSystem, Path, ITerminal, FolderItem, InternalError, Async } from '@rushstack/node-core-library'; + +import { FileSystem, ITerminal, FolderItem, InternalError, Async } from '@rushstack/node-core-library'; import { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; @@ -20,8 +21,7 @@ export interface IProjectBuildCacheOptions { projectOutputFolderNames: ReadonlyArray; additionalProjectOutputFilePaths?: ReadonlyArray; additionalContext?: Record; - command: string; - trackedProjectFiles: string[] | undefined; + configHash: string; projectChangeAnalyzer: ProjectChangeAnalyzer; terminal: ITerminal; phaseName: string; @@ -76,55 +76,10 @@ export class ProjectBuildCache { public static async tryGetProjectBuildCache( options: IProjectBuildCacheOptions ): Promise { - const { terminal, project, projectOutputFolderNames, trackedProjectFiles } = options; - if (!trackedProjectFiles) { - return undefined; - } - - if ( - !ProjectBuildCache._validateProject(terminal, project, projectOutputFolderNames, trackedProjectFiles) - ) { - return undefined; - } - const cacheId: string | undefined = await ProjectBuildCache._getCacheId(options); return new ProjectBuildCache(cacheId, options); } - private static _validateProject( - terminal: ITerminal, - project: RushConfigurationProject, - projectOutputFolderNames: ReadonlyArray, - trackedProjectFiles: string[] - ): boolean { - const normalizedProjectRelativeFolder: string = Path.convertToSlashes(project.projectRelativeFolder); - const outputFolders: string[] = []; - if (projectOutputFolderNames) { - for (const outputFolderName of projectOutputFolderNames) { - outputFolders.push(`${normalizedProjectRelativeFolder}/${outputFolderName}/`); - } - } - - const inputOutputFiles: string[] = []; - for (const file of trackedProjectFiles) { - for (const outputFolder of outputFolders) { - if (file.startsWith(outputFolder)) { - inputOutputFiles.push(file); - } - } - } - - if (inputOutputFiles.length > 0) { - terminal.writeWarningLine( - 'Unable to use build cache. The following files are used to calculate project state ' + - `and are considered project output: ${inputOutputFiles.join(', ')}` - ); - return false; - } else { - return true; - } - } - public async tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise { const cacheId: string | undefined = specifiedCacheId || this._cacheId; if (!cacheId) { @@ -417,39 +372,29 @@ export class ProjectBuildCache { // - A SHA1 hash is created and the following data is fed into it, in order: // 1. The JSON-serialized list of output folder names for this // project (see ProjectBuildCache._projectOutputFolderNames) - // 2. The command that will be run in the project + // 2. The configHash from the operation's runner // 3. Each dependency project hash (from the array constructed in previous steps), // in sorted alphanumerical-sorted order // - A hex digest of the hash is returned const projectChangeAnalyzer: ProjectChangeAnalyzer = options.projectChangeAnalyzer; const projectStates: string[] = []; - const projectsThatHaveBeenProcessed: Set = new Set(); - let projectsToProcess: Set = new Set(); + const projectsToProcess: Set = new Set(); projectsToProcess.add(options.project); - while (projectsToProcess.size > 0) { - const newProjectsToProcess: Set = new Set(); - for (const projectToProcess of projectsToProcess) { - projectsThatHaveBeenProcessed.add(projectToProcess); - - const projectState: string | undefined = await projectChangeAnalyzer._tryGetProjectStateHashAsync( - projectToProcess, - options.terminal - ); - if (!projectState) { - // If we hit any projects with unknown state, return unknown cache ID - return undefined; - } else { - projectStates.push(projectState); - for (const dependency of projectToProcess.dependencyProjects) { - if (!projectsThatHaveBeenProcessed.has(dependency)) { - newProjectsToProcess.add(dependency); - } - } + for (const projectToProcess of projectsToProcess) { + const projectState: string | undefined = await projectChangeAnalyzer._tryGetProjectStateHashAsync( + projectToProcess, + options.terminal + ); + if (!projectState) { + // If we hit any projects with unknown state, return unknown cache ID + return undefined; + } else { + projectStates.push(projectState); + for (const dependency of projectToProcess.dependencyProjects) { + projectsToProcess.add(dependency); } } - - projectsToProcess = newProjectsToProcess; } const sortedProjectStates: string[] = projectStates.sort(); @@ -460,13 +405,13 @@ export class ProjectBuildCache { const serializedOutputFolders: string = JSON.stringify(options.projectOutputFolderNames); hash.update(serializedOutputFolders); hash.update(RushConstants.hashDelimiter); - hash.update(options.command); + hash.update(options.configHash); hash.update(RushConstants.hashDelimiter); if (options.additionalContext) { for (const key of Object.keys(options.additionalContext).sort()) { // Add additional context keys and values. // - // This choice (to modiy the hash for every key regardless of whether a value is set) implies + // This choice (to modify the hash for every key regardless of whether a value is set) implies // that just _adding_ an env var to the list of dependsOnEnvVars will modify its hash. This // seems appropriate, because this behavior is consistent whether or not the env var happens // to have a value. diff --git a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts index afbae532838..ea906bc6991 100644 --- a/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts +++ b/libraries/rush-lib/src/logic/buildCache/test/ProjectBuildCache.test.ts @@ -41,8 +41,7 @@ describe(ProjectBuildCache.name, () => { projectRelativeFolder: 'apps/acme-wizard', dependencyProjects: [] } as unknown as RushConfigurationProject, - command: 'build', - trackedProjectFiles: options.hasOwnProperty('trackedProjectFiles') ? options.trackedProjectFiles : [], + configHash: 'build', projectChangeAnalyzer, terminal, phaseName: 'build' @@ -58,13 +57,5 @@ describe(ProjectBuildCache.name, () => { `"acme-wizard/1926f30e8ed24cb47be89aea39e7efd70fcda075"` ); }); - - it('returns undefined if the tracked file list is undefined', async () => { - expect( - await prepareSubject({ - trackedProjectFiles: undefined - }) - ).toBe(undefined); - }); }); }); diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index c7fc0b95e99..809e4e3a450 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -1,22 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import * as path from 'path'; import * as crypto from 'crypto'; -import { - Async, - ColorValue, - FileSystem, - InternalError, - ITerminal, - JsonFile, - JsonObject, - NewlineKind, - Sort, - Terminal -} from '@rushstack/node-core-library'; +import { Async, InternalError, ITerminal, NewlineKind, Sort, Terminal } from '@rushstack/node-core-library'; import { CollatedTerminal, CollatedWriter } from '@rushstack/stream-collator'; -import { DiscardStdoutTransform, PrintUtilities, TextRewriterTransform } from '@rushstack/terminal'; +import { DiscardStdoutTransform, TextRewriterTransform } from '@rushstack/terminal'; import { SplitterTransform, TerminalWritable } from '@rushstack/terminal'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; @@ -32,7 +20,7 @@ import { DisjointSet } from '../cobuild/DisjointSet'; import { PeriodicCallback } from './PeriodicCallback'; import { NullTerminalProvider } from '../../utilities/NullTerminalProvider'; -import type { Operation } from './Operation'; +import { Operation } from './Operation'; import type { IOperationRunnerContext } from './IOperationRunner'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { @@ -58,118 +46,151 @@ export interface IProjectDeps { export interface IOperationBuildCacheContext { isCacheWriteAllowed: boolean; isCacheReadAllowed: boolean; - isSkipAllowed: boolean; + projectBuildCache: ProjectBuildCache | undefined; + cacheDisabledReason: string | undefined; + operationSettings: IOperationSettings | undefined; + cobuildLock: CobuildLock | undefined; + // The id of the cluster contains the operation, used when acquiring cobuild lock cobuildClusterId: string | undefined; + // Controls the log for the cache subsystem buildCacheTerminal: ITerminal | undefined; buildCacheProjectLogWritable: ProjectLogWritable | undefined; + periodicCallback: PeriodicCallback; - projectDeps: IProjectDeps | undefined; - currentDepsPath: string | undefined; cacheRestored: boolean; } +export interface ICacheableOperationPluginOptions { + allowWarningsInSuccessfulBuild: boolean; + buildCacheConfiguration: BuildCacheConfiguration; + cobuildConfiguration: CobuildConfiguration | undefined; + terminal: ITerminal; +} + export class CacheableOperationPlugin implements IPhasedCommandPlugin { - private _buildCacheContextByOperationExecutionRecord: Map< - OperationExecutionRecord, - IOperationBuildCacheContext - > = new Map(); + private _buildCacheContextByOperation: Map = new Map(); private _createContext: ICreateOperationsContext | undefined; + private readonly _options: ICacheableOperationPluginOptions; + + public constructor(options: ICacheableOperationPluginOptions) { + this._options = options; + } + public apply(hooks: PhasedCommandHooks): void { + const { allowWarningsInSuccessfulBuild, buildCacheConfiguration, cobuildConfiguration, terminal } = + this._options; + hooks.beforeExecuteOperations.tapPromise( PLUGIN_NAME, async ( recordByOperation: Map, context: ICreateOperationsContext ): Promise => { - const { buildCacheConfiguration, isIncrementalBuildAllowed, cobuildConfiguration } = context; - if (!buildCacheConfiguration) { - return; - } + const { isIncrementalBuildAllowed, projectChangeAnalyzer, projectConfigurations } = context; this._createContext = context; - let disjointSet: DisjointSet | undefined; - if (cobuildConfiguration?.cobuildEnabled) { - disjointSet = new DisjointSet(); - } + const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildEnabled + ? new DisjointSet() + : undefined; - const records: IterableIterator = - recordByOperation.values() as IterableIterator; - - for (const record of records) { - disjointSet?.add(record); - const buildCacheContext: IOperationBuildCacheContext = { - // Supports cache writes by default. - isCacheWriteAllowed: true, - isCacheReadAllowed: isIncrementalBuildAllowed, - isSkipAllowed: isIncrementalBuildAllowed, - projectBuildCache: undefined, - cobuildLock: undefined, - cobuildClusterId: undefined, - buildCacheTerminal: undefined, - buildCacheProjectLogWritable: undefined, - periodicCallback: new PeriodicCallback({ - interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 - }), - projectDeps: undefined, - currentDepsPath: undefined, - cacheRestored: false - }; - // Upstream runners may mutate the property of build cache context for downstream runners - this._buildCacheContextByOperationExecutionRecord.set(record, buildCacheContext); - } + await Async.forEachAsync( + recordByOperation.keys(), + async (operation: Operation) => { + const { associatedProject, associatedPhase, runner } = operation; + if (!associatedProject || !associatedPhase || !runner) { + return; + } + + const { name: phaseName } = associatedPhase; + + const projectConfiguration: RushProjectConfiguration | undefined = + projectConfigurations.get(associatedProject); + + // This value can *currently* be cached per-project, but in the future the list of files will vary + // depending on the selected phase. + const fileHashes: Map | undefined = + await projectChangeAnalyzer._tryGetProjectDependenciesAsync(associatedProject, terminal); + + if (!fileHashes) { + throw new Error( + `Build cache is only supported if running in a Git repository. Either disable the build cache or run Rush in a Git repository.` + ); + } + + const operationSettings: IOperationSettings | undefined = + projectConfiguration?.operationSettingsByOperationName.get(phaseName); + const cacheDisabledReason: string | undefined = projectConfiguration + ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName) + : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + + 'or one provided by a rig, so it does not support caching.'; + + disjointSet?.add(operation); + + const buildCacheContext: IOperationBuildCacheContext = { + // Supports cache writes by default. + isCacheWriteAllowed: true, + isCacheReadAllowed: isIncrementalBuildAllowed, + projectBuildCache: undefined, + operationSettings, + cacheDisabledReason, + cobuildLock: undefined, + cobuildClusterId: undefined, + buildCacheTerminal: undefined, + buildCacheProjectLogWritable: undefined, + periodicCallback: new PeriodicCallback({ + interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 + }), + cacheRestored: false + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperation.set(operation, buildCacheContext); + }, + { + concurrency: 10 + } + ); if (disjointSet) { // If disjoint set exists, connect build cache disabled project with its consumers - await Async.forEachAsync( - records, - async (record: OperationExecutionRecord) => { - const { associatedProject: project, associatedPhase: phase } = record; - if (project && phase) { - const buildCacheEnabled: boolean = await this._tryGetProjectBuildCacheEnabledAsync({ - buildCacheConfiguration, - rushProject: project, - commandName: phase.name - }); - if (!buildCacheEnabled) { - /** - * Group the project build cache disabled with its consumers. This won't affect too much in - * a monorepo with high build cache coverage. - * - * The mental model is that if X disables the cache, and Y depends on X, then: - * 1. Y must be built by the same VM that build X; - * 2. OR, Y must be rebuilt on each VM that needs it. - * Approach 1 is probably the better choice. - */ - for (const consumer of record.consumers) { - disjointSet?.union(record, consumer); - } + for (const [operation, { cacheDisabledReason }] of this._buildCacheContextByOperation) { + const { associatedProject: project, associatedPhase: phase } = operation; + if (project && phase) { + if (cacheDisabledReason) { + /** + * Group the project build cache disabled with its consumers. This won't affect too much in + * a monorepo with high build cache coverage. + * + * The mental model is that if X disables the cache, and Y depends on X, then: + * 1. Y must be built by the same VM that build X; + * 2. OR, Y must be rebuilt on each VM that needs it. + * Approach 1 is probably the better choice. + */ + for (const consumer of operation.consumers) { + disjointSet?.union(operation, consumer); } } - }, - { - concurrency: 10 } - ); + } - for (const set of disjointSet.getAllSets()) { + for (const operationSet of disjointSet.getAllSets()) { if (cobuildConfiguration?.cobuildEnabled && cobuildConfiguration.cobuildContextId) { // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. - const groupedRecords: OperationExecutionRecord[] = Array.from(set); - Sort.sortBy(groupedRecords, (record: OperationExecutionRecord) => { - return record.runner.name; + const groupedOperations: Operation[] = Array.from(operationSet); + Sort.sortBy(groupedOperations, (operation: Operation) => { + return operation.name; }); // Generates cluster id, cluster id comes from the project folder and phase name of all operations in the same cluster. const hash: crypto.Hash = crypto.createHash('sha1'); - for (const record of groupedRecords) { - const { associatedPhase: phase, associatedProject: project } = record; + for (const operation of groupedOperations) { + const { associatedPhase: phase, associatedProject: project } = operation; if (project && phase) { hash.update(project.projectRelativeFolder); hash.update(RushConstants.hashDelimiter); @@ -180,9 +201,9 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const cobuildClusterId: string = hash.digest('hex'); // Assign same cluster id to all operations in the same cluster. - for (const record of groupedRecords) { + for (const record of groupedOperations) { const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); + this._getBuildCacheContextByOperationOrThrow(record); buildCacheContext.cobuildClusterId = cobuildClusterId; } } @@ -193,17 +214,22 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { hooks.beforeExecuteOperation.tapPromise( PLUGIN_NAME, - async (runnerContext: IOperationRunnerContext): Promise => { + async ( + runnerContext: IOperationRunnerContext & IOperationExecutionResult + ): Promise => { const { _createContext: createContext } = this; if (!createContext) { return; } - const { - projectChangeAnalyzer, - buildCacheConfiguration, - cobuildConfiguration, - phaseSelection: selectedPhases - } = createContext; + + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperation(runnerContext.operation); + + if (!buildCacheContext) { + return; + } + + const { projectChangeAnalyzer, phaseSelection: selectedPhases } = createContext; const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; const { @@ -216,13 +242,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return; } - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperationExecutionRecord(record); - - if (!buildCacheContext) { - return; - } - const runBeforeExecute = async ({ projectChangeAnalyzer, buildCacheConfiguration, @@ -246,7 +265,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }): Promise => { const buildCacheTerminal: ITerminal = this._getBuildCacheTerminal({ record, - buildCacheConfiguration, + buildCacheContext, + buildCacheEnabled: buildCacheConfiguration?.buildCacheEnabled, rushProject: project, logFilenameIdentifier: phase.logFilenameIdentifier, quietMode: record.quietMode, @@ -254,88 +274,16 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { }); buildCacheContext.buildCacheTerminal = buildCacheTerminal; - const commandToRun: string = record.runner.commandToRun || ''; - - const packageDepsFilename: string = `package-deps_${phase.logFilenameIdentifier}.json`; - const currentDepsPath: string = path.join(project.projectRushTempFolder, packageDepsFilename); - buildCacheContext.currentDepsPath = currentDepsPath; - - let projectDeps: IProjectDeps | undefined; - let trackedProjectFiles: string[] | undefined; - try { - const fileHashes: Map | undefined = - await createContext.projectChangeAnalyzer._tryGetProjectDependenciesAsync( - project, - buildCacheTerminal - ); - - if (fileHashes) { - const files: { [filePath: string]: string } = {}; - trackedProjectFiles = []; - for (const [filePath, fileHash] of fileHashes) { - files[filePath] = fileHash; - trackedProjectFiles.push(filePath); - } - - projectDeps = { - files, - arguments: commandToRun - }; - buildCacheContext.projectDeps = projectDeps; - } - } catch (error) { - // To test this code path: - // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" - buildCacheTerminal.writeLine( - 'Unable to calculate incremental state: ' + (error as Error).toString() - ); - buildCacheTerminal.writeLine({ - text: 'Rush will proceed without incremental execution, caching, and change detection.', - foregroundColor: ColorValue.Cyan - }); - } - - if (!projectDeps && buildCacheContext.isSkipAllowed) { - // To test this code path: - // Remove the `.git` folder then run "rush build --verbose" - buildCacheTerminal.writeLine({ - text: PrintUtilities.wrapWords( - 'This workspace does not appear to be tracked by Git. ' + - 'Rush will proceed without incremental execution, caching, and change detection.' - ), - foregroundColor: ColorValue.Cyan - }); - } - - // If the deps file exists, remove it before starting execution. - FileSystem.deleteFile(currentDepsPath); - - // TODO: Remove legacyDepsPath with the next major release of Rush - const legacyDepsPath: string = path.join(project.projectFolder, 'package-deps.json'); - // Delete the legacy package-deps.json - FileSystem.deleteFile(legacyDepsPath); - - // No-op command - if (!commandToRun) { - // Write deps on success. - if (projectDeps) { - JsonFile.save(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - } - return OperationStatus.NoOp; - } + const configHash: string = record.runner.getConfigHash() || ''; let projectBuildCache: ProjectBuildCache | undefined = await this._tryGetProjectBuildCacheAsync({ - record, + buildCacheContext, buildCacheConfiguration, rushProject: project, phase, - selectedPhases, projectChangeAnalyzer, - commandToRun, + configHash, terminal: buildCacheTerminal, - trackedProjectFiles, operationMetadataManager }); @@ -344,7 +292,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { if (cobuildConfiguration?.cobuildEnabled) { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && - record.consumers.size === 0 && + record.operation.consumers.size === 0 && !projectBuildCache ) { // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get @@ -352,13 +300,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { projectBuildCache = await this._tryGetLogOnlyProjectBuildCacheAsync({ buildCacheConfiguration, cobuildConfiguration, - record, + buildCacheContext, rushProject: project, phase, projectChangeAnalyzer, - commandToRun, + configHash, terminal: buildCacheTerminal, - trackedProjectFiles, operationMetadataManager }); if (projectBuildCache) { @@ -373,7 +320,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } cobuildLock = await this._tryGetCobuildLockAsync({ - record, + buildCacheContext, projectBuildCache, cobuildConfiguration, packageName: project.packageName, @@ -396,11 +343,6 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // incremental builds, and determining whether this project or a dependent // has changed happens inside the hashing logic. // - // - For skipping, "isSkipAllowed" is set to true initially, and during - // the process of running dependents, it will be changed by this plugin to - // false if a dependency wasn't able to be skipped. - // - let buildCacheReadAttempted: boolean = false; const { logPath, errorLogPath } = ProjectLogWritable.getLogFilePaths({ project, @@ -446,38 +388,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } } else if (buildCacheContext.isCacheReadAllowed) { - buildCacheReadAttempted = !!projectBuildCache; const restoreFromCacheSuccess: boolean = await restoreCacheAsync(projectBuildCache); if (restoreFromCacheSuccess) { return OperationStatus.FromCache; } } - if (buildCacheContext.isSkipAllowed && !buildCacheReadAttempted) { - let lastProjectDeps: IProjectDeps | undefined = undefined; - try { - lastProjectDeps = JsonFile.load(currentDepsPath); - } catch (e) { - // Warn and ignore - treat failing to load the file as the project being not built. - buildCacheTerminal.writeWarningLine( - `Warning: error parsing ${packageDepsFilename}: ${e}. Ignoring and ` + - `treating the command "${commandToRun}" as not run.` - ); - } - - const isPackageUnchanged: boolean = !!( - lastProjectDeps && - projectDeps && - projectDeps.arguments === lastProjectDeps.arguments && - _areShallowEqual(projectDeps.files, lastProjectDeps.files) - ); - - if (isPackageUnchanged) { - // Pretend the cache restored when skip - buildCacheContext.cacheRestored = true; - return OperationStatus.Skipped; - } - } if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); @@ -518,17 +434,18 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { PLUGIN_NAME, async (runnerContext: IOperationRunnerContext): Promise => { const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; - const { - status, - consumers, - changedProjectsOnly, - stopwatch, - _operationMetadataManager: operationMetadataManager, - associatedProject: project, - associatedPhase: phase - } = record; + const { status, stopwatch, _operationMetadataManager: operationMetadataManager, operation } = record; - if (!project || !phase) { + const { associatedProject: project, associatedPhase: phase, runner } = operation; + + if (!project || !phase || !runner?.cacheable) { + return; + } + + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperation(operation); + + if (!buildCacheContext) { return; } @@ -543,21 +460,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } } - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperationExecutionRecord(record); - - if (!buildCacheContext) { - return; - } - const { - cobuildLock, - projectBuildCache, - isCacheWriteAllowed, - buildCacheTerminal, - projectDeps, - currentDepsPath, - cacheRestored - } = buildCacheContext; + const { cobuildLock, projectBuildCache, isCacheWriteAllowed, buildCacheTerminal, cacheRestored } = + buildCacheContext; try { if (!cacheRestored) { @@ -610,15 +514,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { status === OperationStatus.Success || (status === OperationStatus.SuccessWithWarning && record.runner.warningsAreAllowed && - !!project.rushConfiguration.experimentsConfiguration.configuration - .buildCacheWithAllowWarningsInSuccessfulBuild); - - if (taskIsSuccessful && projectDeps && currentDepsPath) { - // Write deps on success. - await JsonFile.saveAsync(projectDeps, currentDepsPath, { - ensureFolderExists: true - }); - } + allowWarningsInSuccessfulBuild); // If the command is successful, we can calculate project hash, and no dependencies were skipped, // write a new cache entry. @@ -633,200 +529,121 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { record.status = OperationStatus.SuccessWithWarning; } } + } finally { + buildCacheContext.buildCacheProjectLogWritable?.close(); + buildCacheContext.periodicCallback.stop(); + } + } + ); - // Status changes to direct dependents - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - let blockSkip: boolean = !buildCacheContext?.isSkipAllowed; - - switch (record.status) { - case OperationStatus.Skipped: { - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; - break; - } + hooks.afterExecuteOperation.tap( + PLUGIN_NAME, + (record: IOperationRunnerContext & IOperationExecutionResult): void => { + const { operation } = record; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._buildCacheContextByOperation.get(operation); + // Status changes to direct dependents + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: { - // Legacy incremental build, if asked, prevent skip in dependents if the operation executed. - blockSkip ||= !changedProjectsOnly; - break; - } + switch (record.status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; } + } - // Apply status changes to direct dependents - for (const consumer of consumers) { + // Apply status changes to direct dependents + if (blockCacheWrite) { + for (const consumer of operation.consumers) { const consumerBuildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperationExecutionRecord(consumer); + this._getBuildCacheContextByOperation(consumer); if (consumerBuildCacheContext) { - if (blockCacheWrite) { - consumerBuildCacheContext.isCacheWriteAllowed = false; - } - if (blockSkip) { - consumerBuildCacheContext.isSkipAllowed = false; - } + consumerBuildCacheContext.isCacheWriteAllowed = false; } } - } finally { - buildCacheContext.buildCacheProjectLogWritable?.close(); - buildCacheContext.periodicCallback.stop(); } } ); hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { - this._buildCacheContextByOperationExecutionRecord.clear(); + this._buildCacheContextByOperation.clear(); }); } - private _getBuildCacheContextByOperationExecutionRecord( - record: OperationExecutionRecord - ): IOperationBuildCacheContext | undefined { + private _getBuildCacheContextByOperation(operation: Operation): IOperationBuildCacheContext | undefined { const buildCacheContext: IOperationBuildCacheContext | undefined = - this._buildCacheContextByOperationExecutionRecord.get(record); + this._buildCacheContextByOperation.get(operation); return buildCacheContext; } - private _getBuildCacheContextByOperationExecutionRecordOrThrow( - record: OperationExecutionRecord - ): IOperationBuildCacheContext { + private _getBuildCacheContextByOperationOrThrow(operation: Operation): IOperationBuildCacheContext { const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperationExecutionRecord(record); + this._getBuildCacheContextByOperation(operation); if (!buildCacheContext) { // This should not happen - throw new InternalError(`Build cache context for runner ${record.name} should be defined`); + throw new InternalError(`Build cache context for operation ${operation.name} should be defined`); } return buildCacheContext; } - private async _tryGetProjectBuildCacheEnabledAsync({ - buildCacheConfiguration, - rushProject, - commandName - }: { - buildCacheConfiguration: BuildCacheConfiguration; - rushProject: RushConfigurationProject; - commandName: string; - }): Promise { - const nullTerminalProvider: NullTerminalProvider = new NullTerminalProvider(); - // This is a silent terminal - const terminal: ITerminal = new Terminal(nullTerminalProvider); - - if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); - if (projectConfiguration && projectConfiguration.disableBuildCacheForProject) { - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(commandName); - if (operationSettings && !operationSettings.disableBuildCacheForOperation) { - return true; - } - } - } - return false; - } - private async _tryGetProjectBuildCacheAsync({ buildCacheConfiguration, - record, + buildCacheContext, rushProject, phase, - selectedPhases, projectChangeAnalyzer, - commandToRun, + configHash, terminal, - trackedProjectFiles, operationMetadataManager }: { - record: OperationExecutionRecord; + buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; rushProject: RushConfigurationProject; phase: IPhase; - selectedPhases: Iterable; projectChangeAnalyzer: ProjectChangeAnalyzer; - commandToRun: string; + configHash: string; terminal: ITerminal; - trackedProjectFiles: string[] | undefined; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); if (!buildCacheContext.projectBuildCache) { - if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { - // Disable legacy skip logic if the build cache is in play - buildCacheContext.isSkipAllowed = false; - - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); - if (projectConfiguration) { - const commandName: string = phase.name; - projectConfiguration.validatePhaseConfiguration(selectedPhases, terminal); - if (projectConfiguration.disableBuildCacheForProject) { - terminal.writeVerboseLine('Caching has been disabled for this project.'); - } else { - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(commandName); - if (!operationSettings) { - terminal.writeVerboseLine( - `This project does not define the caching behavior of the "${commandName}" command, so caching has been disabled.` - ); - } else if (operationSettings.disableBuildCacheForOperation) { - terminal.writeVerboseLine( - `Caching has been disabled for this project's "${commandName}" command.` - ); - } else { - const projectOutputFolderNames: ReadonlyArray = - operationSettings.outputFolderNames || []; - const additionalProjectOutputFilePaths: ReadonlyArray = [ - ...(operationMetadataManager?.relativeFilepaths || []) - ]; - const additionalContext: Record = {}; - if (operationSettings.dependsOnEnvVars) { - for (const varName of operationSettings.dependsOnEnvVars) { - additionalContext['$' + varName] = process.env[varName] || ''; - } - } - - if (operationSettings.dependsOnAdditionalFiles) { - const repoState: IRawRepoState | undefined = - await projectChangeAnalyzer._ensureInitializedAsync(terminal); + const { cacheDisabledReason } = buildCacheContext; + if (cacheDisabledReason) { + terminal.writeVerboseLine(cacheDisabledReason); + return; + } - const additionalFiles: Map = await getHashesForGlobsAsync( - operationSettings.dependsOnAdditionalFiles, - rushProject.projectFolder, - repoState - ); + const { operationSettings } = buildCacheContext; + if (!operationSettings || !buildCacheConfiguration) { + // Unreachable, since this will have set `cacheDisabledReason`. + return; + } - terminal.writeDebugLine( - `Including additional files to calculate build cache hash:\n ${Array.from( - additionalFiles.keys() - ).join('\n ')} ` - ); + const projectOutputFolderNames: ReadonlyArray = operationSettings.outputFolderNames || []; + const additionalProjectOutputFilePaths: ReadonlyArray = + operationMetadataManager?.relativeFilepaths || []; + const additionalContext: Record = {}; + + await updateAdditionalContextAsync({ + operationSettings, + additionalContext, + projectChangeAnalyzer, + terminal, + rushProject + }); - for (const [filePath, fileHash] of additionalFiles) { - additionalContext['file://' + filePath] = fileHash; - } - } - buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ - project: rushProject, - projectOutputFolderNames, - additionalProjectOutputFilePaths, - additionalContext, - buildCacheConfiguration, - terminal, - command: commandToRun, - trackedProjectFiles: trackedProjectFiles, - projectChangeAnalyzer: projectChangeAnalyzer, - phaseName: phase.name - }); - } - } - } else { - terminal.writeVerboseLine( - `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + - 'or one provided by a rig, so it does not support caching.' - ); - } - } + // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent + buildCacheContext.projectBuildCache = await ProjectBuildCache.tryGetProjectBuildCache({ + project: rushProject, + projectOutputFolderNames, + additionalProjectOutputFilePaths, + additionalContext, + buildCacheConfiguration, + terminal, + configHash, + projectChangeAnalyzer, + phaseName: phase.name + }); } return buildCacheContext.projectBuildCache; @@ -834,113 +651,86 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Get a ProjectBuildCache only cache/restore log files private async _tryGetLogOnlyProjectBuildCacheAsync({ - record, + buildCacheContext, rushProject, terminal, - commandToRun, + configHash, buildCacheConfiguration, cobuildConfiguration, phase, - trackedProjectFiles, projectChangeAnalyzer, operationMetadataManager }: { - record: OperationExecutionRecord; + buildCacheContext: IOperationBuildCacheContext; buildCacheConfiguration: BuildCacheConfiguration | undefined; cobuildConfiguration: CobuildConfiguration; rushProject: RushConfigurationProject; phase: IPhase; - commandToRun: string; + configHash: string; terminal: ITerminal; - trackedProjectFiles: string[] | undefined; projectChangeAnalyzer: ProjectChangeAnalyzer; operationMetadataManager: OperationMetadataManager | undefined; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); - if (buildCacheConfiguration && buildCacheConfiguration.buildCacheEnabled) { - // Disable legacy skip logic if the build cache is in play - buildCacheContext.isSkipAllowed = false; - const projectConfiguration: RushProjectConfiguration | undefined = - await RushProjectConfiguration.tryLoadForProjectAsync(rushProject, terminal); - - let projectOutputFolderNames: ReadonlyArray = []; - const additionalProjectOutputFilePaths: ReadonlyArray = [ - ...(operationMetadataManager?.relativeFilepaths || []) - ]; - const additionalContext: Record = { - // Force the cache to be a log files only cache - logFilesOnly: '1' - }; - if (cobuildConfiguration.cobuildContextId) { - additionalContext.cobuildContextId = cobuildConfiguration.cobuildContextId; - } - if (projectConfiguration) { - const commandName: string = phase.name; - const operationSettings: IOperationSettings | undefined = - projectConfiguration.operationSettingsByOperationName.get(commandName); - if (operationSettings) { - if (operationSettings.outputFolderNames) { - projectOutputFolderNames = operationSettings.outputFolderNames; - } - if (operationSettings.dependsOnEnvVars) { - for (const varName of operationSettings.dependsOnEnvVars) { - additionalContext['$' + varName] = process.env[varName] || ''; - } - } - - if (operationSettings.dependsOnAdditionalFiles) { - const repoState: IRawRepoState | undefined = await projectChangeAnalyzer._ensureInitializedAsync( - terminal - ); + if (!buildCacheConfiguration?.buildCacheEnabled) { + return; + } - const additionalFiles: Map = await getHashesForGlobsAsync( - operationSettings.dependsOnAdditionalFiles, - rushProject.projectFolder, - repoState - ); + const { operationSettings } = buildCacheContext; + + const projectOutputFolderNames: ReadonlyArray = operationSettings?.outputFolderNames ?? []; + const additionalProjectOutputFilePaths: ReadonlyArray = + operationMetadataManager?.relativeFilepaths || []; + const additionalContext: Record = { + // Force the cache to be a log files only cache + logFilesOnly: '1' + }; + if (cobuildConfiguration.cobuildContextId) { + additionalContext.cobuildContextId = cobuildConfiguration.cobuildContextId; + } - for (const [filePath, fileHash] of additionalFiles) { - additionalContext['file://' + filePath] = fileHash; - } - } - } - } - const projectBuildCache: ProjectBuildCache | undefined = - await ProjectBuildCache.tryGetProjectBuildCache({ - project: rushProject, - projectOutputFolderNames, - additionalProjectOutputFilePaths, - additionalContext, - buildCacheConfiguration, - terminal, - command: commandToRun, - trackedProjectFiles, - projectChangeAnalyzer: projectChangeAnalyzer, - phaseName: phase.name - }); - buildCacheContext.projectBuildCache = projectBuildCache; - return projectBuildCache; + if (operationSettings) { + await updateAdditionalContextAsync({ + operationSettings, + additionalContext, + projectChangeAnalyzer, + terminal, + rushProject + }); } + + const projectBuildCache: ProjectBuildCache | undefined = await ProjectBuildCache.tryGetProjectBuildCache({ + project: rushProject, + projectOutputFolderNames, + additionalProjectOutputFilePaths, + additionalContext, + buildCacheConfiguration, + terminal, + configHash, + projectChangeAnalyzer, + phaseName: phase.name + }); + + // eslint-disable-next-line require-atomic-updates -- This is guaranteed to not be concurrent + buildCacheContext.projectBuildCache = projectBuildCache; + + return projectBuildCache; } private async _tryGetCobuildLockAsync({ cobuildConfiguration, - record, + buildCacheContext, projectBuildCache, packageName, phaseName }: { cobuildConfiguration: CobuildConfiguration | undefined; - record: OperationExecutionRecord; + buildCacheContext: IOperationBuildCacheContext; projectBuildCache: ProjectBuildCache | undefined; packageName: string; phaseName: string; }): Promise { - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); if (!buildCacheContext.cobuildLock) { - if (projectBuildCache && cobuildConfiguration && cobuildConfiguration.cobuildEnabled) { + if (projectBuildCache && cobuildConfiguration?.cobuildEnabled) { if (!buildCacheContext.cobuildClusterId) { // This should not happen throw new InternalError('Cobuild cluster id is not defined'); @@ -960,35 +750,30 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private _getBuildCacheTerminal({ record, - buildCacheConfiguration, + buildCacheContext, + buildCacheEnabled: buildCacheEnabled, rushProject, logFilenameIdentifier, quietMode, debugMode }: { record: OperationExecutionRecord; - buildCacheConfiguration: BuildCacheConfiguration | undefined; + buildCacheContext: IOperationBuildCacheContext; + buildCacheEnabled: boolean | undefined; rushProject: RushConfigurationProject; logFilenameIdentifier: string; quietMode: boolean; debugMode: boolean; }): ITerminal { - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); - if (!buildCacheContext.buildCacheTerminal) { + if ( + !buildCacheContext.buildCacheTerminal || + buildCacheContext.buildCacheProjectLogWritable?.isOpen === false + ) { + // The ProjectLogWritable is does not exist or is closed, re-create one buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ record, - buildCacheConfiguration, - rushProject, - logFilenameIdentifier, - quietMode, - debugMode - }); - } else if (buildCacheContext.buildCacheProjectLogWritable?.isOpen === false) { - // The ProjectLogWritable is closed, re-create one - buildCacheContext.buildCacheTerminal = this._createBuildCacheTerminal({ - record, - buildCacheConfiguration, + buildCacheContext, + buildCacheEnabled, rushProject, logFilenameIdentifier, quietMode, @@ -1001,14 +786,16 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { private _createBuildCacheTerminal({ record, - buildCacheConfiguration, + buildCacheContext, + buildCacheEnabled, rushProject, logFilenameIdentifier, quietMode, debugMode }: { record: OperationExecutionRecord; - buildCacheConfiguration: BuildCacheConfiguration | undefined; + buildCacheContext: IOperationBuildCacheContext; + buildCacheEnabled: boolean | undefined; rushProject: RushConfigurationProject; logFilenameIdentifier: string; quietMode: boolean; @@ -1024,8 +811,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // This creates the writer, only do this if necessary. const collatedWriter: CollatedWriter = record.collatedWriter; const cacheProjectLogWritable: ProjectLogWritable | undefined = this._tryGetBuildCacheProjectLogWritable({ - record, - buildCacheConfiguration, + buildCacheContext, + buildCacheEnabled, rushProject, collatedTerminal: collatedWriter.terminal, logFilenameIdentifier @@ -1065,24 +852,23 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } private _tryGetBuildCacheProjectLogWritable({ - buildCacheConfiguration, + buildCacheEnabled, rushProject, - record, + buildCacheContext, collatedTerminal, logFilenameIdentifier }: { - buildCacheConfiguration: BuildCacheConfiguration | undefined; + buildCacheEnabled: boolean | undefined; rushProject: RushConfigurationProject; - record: OperationExecutionRecord; + buildCacheContext: IOperationBuildCacheContext; collatedTerminal: CollatedTerminal; logFilenameIdentifier: string; }): ProjectLogWritable | undefined { // Only open the *.cache.log file(s) if the cache is enabled. - if (!buildCacheConfiguration?.buildCacheEnabled) { + if (!buildCacheEnabled) { return; } - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationExecutionRecordOrThrow(record); + buildCacheContext.buildCacheProjectLogWritable = new ProjectLogWritable( rushProject, collatedTerminal, @@ -1091,17 +877,44 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { return buildCacheContext.buildCacheProjectLogWritable; } } - -function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { - for (const n in object1) { - if (!(n in object2) || object1[n] !== object2[n]) { - return false; +async function updateAdditionalContextAsync({ + operationSettings, + additionalContext, + projectChangeAnalyzer, + terminal, + rushProject +}: { + operationSettings: IOperationSettings; + additionalContext: Record; + projectChangeAnalyzer: ProjectChangeAnalyzer; + terminal: ITerminal; + rushProject: RushConfigurationProject; +}): Promise { + if (operationSettings.dependsOnEnvVars) { + for (const varName of operationSettings.dependsOnEnvVars) { + additionalContext['$' + varName] = process.env[varName] || ''; } } - for (const n in object2) { - if (!(n in object1)) { - return false; + + if (operationSettings.dependsOnAdditionalFiles) { + const repoState: IRawRepoState | undefined = await projectChangeAnalyzer._ensureInitializedAsync( + terminal + ); + + const additionalFiles: Map = await getHashesForGlobsAsync( + operationSettings.dependsOnAdditionalFiles, + rushProject.projectFolder, + repoState + ); + + terminal.writeDebugLine( + `Including additional files to calculate build cache hash:\n ${Array.from(additionalFiles.keys()).join( + '\n ' + )} ` + ); + + for (const [filePath, fileHash] of additionalFiles) { + additionalContext['file://' + filePath] = fileHash; } } - return true; } diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index a7bb902a3d0..ea7f4476daa 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -3,14 +3,18 @@ import type { StdioSummarizer } from '@rushstack/terminal'; import type { OperationStatus } from './OperationStatus'; -import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { Operation } from './Operation'; +import type { IStopwatchResult } from '../../utilities/Stopwatch'; /** * The `IOperationExecutionResult` interface represents the results of executing an {@link Operation}. * @alpha */ export interface IOperationExecutionResult { + /** + * The operation itself + */ + readonly operation: Operation; /** * The current execution status of an operation. Operations start in the 'ready' state, * but can be 'blocked' if an upstream operation failed. It is 'executing' when diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index 13871c68c19..c17eed4d3ff 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -76,6 +76,11 @@ export interface IOperationRunner { */ readonly name: string; + /** + * Whether or not the operation is cacheable. If false, all cache engines will be disabled for this operation. + */ + cacheable: boolean; + /** * Indicates that this runner's duration has meaning. */ @@ -93,12 +98,12 @@ export interface IOperationRunner { warningsAreAllowed: boolean; /** - * Full shell command string to run by this runner. + * Method to be executed for the operation. */ - commandToRun?: string; + executeAsync(context: IOperationRunnerContext): Promise; /** - * Method to be executed for the operation. + * Return a hash of the configuration that affects the operation. */ - executeAsync(context: IOperationRunnerContext): Promise; + getConfigHash(): string; } diff --git a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts new file mode 100644 index 00000000000..e3ab5eaa36f --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import path from 'node:path'; + +import { + Async, + ColorValue, + FileSystem, + JsonFile, + type ITerminal, + type JsonObject +} from '@rushstack/node-core-library'; +import { PrintUtilities } from '@rushstack/terminal'; + +import { Operation } from './Operation'; +import { OperationStatus } from './OperationStatus'; +import type { + ICreateOperationsContext, + IPhasedCommandPlugin, + PhasedCommandHooks +} from '../../pluginFramework/PhasedCommandHooks'; +import { IOperationRunnerContext } from './IOperationRunner'; +import { IOperationExecutionResult } from './IOperationExecutionResult'; +import { ProjectChangeAnalyzer } from '../ProjectChangeAnalyzer'; + +const PLUGIN_NAME: 'LegacySkipPlugin' = 'LegacySkipPlugin'; + +function _areShallowEqual(object1: JsonObject, object2: JsonObject): boolean { + for (const n in object1) { + if (!(n in object2) || object1[n] !== object2[n]) { + return false; + } + } + for (const n in object2) { + if (!(n in object1)) { + return false; + } + } + return true; +} + +export interface IProjectDeps { + files: { [filePath: string]: string }; + arguments: string; +} + +interface ILegacySkipRecord { + allowSkip: boolean; + packageDeps: IProjectDeps | undefined; + packageDepsPath: string; +} + +export interface ILegacySkipPluginOptions { + terminal: ITerminal; + changedProjectsOnly: boolean; + isIncrementalBuildAllowed: boolean; +} + +/** + * Core phased command plugin that implements the legacy skip detection logic, used when build cache is disabled. + */ +export class LegacySkipPlugin implements IPhasedCommandPlugin { + private readonly _options: ILegacySkipPluginOptions; + + public constructor(options: ILegacySkipPluginOptions) { + this._options = options; + } + + public apply(hooks: PhasedCommandHooks): void { + const stateMap: WeakMap = new WeakMap(); + + let projectChangeAnalyzer!: ProjectChangeAnalyzer; + + const { terminal, changedProjectsOnly, isIncrementalBuildAllowed } = this._options; + + hooks.createOperations.tap( + PLUGIN_NAME, + (operations: Set, context: ICreateOperationsContext): Set => { + projectChangeAnalyzer = context.projectChangeAnalyzer; + + return operations; + } + ); + + hooks.beforeExecuteOperations.tapPromise( + PLUGIN_NAME, + async (operations: ReadonlyMap): Promise => { + let logGitWarning: boolean = false; + + await Async.forEachAsync(operations.values(), async (record: IOperationExecutionResult) => { + const { operation } = record; + const { associatedProject, associatedPhase, runner } = operation; + if (!associatedProject || !associatedPhase || !runner) { + return; + } + + if (!runner.cacheable) { + stateMap.set(operation, { + allowSkip: true, + packageDeps: undefined, + packageDepsPath: '' + }); + return; + } + + const packageDepsFilename: string = `package-deps_${associatedPhase.logFilenameIdentifier}.json`; + + const packageDepsPath: string = path.join( + associatedProject.projectRushTempFolder, + packageDepsFilename + ); + + let packageDeps: IProjectDeps | undefined; + + try { + const fileHashes: Map | undefined = + await projectChangeAnalyzer._tryGetProjectDependenciesAsync(associatedProject, terminal); + + if (!fileHashes) { + logGitWarning = true; + return; + } + + const files: Record = {}; + for (const [filePath, fileHash] of fileHashes) { + files[filePath] = fileHash; + } + + packageDeps = { + files, + arguments: runner.getConfigHash() + }; + } catch (error) { + // To test this code path: + // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" + terminal.writeLine( + `Unable to calculate incremental state for ${record.operation.name}: ` + + (error as Error).toString() + ); + terminal.writeLine({ + text: 'Rush will proceed without incremental execution and change detection.', + foregroundColor: ColorValue.Cyan + }); + } + + stateMap.set(operation, { + packageDepsPath, + packageDeps, + allowSkip: isIncrementalBuildAllowed + }); + }); + + if (logGitWarning) { + // To test this code path: + // Remove the `.git` folder then run "rush build --verbose" + terminal.writeLine({ + text: PrintUtilities.wrapWords( + 'This workspace does not appear to be tracked by Git. ' + + 'Rush will proceed without incremental execution, caching, and change detection.' + ), + foregroundColor: ColorValue.Cyan + }); + } + } + ); + + hooks.beforeExecuteOperation.tapPromise( + PLUGIN_NAME, + async ( + record: IOperationRunnerContext & IOperationExecutionResult + ): Promise => { + const { operation } = record; + const skipRecord: ILegacySkipRecord | undefined = stateMap.get(operation); + if (!skipRecord) { + // This operation doesn't support skip detection. + return; + } + + if (!operation.runner!.cacheable) { + // This operation doesn't support skip detection. + return; + } + + const { associatedProject } = operation; + + const { packageDepsPath, packageDeps, allowSkip } = skipRecord; + + let lastProjectDeps: IProjectDeps | undefined = undefined; + + try { + const lastDepsContents: string = await FileSystem.readFileAsync(packageDepsPath); + lastProjectDeps = JSON.parse(lastDepsContents); + } catch (e) { + if (!FileSystem.isNotExistError(e)) { + // Warn and ignore - treat failing to load the file as the operation being not built. + // TODO: Update this to be the terminal specific to the operation. + terminal.writeWarningLine( + `Warning: error parsing ${packageDepsPath}: ${e}. Ignoring and treating this operation as not run.` + ); + } + } + + if (allowSkip) { + const isPackageUnchanged: boolean = !!( + lastProjectDeps && + packageDeps && + packageDeps.arguments === lastProjectDeps.arguments && + _areShallowEqual(packageDeps.files, lastProjectDeps.files) + ); + + if (isPackageUnchanged) { + return OperationStatus.Skipped; + } + } + + // TODO: Remove legacyDepsPath with the next major release of Rush + const legacyDepsPath: string = path.join(associatedProject!.projectFolder, 'package-deps.json'); + + await Promise.all([ + // Delete the legacy package-deps.json + FileSystem.deleteFileAsync(legacyDepsPath), + + // If the deps file exists, remove it before starting execution. + FileSystem.deleteFileAsync(packageDepsPath) + ]); + } + ); + + hooks.afterExecuteOperation.tapPromise( + PLUGIN_NAME, + async (record: IOperationRunnerContext & IOperationExecutionResult): Promise => { + const { status, operation } = record; + + const skipRecord: ILegacySkipRecord | undefined = stateMap.get(operation); + if (!skipRecord) { + return; + } + + const blockSkip: boolean = + !skipRecord.allowSkip || + (!changedProjectsOnly && + (status === OperationStatus.Success || status === OperationStatus.SuccessWithWarning)); + if (blockSkip) { + for (const consumer of operation.consumers) { + const consumerSkipRecord: ILegacySkipRecord | undefined = stateMap.get(consumer); + if (consumerSkipRecord) { + consumerSkipRecord.allowSkip = false; + } + } + } + + if (!record.operation.runner!.cacheable) { + // This operation doesn't support skip detection. + return; + } + + const { packageDeps, packageDepsPath } = skipRecord; + + if ((packageDeps && status === OperationStatus.Success) || status === OperationStatus.NoOp) { + // Write deps on success. + await JsonFile.saveAsync(packageDeps, packageDepsPath, { + ensureFolderExists: true + }); + } + } + ); + } +} diff --git a/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts b/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts index 8e3f14fac25..d2b31e56a8c 100644 --- a/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/NullOperationRunner.ts @@ -31,10 +31,8 @@ export class NullOperationRunner implements IOperationRunner { // This operation does nothing, so timing is meaningless public readonly reportTiming: boolean = false; public readonly silent: boolean; - // The operation may be skipped; it doesn't do anything anyway - public isSkipAllowed: boolean = true; - // The operation is a no-op, so is cacheable. - public isCacheWriteAllowed: boolean = true; + // The operation is a no-op, so it is faster to not cache it + public cacheable: boolean = false; // Nothing will get logged, no point allowing warnings public readonly warningsAreAllowed: boolean = false; @@ -49,4 +47,8 @@ export class NullOperationRunner implements IOperationRunner { public async executeAsync(context: IOperationRunnerContext): Promise { return this.result; } + + public getConfigHash(): string { + return ''; + } } diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 29e242e01dd..1d025de4d31 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -28,6 +28,11 @@ export interface IOperationExecutionRecordContext { * @internal */ export class OperationExecutionRecord implements IOperationRunnerContext { + /** + * The associated operation. + */ + public readonly operation: Operation; + /** * The current execution status of an operation. Operations start in the 'ready' state, * but can be 'blocked' if an upstream operation failed. It is 'executing' when @@ -106,6 +111,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext { ); } + this.operation = operation; this.runner = runner; this.weight = operation.weight; this.associatedPhase = associatedPhase; diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 4ada66c5b45..f51f0af3c03 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -48,9 +48,10 @@ export class ShellOperationRunner implements IOperationRunner { public readonly reportTiming: boolean = true; public readonly silent: boolean = false; + public readonly cacheable: boolean = true; public readonly warningsAreAllowed: boolean; - public readonly commandToRun: string; + private readonly _commandToRun: string; private readonly _logFilenameIdentifier: string; private readonly _rushProject: RushConfigurationProject; @@ -62,7 +63,7 @@ export class ShellOperationRunner implements IOperationRunner { this.name = options.displayName; this._rushProject = options.rushProject; this._rushConfiguration = options.rushConfiguration; - this.commandToRun = options.commandToRun; + this._commandToRun = options.commandToRun; this.warningsAreAllowed = EnvironmentConfiguration.allowWarningsInSuccessfulBuild || phase.allowWarningsOnSuccess || false; this._logFilenameIdentifier = phase.logFilenameIdentifier; @@ -76,6 +77,10 @@ export class ShellOperationRunner implements IOperationRunner { } } + public getConfigHash(): string { + return this._commandToRun; + } + private async _executeAsync(context: IOperationRunnerContext): Promise { const projectLogWritable: ProjectLogWritable = new ProjectLogWritable( this._rushProject, @@ -132,10 +137,10 @@ export class ShellOperationRunner implements IOperationRunner { const projectFolder: string = this._rushProject.projectFolder; // Run the operation - terminal.writeLine('Invoking: ' + this.commandToRun); + terminal.writeLine('Invoking: ' + this._commandToRun); const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( - this.commandToRun, + this._commandToRun, { rushConfiguration: this._rushConfiguration, workingDirectory: projectFolder, diff --git a/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts b/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts index 58a71c26a21..8f505120b91 100644 --- a/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/test/MockOperationRunner.ts @@ -11,8 +11,7 @@ export class MockOperationRunner implements IOperationRunner { public readonly name: string; public readonly reportTiming: boolean = true; public readonly silent: boolean = false; - public isSkipAllowed: boolean = false; - public isCacheWriteAllowed: boolean = false; + public readonly cacheable: boolean = false; public readonly warningsAreAllowed: boolean; public constructor( @@ -32,4 +31,8 @@ export class MockOperationRunner implements IOperationRunner { } return result || OperationStatus.Success; } + + public getConfigHash(): string { + return 'mock'; + } } diff --git a/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts index 8518e53eb86..c5d86c70a5b 100644 --- a/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts @@ -11,7 +11,6 @@ import { ICommandLineJson } from '../../../api/CommandLineJson'; import { PhasedOperationPlugin } from '../PhasedOperationPlugin'; import { ShellOperationRunnerPlugin } from '../ShellOperationRunnerPlugin'; import { ICreateOperationsContext, PhasedCommandHooks } from '../../../pluginFramework/PhasedCommandHooks'; -import { ShellOperationRunner } from '../ShellOperationRunner'; interface ISerializedOperation { name: string; @@ -21,7 +20,7 @@ interface ISerializedOperation { function serializeOperation(operation: Operation): ISerializedOperation { return { name: operation.name!, - commandToRun: (operation.runner as ShellOperationRunner).commandToRun + commandToRun: operation.runner!.getConfigHash() }; } diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 49c5372b530..abf6ca57546 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -15,6 +15,7 @@ import type { IOperationExecutionResult } from '../logic/operations/IOperationExecutionResult'; import type { CobuildConfiguration } from '../api/CobuildConfiguration'; +import { RushProjectConfiguration } from '../api/RushProjectConfiguration'; import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; import type { ITelemetryData } from '../logic/Telemetry'; import type { OperationStatus } from '../logic/operations/OperationStatus'; @@ -78,6 +79,10 @@ export interface ICreateOperationsContext { * The set of Rush projects selected for the current command execution. */ readonly projectSelection: ReadonlySet; + /** + * All successfully loaded rush-project.json data for selected projects. + */ + readonly projectConfigurations: ReadonlyMap; /** * The set of Rush projects that have not been built in the current process since they were last modified. * When `isInitial` is true, this will be an exact match of `projectSelection`. @@ -127,19 +132,16 @@ export class PhasedCommandHooks { * Hook invoked before executing a operation. */ public readonly beforeExecuteOperation: AsyncSeriesBailHook< - [IOperationRunnerContext], + [IOperationRunnerContext & IOperationExecutionResult], OperationStatus | undefined - > = new AsyncSeriesBailHook<[IOperationRunnerContext], OperationStatus | undefined>( - ['runnerContext'], - 'beforeExecuteOperation' - ); + > = new AsyncSeriesBailHook(['runnerContext'], 'beforeExecuteOperation'); /** * Hook invoked after executing a operation. */ - public readonly afterExecuteOperation: AsyncSeriesHook<[IOperationRunnerContext]> = new AsyncSeriesHook< - [IOperationRunnerContext] - >(['runnerContext'], 'afterExecuteOperation'); + public readonly afterExecuteOperation: AsyncSeriesHook< + [IOperationRunnerContext & IOperationExecutionResult] + > = new AsyncSeriesHook(['runnerContext'], 'afterExecuteOperation'); /** * Hook invoked after a run has finished and the command is watching for changes. diff --git a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap index b79206a7e72..a789f7b16a1 100644 --- a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap +++ b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap @@ -43,6 +43,7 @@ Loaded @microsoft/rush-lib from process.env._RUSH_LIB_PATH 'RushConfigurationProject', 'RushConstants', 'RushLifecycleHooks', + 'RushProjectConfiguration', 'RushSession', 'RushUserConfiguration', 'VersionPolicy', From eb9a21efd9b27db7010a6f302fe8d21fa5f2556d Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 30 Aug 2023 17:04:46 +0800 Subject: [PATCH 094/100] fix: skip cacheable beforeExecution when runner is not cacheable --- .../rush-lib/src/logic/operations/CacheableOperationPlugin.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 809e4e3a450..448284b2242 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -235,10 +235,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { const { associatedProject: project, associatedPhase: phase, + runner, _operationMetadataManager: operationMetadataManager } = record; - if (!project || !phase) { + if (!project || !phase || !runner?.cacheable) { return; } From e65207ce5c61801d35e94a52579aa6ed9acec497 Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 30 Aug 2023 23:41:40 +0800 Subject: [PATCH 095/100] feat: try restore cache once when missing cobuild completed state --- .../src/logic/operations/CacheableOperationPlugin.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 448284b2242..91dc834b36a 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -62,6 +62,7 @@ export interface IOperationBuildCacheContext { periodicCallback: PeriodicCallback; cacheRestored: boolean; + isCacheReadAttempted: boolean; } export interface ICacheableOperationPluginOptions { @@ -147,7 +148,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { periodicCallback: new PeriodicCallback({ interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 }), - cacheRestored: false + cacheRestored: false, + isCacheReadAttempted: false }; // Upstream runners may mutate the property of build cache context for downstream runners this._buildCacheContextByOperation.set(operation, buildCacheContext); @@ -353,6 +355,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { projectBuildCache: ProjectBuildCache | undefined, specifiedCacheId?: string ): Promise => { + buildCacheContext.isCacheReadAttempted = true; const restoreFromCacheSuccess: boolean | undefined = await projectBuildCache?.tryRestoreFromCacheAsync(buildCacheTerminal, specifiedCacheId); if (restoreFromCacheSuccess) { @@ -387,6 +390,12 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } return status; } + } else if (!buildCacheContext.isCacheReadAttempted && buildCacheContext.isCacheReadAllowed) { + const restoreFromCacheSuccess: boolean = await restoreCacheAsync(projectBuildCache); + + if (restoreFromCacheSuccess) { + return OperationStatus.FromCache; + } } } else if (buildCacheContext.isCacheReadAllowed) { const restoreFromCacheSuccess: boolean = await restoreCacheAsync(projectBuildCache); From 996a6b022f9efd57cb9a7712023a787a04bd89aa Mon Sep 17 00:00:00 2001 From: David Michon Date: Wed, 30 Aug 2023 11:39:36 -0700 Subject: [PATCH 096/100] Handle async queue being empty --- .../logic/operations/AsyncOperationQueue.ts | 25 +++++++++++-------- .../test/AsyncOperationQueue.test.ts | 16 +++++++++++- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts index 423fbb4a8c7..5bd47bfd597 100644 --- a/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts +++ b/libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts @@ -86,16 +86,6 @@ export class AsyncOperationQueue public assignOperations(): void { const { _queue: queue, _pendingIterators: waitingIterators } = this; - if (this._isDone) { - for (const resolveAsyncIterator of waitingIterators.splice(0)) { - resolveAsyncIterator({ - value: undefined, - done: true - }); - } - return; - } - // By iterating in reverse order we do less array shuffling when removing operations for (let i: number = queue.length - 1; waitingIterators.length > 0 && i >= 0; i--) { const operation: OperationExecutionRecord = queue[i]; @@ -137,6 +127,21 @@ export class AsyncOperationQueue // Otherwise operation is still waiting } + // Since items only get removed from the queue when they have a final status, this should be safe. + if (queue.length === 0) { + this._isDone = true; + } + + if (this._isDone) { + for (const resolveAsyncIterator of waitingIterators.splice(0)) { + resolveAsyncIterator({ + value: undefined, + done: true + }); + } + return; + } + if (waitingIterators.length > 0) { // returns an unassigned operation to let caller decide when there is at least one // remote executing operation which is not ready to process. diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index 3ba19c3d7b7..7c97cf3f248 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -4,7 +4,12 @@ import { Operation } from '../Operation'; import { IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; import { MockOperationRunner } from './MockOperationRunner'; -import { AsyncOperationQueue, IOperationSortFunction, UNASSIGNED_OPERATION } from '../AsyncOperationQueue'; +import { + AsyncOperationQueue, + IOperationIteratorResult, + IOperationSortFunction, + UNASSIGNED_OPERATION +} from '../AsyncOperationQueue'; import { OperationStatus } from '../OperationStatus'; import { Async } from '@rushstack/node-core-library'; @@ -217,4 +222,13 @@ describe(AsyncOperationQueue.name, () => { expect(actualOrder).toEqual(expectedOrder); }); + + it('handles an empty queue', async () => { + const operations: OperationExecutionRecord[] = []; + + const queue: AsyncOperationQueue = new AsyncOperationQueue(operations, nullSort); + const iterator: AsyncIterator = queue[Symbol.asyncIterator](); + const result: IteratorResult = await iterator.next(); + expect(result.done).toEqual(true); + }); }); From 5c659f62d506432e06d174312e5ddb25b48e4b53 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 31 Aug 2023 19:59:55 -0700 Subject: [PATCH 097/100] Update changefile. --- .../changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json index ac388318c6a..396378277eb 100644 --- a/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json +++ b/common/changes/@microsoft/rush/feat-cobuild_2023-02-17-07-02.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@microsoft/rush", - "comment": "(EXPERIMENTAL) Add a cheap way to get distributed builds called \"cobuild\". See [Rush.js Cobuild Feature Note](https://docs.google.com/document/d/1ydh4dVMpqSk_3mi-NgtWhI_g3TTmvkKsFQuddp8-4dI/edit)", + "comment": "(EXPERIMENTAL) Initial release of the cobuild feature, a cheap way to distribute jobs Rush builds across multiple VMs. (GitHub #3485)", "type": "none" } ], From d96b30b5918d26fa87d59a459e97c5de5ed1452e Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 31 Aug 2023 20:01:08 -0700 Subject: [PATCH 098/100] Rename "cobuildEnabled" to "cobuildFeatureEnabled" in "cobuild.json" --- .../sandbox/repo/common/config/rush/cobuild.json | 2 +- common/reviews/api/rush-lib.api.md | 6 +++--- .../rush-init/common/config/rush/cobuild.json | 2 +- libraries/rush-lib/src/api/CobuildConfiguration.ts | 13 +++++++------ .../rush-lib/src/api/EnvironmentConfiguration.ts | 12 ++++++------ .../logic/operations/CacheableOperationPlugin.ts | 8 ++++---- libraries/rush-lib/src/schemas/cobuild.schema.json | 4 ++-- 7 files changed, 24 insertions(+), 23 deletions(-) diff --git a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json index f16a120b1a6..4626f2211d4 100644 --- a/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json +++ b/build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/repo/common/config/rush/cobuild.json @@ -10,7 +10,7 @@ * RUSH_COBUILD_CONTEXT_ID should always be specified as an environment variable with an non-empty string, * otherwise the cobuild feature will be disabled. */ - "cobuildEnabled": true, + "cobuildFeatureEnabled": true, /** * (Required) Choose where cobuild lock will be acquired. diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 9c0809314af..0f21570fdcd 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -97,7 +97,7 @@ export type CloudBuildCacheProviderFactory = (buildCacheJson: IBuildCacheJson) = // @beta export class CobuildConfiguration { readonly cobuildContextId: string | undefined; - readonly cobuildEnabled: boolean; + readonly cobuildFeatureEnabled: boolean; readonly cobuildLeafProjectLogOnlyAllowed: boolean; // (undocumented) get cobuildLockProvider(): ICobuildLockProvider; @@ -182,7 +182,7 @@ export class EnvironmentConfiguration { static get buildCacheEnabled(): boolean | undefined; static get buildCacheWriteAllowed(): boolean | undefined; static get cobuildContextId(): string | undefined; - static get cobuildEnabled(): boolean | undefined; + static get cobuildFeatureEnabled(): boolean | undefined; static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined; static get cobuildRunnerId(): string | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts @@ -308,7 +308,7 @@ export interface ICobuildContext { // @beta (undocumented) export interface ICobuildJson { // (undocumented) - cobuildEnabled: boolean; + cobuildFeatureEnabled: boolean; // (undocumented) cobuildLockProvider: string; } diff --git a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json index 15a874bebb4..a47fad18d5e 100644 --- a/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json +++ b/libraries/rush-lib/assets/rush-init/common/config/rush/cobuild.json @@ -10,7 +10,7 @@ * RUSH_COBUILD_CONTEXT_ID should always be specified as an environment variable with an non-empty string, * otherwise the cobuild feature will be disabled. */ - "cobuildEnabled": false, + "cobuildFeatureEnabled": false, /** * (Required) Choose where cobuild lock will be acquired. diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 11b330597b5..1e508cf688b 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -15,7 +15,7 @@ import schemaJson from '../schemas/cobuild.schema.json'; * @beta */ export interface ICobuildJson { - cobuildEnabled: boolean; + cobuildFeatureEnabled: boolean; cobuildLockProvider: string; } @@ -46,7 +46,8 @@ export class CobuildConfiguration { * actually turn on for that particular build unless the cobuild context id is provided as an * non-empty string. */ - public readonly cobuildEnabled: boolean; + public readonly cobuildFeatureEnabled: boolean; + /** * Cobuild context id * @@ -75,8 +76,8 @@ export class CobuildConfiguration { const { cobuildJson, cobuildLockProviderFactory } = options; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; - this.cobuildEnabled = this.cobuildContextId - ? EnvironmentConfiguration.cobuildEnabled ?? cobuildJson.cobuildEnabled + this.cobuildFeatureEnabled = this.cobuildContextId + ? EnvironmentConfiguration.cobuildFeatureEnabled ?? cobuildJson.cobuildFeatureEnabled : false; this.cobuildRunnerId = EnvironmentConfiguration.cobuildRunnerId || uuidv4(); this.cobuildLeafProjectLogOnlyAllowed = @@ -144,7 +145,7 @@ export class CobuildConfiguration { } public async createLockProviderAsync(terminal: ITerminal): Promise { - if (this.cobuildEnabled) { + if (this.cobuildFeatureEnabled) { terminal.writeLine(`Running cobuild (runner ${this.cobuildContextId}/${this.cobuildRunnerId})`); const cobuildLockProvider: ICobuildLockProvider = await this._cobuildLockProviderFactory( this._cobuildJson @@ -155,7 +156,7 @@ export class CobuildConfiguration { } public async destroyLockProviderAsync(): Promise { - if (this.cobuildEnabled) { + if (this.cobuildFeatureEnabled) { await this._cobuildLockProvider?.disconnectAsync(); } } diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index 4f27f9a676d..c02327592ca 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -145,11 +145,11 @@ export const EnvironmentVariableNames = { RUSH_BUILD_CACHE_WRITE_ALLOWED: 'RUSH_BUILD_CACHE_WRITE_ALLOWED', /** - * Setting this environment variable overrides the value of `cobuildEnabled` in the `cobuild.json` + * Setting this environment variable overrides the value of `cobuildFeatureEnabled` in the `cobuild.json` * configuration file. * * @remarks - * Specify `1` to enable the cobuild or `0` to disable it. + * Specify `1` to enable the cobuild feature or `0` to disable it. * * If there is no cobuild configured, then this environment variable is ignored. */ @@ -244,7 +244,7 @@ export class EnvironmentConfiguration { private static _buildCacheWriteAllowed: boolean | undefined; - private static _cobuildEnabled: boolean | undefined; + private static _cobuildFeatureEnabled: boolean | undefined; private static _cobuildContextId: string | undefined; @@ -353,9 +353,9 @@ export class EnvironmentConfiguration { * If set, enables or disables the cobuild feature. * See {@link EnvironmentVariableNames.RUSH_COBUILD_ENABLED} */ - public static get cobuildEnabled(): boolean | undefined { + public static get cobuildFeatureEnabled(): boolean | undefined { EnvironmentConfiguration._ensureValidated(); - return EnvironmentConfiguration._cobuildEnabled; + return EnvironmentConfiguration._cobuildFeatureEnabled; } /** @@ -516,7 +516,7 @@ export class EnvironmentConfiguration { } case EnvironmentVariableNames.RUSH_COBUILD_ENABLED: { - EnvironmentConfiguration._cobuildEnabled = + EnvironmentConfiguration._cobuildFeatureEnabled = EnvironmentConfiguration.parseBooleanEnvironmentVariable( EnvironmentVariableNames.RUSH_COBUILD_ENABLED, value diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 91dc834b36a..8f513b11744 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -97,7 +97,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { this._createContext = context; - const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildEnabled + const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled ? new DisjointSet() : undefined; @@ -182,7 +182,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { } for (const operationSet of disjointSet.getAllSets()) { - if (cobuildConfiguration?.cobuildEnabled && cobuildConfiguration.cobuildContextId) { + if (cobuildConfiguration?.cobuildFeatureEnabled && cobuildConfiguration.cobuildContextId) { // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. const groupedOperations: Operation[] = Array.from(operationSet); Sort.sortBy(groupedOperations, (operation: Operation) => { @@ -292,7 +292,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { // Try to acquire the cobuild lock let cobuildLock: CobuildLock | undefined; - if (cobuildConfiguration?.cobuildEnabled) { + if (cobuildConfiguration?.cobuildFeatureEnabled) { if ( cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && record.operation.consumers.size === 0 && @@ -740,7 +740,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { phaseName: string; }): Promise { if (!buildCacheContext.cobuildLock) { - if (projectBuildCache && cobuildConfiguration?.cobuildEnabled) { + if (projectBuildCache && cobuildConfiguration?.cobuildFeatureEnabled) { if (!buildCacheContext.cobuildClusterId) { // This should not happen throw new InternalError('Cobuild cluster id is not defined'); diff --git a/libraries/rush-lib/src/schemas/cobuild.schema.json b/libraries/rush-lib/src/schemas/cobuild.schema.json index 1a3b4720d42..6fe630b89d8 100644 --- a/libraries/rush-lib/src/schemas/cobuild.schema.json +++ b/libraries/rush-lib/src/schemas/cobuild.schema.json @@ -15,13 +15,13 @@ { "type": "object", "additionalProperties": false, - "required": ["cobuildEnabled", "cobuildLockProvider"], + "required": ["cobuildFeatureEnabled", "cobuildLockProvider"], "properties": { "$schema": { "description": "Part of the JSON Schema standard, this optional keyword declares the URL of the schema that the file conforms to. Editors may download the schema and use it to perform syntax highlighting.", "type": "string" }, - "cobuildEnabled": { + "cobuildFeatureEnabled": { "description": "Set this to true to enable the cobuild feature.", "type": "boolean" }, From ff628591ca4d199ce052e29fcf692aea9fc39451 Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 31 Aug 2023 20:09:56 -0700 Subject: [PATCH 099/100] Remove the RUSH_COBUILD_ENABLED env var for now. --- common/reviews/api/rush-lib.api.md | 2 -- .../rush-lib/src/api/CobuildConfiguration.ts | 4 +-- .../src/api/EnvironmentConfiguration.ts | 31 ------------------- 3 files changed, 1 insertion(+), 36 deletions(-) diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 0f21570fdcd..f8d0e1899bf 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -182,7 +182,6 @@ export class EnvironmentConfiguration { static get buildCacheEnabled(): boolean | undefined; static get buildCacheWriteAllowed(): boolean | undefined; static get cobuildContextId(): string | undefined; - static get cobuildFeatureEnabled(): boolean | undefined; static get cobuildLeafProjectLogOnlyAllowed(): boolean | undefined; static get cobuildRunnerId(): string | undefined; // Warning: (ae-forgotten-export) The symbol "IEnvironment" needs to be exported by the entry point index.d.ts @@ -217,7 +216,6 @@ export const EnvironmentVariableNames: { readonly RUSH_BUILD_CACHE_CREDENTIAL: "RUSH_BUILD_CACHE_CREDENTIAL"; readonly RUSH_BUILD_CACHE_ENABLED: "RUSH_BUILD_CACHE_ENABLED"; readonly RUSH_BUILD_CACHE_WRITE_ALLOWED: "RUSH_BUILD_CACHE_WRITE_ALLOWED"; - readonly RUSH_COBUILD_ENABLED: "RUSH_COBUILD_ENABLED"; readonly RUSH_COBUILD_CONTEXT_ID: "RUSH_COBUILD_CONTEXT_ID"; readonly RUSH_COBUILD_RUNNER_ID: "RUSH_COBUILD_RUNNER_ID"; readonly RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED: "RUSH_COBUILD_LEAF_PROJECT_LOG_ONLY_ALLOWED"; diff --git a/libraries/rush-lib/src/api/CobuildConfiguration.ts b/libraries/rush-lib/src/api/CobuildConfiguration.ts index 1e508cf688b..b229506aca9 100644 --- a/libraries/rush-lib/src/api/CobuildConfiguration.ts +++ b/libraries/rush-lib/src/api/CobuildConfiguration.ts @@ -76,9 +76,7 @@ export class CobuildConfiguration { const { cobuildJson, cobuildLockProviderFactory } = options; this.cobuildContextId = EnvironmentConfiguration.cobuildContextId; - this.cobuildFeatureEnabled = this.cobuildContextId - ? EnvironmentConfiguration.cobuildFeatureEnabled ?? cobuildJson.cobuildFeatureEnabled - : false; + this.cobuildFeatureEnabled = this.cobuildContextId ? cobuildJson.cobuildFeatureEnabled : false; this.cobuildRunnerId = EnvironmentConfiguration.cobuildRunnerId || uuidv4(); this.cobuildLeafProjectLogOnlyAllowed = EnvironmentConfiguration.cobuildLeafProjectLogOnlyAllowed ?? false; diff --git a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts index c02327592ca..393b5cefbce 100644 --- a/libraries/rush-lib/src/api/EnvironmentConfiguration.ts +++ b/libraries/rush-lib/src/api/EnvironmentConfiguration.ts @@ -144,17 +144,6 @@ export const EnvironmentVariableNames = { */ RUSH_BUILD_CACHE_WRITE_ALLOWED: 'RUSH_BUILD_CACHE_WRITE_ALLOWED', - /** - * Setting this environment variable overrides the value of `cobuildFeatureEnabled` in the `cobuild.json` - * configuration file. - * - * @remarks - * Specify `1` to enable the cobuild feature or `0` to disable it. - * - * If there is no cobuild configured, then this environment variable is ignored. - */ - RUSH_COBUILD_ENABLED: 'RUSH_COBUILD_ENABLED', - /** * Setting this environment variable opts into running with cobuilds. The context id should be the same across * multiple VMs, but changed when it is a new round of cobuilds. @@ -244,8 +233,6 @@ export class EnvironmentConfiguration { private static _buildCacheWriteAllowed: boolean | undefined; - private static _cobuildFeatureEnabled: boolean | undefined; - private static _cobuildContextId: string | undefined; private static _cobuildRunnerId: string | undefined; @@ -349,15 +336,6 @@ export class EnvironmentConfiguration { return EnvironmentConfiguration._buildCacheWriteAllowed; } - /** - * If set, enables or disables the cobuild feature. - * See {@link EnvironmentVariableNames.RUSH_COBUILD_ENABLED} - */ - public static get cobuildFeatureEnabled(): boolean | undefined { - EnvironmentConfiguration._ensureValidated(); - return EnvironmentConfiguration._cobuildFeatureEnabled; - } - /** * Provides a determined cobuild context id if configured * See {@link EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID} @@ -515,15 +493,6 @@ export class EnvironmentConfiguration { break; } - case EnvironmentVariableNames.RUSH_COBUILD_ENABLED: { - EnvironmentConfiguration._cobuildFeatureEnabled = - EnvironmentConfiguration.parseBooleanEnvironmentVariable( - EnvironmentVariableNames.RUSH_COBUILD_ENABLED, - value - ); - break; - } - case EnvironmentVariableNames.RUSH_COBUILD_CONTEXT_ID: { EnvironmentConfiguration._cobuildContextId = value; break; From fac1898d47c538d565b0ee99746c3c024b9bb5fc Mon Sep 17 00:00:00 2001 From: Ian Clanton-Thuon Date: Thu, 31 Aug 2023 20:10:15 -0700 Subject: [PATCH 100/100] Update the workspace test pnpm-lock.yaml. --- .../workspace/common/pnpm-lock.yaml | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml b/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml index e46bb038b5a..c6698769874 100644 --- a/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml +++ b/build-tests/install-test-workspace/workspace/common/pnpm-lock.yaml @@ -7,13 +7,13 @@ importers: rush-lib-test: specifiers: - '@microsoft/rush-lib': file:microsoft-rush-lib-5.101.0.tgz + '@microsoft/rush-lib': file:microsoft-rush-lib-5.103.0.tgz '@types/node': 14.18.36 colors: ^1.4.0 rimraf: ^4.1.2 typescript: ~5.0.4 dependencies: - '@microsoft/rush-lib': file:../temp/tarballs/microsoft-rush-lib-5.101.0.tgz_@types+node@14.18.36 + '@microsoft/rush-lib': file:../temp/tarballs/microsoft-rush-lib-5.103.0.tgz_@types+node@14.18.36 colors: 1.4.0 devDependencies: '@types/node': 14.18.36 @@ -22,17 +22,17 @@ importers: rush-sdk-test: specifiers: - '@microsoft/rush-lib': file:microsoft-rush-lib-5.101.0.tgz - '@rushstack/rush-sdk': file:rushstack-rush-sdk-5.101.0.tgz + '@microsoft/rush-lib': file:microsoft-rush-lib-5.103.0.tgz + '@rushstack/rush-sdk': file:rushstack-rush-sdk-5.103.0.tgz '@types/node': 14.18.36 colors: ^1.4.0 rimraf: ^4.1.2 typescript: ~5.0.4 dependencies: - '@rushstack/rush-sdk': file:../temp/tarballs/rushstack-rush-sdk-5.101.0.tgz_@types+node@14.18.36 + '@rushstack/rush-sdk': file:../temp/tarballs/rushstack-rush-sdk-5.103.0.tgz_@types+node@14.18.36 colors: 1.4.0 devDependencies: - '@microsoft/rush-lib': file:../temp/tarballs/microsoft-rush-lib-5.101.0.tgz_@types+node@14.18.36 + '@microsoft/rush-lib': file:../temp/tarballs/microsoft-rush-lib-5.103.0.tgz_@types+node@14.18.36 '@types/node': 14.18.36 rimraf: 4.4.1 typescript: 5.0.4 @@ -3737,6 +3737,10 @@ packages: /util-deprecate/1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + /uuid/8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + /v8-compile-cache/2.3.0: resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==} dev: true @@ -3898,11 +3902,11 @@ packages: optionalDependencies: commander: 2.20.3 - file:../temp/tarballs/microsoft-rush-lib-5.101.0.tgz_@types+node@14.18.36: - resolution: {tarball: file:../temp/tarballs/microsoft-rush-lib-5.101.0.tgz} - id: file:../temp/tarballs/microsoft-rush-lib-5.101.0.tgz + file:../temp/tarballs/microsoft-rush-lib-5.103.0.tgz_@types+node@14.18.36: + resolution: {tarball: file:../temp/tarballs/microsoft-rush-lib-5.103.0.tgz} + id: file:../temp/tarballs/microsoft-rush-lib-5.103.0.tgz name: '@microsoft/rush-lib' - version: 5.101.0 + version: 5.103.0 engines: {node: '>=5.6.0'} dependencies: '@pnpm/dependency-path': 2.1.2 @@ -3910,10 +3914,10 @@ packages: '@rushstack/heft-config-file': file:../temp/tarballs/rushstack-heft-config-file-0.13.3.tgz_@types+node@14.18.36 '@rushstack/node-core-library': file:../temp/tarballs/rushstack-node-core-library-3.59.7.tgz_@types+node@14.18.36 '@rushstack/package-deps-hash': file:../temp/tarballs/rushstack-package-deps-hash-4.0.44.tgz_@types+node@14.18.36 - '@rushstack/package-extractor': file:../temp/tarballs/rushstack-package-extractor-0.4.1.tgz_@types+node@14.18.36 + '@rushstack/package-extractor': file:../temp/tarballs/rushstack-package-extractor-0.5.1.tgz_@types+node@14.18.36 '@rushstack/rig-package': file:../temp/tarballs/rushstack-rig-package-0.4.1.tgz - '@rushstack/stream-collator': file:../temp/tarballs/rushstack-stream-collator-4.0.262.tgz_@types+node@14.18.36 - '@rushstack/terminal': file:../temp/tarballs/rushstack-terminal-0.5.37.tgz_@types+node@14.18.36 + '@rushstack/stream-collator': file:../temp/tarballs/rushstack-stream-collator-4.0.263.tgz_@types+node@14.18.36 + '@rushstack/terminal': file:../temp/tarballs/rushstack-terminal-0.5.38.tgz_@types+node@14.18.36 '@rushstack/ts-command-line': file:../temp/tarballs/rushstack-ts-command-line-4.15.2.tgz '@types/node-fetch': 2.6.2 '@yarnpkg/lockfile': 1.0.2 @@ -3940,6 +3944,7 @@ packages: tapable: 2.2.1 tar: 6.1.15 true-case-path: 2.2.1 + uuid: 8.3.2 transitivePeerDependencies: - '@types/node' - encoding @@ -4231,19 +4236,20 @@ packages: transitivePeerDependencies: - '@types/node' - file:../temp/tarballs/rushstack-package-extractor-0.4.1.tgz_@types+node@14.18.36: - resolution: {tarball: file:../temp/tarballs/rushstack-package-extractor-0.4.1.tgz} - id: file:../temp/tarballs/rushstack-package-extractor-0.4.1.tgz + file:../temp/tarballs/rushstack-package-extractor-0.5.1.tgz_@types+node@14.18.36: + resolution: {tarball: file:../temp/tarballs/rushstack-package-extractor-0.5.1.tgz} + id: file:../temp/tarballs/rushstack-package-extractor-0.5.1.tgz name: '@rushstack/package-extractor' - version: 0.4.1 + version: 0.5.1 dependencies: '@pnpm/link-bins': 5.3.25 '@rushstack/node-core-library': file:../temp/tarballs/rushstack-node-core-library-3.59.7.tgz_@types+node@14.18.36 - '@rushstack/terminal': file:../temp/tarballs/rushstack-terminal-0.5.37.tgz_@types+node@14.18.36 + '@rushstack/terminal': file:../temp/tarballs/rushstack-terminal-0.5.38.tgz_@types+node@14.18.36 ignore: 5.1.9 jszip: 3.8.0 minimatch: 3.0.8 npm-packlist: 2.1.5 + semver: 7.5.4 transitivePeerDependencies: - '@types/node' @@ -4255,11 +4261,11 @@ packages: resolve: 1.22.1 strip-json-comments: 3.1.1 - file:../temp/tarballs/rushstack-rush-sdk-5.101.0.tgz_@types+node@14.18.36: - resolution: {tarball: file:../temp/tarballs/rushstack-rush-sdk-5.101.0.tgz} - id: file:../temp/tarballs/rushstack-rush-sdk-5.101.0.tgz + file:../temp/tarballs/rushstack-rush-sdk-5.103.0.tgz_@types+node@14.18.36: + resolution: {tarball: file:../temp/tarballs/rushstack-rush-sdk-5.103.0.tgz} + id: file:../temp/tarballs/rushstack-rush-sdk-5.103.0.tgz name: '@rushstack/rush-sdk' - version: 5.101.0 + version: 5.103.0 dependencies: '@rushstack/node-core-library': file:../temp/tarballs/rushstack-node-core-library-3.59.7.tgz_@types+node@14.18.36 '@types/node-fetch': 2.6.2 @@ -4268,22 +4274,22 @@ packages: - '@types/node' dev: false - file:../temp/tarballs/rushstack-stream-collator-4.0.262.tgz_@types+node@14.18.36: - resolution: {tarball: file:../temp/tarballs/rushstack-stream-collator-4.0.262.tgz} - id: file:../temp/tarballs/rushstack-stream-collator-4.0.262.tgz + file:../temp/tarballs/rushstack-stream-collator-4.0.263.tgz_@types+node@14.18.36: + resolution: {tarball: file:../temp/tarballs/rushstack-stream-collator-4.0.263.tgz} + id: file:../temp/tarballs/rushstack-stream-collator-4.0.263.tgz name: '@rushstack/stream-collator' - version: 4.0.262 + version: 4.0.263 dependencies: '@rushstack/node-core-library': file:../temp/tarballs/rushstack-node-core-library-3.59.7.tgz_@types+node@14.18.36 - '@rushstack/terminal': file:../temp/tarballs/rushstack-terminal-0.5.37.tgz_@types+node@14.18.36 + '@rushstack/terminal': file:../temp/tarballs/rushstack-terminal-0.5.38.tgz_@types+node@14.18.36 transitivePeerDependencies: - '@types/node' - file:../temp/tarballs/rushstack-terminal-0.5.37.tgz_@types+node@14.18.36: - resolution: {tarball: file:../temp/tarballs/rushstack-terminal-0.5.37.tgz} - id: file:../temp/tarballs/rushstack-terminal-0.5.37.tgz + file:../temp/tarballs/rushstack-terminal-0.5.38.tgz_@types+node@14.18.36: + resolution: {tarball: file:../temp/tarballs/rushstack-terminal-0.5.38.tgz} + id: file:../temp/tarballs/rushstack-terminal-0.5.38.tgz name: '@rushstack/terminal' - version: 0.5.37 + version: 0.5.38 peerDependencies: '@types/node': '*' peerDependenciesMeta: