From db1df7bed03f624dca78630830b9bb0395666b20 Mon Sep 17 00:00:00 2001 From: Spencer Date: Tue, 16 Jun 2020 12:57:34 -0700 Subject: [PATCH] [kbn/optimizer] fix windows compatibility (#69304) Co-authored-by: spalger --- .../src/optimizer/observe_worker.ts | 66 +++++++++++++++---- .../src/worker/entry_point_creator.ts | 10 ++- .../kbn-optimizer/src/worker/run_worker.ts | 38 +++++++++-- .../src/worker/webpack.config.ts | 14 ++-- 4 files changed, 102 insertions(+), 26 deletions(-) diff --git a/packages/kbn-optimizer/src/optimizer/observe_worker.ts b/packages/kbn-optimizer/src/optimizer/observe_worker.ts index 4527052fa821a4..fef3efc13a516a 100644 --- a/packages/kbn-optimizer/src/optimizer/observe_worker.ts +++ b/packages/kbn-optimizer/src/optimizer/observe_worker.ts @@ -22,7 +22,7 @@ import { inspect } from 'util'; import execa from 'execa'; import * as Rx from 'rxjs'; -import { map, takeUntil } from 'rxjs/operators'; +import { map, takeUntil, first, ignoreElements } from 'rxjs/operators'; import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle, BundleRefs } from '../common'; @@ -68,19 +68,11 @@ if (inspectFlagIndex !== -1) { function usingWorkerProc( config: OptimizerConfig, - workerConfig: WorkerConfig, - bundles: Bundle[], fn: (proc: execa.ExecaChildProcess) => Rx.Observable ) { return Rx.using( (): ProcResource => { - const args = [ - JSON.stringify(workerConfig), - JSON.stringify(bundles.map((b) => b.toSpec())), - BundleRefs.fromBundles(config.bundles).toSpecJson(), - ]; - - const proc = execa.node(require.resolve('../worker/run_worker'), args, { + const proc = execa.node(require.resolve('../worker/run_worker'), [], { nodeOptions: [ ...(inspectFlag && config.inspectWorkers ? [`${inspectFlag}=${inspectPortCounter++}`] @@ -129,6 +121,51 @@ function observeStdio$(stream: Readable, name: WorkerStdio['stream']) { ); } +/** + * We used to pass configuration to the worker as JSON encoded arguments, but they + * grew too large for argv, especially on Windows, so we had to move to an async init + * where we send the args over IPC. To keep the logic simple we basically mock the + * argv behavior and don't use complicated messages or anything so that state can + * be initialized in the worker before most of the code is run. + */ +function initWorker( + proc: execa.ExecaChildProcess, + config: OptimizerConfig, + workerConfig: WorkerConfig, + bundles: Bundle[] +) { + const msg$ = Rx.fromEvent<[unknown]>(proc, 'message').pipe( + // validate the initialization messages from the process + map(([msg]) => { + if (typeof msg === 'string') { + switch (msg) { + case 'init': + return 'init' as const; + case 'ready': + return 'ready' as const; + } + } + + throw new Error(`unexpected message from worker while initializing: [${inspect(msg)}]`); + }) + ); + + return Rx.concat( + msg$.pipe(first((msg) => msg === 'init')), + Rx.defer(() => { + proc.send({ + args: [ + JSON.stringify(workerConfig), + JSON.stringify(bundles.map((b) => b.toSpec())), + BundleRefs.fromBundles(config.bundles).toSpecJson(), + ], + }); + return []; + }), + msg$.pipe(first((msg) => msg === 'ready')) + ).pipe(ignoreElements()); +} + /** * Start a worker process with the specified `workerConfig` and * `bundles` and return an observable of the events related to @@ -140,10 +177,11 @@ export function observeWorker( workerConfig: WorkerConfig, bundles: Bundle[] ): Rx.Observable { - return usingWorkerProc(config, workerConfig, bundles, (proc) => { - let lastMsg: WorkerMsg; + return usingWorkerProc(config, (proc) => { + const init$ = initWorker(proc, config, workerConfig, bundles); - return Rx.merge( + let lastMsg: WorkerMsg; + const worker$: Rx.Observable = Rx.merge( Rx.of({ type: 'worker started', bundles, @@ -201,5 +239,7 @@ export function observeWorker( ) ) ); + + return Rx.concat(init$, worker$); }); } diff --git a/packages/kbn-optimizer/src/worker/entry_point_creator.ts b/packages/kbn-optimizer/src/worker/entry_point_creator.ts index a613e3e8925a4b..f8f41b2e134229 100644 --- a/packages/kbn-optimizer/src/worker/entry_point_creator.ts +++ b/packages/kbn-optimizer/src/worker/entry_point_creator.ts @@ -17,9 +17,13 @@ * under the License. */ -module.exports = function ({ entries }: { entries: Array<{ importId: string; relPath: string }> }) { - const lines = entries.map(({ importId, relPath }) => [ - `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('./${relPath}'))`, +module.exports = function ({ + entries, +}: { + entries: Array<{ importId: string; requirePath: string }>; +}) { + const lines = entries.map(({ importId, requirePath }) => [ + `__kbnBundles__.define('${importId}', __webpack_require__, require.resolve('${requirePath}'))`, ]); return { diff --git a/packages/kbn-optimizer/src/worker/run_worker.ts b/packages/kbn-optimizer/src/worker/run_worker.ts index 178637d39ab003..781cf83624a1e4 100644 --- a/packages/kbn-optimizer/src/worker/run_worker.ts +++ b/packages/kbn-optimizer/src/worker/run_worker.ts @@ -17,7 +17,10 @@ * under the License. */ +import { inspect } from 'util'; + import * as Rx from 'rxjs'; +import { take, mergeMap } from 'rxjs/operators'; import { parseBundles, @@ -80,15 +83,38 @@ setInterval(() => { } }, 1000).unref(); +function assertInitMsg(msg: unknown): asserts msg is { args: string[] } { + if (typeof msg !== 'object' || !msg) { + throw new Error(`expected init message to be an object: ${inspect(msg)}`); + } + + const { args } = msg as Record; + if (!args || !Array.isArray(args) || !args.every((a) => typeof a === 'string')) { + throw new Error( + `expected init message to have an 'args' property that's an array of strings: ${inspect(msg)}` + ); + } +} + Rx.defer(() => { - const workerConfig = parseWorkerConfig(process.argv[2]); - const bundles = parseBundles(process.argv[3]); - const bundleRefs = BundleRefs.parseSpec(process.argv[4]); + process.send!('init'); + + return Rx.fromEvent<[unknown]>(process as any, 'message').pipe( + take(1), + mergeMap(([msg]) => { + assertInitMsg(msg); + process.send!('ready'); + + const workerConfig = parseWorkerConfig(msg.args[0]); + const bundles = parseBundles(msg.args[1]); + const bundleRefs = BundleRefs.parseSpec(msg.args[2]); - // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers - process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; + // set BROWSERSLIST_ENV so that style/babel loaders see it before running compilers + process.env.BROWSERSLIST_ENV = workerConfig.browserslistEnv; - return runCompilers(workerConfig, bundles, bundleRefs); + return runCompilers(workerConfig, bundles, bundleRefs); + }) + ); }).subscribe( (msg) => { send(msg); diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index e361b186c30e09..11f5544cd9274d 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -100,10 +100,16 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: entries: bundle.publicDirNames.map((name) => { const absolute = Path.resolve(bundle.contextDir, name); const newContext = Path.dirname(ENTRY_CREATOR); - return { - importId: `${bundle.type}/${bundle.id}/${name}`, - relPath: Path.relative(newContext, absolute), - }; + const importId = `${bundle.type}/${bundle.id}/${name}`; + + // relative path from context of the ENTRY_CREATOR, with linux path separators + let requirePath = Path.relative(newContext, absolute).split('\\').join('/'); + if (!requirePath.startsWith('.')) { + // ensure requirePath is identified by node as relative + requirePath = `./${requirePath}`; + } + + return { importId, requirePath }; }), }, },