diff --git a/.changeset/poor-moles-unite.md b/.changeset/poor-moles-unite.md new file mode 100644 index 000000000..1acee21ba --- /dev/null +++ b/.changeset/poor-moles-unite.md @@ -0,0 +1,5 @@ +--- +"synckit": minor +--- + +feat: add more env variables support diff --git a/README.md b/README.md index effd481e6..f229a5137 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Perform async work synchronously in Node.js using `worker_threads`, or `child_pr - [Usage](#usage) - [Install](#install) - [API](#api) + - [Env variables](#env-variables) - [TypeScript](#typescript) - [Benchmark](#benchmark) - [Changelog](#changelog) @@ -70,6 +71,12 @@ You must make sure: 1. if `worker_threads` is enabled (by default), the `result` is serialized by [`Structured Clone Algorithm`](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm) 2. if `child_process` is used, the `result` is serialized by `JSON.stringify` +### Env variables + +1. `SYNCKIT_WORKER_THREADS`: whether or not enable `worker_threads`, it's enabled by default, set as `0` to disable +2. `SYNCKIT_BUFFER_SIZE`: `bufferSize` to create `SharedArrayBuffer` for `worker_threads` (default as `1024`), or `maxBuffer` for `child_process` (no default) +3. `SYNCKIT_TIMEOUT`: `timeout` for performing the async job (no default) + ### TypeScript If you want to use `ts-node` for worker file (a `.ts` file), it is supported out of box! @@ -80,7 +87,7 @@ If you want to integrate with [tsconfig-paths](https://www.npmjs.com/package/tsc ## Benchmark -It is about 20x faster than [`sync-threads`](https://github.com/lambci/sync-threads) but 3x slower than native for reading the file content itself 1000 times during runtime. See [Benchmark](./benchmarks/benchmark.txt) for more details. +It is about 20x faster than [`sync-threads`](https://github.com/lambci/sync-threads) but 3x slower than native for reading the file content itself 1000 times during runtime, and 18x faster than `sync-threads` but 4x slower than native for total time. See [Benchmark](./benchmarks/benchmark.txt) for more details. You can try it with running `yarn benchmark` by yourself. [Here](./benchmarks/benchmark.js) is the benchmark source code. diff --git a/benchmarks/benchmark.js b/benchmarks/benchmark.js index c4a2b9c80..60fc4578a 100644 --- a/benchmarks/benchmark.js +++ b/benchmarks/benchmark.js @@ -22,6 +22,38 @@ const syncFn3 = require('./native') const nativeLoadTime = performance.now() - nativeLoadStartTime +const RUN_TIMES = +process.env.RUN_TIMES || 1000 + +const synckitRunStartTime = performance.now() + +let i = RUN_TIMES + +while (i-- > 0) { + syncFn1() +} + +const synckitRuntime = performance.now() - synckitRunStartTime + +const syncThreadsRunStartTime = performance.now() + +i = RUN_TIMES + +while (i-- > 0) { + syncFn2() +} + +const syncThreadsRuntime = performance.now() - syncThreadsRunStartTime + +const nativeRunStartTime = performance.now() + +i = RUN_TIMES + +while (i-- > 0) { + syncFn3() +} + +const nativeRuntime = performance.now() - nativeRunStartTime + class Benchmark { /** * @param {number} synckit @@ -65,38 +97,6 @@ class Benchmark { } } -const RUN_TIMES = +process.env.RUN_TIMES || 1000 - -const synckitRunStartTime = performance.now() - -let i = RUN_TIMES - -while (i-- > 0) { - syncFn1() -} - -const synckitRuntime = performance.now() - synckitRunStartTime - -const syncThreadsRunStartTime = performance.now() - -i = RUN_TIMES - -while (i-- > 0) { - syncFn2() -} - -const syncThreadsRuntime = performance.now() - syncThreadsRunStartTime - -const nativeRunStartTime = performance.now() - -i = RUN_TIMES - -while (i-- > 0) { - syncFn3() -} - -const nativeRuntime = performance.now() - nativeRunStartTime - console.table({ 'load time': new Benchmark( synckitLoadTime, @@ -104,4 +104,9 @@ console.table({ nativeLoadTime, ), 'run time': new Benchmark(synckitRuntime, syncThreadsRuntime, nativeRuntime), + 'total time': new Benchmark( + synckitLoadTime + synckitRuntime, + syncThreadsLoadTime + syncThreadsRuntime, + nativeLoadTime + nativeRuntime, + ), }) diff --git a/benchmarks/benchmark.txt b/benchmarks/benchmark.txt index 73309582d..17d386ddb 100644 --- a/benchmarks/benchmark.txt +++ b/benchmarks/benchmark.txt @@ -1,7 +1,8 @@ $ node benchmarks/benchmark -┌───────────┬────────────┬──────────────┬───────────┬───────────────────┬──────────────────┐ -│ (index) │ synckit │ sync-threads │ native │ perf sync-threads │ perf native │ -├───────────┼────────────┼──────────────┼───────────┼───────────────────┼──────────────────┤ -│ load time │ '24.71ms' │ '1.34ms' │ '0.22ms' │ '18.41x slower' │ '113.89x slower' │ -│ run time │ '198.46ms' │ '4347.89ms' │ '57.51ms' │ '21.91x faster' │ '3.45x slower' │ -└───────────┴────────────┴──────────────┴───────────┴───────────────────┴──────────────────┘ +┌────────────┬────────────┬──────────────┬───────────┬───────────────────┬──────────────────┐ +│ (index) │ synckit │ sync-threads │ native │ perf sync-threads │ perf native │ +├────────────┼────────────┼──────────────┼───────────┼───────────────────┼──────────────────┤ +│ load time │ '30.19ms' │ '1.61ms' │ '0.26ms' │ '18.80x slower' │ '118.01x slower' │ +│ run time │ '216.49ms' │ '4546.67ms' │ '90.49ms' │ '21.00x faster' │ '2.39x slower' │ +│ total time │ '246.68ms' │ '4548.27ms' │ '90.75ms' │ '18.44x faster' │ '2.72x slower' │ +└────────────┴────────────┴──────────────┴───────────┴───────────────────┴──────────────────┘ diff --git a/package.json b/package.json index f8531211e..89f1850f3 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,13 @@ "benchmark:export": "yarn benchmark > benchmarks/benchmark.txt", "build": "run-p build:*", "build:r": "r -f es2015", - "build:ts": "tsc -P src", + "build:ts": "tsc -p src", "lint": "run-p lint:*", "lint:es": "eslint . --cache -f friendly --max-warnings 10", "lint:tsc": "tsc --noEmit", "prepare": "simple-git-hooks && yarn-deduplicate --strategy fewer || exit 0", "prerelease": "npm run build", - "pretest": "yarn build", + "pretest": "yarn build:ts", "release": "clean-publish && changeset publish", "test": "jest", "test-worker": "ts-node test/test-worker-ts && node test/test-worker", @@ -81,10 +81,12 @@ ] }, "typeCoverage": { - "atLeast": 100, + "atLeast": 98.63, "cache": true, "detail": true, - "ignoreNested": true, + "ignoreAsAssertion": true, + "ignoreNonNullAssertion": true, + "showRelativePath": true, "strict": true, "update": true } diff --git a/src/index.ts b/src/index.ts index c3020a49d..97835d0e2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,17 +29,32 @@ export * from './types' */ export const tmpdir = fs.realpathSync(_tmpdir()) -export const useWorkerThreads = !['0', 'false'].includes( - process.env.SYNCKIT_WORKER_THREADS!, -) +const { SYNCKIT_WORKER_THREADS, SYNCKIT_BUFFER_SIZE, SYNCKIT_TIMEOUT } = + process.env + +export const useWorkerThreads = + !SYNCKIT_WORKER_THREADS || !['0', 'false'].includes(SYNCKIT_WORKER_THREADS) + +export const DEFAULT_BUFFER_SIZE = SYNCKIT_BUFFER_SIZE + ? +SYNCKIT_BUFFER_SIZE + : undefined + +export const DEFAULT_TIMEOUT = SYNCKIT_TIMEOUT ? +SYNCKIT_TIMEOUT : undefined + +export const DEFAULT_WORKER_BUFFER_SIZE = DEFAULT_BUFFER_SIZE || 1024 const syncFnCache = new Map() export function createSyncFn( workerPath: string, bufferSize?: number, + timeout?: number, ): Syncify -export function createSyncFn(workerPath: string, bufferSize?: number) { +export function createSyncFn( + workerPath: string, + bufferSize?: number, + timeout = DEFAULT_TIMEOUT, +) { if (!path.isAbsolute(workerPath)) { throw new Error('`workerPath` must be absolute') } @@ -59,6 +74,7 @@ export function createSyncFn(workerPath: string, bufferSize?: number) { const syncFn = (useWorkerThreads ? startWorkerThread : startChildProcess)( resolvedWorkerPath, bufferSize, + timeout, ) syncFnCache.set(workerPath, syncFn) @@ -66,7 +82,11 @@ export function createSyncFn(workerPath: string, bufferSize?: number) { return syncFn } -function startChildProcess(workerPath: string) { +function startChildProcess( + workerPath: string, + bufferSize = DEFAULT_BUFFER_SIZE, + timeout?: number, +) { const executor = workerPath.endsWith('.ts') ? 'ts-node' : 'node' return (...args: unknown[]): T => { @@ -79,13 +99,15 @@ function startChildProcess(workerPath: string) { try { execSync(command, { stdio: 'inherit', + maxBuffer: bufferSize, + timeout, }) const { result, error } = JSON.parse( fs.readFileSync(filename, 'utf8'), ) as DataMessage if (error) { - throw typeof error === 'object' && error && 'message' in error + throw typeof error === 'object' && 'message' in error! ? // eslint-disable-next-line unicorn/error-message Object.assign(new Error(), error) : error @@ -98,7 +120,11 @@ function startChildProcess(workerPath: string) { } } -function startWorkerThread(workerPath: string, bufferSize = 1024) { +function startWorkerThread( + workerPath: string, + bufferSize = DEFAULT_WORKER_BUFFER_SIZE, + timeout?: number, +) { const { port1: mainPort, port2: workerPort } = new MessageChannel() const isTs = workerPath.endsWith('.ts') @@ -126,10 +152,10 @@ function startWorkerThread(workerPath: string, bufferSize = 1024) { const msg: MainToWorkerMessage = { sharedBuffer, id, args } worker.postMessage(msg) - const status = Atomics.wait(sharedBufferView, 0, 0) + const status = Atomics.wait(sharedBufferView, 0, 0, timeout) /* istanbul ignore if */ - if (status !== 'ok' && status !== 'not-equal') { + if (!['ok', 'not-equal'].includes(status)) { throw new Error('Internal error: Atomics.wait() failed: ' + status) } @@ -149,7 +175,7 @@ function startWorkerThread(workerPath: string, bufferSize = 1024) { // MessagePort doesn't copy the properties of Error objects. We still want // error objects to have extra properties such as "warnings" so implement the // property copying manually. - throw Object.assign(error, properties) + throw typeof error === 'object' ? Object.assign(error, properties) : error } return result! @@ -168,7 +194,7 @@ export const runAsWorker = async (fn: T) => { let msg: DataMessage try { msg = { result: (await fn(...args)) as T } - } catch (err) { + } catch (err: unknown) { msg = { error: err instanceof Error @@ -192,7 +218,7 @@ export const runAsWorker = async (fn: T) => { let msg: WorkerToMainMessage try { msg = { id, result: (await fn(...args)) as T } - } catch (err) { + } catch (err: unknown) { const error = err as Error msg = { id, error, properties: { ...error } } } diff --git a/test/fn.spec.ts b/test/fn.spec.ts index 93837fd18..76c634c83 100644 --- a/test/fn.spec.ts +++ b/test/fn.spec.ts @@ -1,30 +1,40 @@ +/* eslint-disable @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires */ + import fs from 'fs' import path from 'path' import { createSyncFn, tmpdir } from 'synckit' -test('createSyncFn with worker threads', () => { - process.env.SYNCKIT_WORKER_THREADS = '1' +type AsyncWorkerFn = (result: T, timeout?: number) => Promise + +beforeEach(() => { + jest.resetModules() + delete process.env.SYNCKIT_BUFFER_SIZE + delete process.env.SYNCKIT_TIMEOUT + delete process.env.SYNCKIT_WORKER_THREADS +}) + +const workerTsPath = require.resolve('./worker-ts') +const workerPath = require.resolve('./worker') +const workerErrorPath = require.resolve('./worker-error') +test('createSyncFn with worker threads', () => { expect(() => createSyncFn('./fake')).toThrow('`workerPath` must be absolute') expect(() => createSyncFn(require.resolve('eslint'))).not.toThrow() - // eslint-disable-next-line sonarjs/no-duplicate-string - const syncFn1 = createSyncFn(require.resolve('./worker-ts')) - const syncFn2 = createSyncFn(require.resolve('./worker-ts')) + const syncFn1 = createSyncFn(workerTsPath) + const syncFn2 = createSyncFn(workerTsPath) const syncFn3 = createSyncFn(require.resolve('../src')) - // eslint-disable-next-line sonarjs/no-duplicate-string - const errSyncFn = createSyncFn(require.resolve('./worker-error')) + const errSyncFn = createSyncFn<() => Promise>(workerErrorPath) expect(syncFn1).toBe(syncFn2) expect(syncFn1).not.toBe(syncFn3) expect(syncFn1(1)).toBe(1) expect(syncFn1(2)).toBe(2) expect(syncFn1(5, 0)).toBe(5) - // eslint-disable-next-line @typescript-eslint/no-unsafe-return expect(() => errSyncFn()).toThrow('Worker Error') - const syncFn4 = createSyncFn(require.resolve('./worker')) + const syncFn4 = createSyncFn(workerPath) expect(syncFn4(1)).toBe(1) expect(syncFn4(2)).toBe(2) @@ -32,36 +42,55 @@ test('createSyncFn with worker threads', () => { }) test('createSyncFn with child process', () => { - jest.resetModules() - process.env.SYNCKIT_WORKER_THREADS = '0' - // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires const { createSyncFn } = require('synckit') as typeof import('synckit') expect(() => createSyncFn('./fake')).toThrow('`workerPath` must be absolute') expect(() => createSyncFn(require.resolve('eslint'))).not.toThrow() - const syncFn1 = createSyncFn(require.resolve('./worker-ts')) - const syncFn2 = createSyncFn(require.resolve('./worker-ts')) + const syncFn1 = createSyncFn(workerTsPath) + const syncFn2 = createSyncFn(workerTsPath) const syncFn3 = createSyncFn(require.resolve('../src')) - const errSyncFn = createSyncFn(require.resolve('./worker-error')) + const errSyncFn = createSyncFn<() => Promise>(workerErrorPath) expect(syncFn1).toBe(syncFn2) expect(syncFn1).not.toBe(syncFn3) expect(syncFn1(1)).toBe(1) expect(syncFn1(2)).toBe(2) expect(syncFn1(5, 0)).toBe(5) - // eslint-disable-next-line @typescript-eslint/no-unsafe-return expect(() => errSyncFn()).toThrow('Worker Error') - const syncFn4 = createSyncFn(require.resolve('./worker')) + const syncFn4 = createSyncFn(workerPath) expect(syncFn4(1)).toBe(1) expect(syncFn4(2)).toBe(2) expect(syncFn4(5, 0)).toBe(5) }) +test('env with worker threads', () => { + process.env.SYNCKIT_BUFFER_SIZE = '0' + process.env.SYNCKIT_TIMEOUT = '1' + + const { createSyncFn } = require('synckit') as typeof import('synckit') + const syncFn = createSyncFn(workerTsPath) + + expect(() => syncFn(1, 100)).toThrow( + 'Internal error: Atomics.wait() failed: timed-out', + ) +}) + +test('env with child process', () => { + process.env.SYNCKIT_BUFFER_SIZE = '0' + process.env.SYNCKIT_TIMEOUT = '1' + process.env.SYNCKIT_WORKER_THREADS = '0' + + const { createSyncFn } = require('synckit') as typeof import('synckit') + const syncFn = createSyncFn(workerTsPath) + + expect(() => syncFn(1, 100)).toThrow('spawnSync /bin/sh ETIMEDOUT') +}) + /** * It should be covered by above tests, * but the jest coverage does not work for child_process, @@ -69,11 +98,8 @@ test('createSyncFn with child process', () => { * @link https://github.com/facebook/jest/issues/5274 */ test('runAsWorker with child process', async () => { - jest.resetModules() - process.env.SYNCKIT_WORKER_THREADS = '0' - // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires const { runAsWorker } = require('synckit') as typeof import('synckit') const originalArgv = process.argv @@ -81,13 +107,13 @@ test('runAsWorker with child process', async () => { const filename = path.resolve(tmpdir, 'synckit-test.json') fs.writeFileSync(filename, JSON.stringify([])) - process.argv = ['ts-node', require.resolve('./worker'), filename] + process.argv = ['ts-node', workerPath, filename] let result = await runAsWorker(() => Promise.resolve(1)) expect(result).toBe(undefined) expect(fs.readFileSync(filename, 'utf8')).toBe(JSON.stringify({ result: 1 })) fs.writeFileSync(filename, JSON.stringify([])) - process.argv = ['ts-node', require.resolve('./worker-error'), filename] + process.argv = ['ts-node', workerErrorPath, filename] result = await runAsWorker(() => Promise.reject(new Error('Error!'))) expect(result).toBe(undefined) expect(JSON.parse(fs.readFileSync(filename, 'utf8'))).toMatchObject({ diff --git a/test/worker-error.ts b/test/worker-error.ts index c5e164029..dca79fd98 100644 --- a/test/worker-error.ts +++ b/test/worker-error.ts @@ -1,4 +1,4 @@ import { runAsWorker } from 'synckit' // eslint-disable-next-line @typescript-eslint/no-floating-promises -runAsWorker(() => Promise.reject(new Error('Worker Error'))) +runAsWorker(() => Promise.reject(new Error('Worker Error'))) diff --git a/test/worker-ts.ts b/test/worker-ts.ts index 415b52b5c..bfe74e0b1 100644 --- a/test/worker-ts.ts +++ b/test/worker-ts.ts @@ -1,4 +1,7 @@ import { runAsWorker } from 'synckit' // eslint-disable-next-line @typescript-eslint/no-floating-promises -runAsWorker((result: T) => Promise.resolve(result)) +runAsWorker( + (result: T, timeout?: number) => + new Promise(resolve => setTimeout(() => resolve(result), timeout)), +) diff --git a/test/worker.js b/test/worker.js index 57da00d74..f00502587 100644 --- a/test/worker.js +++ b/test/worker.js @@ -1,3 +1,6 @@ const { runAsWorker } = require('../lib') -runAsWorker(result => Promise.resolve(result)) +runAsWorker( + (result, timeout) => + new Promise(resolve => setTimeout(() => resolve(result), timeout)), +)