Skip to content

Commit

Permalink
perf(ts): denoise baseline mem usage (#1209)
Browse files Browse the repository at this point in the history
- TS_NODE_TRANSPILE_ONLY=true
- Use v8 heap stats instead of memory usage (isolate vs process heap)
- Baseline before each bench iter and tracking delta

```
# Current main
McapStreamReader
        3.01±0.08 op/s  Heap Used: 268.92±3.81 MB/op    Heap Total: 297.93±4.13 MB/op   ArrayBuffers: 213.39±2.40 MB/op
McapIndexedReader
        2.13±0.01 op/s  Heap Used: 284.31±3.36 MB/op    Heap Total: 316.23±1.45 MB/op   ArrayBuffers: 107.06±0.00 MB/op
McapIndexedReader_reverse
        2.14±0.02 op/s  Heap Used: 280.16±3.09 MB/op    Heap Total: 315.45±0.79 MB/op   ArrayBuffers: 107.06±0.00 MB/op

# New heap baseline (ts-node transpile only + delta tracking)
McapStreamReader
        3.44±0.03 op/s  Heap Used: 51.80±12.44 MB/op    Heap Total: 42.89±11.53 MB/op   ArrayBuffers: 112.95±6.87 MB/op
McapIndexedReader
        2.16±0.01 op/s  Heap Used: 70.75±2.55 MB/op     Heap Total: 60.15±2.89 MB/op    ArrayBuffers: 17.86±0.76 MB/op
McapIndexedReader_reverse
        2.17±0.03 op/s  Heap Used: 59.98±2.50 MB/op     Heap Total: 40.93±1.26 MB/op    ArrayBuffers: 16.00±0.00 MB/op

# New baseline on first change (bigint pr)
McapStreamReader
        4.20±0.07 op/s  Heap Used: 35.31±2.60 MB/op     Heap Total: 28.69±2.44 MB/op    ArrayBuffers: 118.47±10.33 MB/op
McapIndexedReader
        2.58±0.01 op/s  Heap Used: 23.74±2.33 MB/op     Heap Total: 12.38±1.05 MB/op    ArrayBuffers: 1.42±3.41 MB/op
McapIndexedReader_reverse
        2.60±0.00 op/s  Heap Used: 24.55±0.72 MB/op     Heap Total: 14.05±0.32 MB/op    ArrayBuffers: 5.60±0.98 MB/op
```
  • Loading branch information
AaronO committed Aug 12, 2024
1 parent 00eddd3 commit c95275f
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 9 deletions.
23 changes: 16 additions & 7 deletions typescript/benchmarks/bench.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { hrtime, memoryUsage } from "process";
import { getHeapStatistics } from "v8";

const COUNT = 5;

Expand All @@ -9,10 +10,8 @@ type BenchmarkResult =
samples: {
duration: bigint;
memoryUsage: {
rss: number;
heapTotal: number;
heapUsed: number;
external: number;
usedHeapSize: number;
totalHeapSize: number;
arrayBuffers: number;
};
}[];
Expand Down Expand Up @@ -40,12 +39,22 @@ export async function runBenchmark(name: string, run: () => Promise<void>): Prom
};
for (let i = 0; i < COUNT; i++) {
global.gc();
const baseline = getHeapStatistics();
const baselineArrayBuffers = memoryUsage().arrayBuffers;
const before = hrtime.bigint();

await run();

const after = hrtime.bigint();
const currentMemoryUsage = getHeapStatistics();
const currentArrayBuffers = process.memoryUsage().arrayBuffers;
result.samples.push({
duration: after - before,
memoryUsage: memoryUsage(),
memoryUsage: {
usedHeapSize: currentMemoryUsage.used_heap_size - baseline.used_heap_size,
totalHeapSize: currentMemoryUsage.total_heap_size - baseline.total_heap_size,
arrayBuffers: currentArrayBuffers - baselineArrayBuffers,
},
});
}
} else {
Expand Down Expand Up @@ -84,11 +93,11 @@ function printStats(result: BenchmarkResult) {
let memoryResult = "(use --expose-gc to gather memory statistics)";
if (result.gcExposed) {
const used = humanReadableStatistics(
result.samples.map((sample) => sample.memoryUsage.heapUsed / 2 ** 20),
result.samples.map((sample) => sample.memoryUsage.usedHeapSize / 2 ** 20),
"MB/op",
);
const total = humanReadableStatistics(
result.samples.map((sample) => sample.memoryUsage.heapTotal / 2 ** 20),
result.samples.map((sample) => sample.memoryUsage.totalHeapSize / 2 ** 20),
"MB/op",
);
const arrayBuffers = humanReadableStatistics(
Expand Down
4 changes: 2 additions & 2 deletions typescript/benchmarks/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
"typecheck": "tsc -p tsconfig.json --noEmit",
"lint:ci": "eslint --report-unused-disable-directives .",
"lint": "eslint --report-unused-disable-directives --fix .",
"bench": "TS_NODE_FILES=true TS_NODE_PROJECT=tsconfig.cjs.json node --expose-gc -r 'ts-node/register' index.ts",
"bench:debug": "TS_NODE_FILES=true TS_NODE_PROJECT=tsconfig.cjs.json node --inspect-brk --expose-gc -r 'ts-node/register' index.ts"
"bench": "TS_NODE_TRANSPILE_ONLY=true TS_NODE_FILES=true TS_NODE_PROJECT=tsconfig.cjs.json node --huge-max-old-generation-size --expose-gc -r 'ts-node/register' index.ts",
"bench:debug": "TS_NODE_FILES=true TS_NODE_PROJECT=tsconfig.cjs.json node --huge-max-old-generation-size --inspect-brk --expose-gc -r 'ts-node/register' index.ts"
},
"devDependencies": {
"@foxglove/eslint-plugin": "1.0.1",
Expand Down

0 comments on commit c95275f

Please sign in to comment.