From 77a5e7f515517050c1ace774912f4073a98dd906 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 22:36:43 +0200 Subject: [PATCH 01/18] ci: allow memory-only benchmark experiments --- .github/workflows/client-nav-benchmarks.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index 7eedc222ae..5cb261687f 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -20,6 +20,11 @@ on: - 'packages/**' - 'benchmarks/**' workflow_dispatch: + inputs: + memory-only: + description: Run only the memory benchmarks + type: boolean + default: false concurrency: group: client-nav-benchmarks-${{ github.ref }} @@ -45,11 +50,7 @@ jobs: - react - solid - vue - benchmark: - - client-nav - - ssr - - memory-server - - memory-client + benchmark: ${{ fromJSON(inputs.memory-only && '["memory-server", "memory-client"]' || '["client-nav", "ssr", "memory-server", "memory-client"]') }} include: - benchmark: client-nav mode: simulation From 116a8bf8528edb9ade41f544f7d4427a2ad9bc6e Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 22:42:34 +0200 Subject: [PATCH 02/18] test: apply Node compiler and GC corrections to memory workers --- .github/workflows/client-nav-benchmarks.yml | 11 +--- .../react/vite.config.ts | 2 + .../solid/vite.config.ts | 2 + .../vue/vite.config.ts | 2 + .../react/vite.config.ts | 2 + .../solid/vite.config.ts | 2 + .../loader-data-retention/vue/vite.config.ts | 2 + .../mount-unmount/react/vite.config.ts | 2 + .../mount-unmount/solid/vite.config.ts | 2 + .../mount-unmount/vue/vite.config.ts | 2 + .../navigation-churn/react/vite.config.ts | 2 + .../navigation-churn/solid/vite.config.ts | 2 + .../navigation-churn/vue/vite.config.ts | 2 + .../preload-churn/react/vite.config.ts | 2 + .../preload-churn/solid/vite.config.ts | 2 + .../preload-churn/vue/vite.config.ts | 2 + .../react/vite.config.ts | 2 + .../solid/vite.config.ts | 2 + .../unique-location-churn/vue/vite.config.ts | 2 + benchmarks/memory/runtime.ts | 13 ++++ benchmarks/memory/server/package.json | 18 +++++- benchmarks/memory/server/runtime.test.ts | 62 +++++++++++++++++++ .../aborted-requests/react/vite.config.ts | 2 + .../aborted-requests/solid/vite.config.ts | 2 + .../aborted-requests/vue/vite.config.ts | 2 + .../error-paths/react/vite.config.ts | 2 + .../error-paths/solid/vite.config.ts | 2 + .../scenarios/error-paths/vue/vite.config.ts | 2 + .../peak-large-page/react/vite.config.ts | 2 + .../peak-large-page/solid/vite.config.ts | 2 + .../peak-large-page/vue/vite.config.ts | 2 + .../request-churn/react/vite.config.ts | 2 + .../request-churn/solid/vite.config.ts | 2 + .../request-churn/vue/vite.config.ts | 2 + .../react/vite.config.ts | 2 + .../solid/vite.config.ts | 2 + .../serialization-payload/vue/vite.config.ts | 2 + .../server-fn-churn/react/vite.config.ts | 2 + .../server-fn-churn/solid/vite.config.ts | 2 + .../server-fn-churn/vue/vite.config.ts | 2 + .../streaming-peak/react/vite.config.ts | 2 + .../streaming-peak/solid/vite.config.ts | 2 + .../streaming-peak/vue/vite.config.ts | 2 + benchmarks/memory/server/tsconfig.json | 7 ++- .../memory/server/vitest.unit.config.ts | 9 +++ benchmarks/memory/tsconfig.json | 7 +++ 46 files changed, 193 insertions(+), 12 deletions(-) create mode 100644 benchmarks/memory/runtime.ts create mode 100644 benchmarks/memory/server/runtime.test.ts create mode 100644 benchmarks/memory/server/vitest.unit.config.ts create mode 100644 benchmarks/memory/tsconfig.json diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index 5cb261687f..2045a91a0b 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -51,15 +51,6 @@ jobs: - solid - vue benchmark: ${{ fromJSON(inputs.memory-only && '["memory-server", "memory-client"]' || '["client-nav", "ssr", "memory-server", "memory-client"]') }} - include: - - benchmark: client-nav - mode: simulation - - benchmark: ssr - mode: simulation - - benchmark: memory-server - mode: memory - - benchmark: memory-client - mode: memory runs-on: ubuntu-latest steps: - name: Checkout @@ -81,7 +72,7 @@ jobs: uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 timeout-minutes: 20 with: - mode: ${{ matrix.mode }} + mode: ${{ startsWith(matrix.benchmark, 'memory-') && 'memory' || 'simulation' }} run: >- WITH_INSTRUMENTATION=1 pnpm nx run diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts index 048a9cc833..5910981796 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client interrupted-navigations (react)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts index e710e0fd12..fbdfe6b8ea 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client interrupted-navigations (solid)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts index b2912ce9d1..a9814eace8 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -28,6 +29,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client interrupted-navigations (vue)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts index 555e5f5612..e0e31c1fca 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client loader-data-retention (react)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts index 58d6b77acc..4011ff34ce 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client loader-data-retention (solid)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts index bbf2fd02d8..97c5bd726b 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -28,6 +29,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client loader-data-retention (vue)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts index bb0f3c2e0a..020d1115c5 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client mount-unmount (react)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts index 5f0ab26252..94774af226 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client mount-unmount (solid)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts index 0f7573a408..d5ac1a6793 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -28,6 +29,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client mount-unmount (vue)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts index b05b999297..d2881ea8ac 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client navigation-churn (react)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts index 6b6735891a..32b6b7b583 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client navigation-churn (solid)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts index d26ce23d07..defc633ff4 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -28,6 +29,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client navigation-churn (vue)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts index f9c426656b..0d792a669a 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client preload-churn (react)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts index c86a6fcbb3..23fa994bd9 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client preload-churn (solid)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts index ea7ed7b695..a2d39c12de 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -28,6 +29,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client preload-churn (vue)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts index f424c57cef..4d6359c2ab 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client unique-location-churn (react)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts index d4acf0019a..58abb71a63 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -26,6 +27,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client unique-location-churn (solid)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts index 423126019b..edcfd3fa10 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -28,6 +29,7 @@ export default defineConfig({ }, }, test: { + execArgv: memoryExecArgv(), name: '@benchmarks/memory-client unique-location-churn (vue)', watch: false, environment: 'jsdom', diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts new file mode 100644 index 0000000000..922e59e669 --- /dev/null +++ b/benchmarks/memory/runtime.ts @@ -0,0 +1,13 @@ +/** Worker settings for the CodSpeed memory instrument. */ +export function memoryExecArgv() { + if ( + process.env.CODSPEED_ENV === undefined || + process.env.CODSPEED_RUNNER_MODE !== 'memory' + ) { + return [] + } + + // Node 24 still enables Maglev under CodSpeed's --no-opt. Compile baseline + // code eagerly and keep minor GC tied to allocations instead of tasks. + return ['--no-maglev', '--always-sparkplug', '--no-minor-gc-task'] +} diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index 423dde0eb3..b3774a2778 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -237,7 +237,7 @@ } }, "test:types": { - "executor": "nx:noop", + "executor": "nx:run-commands", "dependsOn": [ { "projects": [ @@ -265,6 +265,22 @@ ], "target": "test:types:ssr" } + ], + "options": { + "command": "tsc -p ./tsconfig.json --noEmit", + "cwd": "benchmarks/memory/server" + } + }, + "test:unit": { + "executor": "nx:run-commands", + "options": { + "command": "vitest run --config ./vitest.unit.config.ts", + "cwd": "benchmarks/memory/server" + }, + "inputs": [ + "default", + "^production", + "{workspaceRoot}/benchmarks/memory/runtime.ts" ] } } diff --git a/benchmarks/memory/server/runtime.test.ts b/benchmarks/memory/server/runtime.test.ts new file mode 100644 index 0000000000..d4bcccd745 --- /dev/null +++ b/benchmarks/memory/server/runtime.test.ts @@ -0,0 +1,62 @@ +import { execFileSync } from 'node:child_process' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { resolveConfig } from 'vite' +import codspeedPlugin from '@codspeed/vitest-plugin' +import { memoryExecArgv } from '../runtime' + +afterEach(() => { + vi.unstubAllEnvs() +}) + +describe('memory worker configuration', () => { + it('preserves CodSpeed instrumentation flags in a runnable worker', async () => { + vi.stubEnv('CODSPEED_ENV', 'test') + vi.stubEnv('CODSPEED_RUNNER_MODE', 'memory') + + const config = await resolveConfig( + { + configFile: false, + plugins: [codspeedPlugin()], + test: { execArgv: memoryExecArgv() }, + }, + 'serve', + 'benchmark', + ) + const flags = config.test!.execArgv! + expect(flags).toContain('--expose-gc') + expect(flags).toContain('--predictable') + expect(flags).toContain('--hash-seed=1') + + const output = execFileSync( + process.execPath, + [ + ...flags, + '--trace-opt', + '--eval', + ` + if (typeof global.gc !== 'function') { + throw new Error('Missing GC instrumentation') + } + function hot(value) { return value + 1 } + let total = 0 + for (let index = 0; index < 100000; index++) { + total += hot(index) + } + console.log(total) + `, + ], + { encoding: 'utf8' }, + ) + expect(output).toContain('5000050000') + expect(output).not.toMatch(/MAGLEV/) + }) + + it.each([undefined, 'simulation', 'instrumentation', 'walltime'])( + 'does not change %s workers', + (mode) => { + vi.stubEnv('CODSPEED_ENV', 'test') + vi.stubEnv('CODSPEED_RUNNER_MODE', mode) + expect(memoryExecArgv()).toEqual([]) + }, + ) +}) diff --git a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts index 2ce925e406..f10c6cd41a 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts index 5ee4e152af..3a363dd1d9 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts index 5ccc9f5d1d..add00614f0 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts index bc67b1cfc7..b1cbcff452 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts index 401d1e5f25..e37373bc1e 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts index 9a0ca28a25..e4c4ecbaf8 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts index adaacda86e..96662f978d 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts index 27d4dd65f7..ef186f2bb4 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts index 0f238bd7f7..17cafa9dd8 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts index 1e956958ec..838e892587 100644 --- a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts index fbf8df9491..fcf356782d 100644 --- a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts index a551a96e2f..2f0bd3e69e 100644 --- a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts index de5549de3b..e15003ab06 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts index 926646ee12..8a31f5ec5d 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts index 41a5596adf..ceda068290 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts index 68d9ed839b..401058bdf3 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts index 8d6d226e97..c0b01336eb 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts index 0e249b9f3b..8e4efb3de2 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts index 5af0f0d138..17401fc9bb 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts index 18bf5b2d04..0fd29a9b79 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts index 7cd481da0b..ea6a2f9ec4 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' +import { memoryExecArgv } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,6 +28,7 @@ export default defineConfig({ // and the mid-measurement recompile injects a multi-MB allocation // burst at a run-dependent time. execArgv: [ + ...memoryExecArgv(), '--no-flush-bytecode', // Pre-size the V8 heap so no space has to grow mid-measurement: // heap-growth events allocate several MB at a run-dependent moment, diff --git a/benchmarks/memory/server/tsconfig.json b/benchmarks/memory/server/tsconfig.json index 5d5dcbf825..f3e80c8b72 100644 --- a/benchmarks/memory/server/tsconfig.json +++ b/benchmarks/memory/server/tsconfig.json @@ -3,5 +3,10 @@ "compilerOptions": { "types": ["node", "vite/client", "vitest/globals"] }, - "include": ["bench-utils.ts"] + "include": [ + "bench-utils.ts", + "../runtime.ts", + "*.test.ts", + "vitest.unit.config.ts" + ] } diff --git a/benchmarks/memory/server/vitest.unit.config.ts b/benchmarks/memory/server/vitest.unit.config.ts new file mode 100644 index 0000000000..19405a6f92 --- /dev/null +++ b/benchmarks/memory/server/vitest.unit.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + watch: false, + environment: 'node', + include: ['*.test.ts'], + }, +}) diff --git a/benchmarks/memory/tsconfig.json b/benchmarks/memory/tsconfig.json new file mode 100644 index 0000000000..5a325f3b6c --- /dev/null +++ b/benchmarks/memory/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["runtime.ts"] +} From 9de9cb1e416099ef18477f1061dddf298bd86406 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 22:51:12 +0200 Subject: [PATCH 03/18] test: replace adaptive memory GC polling with fixed work --- benchmarks/memory/server/bench-utils.test.ts | 73 +++++++++++++++++ benchmarks/memory/server/bench-utils.ts | 81 ++----------------- .../scenarios/peak-large-page/shared.ts | 1 - .../scenarios/serialization-payload/shared.ts | 1 - .../server/scenarios/streaming-peak/shared.ts | 3 +- benchmarks/memory/server/tsconfig.json | 1 + 6 files changed, 81 insertions(+), 79 deletions(-) create mode 100644 benchmarks/memory/server/bench-utils.test.ts diff --git a/benchmarks/memory/server/bench-utils.test.ts b/benchmarks/memory/server/bench-utils.test.ts new file mode 100644 index 0000000000..7d4da06356 --- /dev/null +++ b/benchmarks/memory/server/bench-utils.test.ts @@ -0,0 +1,73 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { runSequentialRequestLoop } from './bench-utils' + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +it('drains each response before collecting and starting the next request', async () => { + vi.useFakeTimers() + const events: Array = [] + vi.stubGlobal('gc', () => events.push('gc')) + let index = 0 + const run = runSequentialRequestLoop( + { + fetch() { + const current = index++ + events.push(`fetch ${current}`) + return new Response( + new ReadableStream({ + start(controller) { + setTimeout(() => { + events.push(`body ${current}`) + controller.enqueue(new Uint8Array([1])) + controller.close() + }, 0) + }, + }), + ) + }, + }, + { + seed: 1, + iterations: 2, + buildRequest: () => new Request('http://localhost/'), + pinGcBetweenIterations: true, + }, + ) + await vi.runAllTimersAsync() + await run + expect(events).toEqual([ + 'fetch 0', + 'body 0', + 'gc', + 'gc', + 'fetch 1', + 'body 1', + 'gc', + 'gc', + ]) +}) + +it('releases a failed response reader and does not start another request', async () => { + const failure = new Error('stream failed') + const body = new ReadableStream({ + start(controller) { + controller.error(failure) + }, + }) + const fetch = vi.fn(() => new Response(body)) + await expect( + runSequentialRequestLoop( + { fetch }, + { + seed: 1, + iterations: 2, + buildRequest: () => new Request('http://localhost/'), + }, + ), + ).rejects.toBe(failure) + expect(body.locked).toBe(false) + expect(fetch).toHaveBeenCalledTimes(1) +}) diff --git a/benchmarks/memory/server/bench-utils.ts b/benchmarks/memory/server/bench-utils.ts index b8a09fdd8e..06fb3ac220 100644 --- a/benchmarks/memory/server/bench-utils.ts +++ b/benchmarks/memory/server/bench-utils.ts @@ -28,13 +28,6 @@ export type RunSequentialRequestLoopOptions = // only removes floating garbage, whose collection timing is the // dominant cross-run noise source. pinGcBetweenIterations?: boolean - // For peak-shape scenarios ONLY (flat inter-iteration heap floor): - // verify each pinned collection actually returned the heap to the - // floor, extending the barrier when a runner's late teardown holds the - // payload past the fixed window. Churn/abort scenarios must not set - // this: their floor legitimately drifts, so the verification caps out - // chronically and the extra collections it fires destabilize the peak. - verifyGcFloor?: boolean } export const memoryBenchOptions = { @@ -66,7 +59,7 @@ export async function drainResponse(response: Response) { } try { - while (true) { + for (;;) { const result = await reader.read() if (result.done) { @@ -87,7 +80,6 @@ export async function runSequentialRequestLoop( buildRequest, validateResponse, pinGcBetweenIterations = false, - verifyGcFloor = false, } = options const random = options.seed !== undefined @@ -103,8 +95,6 @@ export async function runSequentialRequestLoop( } }) - const gcPinState = verifyGcFloor ? createGcPinState() : undefined - for (let index = 0; index < iterations; index++) { const request = buildRequest(random, index) const response = await handler.fetch(request) @@ -114,55 +104,17 @@ export async function runSequentialRequestLoop( await drainResponse(response) if (pinGcBetweenIterations) { - await settleAndPinGc(gcPinState) + await settleAndPinGc() } } } -// Renderer/stream teardown spans several event-loop turns (React and Vue -// need more than the two turns the original barrier allowed for), and a -// too-short settle window leaks a whole payload of garbage past the -// collection point, flipping the measured peak bimodally between runs. -// The settle length is a fixed count on purpose: an adaptive exit (collect -// until the post-GC heap size stops moving) makes the collection points -// land at data-dependent turns, which re-introduces exactly the run-to-run -// variance this barrier exists to remove — heap-size readings never fully -// stabilize under the instrumented CI environment, so the exit point (and -// with it every subsequent GC point) shifted between identical runs. +// Allow trailing stream/renderer teardown to finish before each collection. +// The work is fixed; heap-size readings must not decide how many collections +// a measured workload performs. const settleTurnsBeforeGc = 16 -const maxSettleTurns = 64 -// Below every scenario's payload size, above normal inter-iteration jitter. -const settledHeapSlackBytes = 256 * 1024 -// Tracks the smallest post-collection heap size seen in a workload run: -// the inter-iteration floor the heap must return to before the next -// iteration may start. -export interface GcPinState { - floorHeapUsed: number -} - -export function createGcPinState(): GcPinState { - return { floorHeapUsed: Infinity } -} - -// Settle trailing renderer/stream teardown with a fixed number of 0ms timer -// hops (one full event-loop turn each, microtasks flushing between hops), -// then pin a collection point. The second collection one turn later picks -// up whatever the first collection's finalizers released. gc() is present -// under CodSpeed (--expose-gc); in plain smoke runs there is nothing to -// pin, so only the settle hops run. -// -// The collection is then verified against the inter-iteration heap floor: -// on some runners the response teardown holds the payload past the fixed -// settle window (released only by a later internal timer), and that one -// still-reachable payload survives the collection and bleeds into the next -// iteration's measured peak. When the post-collection heap has not returned -// to the floor, keep hopping and collecting — bounded — until the payload -// is actually released, so the barrier self-heals in exactly the case where -// the fixed-count barrier fails. A workload that genuinely accumulates -// reachable memory raises the floor as it goes (the floor is the minimum -// seen), so accumulation still measures; it only pays the turn cap. -export async function settleAndPinGc(state?: GcPinState) { +export async function settleAndPinGc() { for (let turn = 0; turn < settleTurnsBeforeGc; turn++) { await new Promise((resolve) => setTimeout(resolve, 0)) } @@ -176,25 +128,4 @@ export async function settleAndPinGc(state?: GcPinState) { gc() await new Promise((resolve) => setTimeout(resolve, 0)) gc() - - if (!state) { - return - } - - let heapUsed = process.memoryUsage().heapUsed - - for ( - let turn = settleTurnsBeforeGc; - turn < maxSettleTurns && - heapUsed > state.floorHeapUsed + settledHeapSlackBytes; - turn++ - ) { - await new Promise((resolve) => setTimeout(resolve, 0)) - gc() - heapUsed = process.memoryUsage().heapUsed - } - - if (heapUsed < state.floorHeapUsed) { - state.floorHeapUsed = heapUsed - } } diff --git a/benchmarks/memory/server/scenarios/peak-large-page/shared.ts b/benchmarks/memory/server/scenarios/peak-large-page/shared.ts index 578fc9c5a2..7d6736c470 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/shared.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/shared.ts @@ -55,7 +55,6 @@ export function createWorkloadGroup( buildRequest: buildPeakLargePageRequest, validateResponse: validatePeakLargePageResponse, pinGcBetweenIterations: true, - verifyGcFloor: true, }) return { diff --git a/benchmarks/memory/server/scenarios/serialization-payload/shared.ts b/benchmarks/memory/server/scenarios/serialization-payload/shared.ts index ce095b5e10..f353f43241 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/shared.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/shared.ts @@ -62,7 +62,6 @@ export function createWorkloadGroup( buildRequest: buildPayloadRequest, validateResponse: validatePayloadResponse, pinGcBetweenIterations: true, - verifyGcFloor: true, }) return { diff --git a/benchmarks/memory/server/scenarios/streaming-peak/shared.ts b/benchmarks/memory/server/scenarios/streaming-peak/shared.ts index 6b45470329..7a3e08f7f8 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/shared.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/shared.ts @@ -50,7 +50,7 @@ async function readStreamingBody(response: Response) { let body = '' let chunkCount = 0 - while (true) { + for (;;) { const result = await reader.read() if (result.done) { @@ -99,7 +99,6 @@ export function createWorkloadGroup( buildRequest: buildStreamingRequest, validateResponse: validateStreamingResponse, pinGcBetweenIterations: true, - verifyGcFloor: true, }) return { diff --git a/benchmarks/memory/server/tsconfig.json b/benchmarks/memory/server/tsconfig.json index f3e80c8b72..499ec37192 100644 --- a/benchmarks/memory/server/tsconfig.json +++ b/benchmarks/memory/server/tsconfig.json @@ -5,6 +5,7 @@ }, "include": [ "bench-utils.ts", + "scenarios/*/shared.ts", "../runtime.ts", "*.test.ts", "vitest.unit.config.ts" From 1602c4b61a4b8074231ef6b77830224b99c7232f Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 22:56:30 +0200 Subject: [PATCH 04/18] test: control the memory worker old-generation GC budget --- benchmarks/memory/runtime.ts | 7 ++++++- .../server/scenarios/aborted-requests/react/vite.config.ts | 4 ---- .../server/scenarios/aborted-requests/solid/vite.config.ts | 4 ---- .../server/scenarios/aborted-requests/vue/vite.config.ts | 4 ---- .../server/scenarios/error-paths/react/vite.config.ts | 4 ---- .../server/scenarios/error-paths/solid/vite.config.ts | 4 ---- .../memory/server/scenarios/error-paths/vue/vite.config.ts | 4 ---- .../server/scenarios/peak-large-page/react/vite.config.ts | 4 ---- .../server/scenarios/peak-large-page/solid/vite.config.ts | 4 ---- .../server/scenarios/peak-large-page/vue/vite.config.ts | 4 ---- .../server/scenarios/request-churn/react/vite.config.ts | 4 ---- .../server/scenarios/request-churn/solid/vite.config.ts | 4 ---- .../server/scenarios/request-churn/vue/vite.config.ts | 4 ---- .../scenarios/serialization-payload/react/vite.config.ts | 4 ---- .../scenarios/serialization-payload/solid/vite.config.ts | 4 ---- .../scenarios/serialization-payload/vue/vite.config.ts | 4 ---- .../server/scenarios/server-fn-churn/react/vite.config.ts | 4 ---- .../server/scenarios/server-fn-churn/solid/vite.config.ts | 4 ---- .../server/scenarios/server-fn-churn/vue/vite.config.ts | 4 ---- .../server/scenarios/streaming-peak/react/vite.config.ts | 4 ---- .../server/scenarios/streaming-peak/solid/vite.config.ts | 4 ---- .../server/scenarios/streaming-peak/vue/vite.config.ts | 4 ---- 22 files changed, 6 insertions(+), 85 deletions(-) diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 922e59e669..941de2dfe5 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -9,5 +9,10 @@ export function memoryExecArgv() { // Node 24 still enables Maglev under CodSpeed's --no-opt. Compile baseline // code eagerly and keep minor GC tied to allocations instead of tasks. - return ['--no-maglev', '--always-sparkplug', '--no-minor-gc-task'] + return [ + '--no-maglev', + '--always-sparkplug', + '--no-minor-gc-task', + '--initial-old-space-size=512', + ] } diff --git a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts index f10c6cd41a..7a0d8ea3a1 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts index 3a363dd1d9..6729a10f5b 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts index add00614f0..011e60e6a8 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts index b1cbcff452..eca181b2a0 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts index e37373bc1e..73c8b6fc53 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts index e4c4ecbaf8..3c8a1642b3 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts index 96662f978d..202b646c03 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts index ef186f2bb4..6cf1cfd56d 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts index 17cafa9dd8..8e18d805ed 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts index 838e892587..e29fac4fe9 100644 --- a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts index fcf356782d..6f66d036d1 100644 --- a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts index 2f0bd3e69e..7bb96efffa 100644 --- a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts index e15003ab06..ea8783aa79 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts index 8a31f5ec5d..2d26f8c74a 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts index ceda068290..f7884525ad 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts index 401058bdf3..5fa9cfbb82 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts index c0b01336eb..233d3ef5a3 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts index 8e4efb3de2..c264768167 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts index 17401fc9bb..11639a2665 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts index 0fd29a9b79..8a47faf33a 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], diff --git a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts index ea6a2f9ec4..3dbe508d1c 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts @@ -30,10 +30,6 @@ export default defineConfig({ execArgv: [ ...memoryExecArgv(), '--no-flush-bytecode', - // Pre-size the V8 heap so no space has to grow mid-measurement: - // heap-growth events allocate several MB at a run-dependent moment, - // which flips the measured peak bimodally between identical runs. - '--initial-old-space-size=64', '--min-semi-space-size=16', '--max-semi-space-size=16', ], From a7146e9293005276c54b799bed1655a70190009a Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 23:04:27 +0200 Subject: [PATCH 05/18] test: measure natural churn and single-request memory footprints --- benchmarks/memory/server/bench-utils.test.ts | 15 +------ benchmarks/memory/server/bench-utils.ts | 42 ++----------------- .../scenarios/aborted-requests/shared.ts | 11 ++--- .../server/scenarios/error-paths/shared.ts | 6 --- .../scenarios/peak-large-page/shared.ts | 3 +- .../server/scenarios/request-churn/shared.ts | 1 - .../scenarios/serialization-payload/shared.ts | 3 +- .../scenarios/server-fn-churn/shared.ts | 1 - .../server/scenarios/streaming-peak/shared.ts | 3 +- 9 files changed, 13 insertions(+), 72 deletions(-) diff --git a/benchmarks/memory/server/bench-utils.test.ts b/benchmarks/memory/server/bench-utils.test.ts index 7d4da06356..6ea717ae24 100644 --- a/benchmarks/memory/server/bench-utils.test.ts +++ b/benchmarks/memory/server/bench-utils.test.ts @@ -6,10 +6,9 @@ afterEach(() => { vi.unstubAllGlobals() }) -it('drains each response before collecting and starting the next request', async () => { +it('drains each response before starting the next request', async () => { vi.useFakeTimers() const events: Array = [] - vi.stubGlobal('gc', () => events.push('gc')) let index = 0 const run = runSequentialRequestLoop( { @@ -33,21 +32,11 @@ it('drains each response before collecting and starting the next request', async seed: 1, iterations: 2, buildRequest: () => new Request('http://localhost/'), - pinGcBetweenIterations: true, }, ) await vi.runAllTimersAsync() await run - expect(events).toEqual([ - 'fetch 0', - 'body 0', - 'gc', - 'gc', - 'fetch 1', - 'body 1', - 'gc', - 'gc', - ]) + expect(events).toEqual(['fetch 0', 'body 0', 'fetch 1', 'body 1']) }) it('releases a failed response reader and does not start another request', async () => { diff --git a/benchmarks/memory/server/bench-utils.ts b/benchmarks/memory/server/bench-utils.ts index 06fb3ac220..8893d5be4b 100644 --- a/benchmarks/memory/server/bench-utils.ts +++ b/benchmarks/memory/server/bench-utils.ts @@ -17,17 +17,6 @@ export type RunSequentialRequestLoopOptions = iterations?: number buildRequest: (random: () => number, index: number) => Request validateResponse?: (response: Response, request: Request) => void - // Whether V8 collects iteration i's garbage before iteration i+1 - // allocates its payload is not reproducible run to run — the measured - // peak flips by a whole payload depending on GC timing, which shifts - // with runner hardware. Forcing a collection between iterations pins - // the GC points so peak deterministically measures the largest - // single-iteration footprint plus any reachable accumulation. - // Accumulation signals stay visible: leaked or cached objects are - // still referenced, so a forced collection cannot reclaim them — it - // only removes floating garbage, whose collection timing is the - // dominant cross-run noise source. - pinGcBetweenIterations?: boolean } export const memoryBenchOptions = { @@ -75,12 +64,7 @@ export async function runSequentialRequestLoop( handler: StartRequestHandler, options: RunSequentialRequestLoopOptions, ) { - const { - iterations = 10, - buildRequest, - validateResponse, - pinGcBetweenIterations = false, - } = options + const { iterations = 10, buildRequest, validateResponse } = options const random = options.seed !== undefined ? createDeterministicRandom(options.seed) @@ -102,30 +86,12 @@ export async function runSequentialRequestLoop( validate(response, request) await drainResponse(response) - - if (pinGcBetweenIterations) { - await settleAndPinGc() - } } } -// Allow trailing stream/renderer teardown to finish before each collection. -// The work is fixed; heap-size readings must not decide how many collections -// a measured workload performs. -const settleTurnsBeforeGc = 16 - -export async function settleAndPinGc() { - for (let turn = 0; turn < settleTurnsBeforeGc; turn++) { +// Cancellation can leave deferred loaders and renderer teardown queued. +export async function settle() { + for (let turn = 0; turn < 16; turn++) { await new Promise((resolve) => setTimeout(resolve, 0)) } - - const gc = (globalThis as { gc?: () => void }).gc - - if (!gc) { - return - } - - gc() - await new Promise((resolve) => setTimeout(resolve, 0)) - gc() } diff --git a/benchmarks/memory/server/scenarios/aborted-requests/shared.ts b/benchmarks/memory/server/scenarios/aborted-requests/shared.ts index c98f462538..dfd064cd63 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/shared.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/shared.ts @@ -1,4 +1,4 @@ -import { settleAndPinGc } from '#memory-server/bench-utils' +import { settle } from '#memory-server/bench-utils' import type { StartRequestHandler } from '#memory-server/bench-utils' export type { StartRequestHandler } @@ -96,7 +96,7 @@ async function readShellBeforeDeferred( const reader = response.body.getReader() let text = '' - while (true) { + for (;;) { const result = await reader.read() const value = result.value @@ -184,12 +184,9 @@ async function cancelReader( await reader.cancel() } -// Post-abort settlement barrier. Settles renderer teardown across a fixed -// number of event-loop turns, then (under CodSpeed) pins a collection point -// after every aborted request, so the measured peak cannot drift with how -// much floating garbage the previous iterations happened to leave behind. +// Let deferred loaders and renderer teardown finish after cancellation. async function drainCancellation() { - await settleAndPinGc() + await settle() } async function assertAbortedRequestsSanity( diff --git a/benchmarks/memory/server/scenarios/error-paths/shared.ts b/benchmarks/memory/server/scenarios/error-paths/shared.ts index 57e229b7e3..966505f3a0 100644 --- a/benchmarks/memory/server/scenarios/error-paths/shared.ts +++ b/benchmarks/memory/server/scenarios/error-paths/shared.ts @@ -9,8 +9,6 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' -// Sized to sit just above the 2s measured-run floor on CI (per-iteration -// cost with the pinned collection is ~0.13-0.19s across frameworks). const errorPathsIterations = 18 const redirectSeed = 0xdecafbad const notFoundSeed = 0xdecafb0d @@ -144,7 +142,6 @@ export function createWorkloadGroup( iterations: errorPathsIterations, buildRequest: buildRedirectRequest, validateResponse: validateRedirectResponse, - pinGcBetweenIterations: true, }) const runNotFound = () => @@ -153,7 +150,6 @@ export function createWorkloadGroup( iterations: errorPathsIterations, buildRequest: buildNotFoundRequest, validateResponse: validateNotFoundResponse, - pinGcBetweenIterations: true, }) const runError = () => @@ -162,7 +158,6 @@ export function createWorkloadGroup( iterations: errorPathsIterations, buildRequest: buildErrorRequest, validateResponse: validateErrorResponse, - pinGcBetweenIterations: true, }) const runUnmatched = () => @@ -171,7 +166,6 @@ export function createWorkloadGroup( iterations: errorPathsIterations, buildRequest: buildUnmatchedRequest, validateResponse: validateNotFoundResponse, - pinGcBetweenIterations: true, }) return { diff --git a/benchmarks/memory/server/scenarios/peak-large-page/shared.ts b/benchmarks/memory/server/scenarios/peak-large-page/shared.ts index 7d6736c470..486c025f5a 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/shared.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/shared.ts @@ -6,7 +6,7 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' const benchmarkSeed = 0x5eed_0005 -const peakLargePageIterations = 20 +const peakLargePageIterations = 1 const peakLargePageUrl = 'http://localhost/l1/l2/l3/l4/l5/l6/l7/l8' const levelEightMarker = 'data-bench="peak-large-page-level-8"' @@ -54,7 +54,6 @@ export function createWorkloadGroup( iterations: peakLargePageIterations, buildRequest: buildPeakLargePageRequest, validateResponse: validatePeakLargePageResponse, - pinGcBetweenIterations: true, }) return { diff --git a/benchmarks/memory/server/scenarios/request-churn/shared.ts b/benchmarks/memory/server/scenarios/request-churn/shared.ts index a196eec8b5..32125148cd 100644 --- a/benchmarks/memory/server/scenarios/request-churn/shared.ts +++ b/benchmarks/memory/server/scenarios/request-churn/shared.ts @@ -68,7 +68,6 @@ export function createWorkloadGroup( iterations: requestChurnIterations, buildRequest: buildItemRequest, validateResponse: validateItemResponse, - pinGcBetweenIterations: true, }) return { diff --git a/benchmarks/memory/server/scenarios/serialization-payload/shared.ts b/benchmarks/memory/server/scenarios/serialization-payload/shared.ts index f353f43241..7f40aeb9d2 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/shared.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/shared.ts @@ -9,7 +9,7 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' const benchmarkSeed = 0x51eaa11 -const serializationPayloadIterations = 12 +const serializationPayloadIterations = 1 const payloadPageMarker = 'data-bench="serialization-payload"' const requestInit = { @@ -61,7 +61,6 @@ export function createWorkloadGroup( iterations: serializationPayloadIterations, buildRequest: buildPayloadRequest, validateResponse: validatePayloadResponse, - pinGcBetweenIterations: true, }) return { diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts b/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts index 6e2959e757..2bc224c979 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts @@ -208,7 +208,6 @@ export async function createWorkloadGroup( runSequentialRequestLoop(handler, { seed: benchmarkSeed, iterations: serverFnChurnIterations, - pinGcBetweenIterations: true, buildRequest: (_random, index) => { const fixtureIndex = Math.floor(index / 2) % fixtureCount diff --git a/benchmarks/memory/server/scenarios/streaming-peak/shared.ts b/benchmarks/memory/server/scenarios/streaming-peak/shared.ts index 7a3e08f7f8..148649a854 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/shared.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/shared.ts @@ -9,7 +9,7 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' const benchmarkSeed = 0xdecafbad -const streamingPeakIterations = 20 +const streamingPeakIterations = 1 const fallbackMarker = 'streaming-peak-fallback-0' const requestInit = { @@ -98,7 +98,6 @@ export function createWorkloadGroup( iterations: streamingPeakIterations, buildRequest: buildStreamingRequest, validateResponse: validateStreamingResponse, - pinGcBetweenIterations: true, }) return { From 150225802d4b505b12fc17f268aaa7b37cffd73b Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 23:27:20 +0200 Subject: [PATCH 06/18] test: isolate memory error-path workers --- .../scenarios/error-paths/react/memory.bench.ts | 6 +++--- .../scenarios/error-paths/react/memory.error.bench.ts | 11 +++++++++++ .../error-paths/react/memory.not-found.bench.ts | 11 +++++++++++ .../error-paths/react/memory.unmatched.bench.ts | 11 +++++++++++ .../server/scenarios/error-paths/react/tsconfig.json | 2 +- .../server/scenarios/error-paths/react/vite.config.ts | 1 + .../scenarios/error-paths/solid/memory.bench.ts | 6 +++--- .../scenarios/error-paths/solid/memory.error.bench.ts | 11 +++++++++++ .../error-paths/solid/memory.not-found.bench.ts | 11 +++++++++++ .../error-paths/solid/memory.unmatched.bench.ts | 11 +++++++++++ .../server/scenarios/error-paths/solid/tsconfig.json | 2 +- .../server/scenarios/error-paths/solid/vite.config.ts | 1 + .../server/scenarios/error-paths/vue/memory.bench.ts | 6 +++--- .../scenarios/error-paths/vue/memory.error.bench.ts | 11 +++++++++++ .../error-paths/vue/memory.not-found.bench.ts | 11 +++++++++++ .../error-paths/vue/memory.unmatched.bench.ts | 11 +++++++++++ .../server/scenarios/error-paths/vue/tsconfig.json | 2 +- .../server/scenarios/error-paths/vue/vite.config.ts | 1 + 18 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 benchmarks/memory/server/scenarios/error-paths/react/memory.error.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/react/memory.not-found.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/react/memory.unmatched.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/solid/memory.error.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/solid/memory.not-found.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/solid/memory.unmatched.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/vue/memory.error.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/vue/memory.not-found.bench.ts create mode 100644 benchmarks/memory/server/scenarios/error-paths/vue/memory.unmatched.bench.ts diff --git a/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts b/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts index 9a5c211fee..2780830cb2 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/memory.bench.ts @@ -4,8 +4,8 @@ import { workloadGroup } from './setup' await workloadGroup.sanity() +const workload = workloadGroup.workloads[0]! + describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } + bench(workload.name, workload.run, memoryBenchOptions) }) diff --git a/benchmarks/memory/server/scenarios/error-paths/react/memory.error.bench.ts b/benchmarks/memory/server/scenarios/error-paths/react/memory.error.bench.ts new file mode 100644 index 0000000000..d41877e6ce --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/react/memory.error.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[2]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/react/memory.not-found.bench.ts b/benchmarks/memory/server/scenarios/error-paths/react/memory.not-found.bench.ts new file mode 100644 index 0000000000..42d67b1164 --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/react/memory.not-found.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[1]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/react/memory.unmatched.bench.ts b/benchmarks/memory/server/scenarios/error-paths/react/memory.unmatched.bench.ts new file mode 100644 index 0000000000..6179e469dd --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/react/memory.unmatched.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[3]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/react/tsconfig.json b/benchmarks/memory/server/scenarios/error-paths/react/tsconfig.json index 11ddcce4ea..2dba47fc99 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/tsconfig.json +++ b/benchmarks/memory/server/scenarios/error-paths/react/tsconfig.json @@ -7,7 +7,7 @@ "types": ["node", "vite/client", "vitest/globals"] }, "include": [ - "memory.bench.ts", + "memory*.bench.ts", "memory.flame.ts", "setup.ts", "vite.config.ts", diff --git a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts index eca181b2a0..f1e6b78c30 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts @@ -23,6 +23,7 @@ export default defineConfig({ minify: false, }, test: { + isolate: true, // Keep lazily-compiled code alive for the whole run: the pinned // collections age code fast enough for V8 to flush unused bytecode, // and the mid-measurement recompile injects a multi-MB allocation diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts b/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts index 9a5c211fee..2780830cb2 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/memory.bench.ts @@ -4,8 +4,8 @@ import { workloadGroup } from './setup' await workloadGroup.sanity() +const workload = workloadGroup.workloads[0]! + describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } + bench(workload.name, workload.run, memoryBenchOptions) }) diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/memory.error.bench.ts b/benchmarks/memory/server/scenarios/error-paths/solid/memory.error.bench.ts new file mode 100644 index 0000000000..d41877e6ce --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/solid/memory.error.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[2]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/memory.not-found.bench.ts b/benchmarks/memory/server/scenarios/error-paths/solid/memory.not-found.bench.ts new file mode 100644 index 0000000000..42d67b1164 --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/solid/memory.not-found.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[1]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/memory.unmatched.bench.ts b/benchmarks/memory/server/scenarios/error-paths/solid/memory.unmatched.bench.ts new file mode 100644 index 0000000000..6179e469dd --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/solid/memory.unmatched.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[3]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/tsconfig.json b/benchmarks/memory/server/scenarios/error-paths/solid/tsconfig.json index 4b61264e11..1e8103127a 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/tsconfig.json +++ b/benchmarks/memory/server/scenarios/error-paths/solid/tsconfig.json @@ -7,7 +7,7 @@ "types": ["node", "vite/client", "vitest/globals"] }, "include": [ - "memory.bench.ts", + "memory*.bench.ts", "memory.flame.ts", "setup.ts", "vite.config.ts", diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts index 73c8b6fc53..7e7c59b1d5 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts @@ -23,6 +23,7 @@ export default defineConfig({ minify: false, }, test: { + isolate: true, // Keep lazily-compiled code alive for the whole run: the pinned // collections age code fast enough for V8 to flush unused bytecode, // and the mid-measurement recompile injects a multi-MB allocation diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts b/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts index 9a5c211fee..2780830cb2 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/memory.bench.ts @@ -4,8 +4,8 @@ import { workloadGroup } from './setup' await workloadGroup.sanity() +const workload = workloadGroup.workloads[0]! + describe('memory', () => { - for (const workload of workloadGroup.workloads) { - bench(workload.name, workload.run, memoryBenchOptions) - } + bench(workload.name, workload.run, memoryBenchOptions) }) diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/memory.error.bench.ts b/benchmarks/memory/server/scenarios/error-paths/vue/memory.error.bench.ts new file mode 100644 index 0000000000..d41877e6ce --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/vue/memory.error.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[2]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/memory.not-found.bench.ts b/benchmarks/memory/server/scenarios/error-paths/vue/memory.not-found.bench.ts new file mode 100644 index 0000000000..42d67b1164 --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/vue/memory.not-found.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[1]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/memory.unmatched.bench.ts b/benchmarks/memory/server/scenarios/error-paths/vue/memory.unmatched.bench.ts new file mode 100644 index 0000000000..6179e469dd --- /dev/null +++ b/benchmarks/memory/server/scenarios/error-paths/vue/memory.unmatched.bench.ts @@ -0,0 +1,11 @@ +import { bench, describe } from 'vitest' +import { memoryBenchOptions } from '#memory-server/bench-utils' +import { workloadGroup } from './setup' + +await workloadGroup.sanity() + +const workload = workloadGroup.workloads[3]! + +describe('memory', () => { + bench(workload.name, workload.run, memoryBenchOptions) +}) diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/tsconfig.json b/benchmarks/memory/server/scenarios/error-paths/vue/tsconfig.json index 9ad6481342..af8b83005c 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/tsconfig.json +++ b/benchmarks/memory/server/scenarios/error-paths/vue/tsconfig.json @@ -7,7 +7,7 @@ "types": ["node", "vite/client", "vitest/globals"] }, "include": [ - "memory.bench.ts", + "memory*.bench.ts", "memory.flame.ts", "setup.ts", "vite.config.ts", diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts index 3c8a1642b3..fc43376502 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts @@ -23,6 +23,7 @@ export default defineConfig({ minify: false, }, test: { + isolate: true, // Keep lazily-compiled code alive for the whole run: the pinned // collections age code fast enough for V8 to flush unused bytecode, // and the mid-measurement recompile injects a multi-MB allocation From ca20d26142903d795808a9003bc5f5b1fe2fdca5 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 23:29:59 +0200 Subject: [PATCH 07/18] test: remove code generation from memory measurements --- benchmarks/memory/README.md | 68 +++++++++++-------- benchmarks/memory/runtime.ts | 8 +-- .../aborted-requests/react/vite.config.ts | 11 +-- .../aborted-requests/solid/vite.config.ts | 11 +-- .../aborted-requests/vue/vite.config.ts | 11 +-- .../error-paths/react/vite.config.ts | 11 +-- .../error-paths/solid/vite.config.ts | 11 +-- .../scenarios/error-paths/vue/vite.config.ts | 11 +-- .../peak-large-page/react/vite.config.ts | 11 +-- .../peak-large-page/solid/vite.config.ts | 11 +-- .../peak-large-page/vue/vite.config.ts | 11 +-- .../request-churn/react/vite.config.ts | 11 +-- .../request-churn/solid/vite.config.ts | 11 +-- .../request-churn/vue/vite.config.ts | 11 +-- .../react/vite.config.ts | 11 +-- .../solid/vite.config.ts | 11 +-- .../serialization-payload/vue/vite.config.ts | 11 +-- .../server-fn-churn/react/vite.config.ts | 11 +-- .../server-fn-churn/solid/vite.config.ts | 11 +-- .../server-fn-churn/vue/vite.config.ts | 11 +-- .../streaming-peak/react/vite.config.ts | 11 +-- .../streaming-peak/solid/vite.config.ts | 11 +-- .../streaming-peak/vue/vite.config.ts | 11 +-- 23 files changed, 66 insertions(+), 241 deletions(-) diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index cf871f3439..c3040ab2f8 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -50,33 +50,38 @@ same workload through the Flame profiler. only honors tinybench's `setup`/`teardown` options; the CodSpeed runner does the exact opposite. Client benches therefore register **both** — in any given mode exactly one pair runs. -- The process runs with V8 determinism flags (predictable GC schedule, - `--no-opt`). Never call `global.gc()` manually in **churn** scenarios — - their signal is accumulation across iterations, which a forced collection - masks. **Peak** scenarios do the opposite: they set - `pinGcBetweenIterations` on the request loop so a collection runs between - iterations. Their signal is the footprint of a single request, and without - pinned GC points the measured peak flips by a whole payload depending on - whether iteration i's garbage is collected before iteration i+1 allocates. - Because of `--no-opt`, allocation counts overstate production; numbers are - for regression tracking, not absolute claims. +- Worker flags live in `runtime.ts` and apply only to the CodSpeed memory + instrument. They supplement the integration's predictable execution flags. + Disabling code generation and bytecode flushing keeps compiler allocations + out of the signal. Minor GC follows allocations, and a fixed initial + old-generation budget reduces sensitivity to startup heap policy. +- Do not force GC inside a measured request loop or poll `heapUsed` to choose + how much work to do. Both introduce allocator activity from the harness into + the measurement. Footprint scenarios run exactly one large request per + invocation; churn scenarios retain their fixed sequential request counts. +- Error-path cases use separate isolated workers so one case's module and + allocator history cannot contaminate the next case. - Keep each bench under **~1.5M allocations** (instrument overhead grows past 2M); this is the main constraint when tuning iteration counts. ## Bench shapes and signals -- **Churn (leak detector):** N sequential iterations at steady state. If one - iteration leaks L bytes, peak grows by ~N·L; healthy builds show a flat - timeline floor independent of N. Tuning check: doubling N must leave peak - roughly unchanged. -- **Peak (footprint):** one (or very few) large operations; peak memory - scaling with the workload is the signal. +- **Churn:** a fixed number of sequential operations, measuring native + allocation activity and peak outstanding native allocations. +- **Peak (footprint):** one large operation, measuring its native footprint. + +CodSpeed reports native allocator activity, not individual JavaScript object +allocations or retained JS heap size. V8 can reuse heap pages without a visible +native allocation. These benchmarks can detect native buffering and allocator +regressions, but a flat graph does not establish that route data or JS objects +are collectable. Absolute numbers include Node/framework/jsdom activity under +benchmark-specific V8 settings; compare identical harness versions. ## Scenarios ### Server -| Scenario | Shape | Guards against | +| Scenario | Shape | Exercises | | ----------------------- | ----- | --------------------------------------------------------- | | `request-churn` | churn | cross-request retention in document SSR (unique URLs) | | `server-fn-churn` | churn | retention in the server-function RPC path | @@ -88,7 +93,7 @@ same workload through the Flame profiler. ### Client -| Scenario | Shape | Guards against | +| Scenario | Shape | Exercises | | ------------------------- | ----- | -------------------------------------------------------- | | `navigation-churn` | churn | per-navigation retention at steady state | | `unique-location-churn` | churn | unbounded href/search-keyed caches (never-repeated URLs) | @@ -103,14 +108,10 @@ same workload through the Flame profiler. server response is fully consumed before the next request. Pairing a single navigation with its render signal via `Promise.all([navigate, rendered])` is fine — never overlap distinct work items. -- Randomness only via the seeded LCG in `bench-utils.ts`; no `Math.random`, - `Date.now`, or timers with a nonzero delay — anywhere async ordering must be - staged (`streaming-peak`'s deferred sections, `aborted-requests`' deferred - loader data and post-abort drain), chain **0ms `setTimeout` hops** and count - event-loop turns instead. A wall-clock delay races renderer work differently - depending on runner load and instrumentation overhead, which makes the - single measured run non-reproducible; 0ms hops fire in registration order in - the timers phase, so the interleaving is a pure function of the schedule. +- Randomness only via the seeded LCG in `bench-utils.ts`. Deferred streaming + uses counted zero-delay timer hops to stage work. Timers still depend on the + event loop, so validate repeatability on CI instead of assuming deterministic + ordering from a zero delay alone. - Sanity assertions run once at module load and throw on wrong status/markers, so a bench can never silently measure the wrong thing. - Server requests follow `benchmarks/ssr` conventions: document GETs send @@ -184,7 +185,7 @@ pnpm --filter @benchmarks/memory-client clean:profiles ``` Client memory benches are useful for regression tracking of router/React/jsdom -integration behavior, especially retained route/cache data. They are not pure +integration behavior and native allocation activity. They are not pure browser-memory measurements, and local Flame attribution can include jsdom, React DOM, and profiler shutdown frames. @@ -200,3 +201,16 @@ WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/mem WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:solid WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:vue ``` + +Run only memory jobs on an experiment branch: + +```bash +gh workflow run client-nav-benchmarks.yml --ref -f memory-only=true +``` + +Repeat at the same commit, using distinct branch names if runs should overlap +(the workflow cancels older runs on the same ref). Compare all 48 fresh memory +results: 18 client and 30 server. Exclude inherited CodSpeed results and failed +or incomplete repetitions. For each workload, calculate +`100 * (max / min - 1)` across runs for peak bytes, allocated bytes, and +allocation counts; aggregate averages hide unstable cases. diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 941de2dfe5..07a23b0136 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -7,11 +7,11 @@ export function memoryExecArgv() { return [] } - // Node 24 still enables Maglev under CodSpeed's --no-opt. Compile baseline - // code eagerly and keep minor GC tied to allocations instead of tasks. + // Keep code generation out of native allocation measurements and minor GC + // tied to allocations instead of tasks. return [ - '--no-maglev', - '--always-sparkplug', + '--jitless', + '--no-flush-bytecode', '--no-minor-gc-task', '--initial-old-space-size=512', ] diff --git a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts index 7a0d8ea3a1..92d064c495 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server aborted-requests (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts index 6729a10f5b..662b0f5f9e 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server aborted-requests (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts index 011e60e6a8..ffae3f759f 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server aborted-requests (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts index f1e6b78c30..12552ff18e 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts @@ -24,16 +24,7 @@ export default defineConfig({ }, test: { isolate: true, - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server error-paths (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts index 7e7c59b1d5..8c4ad1b861 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts @@ -24,16 +24,7 @@ export default defineConfig({ }, test: { isolate: true, - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server error-paths (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts index fc43376502..f26210e843 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts @@ -24,16 +24,7 @@ export default defineConfig({ }, test: { isolate: true, - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server error-paths (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts index 202b646c03..4b3b37e1b2 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server peak-large-page (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts index 6cf1cfd56d..f8be2661ff 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server peak-large-page (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts index 8e18d805ed..1a7b678c82 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server peak-large-page (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts index e29fac4fe9..1c11315d30 100644 --- a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server request-churn (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts index 6f66d036d1..3690f10e17 100644 --- a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server request-churn (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts index 7bb96efffa..ba9b0aa240 100644 --- a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server request-churn (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts index ea8783aa79..b196660414 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server serialization-payload (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts index 2d26f8c74a..1261519253 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server serialization-payload (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts index f7884525ad..cb3562bda6 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server serialization-payload (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts index 5fa9cfbb82..9001770c3e 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server server-fn-churn (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts index 233d3ef5a3..dd13851fc9 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server server-fn-churn (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts index c264768167..3c0a47da38 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server server-fn-churn (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts index 11639a2665..2451f4f835 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server streaming-peak (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts index 8a47faf33a..700844a25f 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server streaming-peak (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts index 3dbe508d1c..10b7c9256a 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts @@ -23,16 +23,7 @@ export default defineConfig({ minify: false, }, test: { - // Keep lazily-compiled code alive for the whole run: the pinned - // collections age code fast enough for V8 to flush unused bytecode, - // and the mid-measurement recompile injects a multi-MB allocation - // burst at a run-dependent time. - execArgv: [ - ...memoryExecArgv(), - '--no-flush-bytecode', - '--min-semi-space-size=16', - '--max-semi-space-size=16', - ], + execArgv: memoryExecArgv(), name: '@benchmarks/memory-server streaming-peak (vue)', watch: false, environment: 'node', From 81a47c05f7f5ee2dba2cbfca7c286e204659e62a Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 23:37:20 +0200 Subject: [PATCH 08/18] test: make aborted memory requests follow cancellation --- .../memory/server/deferred-records.test.ts | 45 ++++++++++ .../aborted-requests/deferred-records.ts | 39 ++++++++ .../solid/src/routes/stream.$id.tsx | 89 ++----------------- .../vue/src/routes/stream.$id.tsx | 89 ++----------------- .../server/scenarios/streaming-peak/shared.ts | 7 +- .../memory/server/streaming-sanity.test.ts | 35 ++++++++ 6 files changed, 141 insertions(+), 163 deletions(-) create mode 100644 benchmarks/memory/server/deferred-records.test.ts create mode 100644 benchmarks/memory/server/streaming-sanity.test.ts diff --git a/benchmarks/memory/server/deferred-records.test.ts b/benchmarks/memory/server/deferred-records.test.ts new file mode 100644 index 0000000000..1ad9130545 --- /dev/null +++ b/benchmarks/memory/server/deferred-records.test.ts @@ -0,0 +1,45 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { makeDeferredRecords } from './scenarios/aborted-requests/deferred-records' + +afterEach(() => { + vi.useRealTimers() +}) + +it('keeps abort probes pending without timers until the request is cancelled', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const resolved = vi.fn() + const records = makeDeferredRecords('abort-1', 'alpha', controller.signal) + void records.then(resolved) + await vi.runAllTimersAsync() + expect(resolved).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + controller.abort() + await expect(records).resolves.toEqual([]) +}) + +it('handles an already cancelled request without scheduling work', async () => { + vi.useFakeTimers() + await expect( + makeDeferredRecords('abort-1', 'beta', AbortSignal.abort()), + ).resolves.toEqual([]) + expect(vi.getTimerCount()).toBe(0) +}) + +it('cancels timers for an interrupted full response', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const records = makeDeferredRecords('sanity-full', 'alpha', controller.signal) + controller.abort() + await expect(records).resolves.toEqual([]) + expect(vi.getTimerCount()).toBe(0) +}) + +it('still produces full deferred payloads for an uninterrupted response', async () => { + vi.useFakeTimers() + const controller = new AbortController() + const records = makeDeferredRecords('sanity-full', 'beta', controller.signal) + await vi.runAllTimersAsync() + await expect(records).resolves.toHaveLength(20) + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/benchmarks/memory/server/scenarios/aborted-requests/deferred-records.ts b/benchmarks/memory/server/scenarios/aborted-requests/deferred-records.ts index 4886e3ee80..7a1cd6d254 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/deferred-records.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/deferred-records.ts @@ -16,3 +16,42 @@ export function makeAbortedRequestRecords( label: `deferred-${group}-${id}-${index}`, })) } + +// Abort probes keep their deferred payload pending until the request is +// cancelled. Timer chains can outlive cancellation and spill into the next +// measured request; route-loader signals stop observing the request after load. +export function makeDeferredRecords( + id: string, + group: RecordGroup, + signal: AbortSignal, +): Promise> { + return new Promise((resolve) => { + if (signal.aborted) { + resolve([]) + return + } + + let timeoutId: ReturnType | undefined + const onAbort = () => { + clearTimeout(timeoutId) + resolve([]) + } + signal.addEventListener('abort', onAbort, { once: true }) + + if (id === 'sanity-mid-stream' || id.startsWith('abort-')) { + return + } + + let remaining = group === 'alpha' ? 4 : 6 + const step = () => { + remaining -= 1 + if (remaining === 0) { + signal.removeEventListener('abort', onAbort) + resolve(makeAbortedRequestRecords(id, group)) + return + } + timeoutId = setTimeout(step, 0) + } + timeoutId = setTimeout(step, 0) + }) +} diff --git a/benchmarks/memory/server/scenarios/aborted-requests/solid/src/routes/stream.$id.tsx b/benchmarks/memory/server/scenarios/aborted-requests/solid/src/routes/stream.$id.tsx index b47f25a405..bf6af17f31 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/solid/src/routes/stream.$id.tsx +++ b/benchmarks/memory/server/scenarios/aborted-requests/solid/src/routes/stream.$id.tsx @@ -1,90 +1,17 @@ +import { createServerOnlyFn } from '@tanstack/solid-start' +import { getRequest } from '@tanstack/solid-start/server' import { createFileRoute } from '@tanstack/solid-router' import { Show, Suspense, createResource } from 'solid-js' -import { - makeAbortedRequestRecords, - type DeferredRecord, - type RecordGroup, -} from '../../../deferred-records' +import { makeDeferredRecords } from '../../../deferred-records' +import type { DeferredRecord } from '../../../deferred-records' -const alphaResolveTicks = 4 -const betaResolveTicks = 6 -const abortProbeAlphaResolveTicks = 40 -const abortProbeBetaResolveTicks = 60 - -function isAbortProbeId(id: string) { - return id === 'sanity-mid-stream' || id.startsWith('abort-') -} - -function getResolveTicks(id: string, group: RecordGroup) { - if (isAbortProbeId(id)) { - return group === 'alpha' - ? abortProbeAlphaResolveTicks - : abortProbeBetaResolveTicks - } - - return group === 'alpha' ? alphaResolveTicks : betaResolveTicks -} - -// One tick = one 0ms timers-phase hop. Counting event-loop turns instead of -// milliseconds keeps the resolve/abort interleaving a pure function of the -// event-loop schedule: a wall-clock delay races the abort differently -// depending on runner load and instrumentation overhead, which made this -// benchmark's single measured run swing between runs. -function resolveAfterTicks( - ticks: number, - signal: AbortSignal, - value: () => T, - abortedValue: () => T, -) { - return new Promise((resolve) => { - if (signal.aborted) { - resolve(abortedValue()) - return - } - - let remaining = ticks - let timeoutId: ReturnType - - const onAbort = () => { - clearTimeout(timeoutId) - resolve(abortedValue()) - } - - const step = () => { - remaining -= 1 - - if (remaining <= 0) { - signal.removeEventListener('abort', onAbort) - resolve(value()) - return - } - - timeoutId = setTimeout(step, 0) - } - - signal.addEventListener('abort', onAbort, { once: true }) - timeoutId = setTimeout(step, 0) - }) -} - -function makeDeferredRecords( - id: string, - group: RecordGroup, - signal: AbortSignal, -) { - return resolveAfterTicks( - getResolveTicks(id, group), - signal, - () => makeAbortedRequestRecords(id, group), - () => [], - ) -} +const getRequestSignal = createServerOnlyFn(() => getRequest().signal) export const Route = createFileRoute('/stream/$id')({ - loader: ({ params, abortController }) => ({ + loader: ({ params }) => ({ eager: `eager-${params.id}`, - alpha: makeDeferredRecords(params.id, 'alpha', abortController.signal), - beta: makeDeferredRecords(params.id, 'beta', abortController.signal), + alpha: makeDeferredRecords(params.id, 'alpha', getRequestSignal()), + beta: makeDeferredRecords(params.id, 'beta', getRequestSignal()), }), component: StreamComponent, }) diff --git a/benchmarks/memory/server/scenarios/aborted-requests/vue/src/routes/stream.$id.tsx b/benchmarks/memory/server/scenarios/aborted-requests/vue/src/routes/stream.$id.tsx index b723c7f69e..448355b122 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/vue/src/routes/stream.$id.tsx +++ b/benchmarks/memory/server/scenarios/aborted-requests/vue/src/routes/stream.$id.tsx @@ -1,90 +1,17 @@ +import { createServerOnlyFn } from '@tanstack/vue-start' +import { getRequest } from '@tanstack/vue-start/server' import { Await, createFileRoute } from '@tanstack/vue-router' import { Suspense } from 'vue' -import { - makeAbortedRequestRecords, - type DeferredRecord, - type RecordGroup, -} from '../../../deferred-records' +import { makeDeferredRecords } from '../../../deferred-records' +import type { DeferredRecord } from '../../../deferred-records' -const alphaResolveTicks = 4 -const betaResolveTicks = 6 -const abortProbeAlphaResolveTicks = 40 -const abortProbeBetaResolveTicks = 60 - -function isAbortProbeId(id: string) { - return id === 'sanity-mid-stream' || id.startsWith('abort-') -} - -function getResolveTicks(id: string, group: RecordGroup) { - if (isAbortProbeId(id)) { - return group === 'alpha' - ? abortProbeAlphaResolveTicks - : abortProbeBetaResolveTicks - } - - return group === 'alpha' ? alphaResolveTicks : betaResolveTicks -} - -// One tick = one 0ms timers-phase hop. Counting event-loop turns instead of -// milliseconds keeps the resolve/abort interleaving a pure function of the -// event-loop schedule: a wall-clock delay races the abort differently -// depending on runner load and instrumentation overhead, which made this -// benchmark's single measured run swing between runs. -function resolveAfterTicks( - ticks: number, - signal: AbortSignal, - value: () => T, - abortedValue: () => T, -) { - return new Promise((resolve) => { - if (signal.aborted) { - resolve(abortedValue()) - return - } - - let remaining = ticks - let timeoutId: ReturnType - - const onAbort = () => { - clearTimeout(timeoutId) - resolve(abortedValue()) - } - - const step = () => { - remaining -= 1 - - if (remaining <= 0) { - signal.removeEventListener('abort', onAbort) - resolve(value()) - return - } - - timeoutId = setTimeout(step, 0) - } - - signal.addEventListener('abort', onAbort, { once: true }) - timeoutId = setTimeout(step, 0) - }) -} - -function makeDeferredRecords( - id: string, - group: RecordGroup, - signal: AbortSignal, -) { - return resolveAfterTicks( - getResolveTicks(id, group), - signal, - () => makeAbortedRequestRecords(id, group), - () => [], - ) -} +const getRequestSignal = createServerOnlyFn(() => getRequest().signal) export const Route = createFileRoute('/stream/$id')({ - loader: ({ params, abortController }) => ({ + loader: ({ params }) => ({ eager: `eager-${params.id}`, - alpha: makeDeferredRecords(params.id, 'alpha', abortController.signal), - beta: makeDeferredRecords(params.id, 'beta', abortController.signal), + alpha: makeDeferredRecords(params.id, 'alpha', getRequestSignal()), + beta: makeDeferredRecords(params.id, 'beta', getRequestSignal()), }), component: StreamComponent, }) diff --git a/benchmarks/memory/server/scenarios/streaming-peak/shared.ts b/benchmarks/memory/server/scenarios/streaming-peak/shared.ts index 148649a854..01917c4743 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/shared.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/shared.ts @@ -101,7 +101,12 @@ export function createWorkloadGroup( }) return { - sanity: () => assertStreamingPeakSanity(handler), + sanity: async () => { + // Import the route chunks before checking streaming. Cold disk I/O can + // otherwise outlast the deferred payload and collapse the first response. + await run() + await assertStreamingPeakSanity(handler) + }, workloads: [ { name: `mem server streaming-peak chunked (${framework})`, diff --git a/benchmarks/memory/server/streaming-sanity.test.ts b/benchmarks/memory/server/streaming-sanity.test.ts new file mode 100644 index 0000000000..d4b37aa6ab --- /dev/null +++ b/benchmarks/memory/server/streaming-sanity.test.ts @@ -0,0 +1,35 @@ +import { expect, it, vi } from 'vitest' +import { createWorkloadGroup } from './scenarios/streaming-peak/shared' + +function response(chunks: number) { + return new Response( + new ReadableStream({ + start(controller) { + for (let index = 0; index < chunks; index++) { + controller.enqueue( + new TextEncoder().encode('streaming-peak-fallback-0'), + ) + } + controller.close() + }, + }), + ) +} + +it('checks streaming after the route has loaded', async () => { + const fetch = vi + .fn() + .mockImplementationOnce(() => response(1)) + .mockImplementationOnce(() => response(2)) + await createWorkloadGroup('solid', { fetch }).sanity() + expect(fetch).toHaveBeenCalledTimes(2) +}) + +it('still rejects a buffered response after warmup', async () => { + const fetch = () => response(1) + await expect( + createWorkloadGroup('solid', { fetch }).sanity(), + ).rejects.toThrow( + 'Expected chunked sanity response to produce multiple chunks', + ) +}) From a1fb072d92f5c12ccdbf00b32fd9d884b2c03bf7 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 23:41:46 +0200 Subject: [PATCH 09/18] test: simplify memory cancellation helpers --- benchmarks/memory/README.md | 4 +- benchmarks/memory/runtime.ts | 2 +- .../scenarios/aborted-requests/shared.ts | 73 +++++-------------- 3 files changed, 22 insertions(+), 57 deletions(-) diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index c3040ab2f8..39782b2c11 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -52,7 +52,7 @@ same workload through the Flame profiler. any given mode exactly one pair runs. - Worker flags live in `runtime.ts` and apply only to the CodSpeed memory instrument. They supplement the integration's predictable execution flags. - Disabling code generation and bytecode flushing keeps compiler allocations + Disabling machine-code generation and bytecode flushing keeps compiler allocations out of the signal. Minor GC follows allocations, and a fixed initial old-generation budget reduces sensitivity to startup heap policy. - Do not force GC inside a measured request loop or poll `heapUsed` to choose @@ -209,7 +209,7 @@ gh workflow run client-nav-benchmarks.yml --ref -f memory-on ``` Repeat at the same commit, using distinct branch names if runs should overlap -(the workflow cancels older runs on the same ref). Compare all 48 fresh memory +(the workflow serializes runs on the same ref). Compare all 48 fresh memory results: 18 client and 30 server. Exclude inherited CodSpeed results and failed or incomplete repetitions. For each workload, calculate `100 * (max / min - 1)` across runs for peak bytes, allocated bytes, and diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 07a23b0136..2cbd76d029 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -7,7 +7,7 @@ export function memoryExecArgv() { return [] } - // Keep code generation out of native allocation measurements and minor GC + // Keep machine-code generation out of native allocation measurements and minor GC // tied to allocations instead of tasks. return [ '--jitless', diff --git a/benchmarks/memory/server/scenarios/aborted-requests/shared.ts b/benchmarks/memory/server/scenarios/aborted-requests/shared.ts index dfd064cd63..b5a61594ba 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/shared.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/shared.ts @@ -6,13 +6,6 @@ export type { StartRequestHandler } type Framework = 'react' | 'solid' | 'vue' type AbortedRequestReadMode = 'first-chunk' | 'shell-before-deferred' -type AbortedRequestCancelMode = 'plain' | 'swallow-abort-error' - -type AbortedRequestMode = { - readMode: AbortedRequestReadMode - cancelMode: AbortedRequestCancelMode -} - const abortedRequestIterations = 40 let abortedRequestCounter = 0 const eagerMarker = 'data-bench="aborted-requests-eager"' @@ -24,19 +17,10 @@ const betaFirstRecord = (id: string) => `deferred-beta-${id}-0` const betaLastRecord = (id: string) => `deferred-beta-${id}-19` const textDecoder = new TextDecoder() -const abortedRequestModes: Record = { - react: { - readMode: 'first-chunk', - cancelMode: 'swallow-abort-error', - }, - solid: { - readMode: 'first-chunk', - cancelMode: 'swallow-abort-error', - }, - vue: { - readMode: 'shell-before-deferred', - cancelMode: 'swallow-abort-error', - }, +const abortedRequestModes: Record = { + react: 'first-chunk', + solid: 'first-chunk', + vue: 'shell-before-deferred', } const documentRequestInit = { @@ -137,10 +121,10 @@ async function readShellBeforeDeferred( async function readSanityStream( response: Response, request: Request, - mode: AbortedRequestMode, + mode: AbortedRequestReadMode, id: string, ) { - if (mode.readMode === 'shell-before-deferred') { + if (mode === 'shell-before-deferred') { return readShellBeforeDeferred(response, request, id) } @@ -165,33 +149,19 @@ function readLoopStream( return readFirstChunk(response, request) } -async function cancelReader( - reader: ReadableStreamDefaultReader, - mode: AbortedRequestCancelMode, -) { - if (mode === 'swallow-abort-error') { - try { - await reader.cancel() - } catch (error) { - if (!(error instanceof DOMException && error.name === 'AbortError')) { - throw error - } +async function cancelReader(reader: ReadableStreamDefaultReader) { + try { + await reader.cancel() + } catch (error) { + if (!(error instanceof DOMException && error.name === 'AbortError')) { + throw error } - - return } - - await reader.cancel() -} - -// Let deferred loaders and renderer teardown finish after cancellation. -async function drainCancellation() { - await settle() } async function assertAbortedRequestsSanity( handler: StartRequestHandler, - mode: AbortedRequestMode, + mode: AbortedRequestReadMode, ) { const fullId = 'sanity-full' const fullRequest = buildStreamRequest(fullId) @@ -224,13 +194,13 @@ async function assertAbortedRequestsSanity( // reader.cancel() is the response-stream cancellation path if the handler // does not observe Request.signal for this in-process request. controller.abort() - await cancelReader(reader, mode.cancelMode) - await drainCancellation() + await cancelReader(reader) + await settle() } async function runAbortedRequestLoop( handler: StartRequestHandler, - mode: AbortedRequestMode, + mode: AbortedRequestReadMode, ) { for (let index = 0; index < abortedRequestIterations; index++) { const controller = new AbortController() @@ -239,15 +209,10 @@ async function runAbortedRequestLoop( const response = await handler.fetch(request) validateDocumentResponse(response, request) - const { reader } = await readLoopStream( - mode.readMode, - response, - request, - id, - ) + const { reader } = await readLoopStream(mode, response, request, id) controller.abort() - await cancelReader(reader, mode.cancelMode) - await drainCancellation() + await cancelReader(reader) + await settle() } } From a13237be22603776d770b178a5b22d1c6ec1fa00 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sat, 5 Sep 2026 23:57:07 +0200 Subject: [PATCH 10/18] test: remove incremental marking from memory measurements --- benchmarks/memory/README.md | 5 +++-- benchmarks/memory/runtime.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index 39782b2c11..efa1143c5b 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -53,8 +53,9 @@ same workload through the Flame profiler. - Worker flags live in `runtime.ts` and apply only to the CodSpeed memory instrument. They supplement the integration's predictable execution flags. Disabling machine-code generation and bytecode flushing keeps compiler allocations - out of the signal. Minor GC follows allocations, and a fixed initial - old-generation budget reduces sensitivity to startup heap policy. + out of the signal. GC follows allocation thresholds without incremental + marking or minor-GC tasks; a fixed initial old-generation budget reduces + sensitivity to startup heap policy. - Do not force GC inside a measured request loop or poll `heapUsed` to choose how much work to do. Both introduce allocator activity from the harness into the measurement. Footprint scenarios run exactly one large request per diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 2cbd76d029..8d5ddc5a57 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -7,12 +7,13 @@ export function memoryExecArgv() { return [] } - // Keep machine-code generation out of native allocation measurements and minor GC - // tied to allocations instead of tasks. + // Keep machine-code generation out of native allocation measurements and GC + // tied to allocations instead of event-loop tasks and marking time budgets. return [ '--jitless', '--no-flush-bytecode', '--no-minor-gc-task', + '--no-incremental-marking', '--initial-old-space-size=512', ] } From 8715eb274722dba42cab5efd0f463c3f912c66e7 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 00:07:54 +0200 Subject: [PATCH 11/18] test: track shared memory runtime in type-check cache --- benchmarks/memory/server/package.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index b3774a2778..6312f504e4 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -269,7 +269,14 @@ "options": { "command": "tsc -p ./tsconfig.json --noEmit", "cwd": "benchmarks/memory/server" - } + }, + "inputs": [ + "default", + "^production", + "{workspaceRoot}/eslint.config.js", + "dependentTaskOutputs", + "{workspaceRoot}/benchmarks/memory/runtime.ts" + ] }, "test:unit": { "executor": "nx:run-commands", @@ -280,6 +287,8 @@ "inputs": [ "default", "^production", + "{workspaceRoot}/eslint.config.js", + "dependentTaskOutputs", "{workspaceRoot}/benchmarks/memory/runtime.ts" ] } From de1ae3186e85c8e4264e5fc6f77e4e2f388ab664 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 00:35:36 +0200 Subject: [PATCH 12/18] test: separate memory warmups with an event-loop turn --- benchmarks/memory/README.md | 4 ++ benchmarks/memory/client/package.json | 13 +++++- .../react/vite.config.ts | 5 +-- .../solid/vite.config.ts | 5 +-- .../vue/vite.config.ts | 5 +-- .../react/vite.config.ts | 5 +-- .../solid/vite.config.ts | 5 +-- .../loader-data-retention/vue/vite.config.ts | 5 +-- .../mount-unmount/react/vite.config.ts | 5 +-- .../mount-unmount/solid/vite.config.ts | 5 +-- .../mount-unmount/vue/vite.config.ts | 5 +-- .../navigation-churn/react/vite.config.ts | 5 +-- .../navigation-churn/solid/vite.config.ts | 5 +-- .../navigation-churn/vue/vite.config.ts | 5 +-- .../preload-churn/react/vite.config.ts | 5 +-- .../preload-churn/solid/vite.config.ts | 5 +-- .../preload-churn/vue/vite.config.ts | 5 +-- .../react/vite.config.ts | 5 +-- .../solid/vite.config.ts | 5 +-- .../unique-location-churn/vue/vite.config.ts | 5 +-- benchmarks/memory/client/vitest.setup.ts | 7 ++++ benchmarks/memory/runtime.ts | 19 +++++++-- benchmarks/memory/server/package.json | 4 +- benchmarks/memory/server/runtime.test.ts | 42 ++++++++++++++++++- .../aborted-requests/react/vite.config.ts | 4 +- .../aborted-requests/solid/vite.config.ts | 4 +- .../aborted-requests/vue/vite.config.ts | 4 +- .../error-paths/react/vite.config.ts | 4 +- .../error-paths/solid/vite.config.ts | 4 +- .../scenarios/error-paths/vue/vite.config.ts | 4 +- .../peak-large-page/react/vite.config.ts | 4 +- .../peak-large-page/solid/vite.config.ts | 4 +- .../peak-large-page/vue/vite.config.ts | 4 +- .../request-churn/react/vite.config.ts | 4 +- .../request-churn/solid/vite.config.ts | 4 +- .../request-churn/vue/vite.config.ts | 4 +- .../react/vite.config.ts | 4 +- .../solid/vite.config.ts | 4 +- .../serialization-payload/vue/vite.config.ts | 4 +- .../server-fn-churn/react/vite.config.ts | 4 +- .../server-fn-churn/solid/vite.config.ts | 4 +- .../server-fn-churn/vue/vite.config.ts | 4 +- .../streaming-peak/react/vite.config.ts | 4 +- .../streaming-peak/solid/vite.config.ts | 4 +- .../streaming-peak/vue/vite.config.ts | 4 +- benchmarks/memory/server/tsconfig.json | 3 +- benchmarks/memory/server/vitest.setup.ts | 4 ++ benchmarks/memory/turn.ts | 12 ++++++ 48 files changed, 176 insertions(+), 106 deletions(-) create mode 100644 benchmarks/memory/server/vitest.setup.ts create mode 100644 benchmarks/memory/turn.ts diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index efa1143c5b..35d16992f3 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -56,6 +56,10 @@ same workload through the Flame profiler. out of the signal. GC follows allocation thresholds without incremental marking or minor-GC tasks; a fixed initial old-generation budget reduces sensitivity to startup heap policy. +- Before each CodSpeed invocation, a setup hook yields to the event loop, + outside the measurement marker and before CodSpeed forces GC. Promise-only + warmup chains can keep WeakRef targets alive through a collection in the same + job. The hook leaves live router and application state mounted. - Do not force GC inside a measured request loop or poll `heapUsed` to choose how much work to do. Both introduce allocator activity from the harness into the measurement. Footprint scenarios run exactly one large request per diff --git a/benchmarks/memory/client/package.json b/benchmarks/memory/client/package.json index 5bc11e053b..af1770ba0d 100644 --- a/benchmarks/memory/client/package.json +++ b/benchmarks/memory/client/package.json @@ -231,7 +231,7 @@ } }, "test:types": { - "executor": "nx:noop", + "executor": "nx:run-commands", "dependsOn": [ { "projects": [ @@ -256,6 +256,17 @@ ], "target": "test:types:client" } + ], + "options": { + "command": "tsc -p ./tsconfig.json --noEmit", + "cwd": "benchmarks/memory/client" + }, + "inputs": [ + "default", + "^production", + "{workspaceRoot}/eslint.config.js", + "dependentTaskOutputs", + "{workspaceRoot}/benchmarks/memory/*.ts" ] } } diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts index 5910981796..0f05cedf40 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/react/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client interrupted-navigations (react)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts index fbdfe6b8ea..26ff0b5c8c 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client interrupted-navigations (solid)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts index a9814eace8..02574908d2 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -29,10 +29,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client interrupted-navigations (vue)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts index e0e31c1fca..8630417936 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/react/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client loader-data-retention (react)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts index 4011ff34ce..5dfd5b53fe 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/solid/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client loader-data-retention (solid)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts b/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts index 97c5bd726b..006636eaa0 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -29,10 +29,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client loader-data-retention (vue)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts index 020d1115c5..a08352a837 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/react/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client mount-unmount (react)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts index 94774af226..33699639de 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/solid/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client mount-unmount (solid)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts b/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts index d5ac1a6793..a3f0eeaa8c 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -29,10 +29,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client mount-unmount (vue)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts index d2881ea8ac..eef60a658e 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/react/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client navigation-churn (react)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts index 32b6b7b583..7dfb186b2e 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/solid/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client navigation-churn (solid)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts index defc633ff4..b3bdd39dd2 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -29,10 +29,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client navigation-churn (vue)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts index 0d792a669a..c12995f68e 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/react/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client preload-churn (react)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts index 23fa994bd9..efc75d2b93 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/solid/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client preload-churn (solid)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts index a2d39c12de..3d56089ba7 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -29,10 +29,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client preload-churn (vue)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts index 4d6359c2ab..7ee67bfe30 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/react/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client unique-location-churn (react)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts index 58abb71a63..4db90da2d2 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/solid/vite.config.ts @@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url' import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -27,10 +27,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client unique-location-churn (solid)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts b/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts index edcfd3fa10..d074d571b9 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import vue from '@vitejs/plugin-vue' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -29,10 +29,9 @@ export default defineConfig({ }, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('client'), name: '@benchmarks/memory-client unique-location-churn (vue)', watch: false, environment: 'jsdom', - setupFiles: ['../../../vitest.setup.ts'], }, }) diff --git a/benchmarks/memory/client/vitest.setup.ts b/benchmarks/memory/client/vitest.setup.ts index ce3127275a..5f78f1a7d1 100644 --- a/benchmarks/memory/client/vitest.setup.ts +++ b/benchmarks/memory/client/vitest.setup.ts @@ -1,3 +1,6 @@ +import { beforeEach } from 'vitest' +import { endMemoryTurn, isMemoryInstrumented } from '../turn' + // @ts-expect-error global.IS_REACT_ACT_ENVIRONMENT = true @@ -7,3 +10,7 @@ window.scrollTo = scrollTo globalThis.scrollTo = scrollTo export {} + +if (isMemoryInstrumented()) { + beforeEach(endMemoryTurn) +} diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 8d5ddc5a57..b3d262c82a 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -1,9 +1,9 @@ +import { fileURLToPath } from 'node:url' +import { isMemoryInstrumented } from './turn' + /** Worker settings for the CodSpeed memory instrument. */ export function memoryExecArgv() { - if ( - process.env.CODSPEED_ENV === undefined || - process.env.CODSPEED_RUNNER_MODE !== 'memory' - ) { + if (!isMemoryInstrumented()) { return [] } @@ -17,3 +17,14 @@ export function memoryExecArgv() { '--initial-old-space-size=512', ] } + +export function memoryConfig(side: 'client' | 'server') { + const execArgv = memoryExecArgv() + return { + execArgv, + setupFiles: + side === 'client' || execArgv.length + ? [fileURLToPath(new URL(`./${side}/vitest.setup.ts`, import.meta.url))] + : [], + } +} diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index 6312f504e4..510626bec4 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -275,7 +275,7 @@ "^production", "{workspaceRoot}/eslint.config.js", "dependentTaskOutputs", - "{workspaceRoot}/benchmarks/memory/runtime.ts" + "{workspaceRoot}/benchmarks/memory/*.ts" ] }, "test:unit": { @@ -289,7 +289,7 @@ "^production", "{workspaceRoot}/eslint.config.js", "dependentTaskOutputs", - "{workspaceRoot}/benchmarks/memory/runtime.ts" + "{workspaceRoot}/benchmarks/memory/*.ts" ] } } diff --git a/benchmarks/memory/server/runtime.test.ts b/benchmarks/memory/server/runtime.test.ts index d4bcccd745..a843aed314 100644 --- a/benchmarks/memory/server/runtime.test.ts +++ b/benchmarks/memory/server/runtime.test.ts @@ -1,8 +1,10 @@ import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { stripTypeScriptTypes } from 'node:module' import { afterEach, describe, expect, it, vi } from 'vitest' import { resolveConfig } from 'vite' import codspeedPlugin from '@codspeed/vitest-plugin' -import { memoryExecArgv } from '../runtime' +import { memoryConfig, memoryExecArgv } from '../runtime' afterEach(() => { vi.unstubAllEnvs() @@ -17,7 +19,7 @@ describe('memory worker configuration', () => { { configFile: false, plugins: [codspeedPlugin()], - test: { execArgv: memoryExecArgv() }, + test: memoryConfig('server'), }, 'serve', 'benchmark', @@ -26,6 +28,9 @@ describe('memory worker configuration', () => { expect(flags).toContain('--expose-gc') expect(flags).toContain('--predictable') expect(flags).toContain('--hash-seed=1') + expect(config.test!.setupFiles).toEqual([ + expect.stringMatching(/\/memory\/server\/vitest\.setup\.ts$/), + ]) const output = execFileSync( process.execPath, @@ -57,6 +62,39 @@ describe('memory worker configuration', () => { vi.stubEnv('CODSPEED_ENV', 'test') vi.stubEnv('CODSPEED_RUNNER_MODE', mode) expect(memoryExecArgv()).toEqual([]) + expect(memoryConfig('server')).toEqual({ execArgv: [], setupFiles: [] }) }, ) + + it('allows prior-job WeakRef targets to be collected before measurement', () => { + vi.stubEnv('CODSPEED_ENV', 'test') + vi.stubEnv('CODSPEED_RUNNER_MODE', 'memory') + // Node's TS loader needs Wasm, which --jitless disables. Strip types in + // the parent, as Vite does for actual benchmark workers. + const source = stripTypeScriptTypes( + readFileSync(new URL('../turn.ts', import.meta.url), 'utf8'), + ) + const runtimeUrl = `data:text/javascript,${encodeURIComponent(source)}` + const output = execFileSync( + process.execPath, + [ + ...memoryExecArgv(), + '--expose-gc', + '--input-type=module', + '--eval', + ` + const { endMemoryTurn } = await import(${JSON.stringify(runtimeUrl)}) + const ref = (() => new WeakRef({ value: 1 }))() + await Promise.resolve() + global.gc() + console.log(String(ref.deref() !== undefined)) + await endMemoryTurn() + global.gc() + console.log(String(ref.deref() === undefined)) + `, + ], + { encoding: 'utf8' }, + ) + expect(output.trim()).toBe('true\ntrue') + }) }) diff --git a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts index 92d064c495..eff0d0e043 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server aborted-requests (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts index 662b0f5f9e..d7fd3459ee 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server aborted-requests (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts index ffae3f759f..e4e1eb66d4 100644 --- a/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/aborted-requests/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server aborted-requests (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts index 12552ff18e..c00de7727a 100644 --- a/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -24,7 +24,7 @@ export default defineConfig({ }, test: { isolate: true, - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server error-paths (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts index 8c4ad1b861..d7a0bb51b0 100644 --- a/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -24,7 +24,7 @@ export default defineConfig({ }, test: { isolate: true, - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server error-paths (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts index f26210e843..3dc0762aa0 100644 --- a/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/error-paths/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -24,7 +24,7 @@ export default defineConfig({ }, test: { isolate: true, - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server error-paths (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts index 4b3b37e1b2..dcfcf1f92b 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server peak-large-page (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts index f8be2661ff..15b8c3a571 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server peak-large-page (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts index 1a7b678c82..3fab2aff9e 100644 --- a/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/peak-large-page/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server peak-large-page (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts index 1c11315d30..35adf6a249 100644 --- a/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server request-churn (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts index 3690f10e17..6601309a74 100644 --- a/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server request-churn (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts index ba9b0aa240..7f15c5c7af 100644 --- a/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/request-churn/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server request-churn (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts index b196660414..1ed3bcc335 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server serialization-payload (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts index 1261519253..a2d04b986c 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server serialization-payload (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts index cb3562bda6..b63a561228 100644 --- a/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/serialization-payload/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server serialization-payload (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts index 9001770c3e..f72cc53d6f 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server server-fn-churn (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts index dd13851fc9..4fba196bd2 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server server-fn-churn (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts index 3c0a47da38..5ab5ec37e3 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server server-fn-churn (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts index 2451f4f835..5d555441b6 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/react/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/react-start/plugin/vite' import react from '@vitejs/plugin-react' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server streaming-peak (react)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts index 700844a25f..5fbdb48c23 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/solid/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/solid-start/plugin/vite' import solid from 'vite-plugin-solid' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server streaming-peak (solid)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts index 10b7c9256a..cd848584a5 100644 --- a/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts +++ b/benchmarks/memory/server/scenarios/streaming-peak/vue/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'vitest/config' import codspeedPlugin from '@codspeed/vitest-plugin' import { tanstackStart } from '@tanstack/vue-start/plugin/vite' import vueJsx from '@vitejs/plugin-vue-jsx' -import { memoryExecArgv } from '../../../../runtime' +import { memoryConfig } from '../../../../runtime' const rootDir = fileURLToPath(new URL('.', import.meta.url)) @@ -23,7 +23,7 @@ export default defineConfig({ minify: false, }, test: { - execArgv: memoryExecArgv(), + ...memoryConfig('server'), name: '@benchmarks/memory-server streaming-peak (vue)', watch: false, environment: 'node', diff --git a/benchmarks/memory/server/tsconfig.json b/benchmarks/memory/server/tsconfig.json index 499ec37192..ffb2a11caa 100644 --- a/benchmarks/memory/server/tsconfig.json +++ b/benchmarks/memory/server/tsconfig.json @@ -8,6 +8,7 @@ "scenarios/*/shared.ts", "../runtime.ts", "*.test.ts", - "vitest.unit.config.ts" + "vitest.unit.config.ts", + "vitest.setup.ts" ] } diff --git a/benchmarks/memory/server/vitest.setup.ts b/benchmarks/memory/server/vitest.setup.ts new file mode 100644 index 0000000000..be96c0067a --- /dev/null +++ b/benchmarks/memory/server/vitest.setup.ts @@ -0,0 +1,4 @@ +import { beforeEach } from 'vitest' +import { endMemoryTurn } from '../turn' + +beforeEach(endMemoryTurn) diff --git a/benchmarks/memory/turn.ts b/benchmarks/memory/turn.ts new file mode 100644 index 0000000000..3ee6d53833 --- /dev/null +++ b/benchmarks/memory/turn.ts @@ -0,0 +1,12 @@ +export function isMemoryInstrumented() { + return ( + process.env.CODSPEED_ENV !== undefined && + process.env.CODSPEED_RUNNER_MODE === 'memory' + ) +} + +// CodSpeed calls beforeEach before its forced GC and measurement marker. End +// the previous warmup's job so WeakRef targets can be released by that GC. +export function endMemoryTurn() { + return new Promise((resolve) => setImmediate(resolve)) +} From 01a072c858cb512eba2a5da55c35fb4e8d8a3148 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 01:00:31 +0200 Subject: [PATCH 13/18] test: reset client memory apps between benchmark invocations --- benchmarks/memory/README.md | 11 +-- .../react/memory.bench.ts | 6 +- .../solid/memory.bench.ts | 6 +- .../vue/memory.bench.ts | 6 +- .../react/memory.bench.ts | 6 +- .../solid/memory.bench.ts | 6 +- .../loader-data-retention/vue/memory.bench.ts | 6 +- .../mount-unmount/react/memory.bench.ts | 6 +- .../mount-unmount/solid/memory.bench.ts | 6 +- .../mount-unmount/vue/memory.bench.ts | 6 +- .../navigation-churn/react/memory.bench.ts | 6 +- .../navigation-churn/solid/memory.bench.ts | 6 +- .../navigation-churn/vue/memory.bench.ts | 6 +- .../preload-churn/react/memory.bench.ts | 6 +- .../preload-churn/solid/memory.bench.ts | 6 +- .../preload-churn/vue/memory.bench.ts | 6 +- .../react/memory.bench.ts | 6 +- .../solid/memory.bench.ts | 6 +- .../unique-location-churn/vue/memory.bench.ts | 6 +- .../memory/server/client-lifecycle.test.ts | 69 +++++++++++++++++++ benchmarks/memory/server/package.json | 3 +- 21 files changed, 131 insertions(+), 60 deletions(-) create mode 100644 benchmarks/memory/server/client-lifecycle.test.ts diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index 35d16992f3..e60c0c68d8 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -43,10 +43,11 @@ same workload through the Flame profiler. after a forced GC. Under plain `vitest bench` the suites only smoke-test: timing output is meaningless; real numbers come from CodSpeed. - Under CodSpeed the bench fn runs several warmup invocations plus the - measured one **on the same mount**, so bench fns must be idempotent and - module-level counters/LCGs are used where ids must never repeat across - invocations. -- Plain `vitest bench` never runs suite hooks (`beforeAll`/`afterAll`) and + measured one. Client suite hooks mount a fresh app before each invocation + and tear it down afterward, outside the measurement window. This prevents + warmup navigation history and caches from changing the starting state. + Module-level counters/LCGs keep generated URLs unique across invocations. +- Plain `vitest bench` never runs suite hooks (`beforeEach`/`afterEach`) and only honors tinybench's `setup`/`teardown` options; the CodSpeed runner does the exact opposite. Client benches therefore register **both** — in any given mode exactly one pair runs. @@ -59,7 +60,7 @@ same workload through the Flame profiler. - Before each CodSpeed invocation, a setup hook yields to the event loop, outside the measurement marker and before CodSpeed forces GC. Promise-only warmup chains can keep WeakRef targets alive through a collection in the same - job. The hook leaves live router and application state mounted. + job. Client app setup also runs before the forced GC. - Do not force GC inside a measured request loop or poll `heapUsed` to choose how much work to do. Both introduce allocator activity from the harness into the measurement. Footprint scenarios run exactly one large request per diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/react/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/solid/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/interrupted-navigations/vue/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts b/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/react/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/solid/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/loader-data-retention/vue/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts b/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/react/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/solid/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/mount-unmount/vue/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts b/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/react/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/solid/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/navigation-churn/vue/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts b/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/react/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/solid/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/vue/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts b/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/react/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts b/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/solid/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts b/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts index e645ab38f1..85a1f11ee6 100644 --- a/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts +++ b/benchmarks/memory/client/scenarios/unique-location-churn/vue/memory.bench.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, bench, describe } from 'vitest' +import { afterEach, beforeEach, bench, describe } from 'vitest' import { memoryBenchOptions } from '#memory-client/bench-utils' import { workload } from './setup' @@ -6,8 +6,8 @@ await workload.sanity() describe('memory', () => { if (workload.before && workload.after) { - beforeAll(workload.before) - afterAll(workload.after) + beforeEach(workload.before) + afterEach(workload.after) bench(workload.name, workload.run, { ...memoryBenchOptions, diff --git a/benchmarks/memory/server/client-lifecycle.test.ts b/benchmarks/memory/server/client-lifecycle.test.ts new file mode 100644 index 0000000000..451708ef26 --- /dev/null +++ b/benchmarks/memory/server/client-lifecycle.test.ts @@ -0,0 +1,69 @@ +import { globSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { expect, it, vi } from 'vitest' + +const clientRoot = fileURLToPath(new URL('../client/', import.meta.url)) +const entrypoints = globSync('scenarios/*/*/memory.bench.ts', { + cwd: clientRoot, +}) + +it.each(entrypoints)( + '%s resets the app between CodSpeed invocations', + async (file) => { + const events: Array = [] + const beforeHooks: Array<() => unknown> = [] + const afterHooks: Array<() => unknown> = [] + const workload = { + name: 'lifecycle control', + sanity: vi.fn(), + before: () => events.push('mount'), + run: () => events.push('navigate'), + after: () => events.push('unmount'), + } + let run: (() => unknown) | undefined + let options: { setup?: unknown; teardown?: unknown } | undefined + const entrypoint = resolve(clientRoot, file) + const setup = resolve(dirname(entrypoint), 'setup.ts') + + vi.doMock('vitest', () => ({ + describe: (_name: string, register: () => void) => register(), + beforeEach: (hook: () => unknown) => beforeHooks.push(hook), + afterEach: (hook: () => unknown) => afterHooks.push(hook), + bench: ( + _name: string, + fn: () => unknown, + benchOptions: typeof options, + ) => { + run = fn + options = benchOptions + }, + })) + vi.doMock(setup, () => ({ workload })) + + try { + await import(/* @vite-ignore */ entrypoint) + expect(workload.sanity).toHaveBeenCalledOnce() + expect(run).toBe(workload.run) + // Tinybench uses its own hooks; CodSpeed uses the suite hooks below. + expect(options?.setup).toBe(workload.before) + expect(options?.teardown).toBe(workload.after) + for (let invocation = 0; invocation < 3; invocation++) { + for (const before of beforeHooks) { + await before() + } + await run!() + for (const after of afterHooks) { + await after() + } + } + expect(events).toEqual( + Array(3).fill(['mount', 'navigate', 'unmount']).flat(), + ) + } finally { + vi.doUnmock(setup) + vi.doUnmock('vitest') + vi.resetModules() + } + }, +) diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index 510626bec4..d3812ffc3c 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -289,7 +289,8 @@ "^production", "{workspaceRoot}/eslint.config.js", "dependentTaskOutputs", - "{workspaceRoot}/benchmarks/memory/*.ts" + "{workspaceRoot}/benchmarks/memory/*.ts", + "{workspaceRoot}/benchmarks/memory/client/scenarios/*/*/memory.bench.ts" ] } } From 800848138f985835ed618004b25368a9570ad619 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 01:12:15 +0200 Subject: [PATCH 14/18] chore: remove stale memory benchmark guidance --- .../memory/client/scenarios/preload-churn/shared.ts | 10 +++------- benchmarks/memory/client/tsconfig.json | 7 ++++++- .../memory/server/scenarios/server-fn-churn/shared.ts | 2 -- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/benchmarks/memory/client/scenarios/preload-churn/shared.ts b/benchmarks/memory/client/scenarios/preload-churn/shared.ts index 8cea4aca81..68fdfedd3f 100644 --- a/benchmarks/memory/client/scenarios/preload-churn/shared.ts +++ b/benchmarks/memory/client/scenarios/preload-churn/shared.ts @@ -35,18 +35,14 @@ type PreloadRouter = { subscribe: (event: 'onRendered', listener: () => void) => () => void } -// Fixed id for the eviction navigations interleaved into the bench loop; its -// payload is a constant-size steady-state resident, never part of the signal. +// Reuse one navigation target while evicting the distinct preloaded routes. const evictionItemId = 'nav-evict' const preloadChurnIterations = 200 // A navigation commit is what triggers the router's clearExpiredCache -- // preloaded matches (defaultPreloadGcTime: 0) are only evicted then, never -// during a preload-only loop. Interleaving a navigation every few preloads is -// what makes the flat floor assert "eviction releases preloaded payloads". +// during a preload-only loop. Interleave navigation to exercise eviction. const preloadsPerEvictionNavigation = 10 -// Module-level so ids stay unique across runner invocations on one mount; a -// per-invocation LCG would replay identical ids, and every preload after the -// first invocation would dedupe against cachedMatches instead of doing work. +// Keep generated URLs unique across warmup and measured invocations. const benchmarkRandom = createDeterministicRandom(0x706c6f61) let preloadCounter = 0 diff --git a/benchmarks/memory/client/tsconfig.json b/benchmarks/memory/client/tsconfig.json index 9280d5c42f..eefc5c581e 100644 --- a/benchmarks/memory/client/tsconfig.json +++ b/benchmarks/memory/client/tsconfig.json @@ -3,5 +3,10 @@ "compilerOptions": { "types": ["node", "vite/client", "vitest/globals"] }, - "include": ["bench-utils.ts", "lifecycle.ts", "vitest.setup.ts"] + "include": [ + "bench-utils.ts", + "lifecycle.ts", + "vitest.setup.ts", + "scenarios/*/shared.ts" + ] } diff --git a/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts b/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts index 2bc224c979..5fd1e22632 100644 --- a/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts +++ b/benchmarks/memory/server/scenarios/server-fn-churn/shared.ts @@ -38,8 +38,6 @@ type SerovalNode = const benchmarkSeed = 0xdecafbad const payloadSeed = 0x51f0cafe const fixtureCount = 16 -// Sized to sit just above the 2s measured-run floor on CI (per-iteration -// cost with the pinned collection is ~0.08-0.11s across frameworks). const serverFnChurnIterations = 30 const origin = 'http://localhost' const tssContentTypeFramed = 'application/x-tss-framed' From ca170a917934a482ddb547ac0d0c27e96b2d9973 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 01:54:13 +0200 Subject: [PATCH 15/18] test: control the memory benchmark coordinator runtime --- .github/workflows/client-nav-benchmarks.yml | 13 ++--- benchmarks/memory/README.md | 14 ++--- benchmarks/memory/client/package.json | 1 + benchmarks/memory/run.ts | 55 +++++++++++++++++++ benchmarks/memory/runtime.ts | 10 +++- benchmarks/memory/server/launcher.test.ts | 60 +++++++++++++++++++++ benchmarks/memory/server/package.json | 1 + benchmarks/memory/server/tsconfig.json | 2 + benchmarks/memory/tsconfig.json | 3 +- pnpm-lock.yaml | 6 +++ 10 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 benchmarks/memory/run.ts create mode 100644 benchmarks/memory/server/launcher.test.ts diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index 2045a91a0b..25d53e7fd6 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -61,8 +61,8 @@ jobs: - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - # Run the build outside of the CodSpeed action to avoid being slowed by intrumentation overhead. - # (and then run the task itself with --excludeTaskDependencies) + # Build outside instrumentation. Memory runs launch Vitest directly; + # CPU runs skip the task dependencies already built here. - name: Prepare ${{ matrix.benchmark }}:${{ matrix.framework }} benchmark run: >- pnpm nx run @@ -74,7 +74,8 @@ jobs: with: mode: ${{ startsWith(matrix.benchmark, 'memory-') && 'memory' || 'simulation' }} run: >- - WITH_INSTRUMENTATION=1 - pnpm nx run - @benchmarks/${{ matrix.benchmark }}:test:perf:${{ matrix.framework }} - --excludeTaskDependencies + ${{ + startsWith(matrix.benchmark, 'memory-') && + format('NODE_ENV=production WITH_INSTRUMENTATION=1 node benchmarks/memory/run.ts {0} {1}', matrix.benchmark == 'memory-client' && 'client' || 'server', matrix.framework) || + format('WITH_INSTRUMENTATION=1 pnpm nx run @benchmarks/{0}:test:perf:{1} --excludeTaskDependencies', matrix.benchmark, matrix.framework) + }} diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index e60c0c68d8..343339d594 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -51,6 +51,9 @@ same workload through the Flame profiler. only honors tinybench's `setup`/`teardown` options; the CodSpeed runner does the exact opposite. Client benches therefore register **both** — in any given mode exactly one pair runs. +- CI builds through Nx before instrumentation, then `run.ts` replaces itself + with Vitest. This removes Nx/pnpm from the tracked process tree and applies + CodSpeed's predictable flags to the coordinator as well as its workers. - Worker flags live in `runtime.ts` and apply only to the CodSpeed memory instrument. They supplement the integration's predictable execution flags. Disabling machine-code generation and bytecode flushing keeps compiler allocations @@ -200,14 +203,13 @@ once to install the memory executor, and sudo; **uploads results to the CodSpeed dashboard** — local runs do not affect PR baselines): ```bash -WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-server:test:perf:react -WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-server:test:perf:solid -WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-server:test:perf:vue -WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:react -WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:solid -WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:vue +pnpm nx run @benchmarks/memory-client:build:react +NODE_ENV=production WITH_INSTRUMENTATION=1 codspeed run --mode memory -- node benchmarks/memory/run.ts client react ``` +Replace `client` with `server`, or `react` with `solid`/`vue`, to run another +group. Build the matching target first. + Run only memory jobs on an experiment branch: ```bash diff --git a/benchmarks/memory/client/package.json b/benchmarks/memory/client/package.json index af1770ba0d..f3b9beb0b6 100644 --- a/benchmarks/memory/client/package.json +++ b/benchmarks/memory/client/package.json @@ -22,6 +22,7 @@ "vue": "^3.5.16" }, "devDependencies": { + "@codspeed/core": "^5.5.0", "@codspeed/vitest-plugin": "^5.5.0", "@datadog/pprof": "^5.13.2", "@platformatic/flame": "^1.6.0", diff --git a/benchmarks/memory/run.ts b/benchmarks/memory/run.ts new file mode 100644 index 0000000000..bb4ff1228c --- /dev/null +++ b/benchmarks/memory/run.ts @@ -0,0 +1,55 @@ +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { memoryCoordinatorExecArgv } from './runtime.ts' + +const [side, framework, ...args] = process.argv.slice(2) +if ( + (side !== 'client' && side !== 'server') || + !['react', 'solid', 'vue'].includes(framework ?? '') +) { + throw new Error( + 'Usage: node benchmarks/memory/run.ts ', + ) +} + +if (!process.execve) { + throw new Error( + 'Memory benchmarks require Node 24 with process.execve support', + ) +} + +const directory = fileURLToPath(new URL(`./${side}/`, import.meta.url)) +const require = createRequire(join(directory, 'package.json')) +const { getV8Flags } = (await import( + pathToFileURL(require.resolve('@codspeed/core')).href +)) as { + getV8Flags: () => Array +} +const vitest = join( + dirname(require.resolve('vitest/package.json')), + 'vitest.mjs', +) + +// The memory instrument tracks the process tree, including the coordinator. +// Replace this launcher so no Node/Nx/pnpm parent can collect or compile while +// the worker is measured. Builds remain in the workflow's preceding Nx step. +process.chdir(directory) +process.execve( + process.execPath, + [ + process.execPath, + ...memoryCoordinatorExecArgv(), + ...getV8Flags(), + vitest, + 'bench', + '--config', + `vitest.${framework}.config.ts`, + ...args, + ], + Object.fromEntries( + Object.entries(process.env).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ), +) diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index b3d262c82a..0d6b7625b5 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -1,5 +1,5 @@ import { fileURLToPath } from 'node:url' -import { isMemoryInstrumented } from './turn' +import { isMemoryInstrumented } from './turn.ts' /** Worker settings for the CodSpeed memory instrument. */ export function memoryExecArgv() { @@ -18,6 +18,14 @@ export function memoryExecArgv() { ] } +export function memoryCoordinatorExecArgv() { + // Vite's lexer needs WebAssembly, which --jitless disables. Compile the + // coordinator's baseline code eagerly; the benchmark workers stay jitless. + return memoryExecArgv().flatMap((flag) => + flag === '--jitless' ? ['--no-maglev', '--always-sparkplug'] : [flag], + ) +} + export function memoryConfig(side: 'client' | 'server') { const execArgv = memoryExecArgv() return { diff --git a/benchmarks/memory/server/launcher.test.ts b/benchmarks/memory/server/launcher.test.ts new file mode 100644 index 0000000000..c42c02be2e --- /dev/null +++ b/benchmarks/memory/server/launcher.test.ts @@ -0,0 +1,60 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { expect, it } from 'vitest' + +it('replaces the launcher with a deterministic Vitest coordinator', () => { + const directory = mkdtempSync(join(tmpdir(), 'memory-launcher-')) + const probe = join(directory, 'probe.cjs') + writeFileSync( + probe, + ` + console.log(JSON.stringify({ pid: process.pid, args: process.argv, flags: process.execArgv, wasm: typeof WebAssembly })) + if (process.argv[1].endsWith('/vitest.mjs')) { + process.exit(0) + } + `, + ) + try { + const output = execFileSync( + process.execPath, + [fileURLToPath(new URL('../run.ts', import.meta.url)), 'client', 'vue'], + { + encoding: 'utf8', + env: { + ...process.env, + NODE_OPTIONS: `--require=${probe}`, + CODSPEED_ENV: 'test', + CODSPEED_RUNNER_MODE: 'memory', + }, + }, + ) + const [launcher, coordinator] = output + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + expect(coordinator.pid).toBe(launcher.pid) + expect(coordinator.wasm).toBe('object') + expect(coordinator.flags).not.toContain('--jitless') + expect(coordinator.args.slice(-3)).toEqual([ + 'bench', + '--config', + 'vitest.vue.config.ts', + ]) + expect(coordinator.flags).toEqual( + expect.arrayContaining([ + '--no-maglev', + '--always-sparkplug', + '--no-incremental-marking', + '--no-minor-gc-task', + '--predictable', + '--hash-seed=1', + '--expose-gc', + ]), + ) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index d3812ffc3c..fe5f11ce0f 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -23,6 +23,7 @@ "vue": "^3.5.16" }, "devDependencies": { + "@codspeed/core": "^5.5.0", "@codspeed/vitest-plugin": "^5.5.0", "@datadog/pprof": "^5.13.2", "@platformatic/flame": "^1.6.0", diff --git a/benchmarks/memory/server/tsconfig.json b/benchmarks/memory/server/tsconfig.json index ffb2a11caa..1739dec8fe 100644 --- a/benchmarks/memory/server/tsconfig.json +++ b/benchmarks/memory/server/tsconfig.json @@ -1,12 +1,14 @@ { "extends": "../../../tsconfig.json", "compilerOptions": { + "allowImportingTsExtensions": true, "types": ["node", "vite/client", "vitest/globals"] }, "include": [ "bench-utils.ts", "scenarios/*/shared.ts", "../runtime.ts", + "../run.ts", "*.test.ts", "vitest.unit.config.ts", "vitest.setup.ts" diff --git a/benchmarks/memory/tsconfig.json b/benchmarks/memory/tsconfig.json index 5a325f3b6c..4882af692d 100644 --- a/benchmarks/memory/tsconfig.json +++ b/benchmarks/memory/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../tsconfig.json", "compilerOptions": { + "allowImportingTsExtensions": true, "types": ["node"] }, - "include": ["runtime.ts"] + "include": ["runtime.ts", "run.ts"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4058009971..1760d5c7bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -340,6 +340,9 @@ importers: specifier: ^3.5.16 version: 3.5.25(@typescript/typescript6@6.0.2) devDependencies: + '@codspeed/core': + specifier: ^5.5.0 + version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) '@codspeed/vitest-plugin': specifier: ^5.5.0 version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)(tinybench@2.9.0)(vite@8.0.14(@types/node@25.0.9)(esbuild@0.27.4)(jiti@2.7.0)(sass@1.97.2)(terser@5.37.0)(tsx@4.20.3)(yaml@2.9.0))(vitest@4.1.4) @@ -422,6 +425,9 @@ importers: specifier: ^3.5.16 version: 3.5.25(@typescript/typescript6@6.0.2) devDependencies: + '@codspeed/core': + specifier: ^5.5.0 + version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) '@codspeed/vitest-plugin': specifier: ^5.5.0 version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)(tinybench@2.9.0)(vite@8.0.14(@types/node@25.0.9)(esbuild@0.27.4)(jiti@2.7.0)(sass@1.97.2)(terser@5.37.0)(tsx@4.20.3)(yaml@2.9.0))(vitest@4.1.4) From dfe39be60e37bdafc241739956137898eda35a5a Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 02:17:23 +0200 Subject: [PATCH 16/18] revert: keep the better-tested memory benchmark runtime --- .github/workflows/client-nav-benchmarks.yml | 13 +++-- benchmarks/memory/README.md | 14 +++-- benchmarks/memory/client/package.json | 1 - benchmarks/memory/run.ts | 55 ------------------- benchmarks/memory/runtime.ts | 10 +--- benchmarks/memory/server/launcher.test.ts | 60 --------------------- benchmarks/memory/server/package.json | 1 - benchmarks/memory/server/tsconfig.json | 2 - benchmarks/memory/tsconfig.json | 3 +- pnpm-lock.yaml | 6 --- 10 files changed, 14 insertions(+), 151 deletions(-) delete mode 100644 benchmarks/memory/run.ts delete mode 100644 benchmarks/memory/server/launcher.test.ts diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index 25d53e7fd6..2045a91a0b 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -61,8 +61,8 @@ jobs: - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - # Build outside instrumentation. Memory runs launch Vitest directly; - # CPU runs skip the task dependencies already built here. + # Run the build outside of the CodSpeed action to avoid being slowed by intrumentation overhead. + # (and then run the task itself with --excludeTaskDependencies) - name: Prepare ${{ matrix.benchmark }}:${{ matrix.framework }} benchmark run: >- pnpm nx run @@ -74,8 +74,7 @@ jobs: with: mode: ${{ startsWith(matrix.benchmark, 'memory-') && 'memory' || 'simulation' }} run: >- - ${{ - startsWith(matrix.benchmark, 'memory-') && - format('NODE_ENV=production WITH_INSTRUMENTATION=1 node benchmarks/memory/run.ts {0} {1}', matrix.benchmark == 'memory-client' && 'client' || 'server', matrix.framework) || - format('WITH_INSTRUMENTATION=1 pnpm nx run @benchmarks/{0}:test:perf:{1} --excludeTaskDependencies', matrix.benchmark, matrix.framework) - }} + WITH_INSTRUMENTATION=1 + pnpm nx run + @benchmarks/${{ matrix.benchmark }}:test:perf:${{ matrix.framework }} + --excludeTaskDependencies diff --git a/benchmarks/memory/README.md b/benchmarks/memory/README.md index 343339d594..e60c0c68d8 100644 --- a/benchmarks/memory/README.md +++ b/benchmarks/memory/README.md @@ -51,9 +51,6 @@ same workload through the Flame profiler. only honors tinybench's `setup`/`teardown` options; the CodSpeed runner does the exact opposite. Client benches therefore register **both** — in any given mode exactly one pair runs. -- CI builds through Nx before instrumentation, then `run.ts` replaces itself - with Vitest. This removes Nx/pnpm from the tracked process tree and applies - CodSpeed's predictable flags to the coordinator as well as its workers. - Worker flags live in `runtime.ts` and apply only to the CodSpeed memory instrument. They supplement the integration's predictable execution flags. Disabling machine-code generation and bytecode flushing keeps compiler allocations @@ -203,13 +200,14 @@ once to install the memory executor, and sudo; **uploads results to the CodSpeed dashboard** — local runs do not affect PR baselines): ```bash -pnpm nx run @benchmarks/memory-client:build:react -NODE_ENV=production WITH_INSTRUMENTATION=1 codspeed run --mode memory -- node benchmarks/memory/run.ts client react +WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-server:test:perf:react +WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-server:test:perf:solid +WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-server:test:perf:vue +WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:react +WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:solid +WITH_INSTRUMENTATION=1 codspeed run --mode memory -- pnpm nx run @benchmarks/memory-client:test:perf:vue ``` -Replace `client` with `server`, or `react` with `solid`/`vue`, to run another -group. Build the matching target first. - Run only memory jobs on an experiment branch: ```bash diff --git a/benchmarks/memory/client/package.json b/benchmarks/memory/client/package.json index f3b9beb0b6..af1770ba0d 100644 --- a/benchmarks/memory/client/package.json +++ b/benchmarks/memory/client/package.json @@ -22,7 +22,6 @@ "vue": "^3.5.16" }, "devDependencies": { - "@codspeed/core": "^5.5.0", "@codspeed/vitest-plugin": "^5.5.0", "@datadog/pprof": "^5.13.2", "@platformatic/flame": "^1.6.0", diff --git a/benchmarks/memory/run.ts b/benchmarks/memory/run.ts deleted file mode 100644 index bb4ff1228c..0000000000 --- a/benchmarks/memory/run.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createRequire } from 'node:module' -import { dirname, join } from 'node:path' -import { fileURLToPath, pathToFileURL } from 'node:url' -import { memoryCoordinatorExecArgv } from './runtime.ts' - -const [side, framework, ...args] = process.argv.slice(2) -if ( - (side !== 'client' && side !== 'server') || - !['react', 'solid', 'vue'].includes(framework ?? '') -) { - throw new Error( - 'Usage: node benchmarks/memory/run.ts ', - ) -} - -if (!process.execve) { - throw new Error( - 'Memory benchmarks require Node 24 with process.execve support', - ) -} - -const directory = fileURLToPath(new URL(`./${side}/`, import.meta.url)) -const require = createRequire(join(directory, 'package.json')) -const { getV8Flags } = (await import( - pathToFileURL(require.resolve('@codspeed/core')).href -)) as { - getV8Flags: () => Array -} -const vitest = join( - dirname(require.resolve('vitest/package.json')), - 'vitest.mjs', -) - -// The memory instrument tracks the process tree, including the coordinator. -// Replace this launcher so no Node/Nx/pnpm parent can collect or compile while -// the worker is measured. Builds remain in the workflow's preceding Nx step. -process.chdir(directory) -process.execve( - process.execPath, - [ - process.execPath, - ...memoryCoordinatorExecArgv(), - ...getV8Flags(), - vitest, - 'bench', - '--config', - `vitest.${framework}.config.ts`, - ...args, - ], - Object.fromEntries( - Object.entries(process.env).filter( - (entry): entry is [string, string] => entry[1] !== undefined, - ), - ), -) diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 0d6b7625b5..b3d262c82a 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -1,5 +1,5 @@ import { fileURLToPath } from 'node:url' -import { isMemoryInstrumented } from './turn.ts' +import { isMemoryInstrumented } from './turn' /** Worker settings for the CodSpeed memory instrument. */ export function memoryExecArgv() { @@ -18,14 +18,6 @@ export function memoryExecArgv() { ] } -export function memoryCoordinatorExecArgv() { - // Vite's lexer needs WebAssembly, which --jitless disables. Compile the - // coordinator's baseline code eagerly; the benchmark workers stay jitless. - return memoryExecArgv().flatMap((flag) => - flag === '--jitless' ? ['--no-maglev', '--always-sparkplug'] : [flag], - ) -} - export function memoryConfig(side: 'client' | 'server') { const execArgv = memoryExecArgv() return { diff --git a/benchmarks/memory/server/launcher.test.ts b/benchmarks/memory/server/launcher.test.ts deleted file mode 100644 index c42c02be2e..0000000000 --- a/benchmarks/memory/server/launcher.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { execFileSync } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { fileURLToPath } from 'node:url' -import { expect, it } from 'vitest' - -it('replaces the launcher with a deterministic Vitest coordinator', () => { - const directory = mkdtempSync(join(tmpdir(), 'memory-launcher-')) - const probe = join(directory, 'probe.cjs') - writeFileSync( - probe, - ` - console.log(JSON.stringify({ pid: process.pid, args: process.argv, flags: process.execArgv, wasm: typeof WebAssembly })) - if (process.argv[1].endsWith('/vitest.mjs')) { - process.exit(0) - } - `, - ) - try { - const output = execFileSync( - process.execPath, - [fileURLToPath(new URL('../run.ts', import.meta.url)), 'client', 'vue'], - { - encoding: 'utf8', - env: { - ...process.env, - NODE_OPTIONS: `--require=${probe}`, - CODSPEED_ENV: 'test', - CODSPEED_RUNNER_MODE: 'memory', - }, - }, - ) - const [launcher, coordinator] = output - .trim() - .split('\n') - .map((line) => JSON.parse(line)) - expect(coordinator.pid).toBe(launcher.pid) - expect(coordinator.wasm).toBe('object') - expect(coordinator.flags).not.toContain('--jitless') - expect(coordinator.args.slice(-3)).toEqual([ - 'bench', - '--config', - 'vitest.vue.config.ts', - ]) - expect(coordinator.flags).toEqual( - expect.arrayContaining([ - '--no-maglev', - '--always-sparkplug', - '--no-incremental-marking', - '--no-minor-gc-task', - '--predictable', - '--hash-seed=1', - '--expose-gc', - ]), - ) - } finally { - rmSync(directory, { recursive: true, force: true }) - } -}) diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index fe5f11ce0f..d3812ffc3c 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -23,7 +23,6 @@ "vue": "^3.5.16" }, "devDependencies": { - "@codspeed/core": "^5.5.0", "@codspeed/vitest-plugin": "^5.5.0", "@datadog/pprof": "^5.13.2", "@platformatic/flame": "^1.6.0", diff --git a/benchmarks/memory/server/tsconfig.json b/benchmarks/memory/server/tsconfig.json index 1739dec8fe..ffb2a11caa 100644 --- a/benchmarks/memory/server/tsconfig.json +++ b/benchmarks/memory/server/tsconfig.json @@ -1,14 +1,12 @@ { "extends": "../../../tsconfig.json", "compilerOptions": { - "allowImportingTsExtensions": true, "types": ["node", "vite/client", "vitest/globals"] }, "include": [ "bench-utils.ts", "scenarios/*/shared.ts", "../runtime.ts", - "../run.ts", "*.test.ts", "vitest.unit.config.ts", "vitest.setup.ts" diff --git a/benchmarks/memory/tsconfig.json b/benchmarks/memory/tsconfig.json index 4882af692d..5a325f3b6c 100644 --- a/benchmarks/memory/tsconfig.json +++ b/benchmarks/memory/tsconfig.json @@ -1,8 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "allowImportingTsExtensions": true, "types": ["node"] }, - "include": ["runtime.ts", "run.ts"] + "include": ["runtime.ts"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1760d5c7bf..4058009971 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -340,9 +340,6 @@ importers: specifier: ^3.5.16 version: 3.5.25(@typescript/typescript6@6.0.2) devDependencies: - '@codspeed/core': - specifier: ^5.5.0 - version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) '@codspeed/vitest-plugin': specifier: ^5.5.0 version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)(tinybench@2.9.0)(vite@8.0.14(@types/node@25.0.9)(esbuild@0.27.4)(jiti@2.7.0)(sass@1.97.2)(terser@5.37.0)(tsx@4.20.3)(yaml@2.9.0))(vitest@4.1.4) @@ -425,9 +422,6 @@ importers: specifier: ^3.5.16 version: 3.5.25(@typescript/typescript6@6.0.2) devDependencies: - '@codspeed/core': - specifier: ^5.5.0 - version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) '@codspeed/vitest-plugin': specifier: ^5.5.0 version: 5.5.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2)(tinybench@2.9.0)(vite@8.0.14(@types/node@25.0.9)(esbuild@0.27.4)(jiti@2.7.0)(sass@1.97.2)(terser@5.37.0)(tsx@4.20.3)(yaml@2.9.0))(vitest@4.1.4) From e9a9b10d5f741df0e96cf9ee66f4d891d4260837 Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 09:10:58 +0200 Subject: [PATCH 17/18] test: prepare controlled memory flag removal experiments --- .github/workflows/client-nav-benchmarks.yml | 2 +- benchmarks/memory/runtime.ts | 2 +- benchmarks/memory/server/runtime.test.ts | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index 2045a91a0b..a74af25d29 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -27,7 +27,7 @@ on: default: false concurrency: - group: client-nav-benchmarks-${{ github.ref }} + group: client-nav-benchmarks-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.ref }} cancel-in-progress: false queue: single diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index b3d262c82a..5bdccc0d6e 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -23,7 +23,7 @@ export function memoryConfig(side: 'client' | 'server') { return { execArgv, setupFiles: - side === 'client' || execArgv.length + side === 'client' || isMemoryInstrumented() ? [fileURLToPath(new URL(`./${side}/vitest.setup.ts`, import.meta.url))] : [], } diff --git a/benchmarks/memory/server/runtime.test.ts b/benchmarks/memory/server/runtime.test.ts index a843aed314..4b348a9ce6 100644 --- a/benchmarks/memory/server/runtime.test.ts +++ b/benchmarks/memory/server/runtime.test.ts @@ -53,7 +53,6 @@ describe('memory worker configuration', () => { { encoding: 'utf8' }, ) expect(output).toContain('5000050000') - expect(output).not.toMatch(/MAGLEV/) }) it.each([undefined, 'simulation', 'instrumentation', 'walltime'])( From 69ca72a94a3d271ae79d3f3cdd0cea0c0dfd68ea Mon Sep 17 00:00:00 2001 From: Sheraff Date: Sun, 6 Sep 2026 10:36:26 +0200 Subject: [PATCH 18/18] test: measure memory benchmarks bytecode-no-incremental --- benchmarks/memory/runtime.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/benchmarks/memory/runtime.ts b/benchmarks/memory/runtime.ts index 5bdccc0d6e..ea3f7741ec 100644 --- a/benchmarks/memory/runtime.ts +++ b/benchmarks/memory/runtime.ts @@ -9,13 +9,7 @@ export function memoryExecArgv() { // Keep machine-code generation out of native allocation measurements and GC // tied to allocations instead of event-loop tasks and marking time budgets. - return [ - '--jitless', - '--no-flush-bytecode', - '--no-minor-gc-task', - '--no-incremental-marking', - '--initial-old-space-size=512', - ] + return ['--no-flush-bytecode', '--no-incremental-marking'] } export function memoryConfig(side: 'client' | 'server') {