diff --git a/packages/router-plugin/src/hosts/webpack.ts b/packages/router-plugin/src/hosts/webpack.ts index a172bfd94d..5528e6d944 100644 --- a/packages/router-plugin/src/hosts/webpack.ts +++ b/packages/router-plugin/src/hosts/webpack.ts @@ -238,7 +238,10 @@ export function createRouterCompilerWebpackPlugin( // file events, the route files whose registry entry changed: the // bundler sees no file event for them) are built again in the // compilation that takes it - const drain = new StaleSetDrain(session.caches) + const drain = new StaleSetDrain(session.caches, { + tag: '[tanstack-router]', + root: compiler.context, + }) drain.register(envName, compiler) const unsubscribe = onRoutesChanged( routerPluginContext, @@ -330,7 +333,7 @@ export function createRouterCompilerWebpackPlugin( // what became stale after a compilation's drain (a registry change // published meanwhile, a compile overtaken) gets another one compiler.hooks.afterDone.tap(PLUGIN_NAME, () => { - drain.wake() + drain.compiled(envName) }) // the session lives as long as the compiler: until it shuts down // (`compiler.close()`, after a one-shot `run()` too) or stops watching diff --git a/packages/router-utils/src/compiler/cache.ts b/packages/router-utils/src/compiler/cache.ts index cdc756ed34..8932cb35da 100644 --- a/packages/router-utils/src/compiler/cache.ts +++ b/packages/router-utils/src/compiler/cache.ts @@ -809,6 +809,11 @@ export class StaleSets { return this.byEnv.has(env) } + /** The paths of the environment's set (held ones included), left in it. */ + paths(env: string): Array { + return [...(this.byEnv.get(env)?.keys() ?? [])] + } + /** Whether the environment's set holds `path`. */ holds(env: string, path: string): boolean { return this.byEnv.get(env)?.has(path) ?? false diff --git a/packages/router-utils/src/compiler/hosts/rebuild.ts b/packages/router-utils/src/compiler/hosts/rebuild.ts index c8a09b7e3c..8f83c4b46d 100644 --- a/packages/router-utils/src/compiler/hosts/rebuild.ts +++ b/packages/router-utils/src/compiler/hosts/rebuild.ts @@ -6,6 +6,7 @@ * read no mutable state), in the compilation of the event; a set that fills * while its compiler is idle gets it a compilation. */ +import { displayPath } from '../diagnostics' import { identify } from '../identity' import type { CompilerCaches } from '../host' import type { PlanKey } from '../identity' @@ -20,6 +21,12 @@ export interface RebuildCompilationLike< module: TModule, callback: (error?: Error | null) => void, ) => void + /** + * The modules this compilation built (webpack: a `WeakSet`, without a + * module its snapshot found current; Rspack: a `Set`, which also lists + * modules it did not build again). + */ + builtModules?: { has: (module: TModule) => boolean } /** Rspack's compiler namespace names its version; webpack's does not. */ compiler?: { webpack?: { rspackVersion?: string } } entries?: ReadonlyMap }> @@ -53,17 +60,23 @@ const MAX_ASKED = 3 * cause (`caused`); the files a compilation builds from their own events * leave its set at `make` (`building`); the rest of the set is built at * `finishMake` (`rebuild`). What fills a set after its compilation's - * `rebuild` (another environment's event, a registry change, a compile overtaken) is - * built by another compilation (`wake`). + * `rebuild` (another environment's event, a registry change, a compile + * overtaken) is built by another compilation (`compiled`). */ export class StaleSetDrain { - /** Each environment's compiler; `asked`: the compilations `wake` asked it for since its last cause. */ + /** Each environment's compiler; `asked`: the compilations `wake` asked it for since its last cause, or since one ended with its set empty. */ private readonly compilers = new Map< string, { compiler: IdleCompilerLike; asked: number } >() + /** By environment, the paths `building` took out of its set: built by its compilation, or by `rebuild`. */ + private readonly takenAtMake = new Map>() - constructor(private readonly caches: CompilerCaches) {} + constructor( + private readonly caches: CompilerCaches, + /** The tag of the host's messages and the project root (`HostServices`). */ + private readonly host: { tag: string; root: string }, + ) {} /** The compiler of an environment. */ register(env: string, compiler: IdleCompilerLike): void { @@ -79,20 +92,34 @@ export class StaleSetDrain { } /** - * The files a compilation of `env` builds now from their own events, once - * every invalidation of its events ended: the drain needn't build them - * again (what makes them stale from here on enters the set again). + * The files a compilation of `env` builds from their own events (its + * modified files, once every invalidation of its events ended): they + * leave the set (what makes them stale from here on enters it again). + * `rebuild` builds those the compilation did not (webpack with content + * hashes in its snapshots skips a file written again unchanged). */ building(env: string, files: Iterable): void { - this.caches.stale.drop(env, files) + const { stale } = this.caches + let taken = this.takenAtMake.get(env) + for (const file of files) { + if (stale.holds(env, file)) { + taken ??= new Set() + taken.add(file) + } + } + if (taken) { + this.takenAtMake.set(env, taken) + stale.drop(env, taken) + } } /** * Builds again the modules of the environment's stale set (the held ones * too: a module importing a removed file fails here as it would in the * bundler's own rebuild), once the invalidations running (a registry - * change published during this compilation's `watchRun`) ended. `pathOf`: - * a module resource as the stale sets spell its path. + * change published during this compilation's `watchRun`) ended, and the + * modules of what `building` took that the compilation did not build. + * `pathOf`: a module resource as the stale sets spell its path. */ async rebuild< TModule extends { identifier: () => string; resource?: string }, @@ -106,10 +133,37 @@ export class StaleSetDrain { if (caches.invalidations.size > 0) { await caches.settled() } - const paths = caches.stale.take(env, { held: true }) - if (paths.length > 0) { - await rebuildModules(compilation, new Set(paths), pathOf) + const paths = new Set(caches.stale.take(env, { held: true })) + const taken = this.takenAtMake.get(env) + this.takenAtMake.delete(env) + if (paths.size === 0 && !taken) { + return } + const built = compilation.builtModules + await rebuildModulesWhere(compilation, (m) => { + if (!m.resource) { + return false + } + const path = pathOf(m.resource) + return ( + paths.has(path) || + (taken !== undefined && taken.has(path) && !built?.has(m)) + ) + }) + } + + /** + * A compilation of `env` ended (`afterDone`: in `done` the compiler is not + * idle yet, and a compilation asked for there would start inside it): if + * its set is empty, the compilations asked for so far did their work. + * Then `wake`. + */ + compiled(env: string): void { + const entry = this.compilers.get(env) + if (entry && !this.caches.stale.has(env)) { + entry.asked = 0 + } + this.wake() } /** A registry change (its files are stale in every environment that compiled): every compiler may be asked again, the idle ones now. */ @@ -121,23 +175,28 @@ export class StaleSetDrain { } /** - * Asks each idle compiler whose stale set is not empty for a compilation: - * after every compilation (`afterDone`: in `done` the compiler is not idle - * yet, and a compilation asked for there would start inside it) and after - * a registry change. At most `MAX_ASKED` in a row without a cause: a module - * whose rebuild finds itself stale again (its plan's validation and its - * compile disagreeing) waits for the next event instead of compiling - * forever. + * Asks each idle compiler whose stale set is not empty for a compilation. + * At most `MAX_ASKED` in a row that each end with the set still not empty + * and no cause between: a module whose rebuild finds itself stale again + * (its plan's validation and its compile disagreeing) then waits for the + * next event instead of compiling forever, and a warning names its files. */ - wake(): void { + private wake(): void { for (const [env, entry] of this.compilers) { - if ( - entry.compiler.idle && - entry.asked < MAX_ASKED && - this.caches.stale.has(env) - ) { + if (!entry.compiler.idle || !this.caches.stale.has(env)) { + continue + } + if (entry.asked < MAX_ASKED) { entry.asked++ entry.compiler.watching?.invalidate() + } else if (entry.asked === MAX_ASKED) { + entry.asked++ + const files = this.caches.stale + .paths(env) + .map((file) => displayPath(file, this.host.root)) + console.warn( + `${this.host.tag} ${files.join(', ')} ${files.length === 1 ? 'was' : 'were'} still out of date after ${MAX_ASKED} rebuilds in a row (${env}), so the dev server stops rebuilding ${files.length === 1 ? 'it' : 'them'} until a file changes. Save ${files.length === 1 ? 'the file' : 'one of them'} to build ${files.length === 1 ? 'it' : 'them'} again, and please report this to TanStack.`, + ) } } } @@ -181,9 +240,21 @@ export async function rebuildModules< paths: ReadonlySet, pathOf: (resource: string) => string, ): Promise { - const pending = [...compilation.modules].filter( - (m) => m.resource && paths.has(pathOf(m.resource)), + return rebuildModulesWhere( + compilation, + (m) => m.resource !== undefined && paths.has(pathOf(m.resource)), ) +} + +/** Rebuilds the modules of `compilation` that `wanted` takes, in place. */ +async function rebuildModulesWhere< + TModule extends { identifier: () => string; resource?: string }, + TDependency, +>( + compilation: RebuildCompilationLike, + wanted: (module: TModule) => boolean, +): Promise { + const pending = [...compilation.modules].filter(wanted) if (pending.length === 0) { return } diff --git a/packages/router-utils/tests/rebuild-modules.test.ts b/packages/router-utils/tests/rebuild-modules.test.ts index ba9b707201..59d2aba2c7 100644 --- a/packages/router-utils/tests/rebuild-modules.test.ts +++ b/packages/router-utils/tests/rebuild-modules.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, onTestFinished, vi } from 'vitest' import { StaleSetDrain, rebuildModules } from '../src/compiler' import { MemoryHost } from './helpers' @@ -80,7 +80,10 @@ describe('rebuildModules', () => { describe('StaleSetDrain', () => { const drainOf = () => { const { caches } = new MemoryHost({ files: {} }) - const drain = new StaleSetDrain(caches) + const drain = new StaleSetDrain(caches, { + tag: '[tanstack-router]', + root: '/app', + }) const compiler = { idle: true, asked: 0, @@ -91,46 +94,78 @@ describe('StaleSetDrain', () => { }, } drain.register('client', compiler) - return { caches, drain, compiler } + const warnings: Array = [] + const warn = vi.spyOn(console, 'warn').mockImplementation((message) => { + warnings.push(String(message)) + }) + onTestFinished(() => warn.mockRestore()) + return { caches, drain, compiler, warnings } } - it('asks an idle compiler with a stale set for at most three compilations without a cause', () => { - const { caches, drain, compiler } = drainOf() + it('asks an idle compiler for at most three compilations that each leave its set full, then names the files once', () => { + const { caches, drain, compiler, warnings } = drainOf() // a module whose rebuild finds itself stale again, every time - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 6; i++) { caches.stale.add('client', '/app/a.ts') - drain.wake() + drain.compiled('client') } expect(compiler.asked).toBe(3) + expect(warnings).toHaveLength(1) + expect(warnings[0]).toContain('a.ts') // a file event of its own, or a registry change, may ask again drain.caused('client') - drain.wake() + drain.compiled('client') expect(compiler.asked).toBe(4) drain.registryChanged() expect(compiler.asked).toBe(5) }) + it('keeps asking while each compilation it asked for empties the set', () => { + const { caches, drain, compiler, warnings } = drainOf() + for (let i = 0; i < 6; i++) { + // another environment's event, a compile overtaken: filled while idle + caches.stale.add('client', `/app/m${i}.ts`) + drain.compiled('client') + // the compilation asked for takes the set + caches.stale.take('client', { held: true }) + drain.compiled('client') + } + expect(compiler.asked).toBe(6) + expect(warnings).toEqual([]) + }) + it('asks nothing of a compiling compiler, or of one with nothing stale', () => { const { caches, drain, compiler } = drainOf() - drain.wake() + drain.compiled('client') compiler.idle = false caches.stale.add('client', '/app/a.ts') - drain.wake() + drain.compiled('client') expect(compiler.asked).toBe(0) }) - it("rebuilds the set's modules, held ones too, and none a compilation builds from its own events", async () => { + it('rebuilds the set, held paths too, and of the paths a compilation builds from its own events the modules it did not build', async () => { const { caches, drain } = drainOf() caches.stale.add('client', '/app/a.ts') caches.stale.add('client', '/app/b.ts', true) caches.stale.add('client', '/app/c.ts') - drain.building('client', ['/app/c.ts']) - const modules = new Set(['a.ts', 'b.ts', 'c.ts'].map(moduleOf)) + caches.stale.add('client', '/app/d.ts') + // c.ts and d.ts changed; the bundler builds c.ts, d.ts was written again unchanged + drain.building('client', ['/app/c.ts', '/app/d.ts', '/app/e.ts']) + expect(caches.stale.paths('client').sort()).toEqual([ + '/app/a.ts', + '/app/b.ts', + ]) + const modules = new Set( + ['a.ts', 'b.ts', 'c.ts', 'd.ts', 'e.ts'].map(moduleOf), + ) const rebuilt: Array = [] await drain.rebuild( 'client', { modules, + builtModules: new Set( + [...modules].filter((m) => ['c.ts', 'e.ts'].includes(m.identifier())), + ), rebuildModule(m, callback) { rebuilt.push(m.identifier()) callback() @@ -138,7 +173,7 @@ describe('StaleSetDrain', () => { }, (resource) => resource, ) - expect(rebuilt).toEqual(['a.ts', 'b.ts']) + expect(rebuilt).toEqual(['a.ts', 'b.ts', 'd.ts']) expect(caches.stale.has('client')).toBe(false) }) }) diff --git a/packages/start-plugin-core/src/rsbuild/compiler-session.ts b/packages/start-plugin-core/src/rsbuild/compiler-session.ts index 11594dc44e..f91bb55c11 100644 --- a/packages/start-plugin-core/src/rsbuild/compiler-session.ts +++ b/packages/start-plugin-core/src/rsbuild/compiler-session.ts @@ -208,7 +208,10 @@ export class StartRsbuildCompilerSession { loadVirtual: () => Promise.resolve(null), label: 'start:rsbuild', }) - this.drain = new StaleSetDrain(this.start.caches) + this.drain = new StaleSetDrain(this.start.caches, { + tag: '[tanstack-start]', + root: opts.root, + }) registerCompilerStats( 'start:rsbuild:host', this, diff --git a/packages/start-plugin-core/src/rsbuild/host-plugin.ts b/packages/start-plugin-core/src/rsbuild/host-plugin.ts index 47c8c94176..5c08e1ae2e 100644 --- a/packages/start-plugin-core/src/rsbuild/host-plugin.ts +++ b/packages/start-plugin-core/src/rsbuild/host-plugin.ts @@ -467,7 +467,7 @@ export function createStartHostPlugin(opts: StartHostPluginOptions): { // idle (in `done` Rspack has not marked it so: a compilation asked for there would // start inside it) compiler.hooks.afterDone.tap(PLUGIN_NAME, () => { - session.drain.wake() + session.drain.compiled(environment.name) }) compiler.hooks.thisCompilation.tap(PLUGIN_NAME, (compilation) => {