diff --git a/.changepacks/changepack_log_shared_source_reads.json b/.changepacks/changepack_log_shared_source_reads.json new file mode 100644 index 000000000..f48a22bcd --- /dev/null +++ b/.changepacks/changepack_log_shared_source_reads.json @@ -0,0 +1,9 @@ +{ + "changes": { + "packages/plugin-utils/package.json": "Patch", + "packages/webpack-plugin/package.json": "Patch", + "packages/next-plugin/package.json": "Patch" + }, + "note": "Read each source file once per build in the passes that run before the bundler. The import graph, the class-slot reservation and the prewarm read the same files, and a Next production build runs them once per compiler (server, edge, client): they now share a stamp-checked source cache (`withSourceCache` / `readSource` in plugin-utils), which Next passes to the webpack plugin of each compiler. Watch and dev builds keep reading the disk on every compilation. Output is unchanged.", + "date": "2026-11-19T00:00:00.000Z" +} diff --git a/packages/next-plugin/src/__tests__/plugin.test.ts b/packages/next-plugin/src/__tests__/plugin.test.ts index 349ee6761..8f54fa9c8 100644 --- a/packages/next-plugin/src/__tests__/plugin.test.ts +++ b/packages/next-plugin/src/__tests__/plugin.test.ts @@ -470,10 +470,30 @@ describe('DevupUINextPlugin', () => { singleCss: true, cssDir: resolve('.next/cache', 'devup-ui'), productionPrewarmFiles: [], + sourceCache: expect.any(Map), }) }, ) + it('shares one source cache between the compilers of a production build, none in dev', () => { + // Given + delete process.env.TURBOPACK + const ret = DevupUI({}) + // When: Next configures its server and client compilers, then a dev one + ret.webpack!(createWebpackConfig(), createWebpackContext()) + ret.webpack!(createWebpackConfig(), createWebpackContext()) + ret.webpack!(createWebpackConfig(), { + ...createWebpackContext(), + dev: true, + }) + // Then + const [server, client, dev] = devupUIWebpackPluginSpy.mock.calls.map( + ([options]) => (options as { sourceCache?: unknown }).sourceCache, + ) + expect(server).toBeInstanceOf(Map) + expect(client).toBe(server) + expect(dev).toBeUndefined() + }) it.each([ ['a one-route production build', {}, {}, { singleCss: true }], [ @@ -517,6 +537,7 @@ describe('DevupUINextPlugin', () => { cssDir: resolve('.next/cache', 'devup-ui'), engine: 'native', productionPrewarmFiles: [], + sourceCache: expect.any(Map), }) }) @@ -620,6 +641,7 @@ describe('DevupUINextPlugin', () => { engine: 'native', cssDir: resolve('.next/cache', 'devup-ui'), productionPrewarmFiles: [], + sourceCache: expect.any(Map), }) }) @@ -641,6 +663,7 @@ describe('DevupUINextPlugin', () => { engine: 'native', cssDir: resolve('.next/cache', 'devup-ui'), productionPrewarmFiles: [], + sourceCache: expect.any(Map), }) expect(webpack).toHaveBeenCalled() }) @@ -672,19 +695,11 @@ describe('DevupUINextPlugin', () => { ).mockReturnValue(true) try { // When - const config = DevupUI({}, options) as { - turbopack: { - rules: Record< - string, - { loaders: { options: { singleCss: boolean } }[] } - > - } - } - // Then the loaders extract into the layout chosen - expect( - config.turbopack.rules['*.{tsx,ts,mts,cts,jsx,js,mjs,cjs}'] - .loaders[0].options.singleCss, - ).toBe(expected) + DevupUI({}, options) + // Then the coordinator extracts into the layout chosen + expect(startCoordinatorSpy).toHaveBeenCalledWith( + expect.objectContaining({ singleCss: expected }), + ) } finally { singleRoute.mockRestore() graphSpy.mockRestore() diff --git a/packages/next-plugin/src/plugin.ts b/packages/next-plugin/src/plugin.ts index 96b2a8d00..383266483 100644 --- a/packages/next-plugin/src/plugin.ts +++ b/packages/next-plugin/src/plugin.ts @@ -25,9 +25,12 @@ import { normalizeShorthands, normalizeStylexOptions, readJsxImportSource, + readSource, seedFileNumbers, + type SourceCache, type StaticImportGraph, type WasmImportAliases, + withSourceCache, } from '@devup-ui/plugin-utils' // The published declarations name only these: `next.config.ts` type-checks // against them, so they come from the entry that loads nothing else @@ -486,6 +489,8 @@ export function DevupUI( // `[]` (idle fallback) when no routes are detected or the pre-pass fails. let expectedBaseFiles: string[] = [] let staticGraph: StaticImportGraph | undefined + // The graph and the prewarm read the same sources + const sourceCache: SourceCache = new Map() // A one-route production build gets one stylesheet, unless asked otherwise let singleRoute = false const graphStartedAt = profileStart() @@ -494,7 +499,10 @@ export function DevupUI( const tsconfigPath = resolve(process.cwd(), 'tsconfig.json') const cwd = process.cwd() // One scan+parse of the source tree, shared by all three consumers below. - const graph = buildStaticImportGraph(srcDir, tsconfigPath) + const graph = withSourceCache( + () => buildStaticImportGraph(srcDir, tsconfigPath), + sourceCache, + ) staticGraph = graph singleRoute = !watch && @@ -635,13 +643,18 @@ export function DevupUI( const cwd = process.cwd() const collectStartedAt = prewarmStartedAt === undefined ? undefined : performance.now() - const prewarmFiles = collectProductionPrewarmFiles({ - cwd, - graph: staticGraph, - expectedBaseFiles, - libPackage, - include, - }) + const graph = staticGraph + const prewarmFiles = withSourceCache( + () => + collectProductionPrewarmFiles({ + cwd, + graph, + expectedBaseFiles, + libPackage, + include, + }), + sourceCache, + ) const collectDurationMs = elapsedMs(collectStartedAt) for (const filename of prewarmFiles) { const resourcePath = resolve(cwd, filename) @@ -651,7 +664,10 @@ export function DevupUI( ).replaceAll('\\', '/')}` const readStartedAt = prewarmStartedAt === undefined ? undefined : performance.now() - const source = readFileSync(resourcePath, 'utf-8') + const source = withSourceCache( + () => readSource(resourcePath), + sourceCache, + ) if (readStartedAt !== undefined) { prewarmReadMs += performance.now() - readStartedAt prewarmSourceBytes += Buffer.byteLength(source) @@ -905,6 +921,9 @@ export function DevupUI( const { webpack } = config let productionPrewarmFiles: readonly string[] | undefined + // A production build runs its compilers in turn: their plugins read each + // source once + const sourceCache: SourceCache = new Map() // A one-route production build gets one stylesheet, unless asked otherwise let singleRoute = false config.webpack = (config, _options) => { @@ -921,21 +940,23 @@ export function DevupUI( const cwd = root const srcDir = sourceDir(cwd) const tsconfigPath = resolve(cwd, 'tsconfig.json') - const graph = buildStaticImportGraph(srcDir, tsconfigPath) - singleRoute = - options.singleCss === undefined && isSingleRouteApp(srcDir, graph) - productionPrewarmFiles = collectProductionPrewarmFiles({ - cwd, - graph, - expectedBaseFiles: computeCompiledFiles({ + productionPrewarmFiles = withSourceCache(() => { + const graph = buildStaticImportGraph(srcDir, tsconfigPath) + singleRoute = + options.singleCss === undefined && isSingleRouteApp(srcDir, graph) + return collectProductionPrewarmFiles({ cwd, - srcDir, - tsconfigPath, graph, - }), - libPackage: options.package ?? '@devup-ui/react', - include: options.include ?? [], - }) + expectedBaseFiles: computeCompiledFiles({ + cwd, + srcDir, + tsconfigPath, + graph, + }), + libPackage: options.package ?? '@devup-ui/react', + include: options.include ?? [], + }) + }, sourceCache) } config.plugins.push( new DevupUIWebpackPlugin({ @@ -945,6 +966,7 @@ export function DevupUI( productionPrewarmFiles: _options.dev ? undefined : productionPrewarmFiles, + sourceCache: _options.dev ? undefined : sourceCache, ...(!_options.dev && singleRoute ? { singleCss: true } : {}), }), ) diff --git a/packages/next-plugin/src/prewarm.ts b/packages/next-plugin/src/prewarm.ts index c78877de1..699ecd921 100644 --- a/packages/next-plugin/src/prewarm.ts +++ b/packages/next-plugin/src/prewarm.ts @@ -1,10 +1,11 @@ -import { existsSync, readFileSync } from 'node:fs' +import { existsSync } from 'node:fs' import { extname, isAbsolute, relative, resolve } from 'node:path' import { createModuleResolver, isRuntimePackage, parseImports, + readSource, SOURCE_FILE_REGEX, type StaticImportGraph, } from '@devup-ui/plugin-utils' @@ -102,8 +103,7 @@ export function collectProductionPrewarmFiles({ if (!SOURCE_FILE_REGEX.test(filename) || visited.has(filename)) continue visited.add(filename) files.add(toKey(resolvedCwd, filename)) - const source = - filename === module.path ? module.code : readFileSync(filename, 'utf-8') + const source = filename === module.path ? module.code : readSource(filename) for (const imported of parseImports(filename, source)) { queue.push({ importer: filename, specifier: imported.specifier }) } diff --git a/packages/plugin-utils/src/__tests__/source-cache.test.ts b/packages/plugin-utils/src/__tests__/source-cache.test.ts new file mode 100644 index 000000000..4de17fd6a --- /dev/null +++ b/packages/plugin-utils/src/__tests__/source-cache.test.ts @@ -0,0 +1,103 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' + +import { + fileStamp, + readSource, + type SourceCache, + withSourceCache, +} from '../source-cache' + +describe('readSource', () => { + let dir: string + let file: string + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'devup-ui-source-cache-')) + file = join(dir, 'a.tsx') + writeFileSync(file, 'on disk') + }) + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }) + }) + + /** A cache already holding `code` for the file as it is now */ + function holding(code: string): SourceCache { + return new Map([[resolve(file), { stamp: fileStamp(file) ?? '', code }]]) + } + + it('reads the disk outside a cache', () => { + expect(readSource(file)).toBe('on disk') + expect(globalThis.__devupSourceCache).toBeUndefined() + }) + + it('serves an unchanged file from the cache, under any spelling of its path', () => { + // Given + const cache = holding('cached') + // When + const read = withSourceCache( + () => [readSource(file), readSource(join(dir, '.', 'a.tsx'))], + cache, + ) + // Then + expect(read).toEqual(['cached', 'cached']) + }) + + it('reads a changed file again and keeps it', () => { + // Given + const cache = holding('stale') + // When + writeFileSync(file, 'changed text') + // Then + expect(withSourceCache(() => readSource(file), cache)).toBe('changed text') + expect(cache.get(resolve(file))).toEqual({ + stamp: fileStamp(file) ?? '', + code: 'changed text', + }) + }) + + it('reads a file once into an empty cache', () => { + const cache: SourceCache = new Map() + expect(withSourceCache(() => readSource(file), cache)).toBe('on disk') + expect(cache.get(resolve(file))?.code).toBe('on disk') + }) + + it('reports a deleted file and forgets it', () => { + // Given + const cache = holding('cached') + // When + rmSync(file) + // Then + expect(() => withSourceCache(() => readSource(file), cache)).toThrow() + expect(cache.size).toBe(0) + }) + + it('keeps the outer cache in a nested run and restores none after', () => { + // Given + const outer: SourceCache = new Map() + const inner: SourceCache = new Map() + // When + withSourceCache(() => withSourceCache(() => readSource(file), inner), outer) + // Then + expect(outer.size).toBe(1) + expect(inner.size).toBe(0) + expect(globalThis.__devupSourceCache).toBeUndefined() + }) + + it('restores the outer cache when the run throws', () => { + expect(() => + withSourceCache(() => { + throw new Error('boom') + }, new Map()), + ).toThrow('boom') + expect(globalThis.__devupSourceCache).toBeUndefined() + }) + + it('stamps no missing file', () => { + expect(fileStamp(join(dir, 'missing.tsx'))).toBeUndefined() + }) +}) diff --git a/packages/plugin-utils/src/import-graph.ts b/packages/plugin-utils/src/import-graph.ts index 30c341b0c..246a0a782 100644 --- a/packages/plugin-utils/src/import-graph.ts +++ b/packages/plugin-utils/src/import-graph.ts @@ -18,6 +18,7 @@ import { } from 'node:path' import { PackageManifestShapeError } from './package-manifest' +import { fileStamp, readSource } from './source-cache' import { SOURCE_EXTENSIONS, SOURCE_FILE_REGEX, @@ -175,7 +176,7 @@ export function buildStaticImportGraph( } for (const file of files) { - const imports = parseImports(file, readFileSync(file, 'utf-8')) + const imports = parseImports(file, readSource(file)) for (const importRef of imports) { const target = resolveImport(importRef.specifier, file, context) if (!target) { @@ -1239,11 +1240,11 @@ export function createModuleResolver({ return path } /** The file's code, read again only once its stamp changes. */ - const readSource = (path: string) => { + const readCode = (path: string) => { const stamp = fileStamp(path) const cached = sources.get(path) if (cached && cached.stamp === stamp) return cached.code - const code = readFileSync(path, 'utf-8') + const code = readSource(path) sources.set(path, { stamp, code }) return code } @@ -1262,29 +1263,19 @@ export function createModuleResolver({ ((allowPackage?.(specifier) ?? true) ? resolveCachedPackage(specifier, from) : undefined)) - return path ? { path: toId(path), code: readSource(path) } : undefined + return path ? { path: toId(path), code: readCode(path) } : undefined } } -interface CachedPackage { - manifest: string - stamp: string | undefined - path: string -} - interface CachedSource { stamp: string | undefined code: string } -/** Changes whenever the file is written; `undefined` once it is gone. */ -function fileStamp(path: string): string | undefined { - try { - const { mtimeNs, ctimeNs, size } = statSync(path, { bigint: true }) - return `${mtimeNs}:${ctimeNs}:${size}` - } catch { - return undefined - } +interface CachedPackage { + manifest: string + stamp: string | undefined + path: string } /** `resolveFile`, also completing a name like `theme.css` to `theme.css.ts`. */ diff --git a/packages/plugin-utils/src/index.ts b/packages/plugin-utils/src/index.ts index e542401df..6448be35f 100644 --- a/packages/plugin-utils/src/index.ts +++ b/packages/plugin-utils/src/index.ts @@ -101,6 +101,7 @@ export { ShorthandConfigError, type ShorthandErrorLocation, } from './shorthands' +export { readSource, type SourceCache, withSourceCache } from './source-cache' export { SOURCE_EXTENSIONS, SOURCE_FILE_REGEX } from './source-extensions' export { createStateWriter, diff --git a/packages/plugin-utils/src/reservation-discovery.ts b/packages/plugin-utils/src/reservation-discovery.ts index 07623be44..9a1eb3e37 100644 --- a/packages/plugin-utils/src/reservation-discovery.ts +++ b/packages/plugin-utils/src/reservation-discovery.ts @@ -1,4 +1,3 @@ -import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { contextImports } from './context-imports' @@ -15,6 +14,7 @@ import { } from './reservation-source' import { isRuntimePackage } from './runtime-package' import { createNodeModulesExcludeRegex } from './shared' +import { readSource } from './source-cache' import type { WasmImportAliases } from './types' export type ReservationInput = { @@ -152,7 +152,7 @@ export function collectReservationInputs( const code = overlay?.code ?? sources.get(key) ?? - reservationSource(() => readFileSync(physicalPath, 'utf-8')) + reservationSource(() => readSource(physicalPath)) if (code === undefined) continue const filename = overlay?.filename ?? toId(physicalPath) sources.set(key, code) diff --git a/packages/plugin-utils/src/source-cache.ts b/packages/plugin-utils/src/source-cache.ts new file mode 100644 index 000000000..86fcbd33c --- /dev/null +++ b/packages/plugin-utils/src/source-cache.ts @@ -0,0 +1,65 @@ +import { readFileSync, statSync } from 'node:fs' +import { resolve } from 'node:path' + +interface CachedSource { + readonly stamp: string + readonly code: string +} + +/** The sources one build has read, by absolute path */ +export type SourceCache = Map + +declare global { + var __devupSourceCache: SourceCache | undefined +} + +/** Changes whenever the file is written; `undefined` once it is gone. */ +export function fileStamp(path: string): string | undefined { + try { + const { mtimeNs, ctimeNs, size } = statSync(path, { bigint: true }) + return `${mtimeNs}:${ctimeNs}:${size}` + } catch { + return undefined + } +} + +/** + * Run `run` with `cache` holding the sources it reads through `readSource`, + * so the passes of one build that run before the bundler (the import graph, + * the class-slot reservation, the prewarm; in Next once per compiler) read + * each file from disk once. Only a one-shot build passes one cache to several + * runs: in watch mode each run starts from a fresh one, as without it. A run + * inside another uses the outer cache. The plugins and their loaders load + * separate copies of this module, so the active cache lives on `globalThis`. + */ +export function withSourceCache(run: () => T, cache: SourceCache): T { + const outer = globalThis.__devupSourceCache + globalThis.__devupSourceCache = outer ?? cache + try { + return run() + } finally { + globalThis.__devupSourceCache = outer + } +} + +/** + * The UTF-8 text of `path`. Inside `withSourceCache` it is read from disk + * again only once the file changes: checking its stamp costs a fraction of + * reading it. + */ +export function readSource(path: string): string { + const cache = globalThis.__devupSourceCache + if (cache === undefined) return readFileSync(path, 'utf-8') + const key = resolve(path) + const stamp = fileStamp(key) + if (stamp === undefined) { + // Gone: the read reports why, as it would without the cache + cache.delete(key) + return readFileSync(key, 'utf-8') + } + const cached = cache.get(key) + if (cached?.stamp === stamp) return cached.code + const code = readFileSync(key, 'utf-8') + cache.set(key, { stamp, code }) + return code +} diff --git a/packages/webpack-plugin/src/__tests__/plugin.test.ts b/packages/webpack-plugin/src/__tests__/plugin.test.ts index 540cedf45..1b223ff45 100644 --- a/packages/webpack-plugin/src/__tests__/plugin.test.ts +++ b/packages/webpack-plugin/src/__tests__/plugin.test.ts @@ -1623,6 +1623,32 @@ describe('devupUIWebpackPlugin', () => { } }) + it("prewarms from the build's shared sources, read once by an earlier compiler", () => { + // Given a source an earlier compiler of the build already read. + const file = import.meta.path + const sourceCache: pluginUtils.SourceCache = new Map() + pluginUtils.withSourceCache( + () => pluginUtils.readSource(file), + sourceCache, + ) + const [[key, entry]] = [...sourceCache] + sourceCache.set(key, { ...entry, code: 'read before' }) + const plugin = new DevupUIWebpackPlugin({ + engine: 'wasm', + productionPrewarmFiles: [file], + sourceCache, + }) + const compiler = createCompiler() + // When this compiler prepares its compilation. + plugin.apply(asCompiler(compiler)) + compiler.hooks.run.tap.mock.calls[0][1]() + // Then it extracts what was read, without reading the file again. + expect(codeExtractSpy.mock.calls.map(([, code]) => code)).toEqual([ + 'read before', + ]) + expect(globalThis.__devupSourceCache).toBeUndefined() + }) + it.each([false, true])( 'does not prewarm when the complete set is empty or watch=%s', (watch) => { diff --git a/packages/webpack-plugin/src/plugin.ts b/packages/webpack-plugin/src/plugin.ts index 04c009b2d..25d7b1a31 100644 --- a/packages/webpack-plugin/src/plugin.ts +++ b/packages/webpack-plugin/src/plugin.ts @@ -28,6 +28,7 @@ import { normalizeStylexOptions, planAtomHoist, readJsxImportSource, + readSource, registerMarkerLedger, reservationGraph, type ReservationInventory, @@ -37,10 +38,12 @@ import { seedFileNumbers, setupWarning, SOURCE_FILE_REGEX, + type SourceCache, type StaticImportGraph, type StylexStyleResolution, tailwindCssFiles, type WasmImportAliases, + withSourceCache, withTailwindCss, } from '@devup-ui/plugin-utils' import { type Compiler, type RuleSetRule } from 'webpack' @@ -93,6 +96,12 @@ export interface DevupUIWebpackPluginOptions extends Pick< importAliases?: ImportAliases /** Adapter-owned complete extraction set; undefined keeps generic entry reachability. */ readonly productionPrewarmFiles?: readonly string[] + /** + * Adapter-owned: the sources a one-shot build read, shared by the plugins of + * its compilers so each file is read once. Undefined reads them again for + * every compiler and compilation. + */ + readonly sourceCache?: SourceCache } export class DevupUIWebpackPlugin { @@ -113,6 +122,7 @@ export class DevupUIWebpackPlugin { private root = process.cwd() /** What the scan and the loaders report of component-selector markers */ private readonly markers = new ComponentMarkerLedger() + private readonly sourceCache: SourceCache | undefined constructor({ package: libPackage = '@devup-ui/react', @@ -130,8 +140,10 @@ export class DevupUIWebpackPlugin { stylex, atomHoist = 2, productionPrewarmFiles, + sourceCache, importAliases: userImportAliases, }: Partial = {}) { + this.sourceCache = sourceCache this.normalizedShorthands = normalizeShorthands(shorthands ?? {}) this.styleResolution = normalizeStylexOptions(stylex) this.compiler = loadCompiler(engine) @@ -371,7 +383,7 @@ export class DevupUIWebpackPlugin { '/', ) if (!relCssDir.startsWith('./')) relCssDir = `./${relCssDir}` - const source = readFileSync(file, 'utf-8') + const source = readSource(file) try { this.compiler .codeExtract( @@ -447,7 +459,10 @@ export class DevupUIWebpackPlugin { const endBuild = beginBuild(this.compiler) try { compiler.hooks.shutdown?.tap('DevupUIWebpackPlugin', endBuild) - this.initialize(compiler) + withSourceCache( + () => this.initialize(compiler), + this.sourceCache ?? new Map(), + ) } catch (error) { // Webpack cannot return the compiler when apply throws, so no caller can close it. endBuild() @@ -585,7 +600,8 @@ export class DevupUIWebpackPlugin { this.compiler, ) registerMarkerLedger(compiler, this.markers) - const reserveBeforeCompile = () => { + // The reservation and the prewarm read the same sources + const prepare = () => { this.markers.clear() reserve(compiler.options) this.markers.scanned( @@ -595,6 +611,8 @@ export class DevupUIWebpackPlugin { if (this.options.watch) return this.prewarmExtractor(discovered, compiler.options) } + const reserveBeforeCompile = () => + withSourceCache(prepare, this.sourceCache ?? new Map()) compiler.hooks.run.tap( 'DevupUIWebpackPluginReservation', reserveBeforeCompile,