diff --git a/.changeset/highlight-fewer-grammars.md b/.changeset/highlight-fewer-grammars.md new file mode 100644 index 00000000..0519b6ee --- /dev/null +++ b/.changeset/highlight-fewer-grammars.md @@ -0,0 +1,5 @@ +--- +'@doc-kit/core': minor +--- + +Code is highlighted with fewer grammars by default, as each one takes its own memory on every thread: JavaScript (`js`, `mjs`, `cjs`, `javascript`) with TypeScript's grammar, which already highlights types, and C++ (`cpp`, `c++`) with C's. The Shiki plugin's `langAlias` option takes precedence over these defaults, so `{ cpp: 'cpp', 'c++': 'cpp' }` brings back C++'s own grammar, and aliases now keep working once a grammar with the same aliases is registered. A highlighter's `resolveLanguage` now returns the language code is highlighted as, as its Shiki instance names it (`typescript` for `mjs`): highlight with what it returns. Transformers see that language too. diff --git a/.changeset/hold-less-in-memory.md b/.changeset/hold-less-in-memory.md new file mode 100644 index 00000000..6e581132 --- /dev/null +++ b/.changeset/hold-less-in-memory.md @@ -0,0 +1,7 @@ +--- +'@doc-kit/core': minor +'@doc-kit/cli': patch +'@doc-kit/generator-react': patch +--- + +Builds hold less in memory. The nodes of `metadata`'s entries no longer keep their `position` in the source, so neither does the output of `json-simple`, now about a third of its size. Each chunk of a streamed output is let go once every generator reading it has read that chunk, rather than once they have all read all of it, and `runGenerators(configuration, { results: false })` keeps none of the requested generators' output, which the CLI now uses. A page made of other pages' content, such as Node.js's `all.html`, renders one part at a time, rather than building every part's elements first. diff --git a/.changeset/html-all.md b/.changeset/html-all.md new file mode 100644 index 00000000..d0f719ad --- /dev/null +++ b/.changeset/html-all.md @@ -0,0 +1,7 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +'@node-core/doc-kit': minor +--- + +`all.html` moves out of the `html` generator into a new Node.js generator, `html-all` (`@node-core/doc-kit`), and `html`'s `generateAllPage` option is removed. `html-all` is spliced between `jsx-ast` and `html`, and adds `all.html`, every module page's content on one page, as a page made of theirs. `html` renders such a page, given with `parts`, the pages it is made of, from their compiled content. Builds that want `all.html` add the `html-all` target. diff --git a/.changeset/six-threads.md b/.changeset/six-threads.md new file mode 100644 index 00000000..adc09d4c --- /dev/null +++ b/.changeset/six-threads.md @@ -0,0 +1,5 @@ +--- +'@doc-kit/core': minor +--- + +Builds use up to 6 threads by default, instead of 4: with generators streaming into each other, more threads now pay off. On the Node.js API docs, the site build is ~15% faster at 6 threads than at 4, for ~20% more memory, while 8 threads were no faster than 6. `--threads` still sets any count. diff --git a/.changeset/stream-dependencies.md b/.changeset/stream-dependencies.md new file mode 100644 index 00000000..af34c986 --- /dev/null +++ b/.changeset/stream-dependencies.md @@ -0,0 +1,6 @@ +--- +'@doc-kit/core': minor +'@doc-kit/generator-react': minor +--- + +Generators can start on their dependency's first chunk instead of waiting for all of it: with `streamsInput: true`, a generator receives its dependency's output as an async iterable of chunks (with one thread, still whole), and `worker.stream` accepts one. `worker.stream` now keeps only a few chunks per thread in flight, started in the order they were submitted, so generators running at once share the worker pool. `metadata`, `section-pages`, `jsx-ast` and `html` now stream their input, so with more than one thread, parsing, metadata, page building and rendering overlap: `html` bundles as soon as every page's metadata is known, and renders pages as they are built. Streaming leaves the generated files unchanged, but `jsx-ast` now yields pages in the order their entries arrive, not the order the site lists them, and, once every page has arrived, a `{ manifest }` item listing their heads in that order. To list the pages as the site does, skip the `{ manifest }` item and sort them with `headingSortFn`, now exported from `@doc-kit/generator-react/jsx-ast/utils/getSortedHeadNodes`, as `pages.toSorted((a, b) => headingSortFn(a.data, b.data))`, keeping the synthetic pages (`data.synthetic`) last. Bundler adapters' `compile` can now be called before and while `buildServer` and `buildClient` run. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16103252..f2c83c26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: path: node - name: Build Node.js API docs - run: node packages/cli/bin/cli.mjs generate -t web -i "./node/doc/api/assert.md" -o out --config-file ./e2e/doc-kit.config.mjs + run: node packages/cli/bin/cli.mjs generate -t web -t html-all -i "./node/doc/api/assert.md" -o out --config-file ./e2e/doc-kit.config.mjs - name: Install Playwright browsers run: pnpm exec playwright install --with-deps diff --git a/docs/configuration.md b/docs/configuration.md index a0d750dd..0db17e6c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -134,7 +134,7 @@ these for that generator alone, except `markdown`, which it adds to. Top-level, alongside `target` and `global`: - `threads` {number} Worker threads used for generation. Defaults to your - CPU count. Each worker's heap is limited to 512MB; to raise the limit, run + CPU count, up to 6. Each worker's heap is limited to 512MB; to raise the limit, run Node.js with `--max-old-space-size` (directly or through `NODE_OPTIONS`). - `chunkSize` {number} Items processed per worker thread. **Default:** `10`. @@ -219,12 +219,20 @@ bundles. To configure it, list it with options in dependencies, so your configuration file can resolve it): - `langs` {Array} More languages: grammars, or modules default-exporting them. -- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. +- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. An + alias leads to the language it names, or on to the one that language is an + alias of. By default, JavaScript (`javascript`, `js`, `mjs`, `cjs`) is + highlighted with TypeScript's grammar, and C++ (`cpp`, `c++`) with C's: + every grammar takes its own memory on each thread, and those highlight that + code too. Aliases take precedence over these defaults: + `{ cpp: 'cpp', 'c++': 'cpp' }` highlights C++ with its own grammar, and a + grammar in `langs` for one of these languages needs an alias to it. - `themes` {Object} The `light` and `dark` themes: names of themes Shiki bundles, modules default-exporting themes, or themes. - `transformers` {Array} Modules default-exporting [Shiki transformers](https://shiki.style/guide/transformers), or lists of - them. + them. They see the language code is highlighted as (`options.lang`), such as + `typescript` for `mjs`. These modules are paths relative to the working directory, or URLs, such as the ones `import.meta.resolve()` gives: diff --git a/docs/creating-generators.md b/docs/creating-generators.md index c2912d33..b5c27f0a 100644 --- a/docs/creating-generators.md +++ b/docs/creating-generators.md @@ -330,9 +330,64 @@ export async function* generate(input, worker) { ### Benefits of Streaming - **Reduced memory usage** - Process data in chunks -- **Earlier downstream starts** - Next generator can begin before this one finishes +- **Earlier downstream starts** - A generator that streams its input (see below) + begins on the first chunk, before this one finishes - **Better parallelism** - Multiple generators can work simultaneously +### Streaming Input + +By default, a generator receives its dependency's output once it is complete, +collected into one array. With `streamsInput: true`, it receives it as it is +produced instead: `input` is an async iterable of the dependency's chunks, and +if the dependency streams its results too, the generator starts as soon as the +dependency does. + +```mjs displayName="index.mjs" +import { generate, processChunk } from './generate.mjs'; + +export default { + name: 'per-chunk-generator', + + description: 'Processes each chunk of metadata as it arrives', + + dependsOn: '@doc-kit/core/metadata', + + hasParallelProcessor: true, + + streamsInput: true, + + generate, + processChunk, +}; +``` + +`worker.stream` takes the chunks as they arrive, starting each one's work while +the next ones are still being produced: + +```mjs displayName="generate.mjs" +export async function* generate(input, worker) { + for await (const chunkResult of worker.stream(input, {})) { + yield chunkResult; + } +} +``` + +- A dependency that does not stream arrives as a single chunk, once it is + complete. +- With one thread, nothing would run alongside, so the input arrives whole, as + it does without `streamsInput`: accept both, as `worker.stream` does. +- With more than one thread, `input` can be iterated only once, and passing it + to `worker.stream` iterates it. If you need it twice, collect it into an + array first. +- The chunks are the dependency's own, in the order it yields them. `metadata`'s + hold whole files, but chunks need not hold whole modules: a generator that + needs all of a module's entries at once should regroup them with + `groupWholeModules` from `@doc-kit/core/utils/generators.mjs`. +- Work that needs the whole input, such as sorting or writing one file, has to + wait for the last chunk. +- Other generators reading the same dependency are not affected: each gets + every chunk, from the first. + ### Non-Streaming Generators Some generators must collect all input before processing. @@ -526,7 +581,9 @@ returning the plugin, instead of a default export. The Shiki plugin does, and returns the highlighter of a generator's Shiki plugin, to highlight code as its pipeline does. It registers Shiki's bundled languages as code in them is highlighted, so before using its `shiki` instance directly, pass the language -through its `resolveLanguage`. +through its `resolveLanguage`, and highlight with the language it returns, +which may not be the one the code names (`mjs` gives `typescript`, and an +unknown language `text`). ## File Output diff --git a/docs/generators.md b/docs/generators.md index ab7270bb..2f757616 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -48,6 +48,7 @@ Outputs consumed by the Node.js project's own build and release processes. | [`man-page`](./generators/man-page.md) | The `node.1` manual page. | | [`api-links`](./generators/api-links.md) | A map of API symbols to their source locations. | | [`addon-verify`](./generators/addon-verify.md) | Extracted addon code samples, arranged for testing. | +| [`html-all`](./generators/html-all.md) | `all.html`, every module page on one page. | ## Pipeline stages diff --git a/packages/cli/bin/commands/generate.mjs b/packages/cli/bin/commands/generate.mjs index 4122f6fb..358fcf03 100644 --- a/packages/cli/bin/commands/generate.mjs +++ b/packages/cli/bin/commands/generate.mjs @@ -64,6 +64,6 @@ export default generate.action( const config = await setConfig(opts); assertRunnableOptions(config); - await runGenerators(config); + await runGenerators(config, { results: false }); }) ); diff --git a/packages/cli/bin/commands/preview.mjs b/packages/cli/bin/commands/preview.mjs index 84272fe6..e8b8e4d5 100644 --- a/packages/cli/bin/commands/preview.mjs +++ b/packages/cli/bin/commands/preview.mjs @@ -62,7 +62,7 @@ export default preview.action( const config = await setConfig(opts); assertRunnableOptions(config); - await runGenerators(config); + await runGenerators(config, { results: false }); // With the default coming from the configuration, this is always set output = config.global.output; diff --git a/packages/core/src/__tests__/caching.test.mjs b/packages/core/src/__tests__/caching.test.mjs index 5dc306e8..4989de5f 100644 --- a/packages/core/src/__tests__/caching.test.mjs +++ b/packages/core/src/__tests__/caching.test.mjs @@ -1,8 +1,121 @@ -import { deepStrictEqual, strictEqual } from 'node:assert'; +import { deepStrictEqual, rejects, strictEqual, throws } from 'node:assert'; import { describe, it } from 'node:test'; +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; import { collectAsyncGenerator, createCache } from '../caching.mjs'; +setFlagsFromString('--expose-gc'); + +// Collects garbage, to tell what is let go +const gc = runInNewContext('gc'); + +/** + * Reads every chunk of an async iterable. + * + * @template T + * @param {AsyncIterable} iterable + * @returns {Promise} + */ +async function collectChunks(iterable) { + const chunks = []; + + for await (const chunk of iterable) { + chunks.push(chunk); + } + + return chunks; +} + +/** + * Reads up to `count` chunks of items, keeping only the items' values, so that + * the test holds none of the chunks. + * + * @param {AsyncIterator>} reader + * @param {number} [count] + * @returns {Promise} + */ +async function readValues(reader, count = Infinity) { + const values = []; + + for (let i = 0; i < count; i++) { + const { value, done } = await reader.next(); + + if (done) { + break; + } + + values.push(...value.map(item => item.value)); + } + + return values; +} + +/** + * Makes chunks of one item each that tell when they are let go. + * + * @returns {{ released: Set, chunkOf: (value: number) => Array<{ value: number }> }} + */ +function trackReleases() { + const released = new Set(); + const registry = new FinalizationRegistry(value => released.add(value)); + + return { + released, + + /** + * A chunk holding an item that tells when it is let go. + * + * @param {number} value + * @returns {Array<{ value: number }>} + */ + chunkOf: value => { + const item = { value }; + + registry.register(item, value); + + return [item]; + }, + }; +} + +/** + * Collects garbage until `done` holds, at most ten times. + * + * @param {() => boolean} done + * @returns {Promise} + */ +async function collectGarbage(done) { + for (let i = 0; i < 10 && !done(); i++) { + await new Promise(resolve => setImmediate(resolve)); + gc(); + } +} + +/** + * Runs `fn`, and gives the rejections left unhandled while it ran. + * + * @param {() => Promise} fn + * @returns {Promise} + */ +async function unhandledRejectionsOf(fn) { + const unhandled = []; + const onUnhandled = error => unhandled.push(error); + + process.on('unhandledRejection', onUnhandled); + + try { + await fn(); + + // A rejection left unhandled is reported once the microtasks have run + await new Promise(resolve => setImmediate(resolve)); + } finally { + process.off('unhandledRejection', onUnhandled); + } + + return unhandled; +} + describe('caching', () => { describe('collectAsyncGenerator', () => { it('should collect all chunks into a flat array', async () => { @@ -106,5 +219,391 @@ describe('caching', () => { await cache.consume('a'); strictEqual(cache.has('a'), false); }); + + it( + 'should stream chunks before the generator completes', + // The generator waits for its first chunk to be read, so a stream that + // waited for the generator to complete would never give it + { timeout: 10_000 }, + async () => { + const cache = createCache(); + const gate = Promise.withResolvers(); + + async function* gen() { + yield [1]; + await gate.promise; + yield [2]; + } + + cache.store('a', gen()); + + const stream = await cache.stream('a'); + const reader = stream[Symbol.asyncIterator](); + const first = await reader.next(); + + // The first chunk arrives while the generator still waits on the gate + deepStrictEqual(first.value, [1]); + + gate.resolve(); + + const rest = await collectChunks(reader); + + deepStrictEqual(rest, [[2]]); + } + ); + + it('should run a generator once for streaming and collecting consumers', async () => { + const cache = createCache(); + + let iterations = 0; + + async function* gen() { + iterations++; + yield [1, 2]; + yield [3]; + } + + cache.store('a', gen()); + + const [streamed, collected] = await Promise.all([ + cache.stream('a').then(collectChunks), + cache.consume('a'), + ]); + + deepStrictEqual(streamed, [[1, 2], [3]]); + deepStrictEqual(collected, [1, 2, 3]); + strictEqual(iterations, 1); + }); + + it('should replay every chunk to a reader that starts late', async () => { + const cache = createCache(); + + async function* gen() { + yield [1]; + yield [2]; + } + + cache.store('a', gen()); + + await cache.consume('a'); + + const reader = await cache.stream('a'); + const chunks = await collectChunks(reader); + + deepStrictEqual(chunks, [[1], [2]]); + }); + + it('should stream a non-streaming result as a single chunk', async () => { + const cache = createCache(); + + cache.store('a', Promise.resolve([1, 2])); + + const reader = await cache.stream('a'); + const chunks = await collectChunks(reader); + + deepStrictEqual(chunks, [[1, 2]]); + }); + + it('should throw a failure to streaming readers after the chunks before it', async () => { + const cache = createCache(); + + async function* gen() { + yield [1]; + throw new Error('boom'); + } + + cache.store('a', gen()); + + const chunks = []; + + const unhandled = await unhandledRejectionsOf(() => + rejects(async () => { + const reader = await cache.stream('a'); + + for await (const chunk of reader) { + chunks.push(chunk); + } + }, /boom/) + ); + + deepStrictEqual(chunks, [[1]]); + deepStrictEqual(unhandled, []); + }); + + it('should handle a failure before the stream is read', async () => { + const cache = createCache(); + + const unhandled = await unhandledRejectionsOf(async () => { + cache.store('a', Promise.reject(new Error('boom'))); + + // The reader is asked for as a generator that streams its input + // starts, before it reads anything + await rejects(cache.stream('a'), /boom/); + }); + + deepStrictEqual(unhandled, []); + }); + + it('should start recording a streamed result before it is read', async () => { + const cache = createCache(); + + let started = false; + + async function* gen() { + started = true; + yield [1]; + } + + cache.store('a', gen()); + + const reader = await cache.stream('a'); + + await new Promise(resolve => setImmediate(resolve)); + + strictEqual(started, true); + + const chunks = await collectChunks(reader); + + deepStrictEqual(chunks, [[1]]); + }); + + it('should read a stream only once', async () => { + const cache = createCache(); + + async function* gen() { + yield [1]; + } + + cache.store('a', gen()); + + const stream = await cache.stream('a'); + const chunks = await collectChunks(stream); + + deepStrictEqual(chunks, [[1]]); + throws(() => stream[Symbol.asyncIterator](), /can only be read once/); + }); + + it('should give a reader every chunk even if it starts reading after the result was evicted', async () => { + const cache = createCache(); + + cache.populateConsumerCounts(['a'], () => undefined); + + async function* gen() { + yield [1]; + yield [2]; + yield [3]; + } + + cache.store('a', gen()); + + // Its only read: the result is evicted, and its chunks go once read + const stream = await cache.stream('a'); + + strictEqual(cache.has('a'), false); + + // Every chunk is recorded before the reader reads any + await new Promise(resolve => setImmediate(resolve)); + + const chunks = await collectChunks(stream); + + deepStrictEqual(chunks, [[1], [2], [3]]); + }); + + it('should drain a result, alongside its other readers', async () => { + const cache = createCache(); + const recorded = []; + + cache.populateConsumerCounts(['a', 'b'], name => + name === 'b' ? 'a' : undefined + ); + + async function* gen() { + for (const value of [1, 2, 3]) { + recorded.push(value); + + yield [value]; + } + } + + cache.store('a', gen()); + + const reader = await cache.stream('a'); + + await cache.drain('a'); + + strictEqual(cache.has('a'), false); + deepStrictEqual(recorded, [1, 2, 3]); + + const chunks = await collectChunks(reader); + + deepStrictEqual(chunks, [[1], [2], [3]]); + }); + + it('should let go of the chunks of a drained result as they are recorded', async () => { + const cache = createCache(); + const gate = Promise.withResolvers(); + const { released, chunkOf } = trackReleases(); + + cache.populateConsumerCounts(['a'], () => undefined); + + async function* gen() { + yield chunkOf(1); + yield chunkOf(2); + yield chunkOf(3); + await gate.promise; + } + + cache.store('a', gen()); + + const draining = cache.drain('a'); + + // The generator waits, past its chunks: the first is no longer held + await collectGarbage(() => released.has(1)); + + strictEqual(released.has(1), true); + + gate.resolve(); + await draining; + }); + + it('should let go of the chunks a reader has read, after the generator ended', async () => { + const cache = createCache(); + const { released, chunkOf } = trackReleases(); + + cache.populateConsumerCounts(['a'], () => undefined); + + async function* gen() { + yield chunkOf(1); + yield chunkOf(2); + yield chunkOf(3); + } + + cache.store('a', gen()); + + // Its only read: no more readers can start + const stream = await cache.stream('a'); + + // The generator ends before any chunk is read + await new Promise(resolve => setImmediate(resolve)); + + const reader = stream[Symbol.asyncIterator](); + const read = await readValues(reader, 2); + + await collectGarbage(() => released.has(1)); + + strictEqual(released.has(1), true); + + const rest = await readValues(reader); + + // The last chunk goes once the reader is done, while it is still held + await collectGarbage(() => released.has(3)); + + const end = await reader.next(); + + deepStrictEqual([...read, ...rest], [1, 2, 3]); + strictEqual(end.done, true); + strictEqual(released.has(3), true); + }); + + it('should let go of a chunk only once every streaming reader has read it', async () => { + const cache = createCache(); + const { released, chunkOf } = trackReleases(); + + // `x`, `y` and `z` all read `a` + cache.populateConsumerCounts(['x', 'y', 'z'], name => + name === 'a' ? undefined : 'a' + ); + + async function* gen() { + yield chunkOf(1); + yield chunkOf(2); + yield chunkOf(3); + } + + cache.store('a', gen()); + + const streams = []; + + // The third is the last read: no more readers can start + for (let i = 0; i < 3; i++) { + const stream = await cache.stream('a'); + + streams.push(stream); + } + + // The reader that falls behind started between the other two, so neither + // the first reader nor the last one alone tells what all have read + const [first, middle, last] = streams.map(stream => + stream[Symbol.asyncIterator]() + ); + + const firstRead = await readValues(first, 2); + const lastRead = await readValues(last, 2); + + // `middle` has yet to read the first chunk + await collectGarbage(() => released.has(1)); + + strictEqual(released.has(1), false); + + const middleRead = await readValues(middle, 2); + + await collectGarbage(() => released.has(1)); + + strictEqual(released.has(1), true); + + const firstRest = await readValues(first); + const middleRest = await readValues(middle); + const lastRest = await readValues(last); + + deepStrictEqual([...firstRead, ...firstRest], [1, 2, 3]); + deepStrictEqual([...middleRead, ...middleRest], [1, 2, 3]); + deepStrictEqual([...lastRead, ...lastRest], [1, 2, 3]); + }); + + it('should drain a non-streaming result', async () => { + const cache = createCache(); + + cache.populateConsumerCounts(['a'], () => undefined); + cache.store('a', Promise.resolve({ value: 1 })); + + const drained = await cache.drain('a'); + + strictEqual(drained, undefined); + strictEqual(cache.has('a'), false); + }); + + it('should reject draining a generator that fails', async () => { + const cache = createCache(); + + cache.populateConsumerCounts(['a'], () => undefined); + + async function* gen() { + yield [1]; + + throw new Error('drained failure'); + } + + cache.store('a', gen()); + + await rejects(cache.drain('a'), /drained failure/); + }); + + it('should count a stream as a read for eviction', async () => { + const cache = createCache(); + + cache.populateConsumerCounts(['a'], () => undefined); + + async function* gen() { + yield [1]; + } + + cache.store('a', gen()); + + const reader = await cache.stream('a'); + const chunks = await collectChunks(reader); + + deepStrictEqual(chunks, [[1]]); + strictEqual(cache.has('a'), false); + }); }); }); diff --git a/packages/core/src/__tests__/generators.test.mjs b/packages/core/src/__tests__/generators.test.mjs index d3be80da..6e146cff 100644 --- a/packages/core/src/__tests__/generators.test.mjs +++ b/packages/core/src/__tests__/generators.test.mjs @@ -16,6 +16,9 @@ const streamOf = chunk => yield chunk; })(); +// Lets `gated-source` wait for its streaming reader +const gate = { opened: Promise.withResolvers(), openedBy: undefined }; + // Synthetic generators keyed by specifier (any string works as a specifier) const syntheticGenerators = { // Root streaming generator (no dependency) @@ -99,6 +102,81 @@ const syntheticGenerators = { return { all: input }; }, }, + // A source that only yields its second chunk once its streaming reader has + // received the first, or after a second, recording which happened first + 'gated-source': { + name: 'gated-source', + hasParallelProcessor: true, + generate: async () => { + record('gated-source'); + + return (async function* () { + yield [1]; + + let timer; + + const timeout = new Promise(resolve => { + timer = setTimeout(resolve, 1000, 'timeout'); + }); + + gate.openedBy = await Promise.race([gate.opened.promise, timeout]); + + clearTimeout(timer); + + yield [2]; + })(); + }, + }, + 'stream-reader': { + name: 'stream-reader', + dependsOn: 'gated-source', + streamsInput: true, + async *generate(input) { + record('stream-reader'); + + for await (const chunk of input) { + gate.opened.resolve('reader'); + + yield [{ streamed: chunk }]; + } + }, + }, + // A source that fails before yielding anything, read only by a generator + // that streams its input and starts reading it a turn of the event loop + // later, once a failure left unhandled would have been reported + 'failing-source': { + name: 'failing-source', + hasParallelProcessor: true, + generate: async () => { + throw new Error('source failed'); + }, + }, + 'slow-reader': { + name: 'slow-reader', + dependsOn: 'failing-source', + streamsInput: true, + async *generate(input) { + await new Promise(resolve => setImmediate(resolve)); + + for await (const chunk of input) { + yield chunk; + } + }, + }, + 'shape-reader': { + name: 'shape-reader', + dependsOn: 'metadata', + streamsInput: true, + generate: async input => ({ array: Array.isArray(input) }), + }, + 'collect-reader': { + name: 'collect-reader', + dependsOn: 'gated-source', + generate: async input => { + record('collect-reader'); + return { collected: input }; + }, + }, }; mock.module('../generators/loader.mjs', { @@ -245,4 +323,81 @@ describe('createGenerator orchestration', () => { { name: 'gen-a', markdown, ran: ['ast', 'metadata'] }, ]); }); + + it('streams a dependency to a generator that streams its input', async () => { + const { runGenerators } = createGenerator(); + + const results = await runGenerators({ + target: ['stream-reader', 'collect-reader'], + threads: 2, + }); + + // The reader got the first chunk while the source was still running + assert.equal(gate.openedBy, 'reader'); + assert.equal(runs['gated-source'], 1); + + assert.deepStrictEqual(results, [ + [{ streamed: [1] }, { streamed: [2] }], + { collected: [1, 2] }, + ]); + }); + + it('rejects with the failure of a dependency read as a stream', async () => { + const { runGenerators } = createGenerator(); + const unhandled = []; + const onUnhandled = error => unhandled.push(error); + + process.on('unhandledRejection', onUnhandled); + + try { + await assert.rejects( + runGenerators({ target: ['slow-reader'], threads: 2 }), + /source failed/ + ); + + // A rejection left unhandled is reported once the microtasks have run + await new Promise(resolve => setImmediate(resolve)); + } finally { + process.off('unhandledRejection', onUnhandled); + } + + assert.deepStrictEqual(unhandled, []); + }); + + it('gives a generator that streams its input the whole input with one thread', async () => { + const one = await createGenerator().runGenerators({ + target: ['shape-reader'], + threads: 1, + }); + const two = await createGenerator().runGenerators({ + target: ['shape-reader'], + threads: 2, + }); + + assert.deepStrictEqual(one, [{ array: true }]); + assert.deepStrictEqual(two, [{ array: false }]); + }); + + it('runs every generator but keeps no results with results: false', async () => { + const { runGenerators } = createGenerator(); + + const results = await runGenerators( + { target: ['gen-c', 'gen-c-all'], threads: 1 }, + { results: false } + ); + + assert.equal(results, undefined); + assert.equal(runs['gen-c'], 1); + assert.equal(runs['gen-c-all'], 1); + }); + + it('rejects with a failure with results: false', async () => { + await assert.rejects( + createGenerator().runGenerators( + { target: ['slow-reader'], threads: 2 }, + { results: false } + ), + /source failed/ + ); + }); }); diff --git a/packages/core/src/caching.mjs b/packages/core/src/caching.mjs index 9ad72110..6198d196 100644 --- a/packages/core/src/caching.mjs +++ b/packages/core/src/caching.mjs @@ -2,6 +2,7 @@ import logger from './logger/index.mjs'; import { isAsyncIterable } from './utils/misc.mjs'; +import { createReplay } from './utils/replay.mjs'; const cachingLogger = logger.child('caching'); @@ -36,6 +37,17 @@ export const collectAsyncGenerator = async generator => { return results; }; +/** + * Gives a value as the single chunk of an async generator. + * + * @template T + * @param {T} value + * @returns {AsyncGenerator} + */ +async function* once(value) { + yield value; +} + /** * Creates the cache that backs the generator pipeline. It is the single owner * of every piece of caching the orchestrator needs. @@ -43,17 +55,23 @@ export const collectAsyncGenerator = async generator => { * Generator caching: each generator's pending result, so a generator and its * dependencies are only ever scheduled once. * - * Stream caching: collecting an async generator's chunks into a single array - * exactly once, shared by every consumer. + * Stream caching: running an async generator exactly once, recording its + * chunks for every consumer: those that collect it share a single array, and + * those that stream it read each chunk as soon as it is yielded. * * Dependency caching & eviction: tracking how many consumers still need each - * result and dropping it once the last one has read it, so intermediate results - * can be garbage collected while later generators keep running. + * result and dropping it once the last one has asked for it, and a streamed + * result's chunks once every reader has read them, so intermediate results can + * be garbage collected while later generators keep running. */ export const createCache = () => { /** @type {{ [key: string]: Promise | AsyncGenerator }} */ const results = {}; + // The replays of the results that stream, and those of them collected into + // one array, shared by the consumers that collect them + /** @type {Map>} */ + const replays = new Map(); /** @type {Map>} */ const collections = new Map(); @@ -66,27 +84,40 @@ export const createCache = () => { const remaining = {}; /** - * Collects an async generator's chunks, ensuring a single collection is - * shared across all consumers of the same key. + * Gets the replay of a streamed result, starting it the first time a + * consumer reads the result, so every consumer shares its one run. * - * @param {string} key - Cache key (resolved generator specifier) - * @param {AsyncGenerator} generator - Generator to collect - * @returns {Promise} Promise resolving to the collected items + * @param {string} name - Cache key (resolved generator specifier) + * @param {AsyncGenerator} generator - The result + * @returns {ReturnType} */ - const collect = (key, generator) => { - const hasKey = collections.has(key); + const replayOf = (name, generator) => { + if (!replays.has(name)) { + cachingLogger.debug(`Starting to record "${name}"`); - if (!hasKey) { - collections.set(key, collectAsyncGenerator(generator)); + replays.set(name, createReplay(generator)); } - cachingLogger.debug( - hasKey - ? `Using cached collection for "${key}"` - : `Starting collection for "${key}"` - ); + return replays.get(name); + }; + + /** + * Counts a read of a result, and evicts it once every expected consumer has + * read it. + * + * @param {string} name - Cache key (resolved generator specifier) + */ + const release = name => { + if (name in remaining && --remaining[name] <= 0) { + delete results[name]; + + // Every reader has started, so what they all read can go + replays.get(name)?.forget(); + replays.delete(name); + collections.delete(name); - return collections.get(key); + cachingLogger.debug(`Evicted "${name}" after final consumer`); + } }; return { @@ -165,19 +196,92 @@ export const createCache = () => { consume: async name => { const result = await results[name]; - const value = isAsyncIterable(result) - ? await collect(name, result) - : result; + let value = result; - // Evict once the last consumer has retrieved the result - if (name in remaining && --remaining[name] <= 0) { - delete results[name]; - collections.delete(name); + if (isAsyncIterable(result)) { + // Consumers that collect it share one array, read like a stream + if (!collections.has(name)) { + collections.set( + name, + collectAsyncGenerator(replayOf(name, result).read()) + ); + } - cachingLogger.debug(`Evicted "${name}" after final consumer`); + value = await collections.get(name); } + release(name); + return value; }, + + /** + * Runs a generator to its end, keeping none of its result, which counts as + * one of its reads. + * + * @param {string} name - Cache key (resolved generator specifier) to drain + * @returns {Promise} + */ + drain: async name => { + const result = await results[name]; + + if (!isAsyncIterable(result)) { + release(name); + + return; + } + + const { ended } = replayOf(name, result); + + // Once nothing else reads it, its chunks go as they're recorded + release(name); + + await ended; + }, + + /** + * Gives a reader of a generator's result as it is produced: chunk by chunk + * if it streams, or as a single chunk once it resolves. As with `consume`, + * the result is awaited and its recording started right away, so the + * generator runs and its failure is handled before the reader starts + * reading, and this counts as one of its reads. + * + * @param {string} name - Cache key (resolved generator specifier) to stream + * @returns {Promise>} + */ + stream: async name => { + const result = await results[name]; + + let reader; + + if (isAsyncIterable(result)) { + reader = replayOf(name, result).read(); + } else { + // A result that doesn't stream is read as a single chunk + reader = once(result); + } + + release(name); + + let read = false; + + // Read once: what every reader has read is let go + return { + /** + * Reads every chunk, from the first, once. + * + * @returns {AsyncGenerator} + */ + [Symbol.asyncIterator]: () => { + if (read) { + throw new Error(`The output of "${name}" can only be read once`); + } + + read = true; + + return reader; + }, + }; + }, }; }; diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index b9959f4a..37f2846b 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -28,17 +28,20 @@ const createGenerator = () => { let pool; /** - * Gets the collected input from a dependency generator. + * Gets the input from a dependency generator: collected, or its chunks as the + * dependency yields them. * * @param {string | undefined} dependsOn - Resolved dependency specifier + * @param {boolean} [streamed] - Whether to give its chunks as they are yielded + * rather than collected * @returns {Promise} */ - const getDependencyInput = async dependsOn => { + const getDependencyInput = async (dependsOn, streamed) => { if (!dependsOn) { return undefined; } - return cache.consume(dependsOn); + return streamed ? cache.stream(dependsOn) : cache.consume(dependsOn); }; /** @@ -57,7 +60,7 @@ const createGenerator = () => { } const generator = generators.get(specifier); - const { name, generate, hasParallelProcessor } = generator; + const { name, generate, hasParallelProcessor, streamsInput } = generator; const dependsOn = inputOf.get(specifier); @@ -75,7 +78,12 @@ const createGenerator = () => { cache.store( specifier, (async () => { - const dependencyInput = await getDependencyInput(dependsOn); + // With one thread nothing runs alongside, so streaming would only keep + // every stage's data alive at once: each stage takes its input whole + const dependencyInput = await getDependencyInput( + dependsOn, + streamsInput && configuration.threads > 1 + ); generatorsLogger.debug(`Starting "${name}"`); @@ -91,8 +99,8 @@ const createGenerator = () => { // input while the generator runs, and the generator can let it go return generate(dependencyInput, await worker); })().then(result => { - // For streaming generators, "Completed" is logged when the cache - // finishes collecting, not here when the generator returns + // For streaming generators, the cache logs each chunk as it records + // it, not here when the generator returns if (!isAsyncIterable(result)) { generatorsLogger.debug(`Completed "${name}"`); } @@ -106,9 +114,13 @@ const createGenerator = () => { * Runs all requested generators with their dependencies. * * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options - * @returns {Promise} Results of all requested generators + * @param {{ results?: boolean }} [options] - With `results: false`, none of + * the requested generators' results is kept, so each can be let go as soon + * as it is produced + * @returns {Promise} Results of all requested + * generators, or `undefined` with `results: false` */ - const runGenerators = async configuration => { + const runGenerators = async (configuration, { results = true } = {}) => { const { target, threads } = configuration; // Resolve shorthand names and load the full dependency closure up front, @@ -141,15 +153,14 @@ const createGenerator = () => { scheduleGenerator(specifier, generators, inputOf, configuration); } - // Start all collections in parallel (don't await sequentially). Consuming - // through the shared path lets the final read also trigger eviction. - const results = await Promise.all( - targets.map(specifier => cache.consume(specifier)) - ); + // Start all reads in parallel (don't await sequentially). Reading through + // the shared path lets the final read also trigger eviction. + const read = results ? cache.consume : cache.drain; + const collected = await Promise.all(targets.map(read)); await pool.destroy(); - return results; + return results ? collected : undefined; }; return { runGenerators }; diff --git a/packages/core/src/generators/index.mjs b/packages/core/src/generators/index.mjs index 9ddae661..92c30b4e 100644 --- a/packages/core/src/generators/index.mjs +++ b/packages/core/src/generators/index.mjs @@ -15,6 +15,7 @@ export const publicGenerators = { 'json-simple': '@doc-kit/core/json-simple', 'legacy-html': '@node-core/doc-kit-legacy/legacy-html', 'legacy-html-all': '@node-core/doc-kit-legacy/legacy-html-all', + 'html-all': '@node-core/doc-kit/html-all', 'man-page': '@node-core/doc-kit/man-page', 'legacy-json': '@node-core/doc-kit-legacy/legacy-json', 'legacy-json-all': '@node-core/doc-kit-legacy/legacy-json-all', diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index fcdd965b..64747e68 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -16,6 +16,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { remarkPlugins: [ 'remark-parse', diff --git a/packages/core/src/generators/metadata/types.d.ts b/packages/core/src/generators/metadata/types.d.ts index c84305c3..0e04b1e1 100644 --- a/packages/core/src/generators/metadata/types.d.ts +++ b/packages/core/src/generators/metadata/types.d.ts @@ -3,7 +3,10 @@ import type { Position } from 'unist'; export type Generator = GeneratorMetadata< {}, - Generate, AsyncGenerator>, + Generate< + Array | AsyncIterable>, + AsyncGenerator + >, ProcessChunk> >; diff --git a/packages/core/src/generators/metadata/utils/__tests__/parse.test.mjs b/packages/core/src/generators/metadata/utils/__tests__/parse.test.mjs index 4458f5d2..1b9e0c92 100644 --- a/packages/core/src/generators/metadata/utils/__tests__/parse.test.mjs +++ b/packages/core/src/generators/metadata/utils/__tests__/parse.test.mjs @@ -4,6 +4,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { u } from 'unist-builder'; +import { visit } from 'unist-util-visit'; import { parseApiDoc } from '../parse.mjs'; @@ -338,4 +339,34 @@ describe('parseApiDoc', () => { assert.strictEqual(secondContent[0].type, 'heading'); }); }); + + describe('source positions', () => { + it('drops where each node of an entry came from', () => { + const tree = u('root', [ + h('fs'), + stability('Stability: 2 - Stable'), + u('paragraph', [u('text', 'Content.')]), + ]); + const position = { + start: { line: 1, column: 1 }, + end: { line: 1, column: 1 }, + }; + + visit(tree, node => { + node.position = position; + }); + + const [entry] = parseApiDoc({ path, tree }, typeMap); + const positioned = []; + + visit(entry.content, node => { + if ('position' in node) { + positioned.push(node.type); + } + }); + + assert.strictEqual(entry.content.children.length, 3); + assert.deepStrictEqual(positioned, []); + }); + }); }); diff --git a/packages/core/src/generators/metadata/utils/parse.mjs b/packages/core/src/generators/metadata/utils/parse.mjs index c3eb41fb..8a805012 100644 --- a/packages/core/src/generators/metadata/utils/parse.mjs +++ b/packages/core/src/generators/metadata/utils/parse.mjs @@ -155,5 +155,17 @@ export const parseApiDoc = ({ path, tree, mdx = false }, typeMap) => { return SKIP; }); + // Nothing after this reads where a node came from: letting it go saves + // memory, and copying it to and from the worker threads + for (const { content, heading, stability } of metadataCollection) { + for (const root of [content, heading, stability]) { + if (root) { + visit(root, node => { + delete node.position; + }); + } + } + } + return metadataCollection; }; diff --git a/packages/core/src/generators/types.d.ts b/packages/core/src/generators/types.d.ts index 8bee0399..51e7634c 100644 --- a/packages/core/src/generators/types.d.ts +++ b/packages/core/src/generators/types.d.ts @@ -13,16 +13,18 @@ declare global { */ export interface ParallelWorker { /** - * Processes items in parallel across worker threads and yields results - * as each chunk completes. Enables downstream processing to begin + * Processes items in parallel across worker threads and yields each + * chunk's results in submission order, as soon as it and every chunk + * before it have completed. Enables downstream processing to begin * while upstream chunks are still being processed. * - * @param items - Items to process (determines chunk distribution) + * @param items - Items to process (determines chunk distribution), or + * batches of them as a streamed input yields them * @param opts - Additional options to pass to workers - * @yields Each chunk's results as they complete + * @yields Each chunk's results, in submission order */ stream( - items: T[], + items: T[] | AsyncIterable, opts?: Record ): AsyncGenerator; } @@ -70,6 +72,19 @@ declare global { hasParallelProcessor?: boolean; + /** + * Receive the dependency's output as it is produced, as an async iterable + * of its chunks, instead of collected into one array once it is complete, + * so this generator can start on the first chunk while the rest are still + * being produced. The async iterable can be iterated only once. A + * dependency that doesn't stream arrives as a single chunk, once it is + * complete. + * + * With one thread, the input still arrives collected into one array, as + * without `streamsInput`, so accept both. + */ + streamsInput?: boolean; + /** * The unified plugins this generator processes Markdown with, listed as * in the `markdown` option, with paths relative to its module. Each list diff --git a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs index 030f5d10..7820ad8d 100644 --- a/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs +++ b/packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs @@ -93,20 +93,97 @@ describe('createHighlighter', () => { const loaded = () => highlighter.shiki.getLoadedLanguages(); assert.ok(!loaded().includes('python')); - assert.ok(!loaded().includes('javascript')); + assert.ok(!loaded().includes('rust')); // By its name, an alias of its own, or one of the options - assert.equal(highlighter.resolveLanguage('py'), 'py'); - assert.equal(highlighter.resolveLanguage('mjs'), 'mjs'); + assert.equal(highlighter.resolveLanguage('py'), 'python'); + assert.equal(highlighter.resolveLanguage('rs'), 'rs'); assert.ok(loaded().includes('python')); - assert.ok(loaded().includes('javascript')); - assert.ok(loaded().includes('cjs')); + assert.ok(loaded().includes('rust')); assert.equal(highlighter.resolveLanguage(undefined), 'text'); assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext'); }); + it('highlights JavaScript as TypeScript, and C++ as C', async () => { + const highlighter = await createHighlighter(); + const loaded = () => highlighter.shiki.getLoadedLanguages(); + + for (const language of ['javascript', 'JS', 'mjs', 'cjs']) { + assert.equal(highlighter.resolveLanguage(language), 'typescript'); + } + + assert.equal(highlighter.resolveLanguage('cpp'), 'c'); + assert.equal(highlighter.resolveLanguage('c++'), 'c'); + assert.ok(!loaded().includes('cpp')); + + // Even once a grammar that has them as its own aliases is registered + assert.equal(highlighter.resolveLanguage('coffee'), 'coffee'); + assert.ok(loaded().includes('javascript')); + assert.equal(highlighter.resolveLanguage('js'), 'typescript'); + + // Not the names every object inherits + assert.equal(highlighter.resolveLanguage('constructor'), 'text'); + }); + + it('takes aliases over the default ones', async () => { + const highlighter = await createHighlighter({ + langAlias: { cpp: 'cpp', js: 'javascript' }, + }); + + assert.equal(highlighter.resolveLanguage('cpp'), 'cpp'); + assert.equal(highlighter.resolveLanguage('js'), 'javascript'); + assert.equal(highlighter.resolveLanguage('mjs'), 'typescript'); + assert.ok(highlighter.shiki.getLoadedLanguages().includes('cpp')); + }); + + it('follows configured aliases, and finds languages named in any case', async () => { + const highlighter = await createHighlighter({ + langs: [{ ...grammar, name: 'OxcConf', scopeName: 'source.oxcconf2' }], + langAlias: { + a: 'b', + b: 'python', + loop: 'pool', + pool: 'loop', + oc: 'OxcConf', + conf: 'INI', + }, + }); + + assert.equal(highlighter.resolveLanguage('a'), 'python'); + assert.equal(highlighter.resolveLanguage('loop'), 'text'); + assert.equal(highlighter.resolveLanguage('Python'), 'python'); + assert.equal(highlighter.resolveLanguage('oc'), 'OxcConf'); + assert.equal(highlighter.resolveLanguage('conf'), 'ini'); + assert.equal( + highlighter.highlightToHtml('[section]', 'conf'), + highlighter.highlightToHtml('[section]', 'ini') + ); + }); + + it('highlights with a language of `langs` an alias leads to', async () => { + const highlighter = await createHighlighter({ + langs: [ + { + ...grammar, + name: 'javascript', + aliases: ['js'], + scopeName: 'source.oxc-js', + }, + ], + langAlias: { js: 'javascript' }, + }); + + assert.equal(highlighter.resolveLanguage('js'), 'javascript'); + assert.match( + highlighter.highlightToHtml('minify on', 'js'), + /--shiki-dark/ + ); + assert.ok(!highlighter.shiki.getLoadedLanguages().includes('typescript')); + assert.equal(highlighter.resolveLanguage('mjs'), 'typescript'); + }); + it('lists the bundled languages without registering them', async () => { const highlighter = await createHighlighter({ langs: [grammar] }); diff --git a/packages/core/src/plugins/shiki/constants.mjs b/packages/core/src/plugins/shiki/constants.mjs new file mode 100644 index 00000000..2d5e9cc7 --- /dev/null +++ b/packages/core/src/plugins/shiki/constants.mjs @@ -0,0 +1,17 @@ +'use strict'; + +// What code in an unknown language is highlighted as +export const FALLBACK_LANGUAGE = 'text'; + +// What code in these languages is highlighted as, unless configured otherwise. +// Every grammar takes its own memory on each thread, and time to register, and +// TypeScript's highlights JavaScript (types are highlighted as TypeScript +// anyway), as C's does most of C++ +export const DEFAULT_ALIASES = new Map([ + ['javascript', 'typescript'], + ['js', 'typescript'], + ['mjs', 'typescript'], + ['cjs', 'typescript'], + ['cpp', 'c'], + ['c++', 'c'], +]); diff --git a/packages/core/src/plugins/shiki/highlighter.mjs b/packages/core/src/plugins/shiki/highlighter.mjs index 06dc0a58..bf9f7443 100644 --- a/packages/core/src/plugins/shiki/highlighter.mjs +++ b/packages/core/src/plugins/shiki/highlighter.mjs @@ -12,6 +12,8 @@ import { importFromURL } from '#utils/loaders.mjs'; import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs'; import { lazy } from '#utils/misc.mjs'; +import { DEFAULT_ALIASES, FALLBACK_LANGUAGE } from './constants.mjs'; + const require = createRequire(import.meta.url); /** @@ -25,7 +27,7 @@ const require = createRequire(import.meta.url); * * @typedef {Object} SyntaxHighlighter * @property {import('shiki').HighlighterCore} shiki - The Shiki instance - * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones + * @property {(languageId?: string) => string} resolveLanguage - Resolves the language code is highlighted as, falling back to plain text for unknown ones * @property {(code: string, lang: string, meta?: Record) => string} highlightToHtml - Highlights code, returning the inner HTML of its `` element * @property {(code: string, lang: string, meta?: Record) => ReturnType} highlightToHast - Highlights code, returning a HAST tree * @property {Array} langs - The languages it highlights @@ -64,9 +66,6 @@ let engine; // The highlighters of the options given, by their JSON const highlighters = new Map(); -// What code in an unknown language is highlighted as -const FALLBACK_LANGUAGE = 'text'; - // The languages Shiki bundles. Registering a grammar takes time and memory, // and slows down every highlight after it, so each one is only imported and // registered once code in it is highlighted (see `resolveLanguage`). @@ -220,10 +219,12 @@ const importHighlighter = async ({ engine: regexEngine, // The bundled languages are registered as code uses them langs: withEmbeddedLanguages(importedLangs), - // A copy, as Shiki adds the aliases of the languages it registers to it - langAlias: { ...langAlias }, }; + // Resolved here, as Shiki replaces an alias with a grammar's own as it + // registers + const aliases = new Map(Object.entries(langAlias)); + // Without themes of its own, the highlighter has a default light and dark one if (themes) { const [light, dark] = await Promise.all([ @@ -256,8 +257,9 @@ const importHighlighter = async ({ }); /** - * Resolves a language, falling back to plain text for unknown ones. A - * bundled language is registered the first time it's resolved. + * Resolves the language code is highlighted as, through the aliases, falling + * back to plain text for unknown ones. A bundled language is registered the + * first time it's resolved. * * @param {string} [languageId] * @returns {string} @@ -267,20 +269,39 @@ const importHighlighter = async ({ return FALLBACK_LANGUAGE; } + const followed = new Set(); + + let language = languageId; + + // Each configured alias once, so circular ones end + while (aliases.has(language.toLowerCase()) && !followed.has(language)) { + followed.add(language); + language = aliases.get(language.toLowerCase()); + } + + if (followed.size === 0) { + language = DEFAULT_ALIASES.get(language.toLowerCase()) ?? language; + } + const instance = load().shiki; - const name = instance.resolveLangAlias(languageId.toLowerCase()); + const loaded = instance.getLoadedLanguages(); - if (isSpecialLang(name) || instance.getLoadedLanguages().includes(name)) { - return languageId; + // Shiki has bundled languages' names and aliases in lowercase + for (const name of [language, language.toLowerCase()]) { + if (isSpecialLang(name) || loaded.includes(name)) { + return name; + } } + const name = language.toLowerCase(); + if (!BUNDLED_NAMES.has(name)) { return FALLBACK_LANGUAGE; } instance.loadLanguageSync(importBundledLanguage(name)); - return languageId; + return name; }; /** diff --git a/packages/core/src/threading/__tests__/index.test.mjs b/packages/core/src/threading/__tests__/index.test.mjs index 20327884..127c0edc 100644 --- a/packages/core/src/threading/__tests__/index.test.mjs +++ b/packages/core/src/threading/__tests__/index.test.mjs @@ -128,4 +128,14 @@ describe('createWorkerPool', () => { await pool.destroy(); } }); + + it('should start tasks in the order they were submitted', async () => { + const pool = createWorkerPool(1); + + try { + strictEqual(pool.options.stricterFIFO, true); + } finally { + await pool.destroy(); + } + }); }); diff --git a/packages/core/src/threading/__tests__/parallel.test.mjs b/packages/core/src/threading/__tests__/parallel.test.mjs index 4a2c3653..4c87e871 100644 --- a/packages/core/src/threading/__tests__/parallel.test.mjs +++ b/packages/core/src/threading/__tests__/parallel.test.mjs @@ -2,6 +2,7 @@ import { deepStrictEqual, ok, rejects, strictEqual } from 'node:assert'; import { describe, it } from 'node:test'; import { loadGenerator } from '../../generators/loader.mjs'; +import { CHUNKS_PER_THREAD } from '../constants.mjs'; import createWorkerPool from '../index.mjs'; import createParallelWorker from '../parallel.mjs'; @@ -42,6 +43,52 @@ async function collectChunks(generator) { return chunks; } +/** + * Streams three times as many one-item chunks as two threads can run at once + * through a pool that holds each task until it is settled, newest first, and + * checks before each that as many run as there is room for: no more, and no + * fewer while any are left to start. Settling the newest first holds back the + * oldest chunk, so no result can be emitted until the last: the room must come + * from the settling alone. The results still come out in submission order. + * A stream that lost count of the held chunks would never end, so its tests + * need a timeout. + * + * @param {(items: number[]) => number[] | AsyncIterable} toInput + * @returns {Promise} + */ +async function checkChunksPerThread(toInput) { + const limit = 2 * CHUNKS_PER_THREAD; + const items = Array.from({ length: limit * 3 }, (_, index) => index); + const started = []; + const running = []; + const pool = { + run: task => + new Promise(resolve => { + started.push(...task.input); + running.push(() => resolve(task.input)); + }), + }; + const worker = createParallelWorker('ast-js', astJsGenerator, pool, { + threads: 2, + chunkSize: 1, + }); + + const result = collectStream(worker.stream(toInput(items), {})); + + for (let settled = 0; settled < items.length; settled++) { + await new Promise(resolve => setImmediate(resolve)); + + strictEqual(running.length, Math.min(limit, items.length - settled)); + + running.pop()(); + } + + const results = await result; + + deepStrictEqual(started, items); + deepStrictEqual(results, items); +} + describe('createParallelWorker', () => { it('should create a ParallelWorker with stream method', async () => { const pool = createWorkerPool(2); @@ -230,6 +277,12 @@ describe('createParallelWorker', () => { await rejects(result, /chunk failed/); }); + it( + 'should start a few chunks per thread at a time, and the next as each settles', + { timeout: 10_000 }, + () => checkChunksPerThread(items => items) + ); + it('should process one chunk at a time, in order, with a single thread', async () => { let running = 0; let mostRunning = 0; @@ -257,4 +310,167 @@ describe('createParallelWorker', () => { strictEqual(mostRunning, 1); deepStrictEqual(chunks, [[1, 2], [3, 4], [5]]); }); + + describe('with streamed batches', () => { + it( + 'starts a batch before the next one arrives', + // The second batch only arrives once the first result is read, so a + // stream that waited for it would never give one + { timeout: 10_000 }, + async () => { + const started = []; + const pool = { + run: async task => { + started.push(task.input); + + return task.input; + }, + }; + const worker = createParallelWorker('ast-js', astJsGenerator, pool, { + threads: 2, + chunkSize: 2, + }); + const gate = Promise.withResolvers(); + + async function* batches() { + yield [1, 2, 3]; + await gate.promise; + yield [4]; + } + + const reader = worker.stream(batches(), {}); + const first = await reader.next(); + + // The second batch hasn't arrived, but the first one's chunks ran + deepStrictEqual(first.value, [1, 2]); + deepStrictEqual(started, [[1, 2], [3]]); + + gate.resolve(); + + const rest = await collectChunks(reader); + + deepStrictEqual(rest, [[3], [4]]); + } + ); + + it( + 'yields in submission order across batches', + // The test waits for every chunk to start, so it would wait forever on a + // stream that held one back + { timeout: 10_000 }, + async () => { + const resolvers = []; + const pool = { + run: task => + new Promise(resolve => { + resolvers.push(() => resolve(task.input)); + }), + }; + const worker = createParallelWorker('ast-js', astJsGenerator, pool, { + threads: 2, + chunkSize: 1, + }); + + async function* batches() { + yield [0, 1]; + yield [2]; + } + + const result = collectStream(worker.stream(batches(), {})); + + while (resolvers.length < 3) { + await new Promise(resolve => setImmediate(resolve)); + } + + for (const resolve of resolvers.toReversed()) { + resolve(); + } + + const results = await result; + + deepStrictEqual(results, [0, 1, 2]); + } + ); + + it('throws a failure reading the batches after the chunks before it', async () => { + const pool = { run: async task => task.input }; + const worker = createParallelWorker('ast-js', astJsGenerator, pool, { + threads: 2, + chunkSize: 1, + }); + + async function* batches() { + yield [1, 2]; + throw new Error('source failed'); + } + + const chunks = []; + + await rejects(async () => { + for await (const chunk of worker.stream(batches(), {})) { + chunks.push(chunk); + } + }, /source failed/); + + deepStrictEqual(chunks, [[1], [2]]); + }); + + it( + 'starts a few chunks per thread at a time, across batches', + { timeout: 10_000 }, + () => + checkChunksPerThread(async function* (items) { + for (let index = 0; index < items.length; index += 5) { + yield items.slice(index, index + 5); + } + }) + ); + + it('runs each batch on this thread as it arrives with a single thread', async () => { + const events = []; + const generator = { + name: 'sequential', + async processChunk(items, indices) { + events.push(`chunk ${indices.map(index => items[index])}`); + + return indices.map(index => items[index]); + }, + }; + const worker = createParallelWorker('sequential', generator, null, { + threads: 1, + chunkSize: 2, + }); + + async function* batches() { + events.push('batch 1'); + yield [1, 2, 3]; + events.push('batch 2'); + yield [4]; + } + + const chunks = await collectChunks(worker.stream(batches(), {})); + + deepStrictEqual(chunks, [[1, 2], [3], [4]]); + deepStrictEqual(events, [ + 'batch 1', + 'chunk 1,2', + 'chunk 3', + 'batch 2', + 'chunk 4', + ]); + }); + + it('yields nothing for no batches', async () => { + const worker = createParallelWorker('ast-js', astJsGenerator, null, { + threads: 2, + chunkSize: 1, + }); + + async function* batches() {} + + const chunks = await collectChunks(worker.stream(batches(), {})); + + deepStrictEqual(chunks, []); + }); + }); }); diff --git a/packages/core/src/threading/constants.mjs b/packages/core/src/threading/constants.mjs index 9872c668..fd923dab 100644 --- a/packages/core/src/threading/constants.mjs +++ b/packages/core/src/threading/constants.mjs @@ -3,9 +3,10 @@ /** * The default `threads` ceiling. Each worker holds a heap of its own, with the * libraries and the pages it is working on, so past a few threads memory, not - * CPU, is what runs out. `--threads` raises it. + * CPU, is what runs out: on the Node.js docs, 8 threads were no faster than 6, + * for ~15% more memory. `--threads` raises it. */ -export const DEFAULT_MAX_THREADS = 4; +export const DEFAULT_MAX_THREADS = 6; /** * The heap size limit of each worker thread (V8's old space), in MB. V8 lets a @@ -22,3 +23,10 @@ export const WORKER_HEAP_SIZE = 512; * over half of a worker's heap. Node.js's `--max-semi-space-size` replaces it. */ export const WORKER_GEN_SIZE = 96; + +/** + * How many chunks of one generator can be started, running or queued, per + * thread at a time: enough to keep every thread busy while more are submitted, + * few enough that generators running at once share the pool. + */ +export const CHUNKS_PER_THREAD = 3; diff --git a/packages/core/src/threading/index.mjs b/packages/core/src/threading/index.mjs index 0a2ed537..7cbf5b2b 100644 --- a/packages/core/src/threading/index.mjs +++ b/packages/core/src/threading/index.mjs @@ -32,6 +32,10 @@ export default function createWorkerPool(threads) { maxYoungGenerationSizeMb: WORKER_GEN_SIZE, }, recordTiming: false, + // A task taken from the queue that no worker can run yet goes back to the + // front, not the back, so tasks start in the order they were submitted + // and results, yielded in that order, are not held up + stricterFIFO: true, workerData: { logLevel: logger.getLogLevel() }, }); } diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index 136bac69..4b0874b3 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -1,6 +1,10 @@ 'use strict'; import logger from '#logger/index.mjs'; +import { toBatches } from '#utils/generators.mjs'; +import { createSignal, isAsyncIterable } from '#utils/misc.mjs'; + +import { CHUNKS_PER_THREAD } from './constants.mjs'; const parallelLogger = logger.child('parallel'); @@ -71,7 +75,8 @@ export default function createParallelWorker( pool, configuration ) { - const { threads, chunkSize } = configuration; + const { threads: defaultThreads = 1, chunkSize } = configuration; + const threads = Math.max(Number(defaultThreads), 1); const { name } = generator; @@ -79,54 +84,171 @@ export default function createParallelWorker( /** * Processes items in parallel, yielding results in submission order. * + * The items can also arrive in batches, as a dependency streams them: each + * batch's chunks start as soon as it arrives, while earlier ones still run. + * * @template T - * @param {T[]} items - Items to process + * @param {T[] | AsyncIterable} items - Items to process * @param {object} extra - Extra options * @yields {R[]} Chunk results in input order */ async *stream(items, extra) { - if (items.length === 0) { + const streamed = isAsyncIterable(items); + + if (!streamed && items.length === 0) { return; } - const chunks = createChunks(items.length, chunkSize); + const batches = toBatches(items); + + /** + * Splits a batch into chunks. + * + * @param {T[]} batch - Items to split + * @returns {number[][]} Each chunk's indices into the batch + */ + const distribute = batch => { + const chunks = createChunks(batch.length, chunkSize); - parallelLogger.debug( - `Distributing ${items.length} items across ${chunks.length} chunks`, - { generator: name, chunks: chunks.length, chunkSize, threads } - ); + parallelLogger.debug( + `Distributing ${batch.length} items across ${chunks.length} chunks`, + { generator: name, chunks: chunks.length, chunkSize, threads } + ); + + return chunks; + }; // On this thread, one chunk at a time, so their work isn't all held at once - if (threads <= 1 || items.length <= 2) { - for (const indices of chunks) { - yield await generator.processChunk(items, indices, extra); + if (threads <= 1 || (!streamed && items.length <= 2)) { + for await (const batch of batches) { + for (const indices of distribute(batch)) { + const result = await generator.processChunk(batch, indices, extra); + + yield result; + } } return; } - // Start every chunk immediately, but emit results in submission order. - // Handle errors as each task settles so a later failure cannot become an - // unhandled rejection while earlier chunks are still running. - const tasks = chunks.map(indices => { - const task = pool.run( - createTask(items, indices, extra, configuration, specifier, name) - ); + // Read every batch as it arrives, but start only a few chunks per thread + // at a time, so that generators running at once share the pool instead + // of one's chunks waiting behind all of another's. Emit results in + // submission order. Handle errors as each task settles so a later failure + // cannot become an unhandled rejection while earlier chunks are still + // running. + const tasks = []; + const waiting = []; + const limit = threads * CHUNKS_PER_THREAD; - return task.then( - result => ({ result }), - error => ({ error }) - ); - }); + let running = 0; + let started = 0; + let doneReading = false; + + // Notified as chunks start, and once every batch is read + const changed = createSignal(); + + /** + * Settles a chunk's task, making room for the next one. + * + * @param {{ result: unknown } | { error: unknown }} outcome - The task's + * result or error + * @returns {{ result: unknown } | { error: unknown }} The same outcome + */ + const settle = outcome => { + running--; + start(); + + return outcome; + }; + + /** + * Starts the waiting chunks there is room for. + */ + const start = () => { + while (running < limit && started < waiting.length) { + const { batch, indices } = waiting[started]; + + // A started chunk no longer needs to hold its batch here + waiting[started++] = undefined; + running++; + + const task = pool.run( + createTask(batch, indices, extra, configuration, specifier, name) + ); + + tasks.push( + task.then( + result => settle({ result }), + error => settle({ error }) + ) + ); + } + + changed.notify(); + }; + + /** + * Queues every chunk of a batch, and starts those there is room for. + * + * @param {T[]} batch - Items to queue + */ + const add = batch => { + for (const indices of distribute(batch)) { + waiting.push({ batch, indices }); + } + + start(); + }; + + /** + * Queues every batch as it arrives: an array's at once, so its first + * chunks start right away. + */ + const readBatches = async () => { + try { + if (!streamed) { + add(items); + + return; + } + + for await (const batch of items) { + add(batch); + } + } finally { + doneReading = true; + changed.notify(); + } + }; + + const reading = readBatches(); + + // A failure reading the batches is thrown below, once every chunk + // submitted before it has been emitted + reading.catch(() => {}); + + for (let index = 0; ; index++) { + while (index >= tasks.length) { + if (doneReading && started === waiting.length) { + await reading; + + return; + } + + await changed.wait(); + } + + const outcome = await tasks[index]; - for (const [index, task] of tasks.entries()) { - const outcome = await task; + // Emitted chunks are the consumer's to keep or let go + tasks[index] = undefined; if ('error' in outcome) { throw outcome.error; } - parallelLogger.debug(`Chunk ${index + 1}/${chunks.length} completed`, { + parallelLogger.debug(`Chunk ${index + 1}/${waiting.length} completed`, { generator: name, }); diff --git a/packages/core/src/utils/__tests__/channel.test.mjs b/packages/core/src/utils/__tests__/channel.test.mjs new file mode 100644 index 00000000..7f5a9d4c --- /dev/null +++ b/packages/core/src/utils/__tests__/channel.test.mjs @@ -0,0 +1,63 @@ +import { deepStrictEqual } from 'node:assert'; +import { describe, it } from 'node:test'; + +import { createChannel } from '../channel.mjs'; + +/** + * Reads every batch of an async iterable. + * + * @template T + * @param {AsyncIterable} iterable + * @returns {Promise} + */ +async function readBatches(iterable) { + const batches = []; + + for await (const batch of iterable) { + batches.push(batch); + } + + return batches; +} + +describe('createChannel', () => { + it('gives the batches as they are added, until it ends', async () => { + const channel = createChannel(); + const reading = readBatches(channel); + + channel.push([1]); + await new Promise(resolve => setImmediate(resolve)); + channel.push([2]); + channel.end(); + + const batches = await reading; + + deepStrictEqual(batches, [[1], [2]]); + }); + + it('gives a prepended batch before those not read yet', async () => { + const channel = createChannel(); + + channel.push([1]); + channel.push([2]); + channel.prepend([0]); + channel.end(); + + const batches = await readBatches(channel); + + deepStrictEqual(batches, [[0], [1], [2]]); + }); + + it('drops the batches not read yet when cleared', async () => { + const channel = createChannel(); + + channel.push([1]); + channel.clear(); + channel.push([2]); + channel.end(); + + const batches = await readBatches(channel); + + deepStrictEqual(batches, [[2]]); + }); +}); diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index 6fe290db..210f6bcc 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -4,9 +4,12 @@ import { describe, it } from 'node:test'; import { getEntryDescription, groupNodesByModule, + groupWholeModules, getVersionFromSemVer, coerceSemVer, getCompatibleVersions, + merge, + toBatches, } from '../generators.mjs'; describe('getEntryDescription', () => { @@ -86,6 +89,97 @@ describe('groupNodesByModule', () => { }); }); +describe('groupWholeModules', () => { + /** + * Collects the batches regrouped from the given input, each entry as its + * module and number. + * + * @param {Array<{ api: string }> | AsyncIterable>} input + * @returns {Promise} + */ + const collect = async input => { + const output = []; + + for await (const batch of groupWholeModules(input)) { + output.push(batch.map(({ api, n }) => `${api}${n}`)); + } + + return output; + }; + + /** + * Regroups the given batches, streamed in, and collects the batches it + * yields. + * + * @param {Array>} batches + * @returns {Promise} + */ + const regroup = batches => { + async function* input() { + yield* batches; + } + + return collect(input()); + }; + + /** + * An entry of a module, told apart from the module's others by its number. + * + * @param {string} api + * @param {number} n + * @returns {{ api: string, n: number }} + */ + const entry = (api, n) => ({ api, n }); + + it('passes the entries on in the same order, each batch ending once its last module is whole', async () => { + const output = await regroup([ + [entry('a', 1), entry('a', 2), entry('b', 1)], + [entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1'], ['c1']]); + }); + + it('holds back a module that continues in the next batch', async () => { + const output = await regroup([ + [entry('a', 1), entry('b', 1)], + [entry('b', 2), entry('b', 3)], + [entry('b', 4), entry('c', 1)], + ]); + + assert.deepEqual(output, [['a1'], ['b1', 'b2', 'b3', 'b4'], ['c1']]); + }); + + it('skips empty batches', async () => { + const output = await regroup([[], [entry('a', 1)], []]); + + assert.deepEqual(output, [['a1']]); + }); + + it('takes entries given as one array as a single batch', async () => { + const output = await collect([entry('a', 1), entry('a', 2), entry('b', 1)]); + + assert.deepEqual(output, [['a1', 'a2'], ['b1']]); + }); + + it('rejects a module whose entries arrive apart', async () => { + await assert.rejects( + regroup([ + [entry('a', 1), entry('b', 1)], + [entry('c', 1), entry('a', 2)], + ]), + /"a" arrived apart/ + ); + }); + + it('rejects a module whose entries arrive apart in one batch too', async () => { + await assert.rejects( + regroup([[entry('a', 1), entry('b', 1), entry('a', 2)]]), + /"a" arrived apart/ + ); + }); +}); + describe('getVersionFromSemVer', () => { it('returns major.x for minor 0', () => { const version = { major: 18, minor: 0, patch: 0 }; @@ -138,3 +232,69 @@ describe('getCompatibleVersions', () => { assert.equal(result.length, 2); }); }); + +describe('toBatches', () => { + it('gives an array as a single batch, and batches as they are', async () => { + async function* batches() { + yield [1]; + } + + const streamed = batches(); + + assert.deepEqual(toBatches([1, 2]), [[1, 2]]); + assert.equal(toBatches(streamed), streamed); + }); +}); + +describe('merge', () => { + it('yields each item as soon as it arrives, until every source ends', async () => { + const gate = Promise.withResolvers(); + + async function* slow() { + await gate.promise; + + yield 'slow'; + } + + async function* fast() { + yield 'fast'; + yield 'faster'; + } + + const items = []; + + for await (const item of merge(slow(), fast())) { + items.push(item); + + if (item === 'faster') { + gate.resolve(); + } + } + + assert.deepEqual(items, ['fast', 'faster', 'slow']); + }); + + it('throws a failure without waiting for the other sources', async () => { + async function* never() { + await new Promise(() => {}); + + yield 'never'; + } + + async function* failing() { + yield 'before'; + + throw new Error('merged failure'); + } + + const items = []; + + await assert.rejects(async () => { + for await (const item of merge(never(), failing())) { + items.push(item); + } + }, /merged failure/); + + assert.deepEqual(items, ['before']); + }); +}); diff --git a/packages/core/src/utils/__tests__/replay.test.mjs b/packages/core/src/utils/__tests__/replay.test.mjs new file mode 100644 index 00000000..2225e4be --- /dev/null +++ b/packages/core/src/utils/__tests__/replay.test.mjs @@ -0,0 +1,86 @@ +import { deepStrictEqual, rejects } from 'node:assert'; +import { describe, it } from 'node:test'; + +import { createReplay } from '../replay.mjs'; + +/** + * Reads every chunk of an async iterable. + * + * @template T + * @param {AsyncIterable} iterable + * @returns {Promise} + */ +async function collectChunks(iterable) { + const chunks = []; + + for await (const chunk of iterable) { + chunks.push(chunk); + } + + return chunks; +} + +describe('createReplay', () => { + it('should give every reader every chunk, in order', async () => { + const gate = Promise.withResolvers(); + + async function* gen() { + yield [1]; + await gate.promise; + yield [2]; + yield [3]; + } + + const replay = createReplay(gen()); + + const early = collectChunks(replay.read()); + + await new Promise(resolve => setImmediate(resolve)); + gate.resolve(); + + const late = collectChunks(replay.read()); + + const earlyChunks = await early; + const lateChunks = await late; + + deepStrictEqual(earlyChunks, [[1], [2], [3]]); + deepStrictEqual(lateChunks, [[1], [2], [3]]); + }); + + it('should throw rather than give a reader a chunk that was let go', async () => { + async function* gen() { + yield [1]; + yield [2]; + } + + const replay = createReplay(gen()); + + await replay.ended; + + // With no reader left, every chunk goes + replay.forget(); + + await rejects(collectChunks(replay.read()), /Chunk 0 was let go/); + }); + + it('should run the generator to its end, and throw its failure to readers', async () => { + async function* gen() { + yield [1]; + + throw new Error('replayed failure'); + } + + const replay = createReplay(gen()); + const chunks = []; + + await rejects(replay.ended, /replayed failure/); + + await rejects(async () => { + for await (const chunk of replay.read()) { + chunks.push(chunk); + } + }, /replayed failure/); + + deepStrictEqual(chunks, [[1]]); + }); +}); diff --git a/packages/core/src/utils/channel.mjs b/packages/core/src/utils/channel.mjs new file mode 100644 index 00000000..1aaeaa35 --- /dev/null +++ b/packages/core/src/utils/channel.mjs @@ -0,0 +1,75 @@ +'use strict'; + +import { createSignal } from './misc.mjs'; + +/** + * Creates a queue of batches that can be read as an async iterable while it is + * still being filled. + * + * @template T + */ +export const createChannel = () => { + /** @type {Array} */ + const batches = []; + + // Notified as batches are added, and at the end + const added = createSignal(); + + let ended = false; + + return { + /** + * Adds a batch, after those not read yet. + * + * @param {T[]} batch + */ + push(batch) { + batches.push(batch); + added.notify(); + }, + + /** + * Adds a batch, before those not read yet. + * + * @param {T[]} batch + */ + prepend(batch) { + batches.unshift(batch); + added.notify(); + }, + + /** + * Drops the batches not read yet. + */ + clear() { + batches.length = 0; + }, + + /** + * Ends the channel: the reader stops once it has read every batch. + */ + end() { + ended = true; + added.notify(); + }, + + /** + * Reads the batches as they are added, until the channel ends. + * + * @returns {AsyncGenerator} + */ + async *[Symbol.asyncIterator]() { + for (;;) { + while (batches.length === 0) { + if (ended) { + return; + } + + await added.wait(); + } + + yield batches.shift(); + } + }, + }; +}; diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 516113bc..d64fdb7d 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -2,6 +2,8 @@ import { coerce, major } from 'semver'; +import { createChannel } from './channel.mjs'; +import { isAsyncIterable } from './misc.mjs'; import { transformNodeToString } from './unist.mjs'; /** @@ -42,6 +44,131 @@ export const groupNodesByModule = nodes => { return Map.groupBy(nodes, node => node.api); }; +/** + * Gives a generator's input as batches: as they arrive if it streams, or as a + * single batch if it arrives whole, as it does with one thread. + * + * @template T + * @param {Array | AsyncIterable>} input + * @returns {Array> | AsyncIterable>} + */ +export const toBatches = input => { + if (isAsyncIterable(input)) { + return input; + } + + return [input]; +}; + +/** + * Reads several async iterables at once, yielding each item as soon as it + * arrives, until they have all ended. A failure is thrown once the items that + * arrived before it are yielded, without waiting for the other iterables. + * + * @template T + * @param {...AsyncIterable} sources + * @returns {AsyncGenerator} + */ +export async function* merge(...sources) { + const channel = createChannel(); + + let pending = sources.length; + let failure; + + /** + * Passes a source's items on as they arrive, and ends the channel once every + * source has ended, or one fails. + * + * @param {AsyncIterable} source + */ + const pipe = async source => { + try { + for await (const item of source) { + channel.push(item); + } + } catch (error) { + failure ??= { error }; + } finally { + pending--; + + if (failure || pending === 0) { + channel.end(); + } + } + }; + + for (const source of sources) { + pipe(source); + } + + yield* channel; + + if (failure) { + throw failure.error; + } +} + +/** + * Regroups streamed entries so that each batch holds whole modules: a module + * whose entries continue in the next batch is held back and passed on with + * it. A module's entries must arrive together: one that comes back after + * another module's is an error, wherever the batches end, as its pages would + * be built without it. Entries given as one array, as a generator that streams + * its input gets them with one thread, are taken as a single batch. + * + * @template {{ api: string }} T + * @param {Array | AsyncIterable>} batches - The entries, in + * batches as they are streamed, or in one array + * @returns {AsyncGenerator>} The same entries, in batches of whole + * modules + */ +export async function* groupWholeModules(batches) { + // The modules whose entries have all arrived + const finished = new Set(); + + // The module whose entries are arriving, and those of them held back + let current; + let held = []; + + for await (const batch of toBatches(batches)) { + for (const { api } of batch) { + if (api !== current) { + if (finished.has(api)) { + throw new Error( + `The entries of "${api}" arrived apart: a module's entries must ` + + 'arrive together, in one chunk or in consecutive ones' + ); + } + + if (current !== undefined) { + finished.add(current); + } + + current = api; + } + } + + const entries = held.length > 0 ? [...held, ...batch] : batch; + + // The module the batch ends with may continue in the next one + let end = entries.length; + + while (end > 0 && entries[end - 1].api === current) { + end--; + } + + held = entries.slice(end); + + if (end > 0) { + yield entries.slice(0, end); + } + } + + if (held.length > 0) { + yield held; + } +} + /** * Parses the SemVer string into a Node.js-alike version * diff --git a/packages/core/src/utils/misc.mjs b/packages/core/src/utils/misc.mjs index 92d91e01..6b73c196 100644 --- a/packages/core/src/utils/misc.mjs +++ b/packages/core/src/utils/misc.mjs @@ -70,3 +70,29 @@ export const deepMerge = (...objects) => { return result; }, {}); }; + +/** + * Creates a signal: waiting on it resolves the next time it is notified. + * + * @returns {{ wait: () => Promise, notify: () => void }} + */ +export const createSignal = () => { + let next = Promise.withResolvers(); + + return { + /** + * Waits for the next notification. + * + * @returns {Promise} + */ + wait: () => next.promise, + + /** + * Wakes what waits. + */ + notify: () => { + next.resolve(); + next = Promise.withResolvers(); + }, + }; +}; diff --git a/packages/core/src/utils/replay.mjs b/packages/core/src/utils/replay.mjs new file mode 100644 index 00000000..52b555b3 --- /dev/null +++ b/packages/core/src/utils/replay.mjs @@ -0,0 +1,152 @@ +'use strict'; + +import { createSignal } from './misc.mjs'; + +/** + * Runs an async generator to its end, recording its chunks so that any number + * of readers can each read all of them, from the first, while it still runs. + * + * Once no more readers can start (see `forget`), a chunk every reader has read + * is let go. + * + * @template T + * @param {AsyncGenerator} generator - Generator to run + * @returns {{ ended: Promise, read: () => AsyncGenerator, forget: () => void }} + */ +export const createReplay = generator => { + /** @type {Array} */ + const chunks = []; + + // Where each reader is, by the index of the next chunk it reads + const readers = new Set(); + + let done = false; + let failure; + let forgotten = false; + let trimmed = 0; + + // Notified as each chunk is recorded, and at the end + const recorded = createSignal(); + + /** + * Lets go of the chunks every reader has read, once no more can start. + */ + const trim = () => { + if (!forgotten) { + return; + } + + let read = chunks.length; + + for (const reader of readers) { + read = Math.min(read, reader.index); + } + + for (; trimmed < read; trimmed++) { + chunks[trimmed] = undefined; + } + }; + + /** + * Runs the generator to its end, recording each chunk, whether or not its + * readers keep up. + */ + const record = async () => { + try { + for await (const chunk of generator) { + chunks.push(chunk); + recorded.notify(); + trim(); + } + } catch (error) { + failure = { error }; + + throw error; + } finally { + done = true; + recorded.notify(); + + // Readers share this closure: let the finished generator, and all it + // holds, go + generator = undefined; + } + }; + + /** + * Reads every chunk from where a reader is, those recorded and those still + * to come, then throws if the generator failed. + * + * @param {{ index: number }} reader + * @returns {AsyncGenerator} + */ + async function* readFrom(reader) { + try { + for (; ; reader.index++) { + while (reader.index >= chunks.length) { + if (failure) { + throw failure.error; + } + + if (done) { + return; + } + + await recorded.wait(); + } + + // A reader started after `forget` can be behind what was let go: fail + // rather than give it `undefined` for a chunk + if (reader.index < trimmed) { + throw new Error( + `Chunk ${reader.index} was let go before this reader read it` + ); + } + + const chunk = chunks[reader.index]; + + trim(); + + yield chunk; + } + } finally { + readers.delete(reader); + trim(); + } + } + + const ended = record(); + + // Its readers get a failure as they read; this keeps one with none from + // being reported as unhandled + ended.catch(() => {}); + + return { + /** + * Resolves once the generator has ended, or rejects with its failure. + */ + ended, + + /** + * Reads every chunk, from the first. The reader counts from when this is + * called, not from when it is first read, so nothing it hasn't read yet + * goes in the meantime. + * + * @returns {AsyncGenerator} + */ + read() { + const reader = { index: 0 }; + + readers.add(reader); + + return readFrom(reader); + }, + + /** + * Says no more readers will start, so chunks every reader has read can go. + */ + forget() { + forgotten = true; + trim(); + }, + }; +}; diff --git a/packages/node/README.md b/packages/node/README.md index 0de94631..dac631c0 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -20,6 +20,7 @@ npm install --save-dev @doc-kit/core @node-core/doc-kit | `man-page` | The `node.1` manual page, from the CLI options documented in `cli.md`. | | `api-links` | A mapping of API symbols to their source locations, powering source links on nodejs.org. | | `addon-verify` | Extracted C++ addon code samples from `addons.md`, arranged for compilation by the tests. | +| `html-all` | `all.html`, every module page's content on one page, rendered by the `html` generator. | ## Usage diff --git a/packages/node/package.json b/packages/node/package.json index 9e42d740..fd559574 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -13,6 +13,7 @@ "./addon-verify": "./src/addon-verify/index.mjs", "./api-links": "./src/api-links/index.mjs", "./config": "./src/config/index.mjs", + "./html-all": "./src/html-all/index.mjs", "./man-page": "./src/man-page/index.mjs", "./package.json": "./package.json" }, @@ -26,6 +27,7 @@ ], "dependencies": { "@doc-kit/core": "workspace:*", + "@doc-kit/generator-react": "workspace:*", "dedent": "^1.7.2", "estree-util-visit": "^2.0.0", "unist-util-visit": "^5.1.0" diff --git a/packages/node/src/html-all/README.md b/packages/node/src/html-all/README.md new file mode 100644 index 00000000..e91823c6 --- /dev/null +++ b/packages/node/src/html-all/README.md @@ -0,0 +1,19 @@ +# `html-all` Generator + +The `html-all` generator adds `all.html`, every module page's content on one +page, to the pages the [`html`](./html.md) generator renders. It builds nothing +itself: it is spliced between `jsx-ast` and `html`, passes the pages on, adds +`all.html`'s metadata to the manifest, and once every module page has passed, +adds a page made of theirs. `html` assembles it from their compiled content, +so no module is built twice. + +The module pages are its parts, in the order the sidebar lists them; the +index, section pages and the other synthetic pages are left out. + +```sh +npx @doc-kit/cli generate -t web -t html-all -i "doc/api/*.md" -o out +``` + +## Configuring + +It has no options of its own. diff --git a/packages/node/src/html-all/__tests__/generate.test.mjs b/packages/node/src/html-all/__tests__/generate.test.mjs new file mode 100644 index 00000000..633a30a7 --- /dev/null +++ b/packages/node/src/html-all/__tests__/generate.test.mjs @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { generate } from '../generate.mjs'; + +/** + * A page as `jsx-ast` gives it. + * + * @param {string} api + * @param {string} name + * @param {object} [extra] - More of its metadata + */ +const createPage = (api, name, extra = {}) => ({ + data: { api, heading: { depth: 1, data: { name } }, ...extra }, + headings: [{ slug: api }], + content: '<>', +}); + +/** + * Reads every batch the generator yields. + * + * @param {AsyncIterable> | Array} input + */ +const run = async input => { + const batches = []; + + for await (const batch of generate(input)) { + batches.push(batch); + } + + return batches; +}; + +describe('html-all', () => { + const zlib = createPage('zlib', 'Zlib'); + const fs = createPage('fs', 'File system'); + const index = createPage('index', 'Index'); + const chunk = createPage('fs/promises', 'fs.promises', { chunk: {} }); + const notFound = createPage('404', 'Page not found', { synthetic: true }); + + it('adds all.html right after its last part, in sidebar order', async () => { + async function* input() { + yield [zlib]; + yield [{ manifest: [zlib.data, fs.data, index.data, notFound.data] }]; + yield [index, fs]; + yield [chunk, notFound]; + } + + const batches = await run(input()); + + assert.deepEqual(batches[0], [zlib]); + + const [{ manifest }] = batches[1]; + + assert.deepEqual(manifest.slice(0, -1), [ + zlib.data, + fs.data, + index.data, + notFound.data, + ]); + assert.equal(manifest.at(-1).api, 'all'); + + const [, , all] = batches[2]; + + assert.deepEqual(batches[2].slice(0, 2), [index, fs]); + assert.equal(all.data.api, 'all'); + assert.deepEqual(all.parts, ['fs', 'zlib']); + assert.deepEqual(all.headings, [{ slug: 'fs' }, { slug: 'zlib' }]); + + assert.deepEqual(batches[3], [chunk, notFound]); + }); + + it('adds all.html after every page given at once, as with one thread', async () => { + const batches = await run([zlib, fs, { manifest: [zlib.data, fs.data] }]); + + const pages = batches.flat(); + + assert.equal(pages.at(-1).data.api, 'all'); + assert.deepEqual(pages.at(-1).parts, ['fs', 'zlib']); + }); +}); diff --git a/packages/node/src/html-all/generate.mjs b/packages/node/src/html-all/generate.mjs new file mode 100644 index 00000000..136f3d6a --- /dev/null +++ b/packages/node/src/html-all/generate.mjs @@ -0,0 +1,95 @@ +'use strict'; + +import { toBatches } from '@doc-kit/core/utils/generators.mjs'; +import { headingSortFn } from '@doc-kit/generator-react/jsx-ast/utils/getSortedHeadNodes'; + +// The metadata of `all.html` +const HEAD = { + api: 'all', + path: '/all', + basename: 'all', + synthetic: true, + heading: { + type: 'heading', + depth: 1, + children: [{ type: 'text', value: 'All' }], + data: { name: 'All', text: 'All', slug: 'all' }, + }, +}; + +/** + * Whether a page is part of `all.html`: the module pages, minus the index. + * Chunk pages and the other synthetic pages are left out, as the module pages + * carry their content. + * + * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} data + * @returns {boolean} + */ +const isPart = data => !data.synthetic && !data.chunk && data.api !== 'index'; + +/** + * Builds `all.html` from its parts: their content one after the other, and + * their tables of contents, in the order the sidebar lists them. + * + * @param {Array>} parts + */ +const buildAllPage = parts => { + const pages = parts.toSorted((a, b) => headingSortFn(a.data, b.data)); + + return { + data: HEAD, + headings: pages.flatMap(({ headings }) => headings), + parts: pages.map(({ data }) => data.api), + }; +}; + +/** + * Passes `jsx-ast`'s pages on, with `all.html`'s metadata in the manifest, and + * `all.html` itself right after the last of its parts. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input) { + const parts = []; + + // The parts still to come, once every page's metadata is known + let missing; + let added = false; + + for await (const batch of toBatches(input)) { + const pages = []; + + for (const item of batch) { + if ('manifest' in item) { + missing = new Set(item.manifest.filter(isPart).map(({ api }) => api)); + + for (const { data } of parts) { + missing.delete(data.api); + } + + pages.push({ manifest: [...item.manifest, HEAD] }); + + continue; + } + + pages.push(item); + + if (isPart(item.data)) { + parts.push({ data: item.data, headings: item.headings }); + missing?.delete(item.data.api); + } + } + + if (!added && missing?.size === 0) { + pages.push(buildAllPage(parts)); + added = true; + } + + yield pages; + } + + // Without a manifest, once every page has passed + if (!added) { + yield [buildAllPage(parts)]; + } +} diff --git a/packages/node/src/html-all/index.mjs b/packages/node/src/html-all/index.mjs new file mode 100644 index 00000000..e024a699 --- /dev/null +++ b/packages/node/src/html-all/index.mjs @@ -0,0 +1,26 @@ +'use strict'; + +import { generate } from './generate.mjs'; + +/** + * Adds `all.html`, every module page's content on one page, to the pages the + * `html` generator renders. It builds nothing itself: it passes `jsx-ast`'s + * pages on to `html`, and, once every module page has passed, a page made of + * theirs, which `html` assembles from their compiled content. + * + * @type {import('./types').Generator} + */ +export default { + name: 'html-all', + + description: + 'Adds all.html, every module page on one page, to the html generator', + + dependsOn: '@doc-kit/generator-react/jsx-ast', + + streamsInput: true, + + dependent: '@doc-kit/generator-react/html', + + generate, +}; diff --git a/packages/node/src/html-all/types.d.ts b/packages/node/src/html-all/types.d.ts new file mode 100644 index 00000000..7059190b --- /dev/null +++ b/packages/node/src/html-all/types.d.ts @@ -0,0 +1,8 @@ +import type { PageCode } from '@doc-kit/generator-react/jsx-ast/types'; + +type Pages = Array }>; + +export type Generator = GeneratorMetadata< + {}, + Generate, AsyncGenerator> +>; diff --git a/packages/react/package.json b/packages/react/package.json index d2eb7c32..07196351 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -15,6 +15,7 @@ "./html/ui/*": "./src/html/ui/*", "./html/bundlers/vite": "./src/html/bundlers/vite.mjs", "./jsx-ast": "./src/jsx-ast/index.mjs", + "./jsx-ast/utils/getSortedHeadNodes": "./src/jsx-ast/utils/getSortedHeadNodes.mjs", "./llms-txt": "./src/llms-txt/index.mjs", "./llms-txt-full": "./src/llms-txt-full/index.mjs", "./orama-db": "./src/orama-db/index.mjs", diff --git a/packages/react/src/html/README.md b/packages/react/src/html/README.md index caa8610f..2ebbf64c 100644 --- a/packages/react/src/html/README.md +++ b/packages/react/src/html/README.md @@ -6,12 +6,20 @@ assets, written to `output`. Vite is the default bundler adapter, but projects can supply an adapter for webpack or another bundler. The generator is output-only and does not return an in-memory copy of its HTML or CSS. -The site is built in pieces that are each as small as they can be. The bundler -builds the component library once and the client assets once. Each page's -program is then compiled — JSX to a plain module — and the worker pool imports, -renders, templates, minifies and writes the pages one at a time, so memory -scales with the largest page rather than with the site. `all.html` is assembled -from the module pages' compiled content rather than built again from scratch. +The site is built in pieces that are each as small as they can be, each as +soon as what it needs is there. Each page's program is compiled — JSX to a +plain module — as `jsx-ast` builds the page. The bundler builds the component +library once and the client assets once, as soon as every page's metadata is +known, and the worker pool imports, renders, templates, minifies and writes +the pages one at a time as they are compiled, so memory scales with the largest +page rather than with the site. With one thread, each step waits for the one +before it to finish. + +A page can be made of other pages' content: an input item with `parts`, the +`api`s of the pages it is made of, which come before it in the input, and its +`headings`. Its program imports theirs, so they are not built again, and it is +rendered first, as it takes the longest. Node.js's `html-all` generator +(`@node-core/doc-kit`) adds `all.html` this way. ## Configuring @@ -47,10 +55,6 @@ from the module pages' compiled content rather than built again from scratch. JSX-in-MDX. See [`components`](#components). **Default:** `{}`. - `navigation` {Object} Sidebar groups and navigation bar items. See [`navigation`](#navigation). **Default:** `{}`. -- `generateAllPage` {boolean} When `true`, writes `all.html`: every module - page's content on one page, in sidebar order, assembled from the module pages - rather than built again. Chunk pages and the index are left out. - **Default:** `true`. - `bundler` {WebBundler} Adapter that bundles the component library and the client assets, and compiles page programs. See [Bundler adapters](#bundler-adapters). **Default:** `createViteBundler()`. @@ -201,16 +205,21 @@ for a particular build system. `buildServer` receives `{ entry, virtualImports, outDir, config }`. The `entry` is the component library's source: re-exports of every component a page may render, Preact's `h` and `Fragment`, and `renderToStringAsync`. It must be -bundled into one self-contained module written under `outDir` (a temporary -directory the generator removes afterwards), since the page programs import it -from wherever they are compiled to. +bundled into one self-contained module written under `outDir` (an empty +temporary directory the generator removes afterwards), since the page programs +import it from wherever they are compiled to. `compile(code, fileName)` receives one page program: a module that imports the library and exports the page's `content` and a default function rendering the -page from its layout props, written in JSX. It must return plain JavaScript. The JSX must compile with the classic -runtime to calls of the `_jsx` and `_Fragment` bindings the program imports -(these names are exported as `JSX_PRAGMA` and `JSX_PRAGMA_FRAG` from the -generator's `constants.mjs`), so that the page and the library share one Preact. +page from its layout props, written in JSX. It must return plain JavaScript. +Pages are compiled as they are built, so `compile` can be called before +`buildServer`, and while `buildServer` or `buildClient` still runs: it must +leave the program's imports as they are, since the library they point to may +not exist yet, and must not depend on the builds. The JSX must compile with the +classic runtime to calls of the `_jsx` and `_Fragment` bindings the program +imports (these names are exported as `JSX_PRAGMA` and `JSX_PRAGMA_FRAG` from +the generator's `constants.mjs`), so that the page and the library share one +Preact. `buildClient` receives `{ entry, virtualImports, config }`. The client `entry` is a single program shared by every page. It must be bundled into @@ -474,7 +483,7 @@ export default ({ metadata }) => ( - `headings` {Array} Pre-computed table of contents heading entries. - `readingTime` {string|undefined} Estimated reading time (e.g. `'5 min read'`). Only present when the `jsx-ast` generator's `showReadingTime` option is - enabled. `all.html` has none. + enabled. A page made of others has none. - `children` {ComponentChildren} Processed page content. The `Layout` component receives the props above. Custom Layout components can use diff --git a/packages/react/src/html/__tests__/generate.test.mjs b/packages/react/src/html/__tests__/generate.test.mjs index a2198925..62d00fb7 100644 --- a/packages/react/src/html/__tests__/generate.test.mjs +++ b/packages/react/src/html/__tests__/generate.test.mjs @@ -3,8 +3,10 @@ import { mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; +import { setTimeout } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; +import { collectAsyncGenerator } from '@doc-kit/core/caching.mjs'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; @@ -12,10 +14,60 @@ import { jsx, toJs } from 'estree-util-to-js'; import buildContent from '../../jsx-ast/utils/buildContent.mjs'; import { buildNotFoundPage } from '../../jsx-ast/utils/synthetic/404.mjs'; -import { generate as chunk } from '../../section-pages/generate.mjs'; +import { generate as sectionPages } from '../../section-pages/generate.mjs'; import { compile, createViteBundler } from '../bundlers/vite.mjs'; import { FONTS } from '../constants.mjs'; -import { generate } from '../generate.mjs'; +import { generate as generatePages } from '../generate.mjs'; +import { processChunk } from '../utils/render.mjs'; + +/** + * Yields the given items as one batch, as a dependency streaming them does. + * + * @param {Array} items + */ +async function* streamInOneBatch(items) { + yield items; +} + +/** + * A page made of the given pages' content, in that order, as Node.js's + * `html-all` generator gives `all.html`. + * + * @param {Array<{ data: { api: string }, headings: Array }>} pages + */ +const createAllPage = pages => ({ + data: { + api: 'all', + path: '/all', + basename: 'all', + synthetic: true, + heading: { + type: 'heading', + depth: 1, + children: [{ type: 'text', value: 'All' }], + data: { name: 'All', text: 'All', slug: 'all' }, + }, + }, + headings: pages.flatMap(({ headings }) => headings), + parts: pages.map(({ data }) => data.api), +}); + +/** + * Runs the generator over the given pages, streamed in as one batch, as + * `jsx-ast` yields them. + * + * @param {Array} pages + */ +const generate = pages => generatePages(streamInOneBatch(pages)); + +/** + * Runs `section-pages` over the given entries, streamed in as one batch, and + * collects its pages. + * + * @param {Array} entries + */ +const chunk = entries => + collectAsyncGenerator(sectionPages(streamInOneBatch(entries))); // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( @@ -76,12 +128,104 @@ const createTestConfiguration = async (context, target = ['html']) => { }, }); - // The pages under test are the module pages themselves - config.html.generateAllPage = false; - return { config, output }; }; +/** + * A page as `jsx-ast` emits it, its content a plain paragraph. + * + * @param {string} api + * @param {string} text + */ +const createPage = (api, text) => ({ + data: createEntry(api, text), + headings: [{ depth: 1, value: text, slug: api }], + readingTime: undefined, + content: `<>

${text} body

`, +}); + +/** + * A stand-in for the component library: the JSX runtime, and a renderer that + * writes the elements out as plain markup. + */ +const STAND_IN_LIBRARY = [ + 'export const h = (type, props, ...children) => ({ type, props: { ...props, children } });', + 'export const Fragment = ({ children }) => children;', + 'export const Layout = ({ children }) => children;', + 'const render = node =>', + ' Array.isArray(node) ? node.map(render).join("")', + ' : typeof node?.type === "function" ? render(node.type(node.props))', + ' : typeof node?.type === "string" ? `<${node.type}>${render(node.props.children)}`', + ' : String(node ?? "");', + 'export const renderToStringAsync = async node => render(node);', +].join('\n'); + +/** + * A stand-in bundler: it builds the stand-in library, compiles the pages + * with the real compiler, and has no client assets. It records each program + * it compiles, by file name, and each bundle it builds. + * + * @param {Array} calls + * @param {{ bundled?: PromiseWithResolvers, slow?: boolean }} [options] + * `bundled` is resolved once both bundles are built; `slow` makes the + * library take a while, as a real one does, so that a generator that didn't + * wait for it would remove its directory while it is being built. + */ +const createStandInBundler = (calls, { bundled, slow = false } = {}) => ({ + async buildServer({ outDir }) { + calls.push('server'); + + if (slow) { + await setTimeout(50); + } + + const library = join(outDir, 'library.mjs'); + + await writeFile(library, STAND_IN_LIBRARY); + + return pathToFileURL(library).href; + }, + + compile(code, fileName) { + calls.push(fileName); + + return compile(code, fileName); + }, + + async buildClient() { + calls.push('client'); + bundled?.resolve(); + + return { scripts: [], preloads: [], stylesheets: [] }; + }, +}); + +/** + * The temporary directories the generator builds the site in. + */ +const listBuildDirectories = async () => { + const names = await readdir(tmpdir()); + + return names.filter(name => name.startsWith('doc-kit-html-')); +}; + +/** + * Every page written to the output directory, by file name. + * + * @param {string} output + */ +const readPages = async output => { + const names = await readdir(output, { recursive: true }); + const pages = names.filter(name => name.endsWith('.html')).sort(); + const contents = await Promise.all( + pages.map(name => readFile(join(output, name), 'utf8')) + ); + + return Object.fromEntries( + pages.map((name, index) => [name, contents[index]]) + ); +}; + describe('web generate', () => { it('writes bundled HTML and omits View As links for synthetic pages', async context => { const { config, output } = await createTestConfiguration(context); @@ -127,9 +271,8 @@ describe('web generate', () => { } }); - it('assembles all.html from the module pages, in sidebar order', async context => { - const { config, output } = await createTestConfiguration(context); - config.html.generateAllPage = true; + it('assembles a page made of other pages from their content', async context => { + const { output } = await createTestConfiguration(context); const entries = [ createEntry('zlib', 'Zlib'), @@ -137,16 +280,18 @@ describe('web generate', () => { createEntry('index', 'Index'), ]; - await generate( - await Promise.all( - entries.map(entry => buildContent([entry], entry)) - ).then(contents => contents.map(toPage)) + const contents = await Promise.all( + entries.map(entry => buildContent([entry], entry)) ); + const [zlib, fs, index] = contents.map(toPage); + + await generate([zlib, fs, index, createAllPage([fs, zlib])]); + const html = await readFile(join(output, 'all.html'), 'utf8'); assert.match(html, /All \|/); - // Both modules' content, file system first, the index left out + // Its parts' content, in its order, and nothing else assert.match(html, /File system body[\s\S]*Zlib body/); assert.doesNotMatch(html, /Index body/); // Their tables of contents, concatenated @@ -154,6 +299,38 @@ describe('web generate', () => { assert.doesNotMatch(html, /View As/); }); + it('numbers the ids of a page made of others across its parts', async context => { + const { output } = await createTestConfiguration(context); + + // Each module has code tabs, whose elements take ids + const entries = ['zlib', 'fs'].map(api => { + const entry = createEntry(api, api); + + entry.content.children.push( + { type: 'code', lang: 'mjs', value: `import ${api};` }, + { type: 'code', lang: 'cjs', value: `require('${api}');` } + ); + + return entry; + }); + + const contents = await Promise.all( + entries.map(entry => buildContent([entry], entry)) + ); + + const pages = contents.map(toPage); + + await generate([...pages, createAllPage(pages)]); + + const html = await readFile(join(output, 'all.html'), 'utf8'); + const ids = [...html.matchAll(/ id="([^"]*P0-\d+[^"]*)"/g)].map( + ([, id]) => id + ); + + assert.ok(ids.length >= 4); + assert.equal(new Set(ids).size, ids.length); + }); + it('renders chunk pages with navigation back to their module', async context => { const { config, output } = await createTestConfiguration(context, [ 'section-pages', @@ -380,6 +557,216 @@ describe('web generate', () => { html, /<link rel="stylesheet" crossorigin href="\.\/custom\/index\.css" \/>/ ); - assert.deepStrictEqual(calls, ['server', 'client', 'compile']); + // The page is compiled as it arrives, before the library is built + assert.deepStrictEqual(calls, ['compile', 'server', 'client']); + }); + + describe('with the manifest', () => { + const zlib = createPage('zlib', 'Zlib'); + const fs = createPage('fs', 'File system'); + const index = createPage('index', 'Index'); + const all = createAllPage([fs, zlib]); + const manifest = [zlib, fs, index, all].map(({ data }) => data); + + it( + 'bundles once the manifest arrives, and compiles each page as it arrives', + // The input waits for the bundles, so bundles that wait for the whole + // input would never be built + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const calls = []; + const bundled = Promise.withResolvers(); + + config.threads = 2; + config.html.bundler = createStandInBundler(calls, { bundled }); + + async function* input() { + yield [zlib]; + yield [{ manifest }]; + + // The bundles are built without waiting for the pages still to come + await bundled.promise; + + yield [fs, all]; + yield [index]; + } + + await generatePages(input()); + + // The page made of others comes right after its last part + assert.deepStrictEqual(calls, [ + 'zlib.jsx', + 'server', + 'client', + 'fs.jsx', + 'all.jsx', + 'index.jsx', + ]); + + const html = await readFile(join(output, 'all.html'), 'utf8'); + + // In its order, the index left out + assert.match(html, /File system body[\s\S]*Zlib body/); + assert.doesNotMatch(html, /Index body/); + } + ); + + it('builds the same site from its whole input, as it gets it with one thread', async context => { + /** + * Builds the site from the given input and reads its pages. + * + * @param {number} threads + * @param {Array<object> | AsyncIterable<Array<object>>} input + */ + const build = async (threads, input) => { + const { config, output } = await createTestConfiguration(context); + + config.threads = threads; + config.html.bundler = createStandInBundler([]); + + await generatePages(input); + + return readPages(output); + }; + + async function* batches() { + yield [zlib]; + yield [{ manifest }]; + yield [fs, all]; + yield [index]; + } + + const streamed = await build(2, batches()); + const whole = await build(1, [zlib, fs, all, { manifest }, index]); + + assert.deepStrictEqual(Object.keys(whole), [ + 'all.html', + 'fs.html', + 'index.html', + 'zlib.html', + ]); + assert.deepStrictEqual(whole, streamed); + }); + + it( + 'rejects when its input fails before the manifest, leaving no temporary files', + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const before = await listBuildDirectories(); + + config.threads = 2; + config.html.bundler = createStandInBundler([]); + + async function* input() { + yield [zlib]; + + throw new Error('jsx-ast failed'); + } + + await assert.rejects(generatePages(input()), /jsx-ast failed/); + + const after = await listBuildDirectories(); + const pages = await readPages(output); + + assert.deepStrictEqual( + after.filter(name => !before.includes(name)), + [] + ); + assert.deepStrictEqual(pages, {}); + } + ); + + it( + 'writes no page once its input has failed after the manifest, and leaves no temporary files', + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const before = await listBuildDirectories(); + const calls = []; + + config.threads = 2; + // Still bundling when the input fails + config.html.bundler = createStandInBundler(calls, { slow: true }); + + async function* input() { + yield [zlib]; + yield [{ manifest }]; + + throw new Error('jsx-ast failed'); + } + + await assert.rejects(generatePages(input()), /jsx-ast failed/); + + const after = await listBuildDirectories(); + const pages = await readPages(output); + + // It waited for the bundles before removing the directory they are + // written to + assert.deepStrictEqual(calls, ['zlib.jsx', 'server', 'client']); + assert.deepStrictEqual( + after.filter(name => !before.includes(name)), + [] + ); + assert.deepStrictEqual(pages, {}); + } + ); + + it( + 'writes none of the pages still waiting for a worker once its input has failed', + // The worker waits for the input to fail + { timeout: 10_000 }, + async context => { + const { config, output } = await createTestConfiguration(context); + const rendering = Promise.withResolvers(); + const failed = Promise.withResolvers(); + + config.threads = 2; + config.html.bundler = createStandInBundler([]); + + // Renders on this thread, a batch at a time, as the generator does + // without a pool, but the first only once the input has failed + const worker = { + async *stream(batches, extra) { + for await (const tasks of batches) { + rendering.resolve(); + + await failed.promise; + + const written = await processChunk( + tasks, + [...tasks.keys()], + extra + ); + + yield written; + } + }, + }; + + async function* input() { + yield [zlib]; + yield [{ manifest }]; + + // While the zlib page is being rendered, so this one waits for it + await rendering.promise; + + yield [fs]; + + // The failure reaches the generator within this turn of the event + // loop + setImmediate(failed.resolve); + + throw new Error('jsx-ast failed'); + } + + await assert.rejects(generatePages(input(), worker), /jsx-ast failed/); + + const pages = await readPages(output); + + assert.deepStrictEqual(Object.keys(pages), ['zlib.html']); + } + ); }); }); diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index 83f7ba23..04a89fe3 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -1,15 +1,18 @@ 'use strict'; +import { rmSync } from 'node:fs'; import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import logger from '@doc-kit/core/logger/index.mjs'; +import { createChannel } from '@doc-kit/core/utils/channel.mjs'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; +import { toBatches } from '@doc-kit/core/utils/generators.mjs'; +import { headingSortFn } from '../jsx-ast/utils/getSortedHeadNodes.mjs'; import { resolveBundler } from './bundlers/index.mjs'; -import { buildAllPage } from './utils/all.mjs'; import { copyStaticAssets } from './utils/copying.mjs'; import createProgramBuilder, { moduleFileName } from './utils/generate.mjs'; import { createPageWriter } from './utils/render.mjs'; @@ -19,18 +22,14 @@ const htmlLogger = logger.child('html'); /** * Main generation function: turns the pages' JSX into the static site. * - * Receives `jsx-ast`'s output as `{ data, headings, readingTime, content }` - * items, `content` being each page's JSX code. The site is then built in - * pieces that are each as small as they can be: - * - * 1. The component library is bundled once, for the server. - * 2. The client assets are bundled once; every page loads the same ones. - * 3. Each page's program is compiled (JSX to a plain module) and written to a - * temporary directory, one at a time, so no page is held longer than that. - * 4. `all.html`, when enabled, is a program that imports the module pages' - * content, so it is compiled from what was already compiled. - * 5. The worker pool imports, renders, templates, minifies and writes the - * pages, one page in memory per worker. + * Receives `jsx-ast`'s pages as it builds them, `content` being each page's JSX + * code, and a `{ manifest }` of every page's head once it knows them all. Each + * page's program is compiled to a plain module as it arrives, the component + * library and the client assets are bundled once every page's metadata is + * known, and the worker pool renders the pages as they are compiled, one in + * memory per worker. A page made of other pages' content (with `parts`, which + * come before it) imports their compiled programs, and is rendered first. With + * one thread, the input arrives whole. * * @type {import('./types').Generator['generate']} */ @@ -43,18 +42,23 @@ export async function generate(input, worker) { // page is written; the directory is removed afterwards const outDir = await mkdtemp(join(tmpdir(), 'doc-kit-html-')); + /** + * Removes the directory if another generator's failure ends the process + * before this one returns. + */ + const removeOnExit = () => rmSync(outDir, { recursive: true, force: true }); + + process.once('exit', removeOnExit); + try { - const { assets, tasks } = await buildPrograms({ - pages: [...input], - outDir, - config, - }); + const building = buildSite(input, { worker, outDir, config, template }); - // The pages' programs are compiled, so rendering doesn't need their code input = undefined; - await createPageWriter(worker)(tasks, { template, assets }); + await building; } finally { + process.off('exit', removeOnExit); + await rm(outDir, { recursive: true, force: true }); } @@ -62,24 +66,23 @@ export async function generate(input, worker) { } /** - * Bundles the component library and the client assets, and compiles every - * page's program into `outDir`, for the worker pool to render. - * - * Everything it sets up for that, the bundler included, is let go once it - * returns: with one thread, the pages are rendered on this thread next. + * Compiles every page's program into `outDir` as the pages arrive, bundles the + * component library and the client assets as soon as every page's metadata is + * known, and has the worker pool render the pages as they are compiled. * + * @param {Parameters<import('./types').Generator['generate']>[0]} input - The pages and the manifest, as they arrive, or all at once with one thread * @param {object} options - * @param {Array<import('./types').Page>} options.pages - Every page, in render order + * @param {ParallelWorker} [options.worker] - Renders the pages; without it, they are rendered on this thread * @param {string} options.outDir - Where the library and the programs are written * @param {import('./types').ResolvedWebConfiguration} options.config - * @returns {Promise<{ assets: import('./types').ClientAssets, tasks: Array<import('./types').PageTask> }>} + * @param {string} options.template - The page template + * @returns {Promise<void>} */ -const buildPrograms = async ({ pages, outDir, config }) => { - const all = config.generateAllPage ? buildAllPage(pages) : undefined; - - // Every page's metadata, in render order — the sidebar, the index and the - // cross links need the whole set. - const datas = [...pages, ...(all ? [all] : [])].map(({ data }) => data); +const buildSite = async (input, options) => { + // Destructured here rather than in the parameters: with a destructuring + // parameter, the arguments keep their own copy of `input`, which letting go + // of `input` below would not let go of + const { worker, outDir, config, template } = options; // Loaded here rather than with the generator, so the threads rendering // pages, which load this module too, never load what only this needs @@ -89,26 +92,13 @@ const buildPrograms = async ({ pages, outDir, config }) => { const { buildLibraryProgram, buildPageProgram, clientProgram } = createProgramBuilder(); - const libraryURL = await bundler.buildServer({ - entry: buildLibraryProgram(), - virtualImports: createVirtualImports(datas, config.virtualImports, true), - outDir, - config, - }); - - htmlLogger.debug('Built the component library'); - - const assets = await bundler.buildClient({ - entry: clientProgram, - virtualImports: createVirtualImports(datas, config.virtualImports, false), - config, - }); - - htmlLogger.debug('Built the client assets', assets); - const modulesDir = join(outDir, 'pages'); await mkdir(modulesDir); + // Pages are compiled before the library is built, so they import it through + // this module, which re-exports it once it is + const libraryURL = pathToFileURL(join(outDir, 'page-library.mjs')).href; + /** * Compiles a page's program to disk and describes it for the workers. * @@ -136,25 +126,141 @@ const buildPrograms = async ({ pages, outDir, config }) => { }; }; - const tasks = []; + /** + * Bundles the component library and the client assets. + * + * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} heads - Every page's metadata, in the order the pages arrived + * @returns {Promise<import('./types').ClientAssets>} + */ + const bundle = async heads => { + // Every page's metadata, in the order the site lists the pages — the + // sidebar, the index and the cross links need the whole set + const datas = [ + ...heads.filter(data => !data.synthetic).toSorted(headingSortFn), + ...heads.filter(data => data.synthetic), + ]; - for (const page of pages) { - tasks.push(await compile(page)); - } + // An empty directory of its own: the compiled pages are already in `outDir` + const libraryDir = join(outDir, 'library'); + await mkdir(libraryDir); + + const builtLibraryURL = await bundler.buildServer({ + entry: buildLibraryProgram(), + virtualImports: createVirtualImports(datas, config.virtualImports, true), + outDir: libraryDir, + config, + }); - // The composed page imports the other pages' compiled programs, which - // exist from here on, so it is rendered by the worker pool alongside them. - // Rendering it on this thread instead would hold the whole site in it and - // block the pool from shutting down its idle workers until it is done. - // It goes first: it takes by far the longest, so the other pages are - // rendered while it is. - if (all) { - const allTask = await compile(all); + await writeFile( + fileURLToPath(libraryURL), + `export * from ${JSON.stringify(builtLibraryURL)};\n` + ); - tasks.unshift(allTask); - } + htmlLogger.debug('Built the component library'); + + const assets = await bundler.buildClient({ + entry: clientProgram, + virtualImports: createVirtualImports(datas, config.virtualImports, false), + config, + }); + + htmlLogger.debug('Built the client assets', assets); + + return assets; + }; + + // The compiled pages, rendered once the bundles are built + const rendering = createChannel(); + + let writing; - htmlLogger.debug(`Compiled ${tasks.length} page programs`); + /** + * Starts bundling, then rendering the compiled pages. + * + * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} heads - Every page's metadata + */ + const startWriting = heads => { + writing = bundle(heads).then(assets => + createPageWriter(worker)(rendering, { template, assets }) + ); + + // Thrown once every compiled page has been handed over + writing.catch(() => {}); + }; + + const compiled = []; + + // With one thread, pages are rendered on this thread once all are compiled, + // so the manifest is kept until then + let manifest; + + const alongside = getConfig().threads > 1; + + /** + * Compiles each page as it arrives, and hands it over. A function of its + * own, so that once it returns nothing holds the pages' code. + * + * @param {typeof input} pages + */ + const compilePages = async pages => { + for await (const items of toBatches(pages)) { + const tasks = []; + + for (const item of items) { + if ('manifest' in item) { + if (alongside) { + startWriting(item.manifest); + } else { + manifest = item.manifest; + } + + continue; + } + + const task = await compile(item); + + compiled.push(task); + + // A page made of others' content takes by far the longest to render + if ('parts' in item) { + rendering.prepend([task]); + } else { + tasks.push(task); + } + } + + if (tasks.length > 0) { + rendering.push(tasks); + } + } + }; + + try { + const compiling = compilePages(input); + + input = undefined; + + await compiling; + + // With one thread, or without a manifest + if (!writing) { + startWriting(manifest ?? compiled.map(({ data }) => data)); + + manifest = undefined; + } + + htmlLogger.debug(`Compiled ${compiled.length} page programs`); + } catch (error) { + // Pages still waiting for a worker aren't written + rendering.clear(); + + throw error; + } finally { + rendering.end(); + + // The bundler and the workers may still use `outDir`, removed after this + await writing?.catch(() => {}); + } - return { assets, tasks }; + await writing; }; diff --git a/packages/react/src/html/index.mjs b/packages/react/src/html/index.mjs index 607592a3..7c469a49 100644 --- a/packages/react/src/html/index.mjs +++ b/packages/react/src/html/index.mjs @@ -13,11 +13,13 @@ import { processChunk } from './utils/render.mjs'; * - Client-side JavaScript with code splitting * - Bundled CSS styles * - * The configured bundler builds the component library and the client assets - * once each; the pages are then compiled, rendered, templated, minified and - * written one at a time by the worker pool, so memory scales with the largest - * page rather than with the site. `all.html` is assembled from the module - * pages' compiled content instead of being built again from scratch. + * Each page is compiled as `jsx-ast` builds it. Once every page's metadata is + * known, the configured bundler builds the component library and the client + * assets once each, and the worker pool renders, templates, minifies and writes + * the pages one at a time as they are compiled, so memory scales with the + * largest page rather than with the site. A page made of other pages' content, + * such as Node.js's `all.html` (see its `html-all` generator), is assembled from + * their compiled content instead of being built again. * * This terminal generator writes to the output directory and does not return * an in-memory copy. @@ -31,6 +33,8 @@ export default { dependsOn: '@doc-kit/generator-react/jsx-ast', + streamsInput: true, + /** * @param {import('@doc-kit/core/utils/configuration/types').Configuration} config */ @@ -82,10 +86,6 @@ export default { showCrossLinks: false, }, - // Whether to write `all.html`: every module page's content on one page, - // assembled from the module pages rather than built again. - generateAllPage: true, - // When omitted, the Vite adapter is loaded lazily during generation. bundler: undefined, }), diff --git a/packages/react/src/html/types.d.ts b/packages/react/src/html/types.d.ts index 5bfb7251..87634a2f 100644 --- a/packages/react/src/html/types.d.ts +++ b/packages/react/src/html/types.d.ts @@ -1,3 +1,4 @@ +import type { MetadataEntry } from '@doc-kit/core/generators/metadata/types'; import type { GlobalConfiguration } from '@doc-kit/core/utils/configuration/types'; import type NavBar from '@node-core/ui-components/Containers/NavBar'; import type SideBar from '@node-core/ui-components/Containers/Sidebar'; @@ -31,8 +32,8 @@ export type HeadConfig = { export type ResolvedWebConfiguration = Configuration & GlobalConfiguration; -// A page assembled from other pages' content (`all.html`): `parts` lists the -// `api`s whose compiled programs it imports, in order. +// A page assembled from other pages' content: `parts` lists the `api`s whose +// compiled programs it imports, in order. They come before it in the input. export type ComposedPage = Omit<PageCode, 'content' | 'readingTime'> & { parts: Array<string>; }; @@ -114,14 +115,16 @@ export type Configuration = { navbar?: ComponentProps<typeof NavBar>['navItems']; showCrossLinks?: boolean; }; - // Whether to write `all.html`, every module page's content on one page. - generateAllPage: boolean; // Optional bundler adapter. When omitted, the Vite adapter is loaded lazily. bundler?: WebBundler; }; export type Generator = GeneratorMetadata< Configuration, - Generate<Array<PageCode>, Promise<void>>, + Generate< + | Array<PageCode | { manifest: Array<MetadataEntry> }> + | AsyncIterable<Array<PageCode | { manifest: Array<MetadataEntry> }>>, + Promise<void> + >, ProcessChunk<PageTask, string, { template: string; assets: ClientAssets }> >; diff --git a/packages/react/src/html/utils/__tests__/all.test.mjs b/packages/react/src/html/utils/__tests__/all.test.mjs deleted file mode 100644 index 7c0f23ab..00000000 --- a/packages/react/src/html/utils/__tests__/all.test.mjs +++ /dev/null @@ -1,43 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; - -import { buildAllPage } from '../all.mjs'; - -const createPage = (api, name, { depth = 1, chunk, synthetic } = {}) => ({ - data: { - api, - path: `/${api}`, - basename: api, - chunk, - synthetic, - heading: { depth, data: { name, text: name, slug: api } }, - }, - headings: [{ depth, value: name, slug: api }], - readingTime: '1 min read', - content: `<><h1>${name}</h1></>`, -}); - -describe('buildAllPage', () => { - it('returns a synthetic `all` page made of the module pages, in sidebar order', () => { - const page = buildAllPage([ - createPage('zlib', 'Zlib'), - createPage('index', 'Index'), - createPage('fs', 'File system'), - createPage('404', 'Page Not Found', { synthetic: true }), - createPage('fs-readfile', 'readFile', { chunk: { api: 'fs' } }), - ]); - - assert.equal(page.data.api, 'all'); - assert.equal(page.data.path, '/all'); - assert.equal(page.data.heading.data.name, 'All'); - assert.equal(page.data.synthetic, true); - // The index, the other synthetic pages and the chunk pages are left out - assert.deepEqual(page.parts, ['fs', 'zlib']); - assert.deepEqual( - page.headings.map(({ value }) => value), - ['File system', 'Zlib'] - ); - // A reading time makes no sense for the whole reference - assert.equal('readingTime' in page, false); - }); -}); diff --git a/packages/react/src/html/utils/__tests__/generate.test.mjs b/packages/react/src/html/utils/__tests__/generate.test.mjs new file mode 100644 index 00000000..65834f9c --- /dev/null +++ b/packages/react/src/html/utils/__tests__/generate.test.mjs @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; +import { pathToFileURL } from 'node:url'; + +import { setConfig } from '@doc-kit/core/utils/configuration/index.mjs'; + +import { compile } from '../../bundlers/vite.mjs'; +import createProgramBuilder, { moduleFileName } from '../generate.mjs'; + +describe('buildPageProgram', () => { + it('imports no component into a composed page, whatever the components are named', async context => { + const config = await setConfig({ + target: ['html'], + version: 'v22.0.0', + changelog: [], + generators: { html: {} }, + }); + + // Named like the composed page's own wrapper + config.html.components = { ComposedPart: '/components/ComposedPart.jsx' }; + + const dir = await mkdtemp(join(tmpdir(), 'doc-kit-program-test-')); + context.after(() => rm(dir, { recursive: true, force: true })); + + // A stand-in for the built library, which exports every component, and + // the part the page is made of + const library = join(dir, 'library.mjs'); + + await writeFile( + library, + [ + 'export const h = (type, props, ...children) => ({ type, props, children });', + 'export const Fragment = "Fragment";', + 'export const Layout = "Layout";', + 'export const renderToStringAsync = async () => "";', + 'export const ComposedPart = () => null;', + ].join('\n') + ); + + await writeFile( + join(dir, moduleFileName('fs')), + 'export const content = () => "File system";' + ); + + const { buildPageProgram } = createProgramBuilder(); + const libraryURL = pathToFileURL(library).href; + const program = buildPageProgram( + { data: { api: 'all' }, headings: [], parts: ['fs'] }, + libraryURL + ); + + const imports = program + .split('\n') + .find(line => line.includes(JSON.stringify(libraryURL))); + + assert.doesNotMatch(imports, /ComposedPart/); + + const file = join(dir, moduleFileName('all')); + const code = await compile(program, 'all.jsx'); + + await writeFile(file, code); + + const { content } = await import(pathToFileURL(file).href); + + assert.equal(typeof content, 'function'); + }); +}); diff --git a/packages/react/src/html/utils/all.mjs b/packages/react/src/html/utils/all.mjs deleted file mode 100644 index 0c254f0f..00000000 --- a/packages/react/src/html/utils/all.mjs +++ /dev/null @@ -1,38 +0,0 @@ -'use strict'; - -import { getSortedHeadNodes } from '../../jsx-ast/utils/getSortedHeadNodes.mjs'; -import { createSyntheticHead } from '../../jsx-ast/utils/synthetic/synthetic.mjs'; - -/** - * Whether a page's content belongs on `all.html`: the module pages, minus the - * index. Chunk pages and the other synthetic pages are left out, since the full - * module pages already carry their content. - * - * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} data - */ -const isModulePage = data => - !data.synthetic && !data.chunk && data.api !== 'index'; - -/** - * Builds the `all.html` page from the module pages already generated. - * - * Its content is exactly the module pages' content, one after the other, so - * nothing is rebuilt: the page program imports each module's `content` export - * (see `buildPageProgram`) and the table of contents is the modules' own, - * concatenated. Modules are ordered as the sidebar lists them. - * - * @param {Array<import('../../jsx-ast/types').PageCode>} pages - Every generated page - * @returns {import('../types').ComposedPage} - */ -export const buildAllPage = pages => { - const byApi = new Map(pages.map(page => [page.data.api, page])); - - const modules = pages.map(({ data }) => data).filter(isModulePage); - const parts = getSortedHeadNodes(modules).map(({ api }) => byApi.get(api)); - - return { - data: createSyntheticHead('all', 'All'), - headings: parts.flatMap(({ headings }) => headings), - parts: parts.map(({ data }) => data.api), - }; -}; diff --git a/packages/react/src/html/utils/generate.mjs b/packages/react/src/html/utils/generate.mjs index c8b0def5..17d22040 100644 --- a/packages/react/src/html/utils/generate.mjs +++ b/packages/react/src/html/utils/generate.mjs @@ -114,34 +114,48 @@ export default () => { * metadata, table of contents and reading time reach it as those props at * render time — only code. * - * A composed page (`all.html`) has no content of its own: it imports the - * `content` of the pages it is made of, so their JSX is never rebuilt. + * A composed page has no content of its own: it imports the + * `content` of the pages it is made of, so their JSX is never rebuilt. Each + * part's elements are only created as it is rendered, so they can go once + * it is: an element rendering a single element doesn't hold on to it, as + * one rendering a fragment does to its children. * * @param {import('../types').Page} page - * @param {string} libraryURL - Where the built component library was written. + * @param {string} libraryURL - The `file:` URL the page imports the component library from. * @returns {string} The program, as JSX. */ const buildPageProgram = (page, libraryURL) => { - const { imports, content } = + const { imports, declarations, content } = 'parts' in page ? { imports: page.parts.map( (api, index) => `import { content as content${index} } from ${quote(`./${moduleFileName(api)}`)};` ), - content: `<>${page.parts.map((_, index) => `{content${index}()}`).join('')}</>`, + declarations: [ + 'const ComposedPartContent = ({ part }) => part();', + 'const ComposedPart = ({ part }) => <ComposedPartContent part={part} />;', + ], + content: `<>${page.parts.map((_, index) => `<ComposedPart part={content${index}} />`).join('')}</>`, } - : { imports: [], content: page.content }; + : { imports: [], declarations: [], content: page.content }; - // Import only the components the page actually renders - const used = componentImports - .filter(({ name }) => content.includes(`<${name}`)) - .map(({ name }) => name) - .filter(name => name !== JSX_IMPORTS.Layout.name); + // Import only the components the page actually renders. A composed page + // renders none of its own (its parts import theirs), and scanning it would + // take its wrappers for configured components of the same name + let used = []; + + if (!('parts' in page)) { + used = componentImports + .filter(({ name }) => content.includes(`<${name}`)) + .map(({ name }) => name) + .filter(name => name !== JSX_IMPORTS.Layout.name); + } return [ `import { ${[...RUNTIME_IMPORTS, ...used].join(', ')} } from ${quote(libraryURL)};`, ...imports, + ...declarations, `export const content = () => ${content};`, `export default props => renderToStringAsync(<${JSX_IMPORTS.Layout.name} {...props}>{content()}</${JSX_IMPORTS.Layout.name}>);`, ].join('\n'); diff --git a/packages/react/src/html/utils/render.mjs b/packages/react/src/html/utils/render.mjs index 4758e9c6..25787ed7 100644 --- a/packages/react/src/html/utils/render.mjs +++ b/packages/react/src/html/utils/render.mjs @@ -74,24 +74,39 @@ export const processChunk = async (tasks, indices, { template, assets }) => { }; /** - * Creates the page writer: it spreads the pages across the worker pool and - * waits for every one to be written. Without a pool — the generator was called - * directly rather than by the orchestrator — the pages are rendered on the - * calling thread instead. + * Renders each batch of pages on the calling thread, as it arrives. + * + * @param {AsyncIterable<Array<import('../types').PageTask>>} batches + * @param {{ template: string, assets: import('../types').ClientAssets }} extra + * @yields {Array<string>} Each batch's written file names + */ +async function* renderOnThisThread(batches, extra) { + for await (const tasks of batches) { + const written = await processChunk(tasks, [...tasks.keys()], extra); + + yield written; + } +} + +/** + * Creates the page writer: it spreads the pages across the worker pool as they + * arrive, in batches, and waits for every one to be written. Without a pool — + * the generator was called directly rather than by the orchestrator — the + * pages are rendered on the calling thread instead. * * @param {ParallelWorker} [worker] - * @returns {(tasks: Array<import('../types').PageTask>, extra: { template: string, assets: import('../types').ClientAssets }) => Promise<void>} + * @returns {(batches: AsyncIterable<Array<import('../types').PageTask>>, extra: { template: string, assets: import('../types').ClientAssets }) => Promise<void>} */ -export const createPageWriter = worker => async (tasks, extra) => { +export const createPageWriter = worker => async (batches, extra) => { const chunks = worker - ? worker.stream(tasks, extra) - : [await processChunk(tasks, [...tasks.keys()], extra)]; + ? worker.stream(batches, extra) + : renderOnThisThread(batches, extra); let count = 0; for await (const chunk of chunks) { count += chunk.length; - renderLogger.debug(`Wrote ${count}/${tasks.length} pages`); + renderLogger.debug(`Wrote ${count} pages`); } }; diff --git a/packages/react/src/jsx-ast/README.md b/packages/react/src/jsx-ast/README.md index aab9b9f8..64424f0a 100644 --- a/packages/react/src/jsx-ast/README.md +++ b/packages/react/src/jsx-ast/README.md @@ -19,8 +19,20 @@ Each page is emitted as `{ data, headings, readingTime, content }`: the page's head entry, its table of contents, the optional reading time, and the processed content serialized to JSX code as one fragment. The page layout is not part of the content — the `html` generator wraps each page in `<Layout>`, and assembles -`all.html` from the module pages' content (see its `generateAllPage` option) -rather than building every module a second time here. +pages made of other pages' content, such as Node.js's `all.html`, rather than +building them a second time here. + +Pages are emitted as they are built, in the order their entries arrived, the +synthetic pages last. Once every page has arrived, it also emits +`{ manifest }`, every page's head in that same order, so that the `html` +generator can build what needs the whole site while pages are still being +built. + +To list the pages as the site does, leave out the `{ manifest }` item and sort +the pages by their head with `headingSortFn`, from +`@doc-kit/generator-react/jsx-ast/utils/getSortedHeadNodes`, as +`pages.toSorted((a, b) => headingSortFn(a.data, b.data))`, keeping the +synthetic pages (`data.synthetic`) last. ## Index page diff --git a/packages/react/src/jsx-ast/__tests__/generate.test.mjs b/packages/react/src/jsx-ast/__tests__/generate.test.mjs index d0324af4..51a0ecb9 100644 --- a/packages/react/src/jsx-ast/__tests__/generate.test.mjs +++ b/packages/react/src/jsx-ast/__tests__/generate.test.mjs @@ -1,5 +1,7 @@ import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; import { loadGenerator } from '@doc-kit/core/generators/loader.mjs'; import getConfig, { @@ -9,6 +11,11 @@ import { loadMarkdownPlugins } from '@doc-kit/core/utils/markdown/plugins.mjs'; import { generate, processChunk } from '../generate.mjs'; +setFlagsFromString('--expose-gc'); + +// Collects garbage, to tell what is let go +const gc = runInNewContext('gc'); + // Pages are rendered with the pipeline of `jsx-ast` await loadMarkdownPlugins( await loadGenerator(import.meta.resolve('../index.mjs')) @@ -49,23 +56,54 @@ const createEntry = (api, name, { stabilityIndex = '2' } = {}) => { }; }; -const collect = async generator => { - const results = []; +/** + * Reads everything the generator emits, keeping the pages apart from the + * manifests. + * + * @param {AsyncIterable<Array<object>>} output + * @returns {Promise<{ pages: Array<object>, manifests: Array<Array<object>> }>} + */ +const collect = async output => { + const pages = []; + const manifests = []; - for await (const chunk of generator) { - results.push(...chunk); + for await (const items of output) { + for (const item of items) { + if ('manifest' in item) { + manifests.push(item.manifest); + } else { + pages.push(item); + } + } } - return results; + return { pages, manifests }; }; +/** + * Stands in for the worker pool: builds each page from its head alone, as its + * batch arrives, and records every item it is given. + * + * @param {Array<object>} seenItems - Where the items it is given go + */ const createWorker = seenItems => ({ - async *stream(items) { - seenItems.push(...items); - yield items.map(({ head }) => ({ type: 'JSXElement', data: head })); + async *stream(batches) { + for await (const items of batches) { + seenItems.push(...items); + yield items.map(({ head }) => ({ type: 'JSXElement', data: head })); + } }, }); +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array<Array<object>>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + describe('jsx-ast generate', () => { it('returns the page content as a JSX fragment alongside its ToC', async () => { await setConfig({ target: ['jsx-ast'] }); @@ -92,9 +130,11 @@ describe('jsx-ast generate', () => { getConfig('jsx-ast').generateNotFoundPage = false; const seenItems = []; - const results = await collect( + const { pages, manifests } = await collect( generate( - [createEntry('index', 'Index'), createEntry('fs', 'File system')], + streamOf([ + [createEntry('index', 'Index'), createEntry('fs', 'File system')], + ]), createWorker(seenItems) ) ); @@ -104,11 +144,181 @@ describe('jsx-ast generate', () => { ['index', 'fs'] ); assert.deepEqual( - results.map(({ data }) => data.api), + pages.map(({ data }) => data.api), ['index', 'fs'] ); + + // Without the 404 page, the manifest lists only the input's pages + assert.deepEqual( + manifests.map(heads => heads.map(({ api }) => api)), + [['index', 'fs']] + ); + }); + + it('starts pages as they arrive and emits them as they are built', async () => { + await setConfig({ target: ['jsx-ast'] }); + + const seenItems = []; + const { pages, manifests } = await collect( + generate( + streamOf([ + [createEntry('fs', 'File system')], + [createEntry('buffer', 'Buffer'), createEntry('index', 'Index')], + ]), + createWorker(seenItems) + ) + ); + + // Built in the order they arrived, the 404 page last + assert.deepEqual( + seenItems.map(({ head }) => head.api), + ['fs', 'buffer', 'index', '404'] + ); + + // Emitted as built: the html generator puts them in the site's order + assert.deepEqual( + pages.map(({ data }) => data.api), + ['fs', 'buffer', 'index', '404'] + ); + + // Once every page has arrived, every page's head, in that order + assert.deepEqual( + manifests.map(heads => heads.map(({ api }) => api)), + [['fs', 'buffer', 'index', '404']] + ); + }); + + it('builds the same pages from its whole input, as it gets it with one thread', async () => { + await setConfig({ target: ['jsx-ast'] }); + + const entries = () => [ + createEntry('fs', 'File system'), + createEntry('buffer', 'Buffer'), + ]; + + const streamed = await collect( + generate(streamOf([entries()]), createWorker([])) + ); + const whole = await collect(generate(entries(), createWorker([]))); + + assert.deepEqual(whole, streamed); + + // The manifest still comes with them + assert.equal(whole.manifests.length, 1); }); + it( + 'emits the manifest without waiting for the pages still being built', + // The first page is built only once the manifest has arrived, so a + // manifest that waited for it would never come + { timeout: 10_000 }, + async () => { + await setConfig({ target: ['jsx-ast'] }); + + const gate = Promise.withResolvers(); + const worker = { + async *stream(batches) { + const all = []; + + // Like the pool, every batch is taken as it arrives + for await (const items of batches) { + all.push(items); + } + + for (const items of all) { + // The first page takes until the test lets it finish + if (items.some(({ head }) => head.api === 'fs')) { + await gate.promise; + } + + yield items.map(({ head }) => ({ data: head })); + } + }, + }; + + const output = generate( + streamOf([[createEntry('fs', 'File system')]]), + worker + ); + + const { value } = await output.next(); + + assert.deepEqual( + value[0].manifest.map(({ api }) => api), + ['fs', '404'] + ); + + gate.resolve(); + + const { pages } = await collect(output); + + assert.deepEqual( + pages.map(({ data }) => data.api), + ['fs', '404'] + ); + } + ); + + it( + 'lets go of the pages it emits before the manifest', + // The input only ends once the test lets it, so pages that waited for the + // manifest would never come + { timeout: 10_000 }, + async () => { + await setConfig({ target: ['jsx-ast'] }); + + const released = new Set(); + const registry = new FinalizationRegistry(api => released.add(api)); + const end = Promise.withResolvers(); + + // `metadata` is still running, so the manifest waits for it + async function* input() { + yield [createEntry('fs', 'File system')]; + yield [createEntry('zlib', 'Zlib')]; + yield [createEntry('buffer', 'Buffer')]; + await end.promise; + } + + const worker = { + async *stream(batches) { + for await (const items of batches) { + yield items.map(({ head }) => { + const page = { data: head }; + + registry.register(page, head.api); + + return page; + }); + } + }, + }; + + const output = generate(input(), worker); + + // Each module's page is built once the next module arrives, and read as + // `html` reads it, letting go of it once it has it. The last one read may + // still be in a register: the first one must not be held. + await output.next(); + await output.next(); + + const reading = output.next(); + + for (let i = 0; i < 10 && !released.has('fs'); i++) { + await new Promise(resolve => setImmediate(resolve)); + gc(); + } + + const releasedEarly = released.has('fs'); + + end.resolve(); + + await reading; + await collect(output); + + assert.equal(releasedEarly, true); + } + ); + it('does not change its input on the main thread', async () => { await setConfig({ target: ['jsx-ast'] }); diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index 62e51852..037e69ce 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -1,10 +1,13 @@ import { isMainThread } from 'node:worker_threads'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + groupWholeModules, + merge, +} from '@doc-kit/core/utils/generators.mjs'; import { jsx, toJs } from 'estree-util-to-js'; -import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; import { buildNotFoundPage } from './utils/synthetic/404.mjs'; /** @@ -43,25 +46,66 @@ export async function processChunk(slicedInput, itemIndices) { /** * Generates per-page JSX code from API metadata. * + * Each page starts building once all of its module's entries have arrived, and + * is emitted once built, in the order the pages arrived, the synthetic pages + * last: the `html` generator puts them in the order the site lists them. + * + * Once every page has arrived, without waiting for the pages still being + * built, it also emits `{ manifest }`: every page's head, in that same order, + * so that `html` can prepare what needs the whole site in the meantime. + * * @type {import('./types').Generator['generate']} */ export async function* generate(input, worker) { - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4700+ entries to every worker - const groupedModules = groupNodesByModule(input); - const descriptors = getSortedHeadNodes(input).map(head => ({ - head, - entries: groupedModules.get(head.api), - })); - - // `all.html` is not built here: it is the module pages concatenated, so the - // `html` generator assembles it from their content instead of building - // every module a second time. - if (getConfig('jsx-ast').generateNotFoundPage) { - descriptors.push(buildNotFoundPage()); + // Pages made of other pages' content are not built here: the `html` + // generator assembles them from that content. + const synthetic = getConfig('jsx-ast').generateNotFoundPage + ? [buildNotFoundPage()] + : []; + + const manifest = Promise.withResolvers(); + + /** + * Describes each arriving page by its head and its module's entries, so a + * worker gets only what its page needs, not all 4700+ entries. Once every + * page has arrived, it resolves the manifest and describes the synthetic + * pages. + * + * @returns {AsyncGenerator<Array<{ head: object, entries: Array<object> }>>} + */ + async function* describe() { + const heads = []; + + // A page needs all of its module's entries + for await (const entries of groupWholeModules(input)) { + const groupedModules = groupNodesByModule(entries); + const descriptors = entries + .filter(node => node.heading.depth === 1) + .map(head => ({ head, entries: groupedModules.get(head.api) })); + + heads.push(...descriptors.map(({ head }) => head)); + + yield descriptors; + } + + manifest.resolve([...heads, ...synthetic.map(({ head }) => head)]); + + if (synthetic.length > 0) { + yield synthetic; + } } - for await (const chunkResult of worker.stream(descriptors)) { - yield chunkResult; + /** + * Gives the manifest, once every page has arrived. + * + * @returns {AsyncGenerator<Array<{ manifest: Array<object> }>>} + */ + async function* announce() { + const heads = await manifest.promise; + + yield [{ manifest: heads }]; } + + // The manifest goes out as soon as it's ready, not after the next pages + yield* merge(worker.stream(describe()), announce()); } diff --git a/packages/react/src/jsx-ast/index.mjs b/packages/react/src/jsx-ast/index.mjs index 59a9ff29..01c17339 100644 --- a/packages/react/src/jsx-ast/index.mjs +++ b/packages/react/src/jsx-ast/index.mjs @@ -26,6 +26,8 @@ export default { hasParallelProcessor: true, + streamsInput: true, + markdown: { // The configured remark plugins run before the alerts, so they can make // alerts too diff --git a/packages/react/src/jsx-ast/types.d.ts b/packages/react/src/jsx-ast/types.d.ts index de5771ab..9e98b395 100644 --- a/packages/react/src/jsx-ast/types.d.ts +++ b/packages/react/src/jsx-ast/types.d.ts @@ -11,6 +11,9 @@ export type Generator = GeneratorMetadata< generateNotFoundPage: boolean; showReadingTime: boolean; }, - Generate<Array<MetadataEntry>, AsyncGenerator<PageCode>>, + Generate< + Array<MetadataEntry> | AsyncIterable<Array<MetadataEntry>>, + AsyncGenerator<Array<PageCode | { manifest: Array<MetadataEntry> }>> + >, ProcessChunk<{ head: MetadataEntry; entries: Array<MetadataEntry> }, PageCode> >; diff --git a/packages/react/src/jsx-ast/utils/buildContent.mjs b/packages/react/src/jsx-ast/utils/buildContent.mjs index 1d2434ee..b86b6cca 100644 --- a/packages/react/src/jsx-ast/utils/buildContent.mjs +++ b/packages/react/src/jsx-ast/utils/buildContent.mjs @@ -330,8 +330,8 @@ export const processEntry = entry => { * * The layout itself (`<Layout>`) is not part of the content. The `html` * generator wraps each page in it, which lets a page be assembled from other - * pages' content — `all.html` is the module pages concatenated — without - * building those modules a second time. + * pages' content, such as Node.js's `all.html`, without building those pages a + * second time. * * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} entries - API documentation metadata entries */ diff --git a/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs b/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs index 3c5d7f04..5623c515 100644 --- a/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs +++ b/packages/react/src/jsx-ast/utils/getSortedHeadNodes.mjs @@ -3,12 +3,13 @@ import { OVERRIDDEN_POSITIONS } from '../constants.mjs'; /** - * Sorts entries by OVERRIDDEN_POSITIONS and then heading name. + * Compares two entries to sort them: those in OVERRIDDEN_POSITIONS first, in + * that order, then the rest by heading name. * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} a * @param {import('@doc-kit/core/generators/metadata/types').MetadataEntry} b * @returns {number} */ -const headingSortFn = (a, b) => { +export const headingSortFn = (a, b) => { const ai = OVERRIDDEN_POSITIONS.indexOf(a.api); const bi = OVERRIDDEN_POSITIONS.indexOf(b.api); diff --git a/packages/react/src/section-pages/__tests__/generate.test.mjs b/packages/react/src/section-pages/__tests__/generate.test.mjs index 8605ba1c..48e02f6e 100644 --- a/packages/react/src/section-pages/__tests__/generate.test.mjs +++ b/packages/react/src/section-pages/__tests__/generate.test.mjs @@ -7,6 +7,40 @@ import getConfig, { import { generate } from '../generate.mjs'; +/** + * Collects every entry the generator yields. + * + * @param {AsyncIterable<Array<object>>} output + * @returns {Promise<Array<object>>} + */ +const collect = async output => { + const entries = []; + + for await (const batch of output) { + entries.push(...batch); + } + + return entries; +}; + +/** + * Runs the generator over the given batches of entries, streamed in as + * `metadata` yields them, and collects what it yields. + * + * @param {...Array<object>} batches + * @returns {Promise<Array<object>>} + */ +const run = (...batches) => collect(generate(streamOf(batches))); + +/** + * Yields each batch in turn, as a streamed input. + * + * @param {Array<Array<object>>} batches + */ +async function* streamOf(batches) { + yield* batches; +} + /** * Builds a metadata-shaped entry whose heading node is shared between * `heading` and `content`, exactly like the metadata generator produces. @@ -72,7 +106,7 @@ describe('section-pages generate', () => { it('keeps every original entry and appends the chunk pages', async () => { const input = createModule(); - const output = await generate(input); + const output = await run(input); assert.deepEqual(output.slice(0, input.length), input); @@ -97,7 +131,7 @@ describe('section-pages generate', () => { }); it('describes where each chunk came from', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const readFile = output.find(e => e.path === '/fs/readFile'); assert.deepEqual(readFile.chunk, { @@ -111,7 +145,7 @@ describe('section-pages generate', () => { }); it('promotes headings so the chunk reads as its own page', async () => { - const output = await generate(createModule()); + const output = await run(createModule()); const dir = output.filter(e => e.api === 'fs-Dir'); assert.deepEqual( @@ -134,13 +168,13 @@ describe('section-pages generate', () => { const input = createModule(); const snapshot = structuredClone(input); - await generate(input); + await run(input); assert.deepEqual(input, snapshot); }); it('leaves modules with a single section alone', async () => { - const output = await generate([ + const output = await run([ createEntry('tiny', 1, 'Tiny'), createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), ]); @@ -149,7 +183,7 @@ describe('section-pages generate', () => { }); it('leaves modules that document no API entry alone', async () => { - const output = await generate([ + const output = await run([ createEntry('guide', 1, 'About this documentation'), createEntry('guide', 2, 'Contributing'), createEntry('guide', 2, 'Stability index'), @@ -172,7 +206,7 @@ describe('section-pages generate', () => { link('https://example.com') ); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -206,7 +240,7 @@ describe('section-pages generate', () => { }, }); - const output = await generate(input); + const output = await run(input); const chunk = output.find(e => e.path === '/fs/readFile'); assert.deepEqual( @@ -225,7 +259,9 @@ describe('section-pages generate', () => { getConfig('section-pages').exclude = ['fs']; try { - assert.equal((await generate(createModule())).length, 6); + const output = await run(createModule()); + + assert.equal(output.length, 6); } finally { getConfig('section-pages').exclude = []; } @@ -235,7 +271,7 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 2; try { - const output = await generate(createModule()); + const output = await run(createModule()); assert.deepEqual( output.filter(e => e.chunk && e.heading.depth === 1).map(e => e.path), @@ -245,4 +281,73 @@ describe('section-pages generate', () => { getConfig('section-pages').maxDepth = 3; } }); + + it('builds the same output from several batches as from one', async () => { + const tiny = [ + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await run([...createModule(), ...tiny]); + const batched = await run(createModule(), tiny); + + assert.deepEqual(batched, whole); + }); + + it('builds the same output from its whole input, as it gets it with one thread', async () => { + const input = () => [ + ...createModule(), + createEntry('tiny', 1, 'Tiny'), + createEntry('tiny', 2, '`tiny.only()`', { name: 'only', type: 'method' }), + ]; + + const whole = await collect(generate(input())); + const streamed = await run(input()); + + assert.deepEqual(whole, streamed); + }); + + it( + 'passes whole modules on before the next batch arrives', + // The next batch waits for the test, so modules held back until it arrives + // would never be passed on + { timeout: 10_000 }, + async () => { + const gate = Promise.withResolvers(); + const fs = createModule(); + + async function* batches() { + yield [...fs, createEntry('tiny', 1, 'Tiny')]; + await gate.promise; + yield [createEntry('zlib', 1, 'Zlib')]; + } + + const output = generate(batches()); + const { value } = await output.next(); + + // `fs` is whole, `tiny` may continue in the next batch + assert.deepEqual(value, fs); + + gate.resolve(); + + const rest = await collect(output); + + assert.deepEqual( + rest.filter(e => !e.chunk).map(e => e.api), + ['tiny', 'zlib'] + ); + assert.equal( + rest.filter(e => e.chunk && e.heading.depth === 1).length, + 3 + ); + } + ); + + it('builds a module split across batches as one', async () => { + const whole = await run(createModule()); + const module = createModule(); + const split = await run(module.slice(0, 2), module.slice(2)); + + assert.deepEqual(split, whole); + }); }); diff --git a/packages/react/src/section-pages/generate.mjs b/packages/react/src/section-pages/generate.mjs index 12e134cd..eb50383f 100644 --- a/packages/react/src/section-pages/generate.mjs +++ b/packages/react/src/section-pages/generate.mjs @@ -1,7 +1,10 @@ 'use strict'; import getConfig from '@doc-kit/core/utils/configuration/index.mjs'; -import { groupNodesByModule } from '@doc-kit/core/utils/generators.mjs'; +import { + groupNodesByModule, + groupWholeModules, +} from '@doc-kit/core/utils/generators.mjs'; import { visit } from 'unist-util-visit'; import { headingLabel } from '../jsx-ast/utils/buildBarProps.mjs'; @@ -100,14 +103,14 @@ const buildChunkEntries = (module, chunk, index, anchors) => { }; /** - * Adds per-section chunk pages for every module, next to the full pages. + * Builds the chunk pages of every module among the given entries. * - * @type {import('./types').Generator['generate']} + * @param {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} input - Whole modules' entries + * @param {import('./types').Configuration} config + * @returns {Array<import('@doc-kit/core/generators/metadata/types').MetadataEntry>} */ -export async function generate(input) { - const { maxDepth, exclude } = getConfig('section-pages'); - - const output = [...input]; +const buildChunkPages = (input, { maxDepth, exclude }) => { + const pages = []; for (const entries of groupNodesByModule(input).values()) { const module = entries.find(entry => entry.heading.depth === 1); @@ -135,9 +138,35 @@ export async function generate(input) { const anchors = buildAnchorMap(chunks); chunks.forEach((chunk, index) => { - output.push(...buildChunkEntries(module, chunk, index, anchors)); + pages.push(...buildChunkEntries(module, chunk, index, anchors)); }); } - return output; + return pages; +}; + +/** + * Adds per-section chunk pages for every module, next to the full pages. + * + * The entries are passed on as their modules arrive, so the pages built from + * them can start while `metadata` still runs, and the chunk pages follow them + * all, in module order, as if built from the whole input at once. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input) { + const config = getConfig('section-pages'); + + const chunkPages = []; + + // A module's chunk pages need all of its entries + for await (const entries of groupWholeModules(input)) { + yield entries; + + chunkPages.push(...buildChunkPages(entries, config)); + } + + if (chunkPages.length > 0) { + yield chunkPages; + } } diff --git a/packages/react/src/section-pages/index.mjs b/packages/react/src/section-pages/index.mjs index b1043d11..380d1883 100644 --- a/packages/react/src/section-pages/index.mjs +++ b/packages/react/src/section-pages/index.mjs @@ -29,6 +29,8 @@ export default { dependsOn: '@doc-kit/core/metadata', + streamsInput: true, + dependent: [ '@doc-kit/generator-react/html', '@doc-kit/generator-react/sitemap', diff --git a/packages/react/src/section-pages/types.d.ts b/packages/react/src/section-pages/types.d.ts index bf57e1ee..9d6c579f 100644 --- a/packages/react/src/section-pages/types.d.ts +++ b/packages/react/src/section-pages/types.d.ts @@ -24,5 +24,8 @@ export type Chunk = { export type Generator = GeneratorMetadata< Configuration, - Generate<Array<MetadataEntry>, Promise<Array<MetadataEntry>>> + Generate< + Array<MetadataEntry> | AsyncIterable<Array<MetadataEntry>>, + AsyncGenerator<Array<MetadataEntry>> + > >; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e7ec9d74..b4df6356 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,6 +174,9 @@ importers: '@doc-kit/core': specifier: workspace:* version: link:../core + '@doc-kit/generator-react': + specifier: workspace:* + version: link:../react dedent: specifier: ^1.7.2 version: 1.7.2 diff --git a/www/doc-kit.config.mjs b/www/doc-kit.config.mjs index 275b717d..45834e35 100644 --- a/www/doc-kit.config.mjs +++ b/www/doc-kit.config.mjs @@ -28,6 +28,7 @@ const PUBLIC_GENERATORS = [ 'legacy-json-all', 'man-page', 'api-links', + 'html-all', 'addon-verify', ];