feat(react): support chunking pages

.changeset/chunked-generator.md

@@ -0,0 +1,6 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+---
+
+feat: `dependent` generators and the `chunked` generator

.changeset/html-parallel-minify.md

@@ -0,0 +1,5 @@
+---
+'@doc-kit/generator-react': minor
+---
+
+perf(html): one client entry chunk, HTML minification in the worker pool

docs/creating-generators.md

@@ -411,6 +411,63 @@ export async function generate(input, worker) {
 }
 ```
 
+### Declaring a dependent
+
+`dependsOn` pulls another generator's output _in_. The inverse, `dependent`,
+pushes a generator's output _into_ another generator's pipeline:
+
+```mjs displayName="index.mjs"
+export default {
+  name: 'chunked',
+
+  dependsOn: '@doc-kit/core/metadata',
+
+  // Deliver this generator's output through `html`
+  dependent: '@doc-kit/generator-react/html',
+
+  generate,
+};
+```
+
+The pipeline splices such a generator in front of the first generator on the
+way to its dependent that consumes the same `dependsOn`. Here `html` depends on
+`jsx-ast`, which depends on `metadata` — so `jsx-ast` is rewired to read from
+`chunked` instead:
+
+```text
+ast → metadata → chunked → jsx-ast → html
+```
+
+Requesting a generator that declares a dependent runs the dependent's whole
+pipeline, and the run's result is the dependent's output — `-t chunked`
+produces the `html` site. Several generators may splice in at the same point;
+they form a chain. A generator whose dependent pipeline never consumes its
+`dependsOn` is an error.
+
+`dependent` also accepts an array. The generator is spliced into every listed
+pipeline; when requested, it is delivered through the dependents that are
+already part of the run, or through all of them when none is:
+
+```mjs displayName="index.mjs"
+export default {
+  name: 'chunked',
+  dependsOn: '@doc-kit/core/metadata',
+  dependent: [
+    '@doc-kit/generator-react/html',
+    '@doc-kit/generator-react/sitemap',
+  ],
+  generate,
+};
+```
+
+With this, `-t chunked` builds the site and the sitemap, while
+`-t html -t chunked` builds just the site.
+
+Use a dependent when a generator transforms an intermediate representation
+(adding, filtering, or rewriting entries) rather than producing a new output
+format of its own. See the [`chunked`](./generators/chunked.md) generator for
+a worked example.
+
 ## File Output
 
 ### Writing Output Files

docs/generators.md

@@ -16,6 +16,7 @@ npx @doc-kit/cli generate -t html -t orama-db -t sitemap -i "docs/**/*.md" -o ou
 | [`orama-db`](./generators/orama-db.md) | The search index behind the `html` site's search box.                |
 | [`llms-txt`](./generators/llms-txt.md) | An [`llms.txt`](https://llmstxt.org/) index for language models.     |
 | [`sitemap`](./generators/sitemap.md)   | A `sitemap.xml` for search engines.                                  |
+| [`chunked`](./generators/chunked.md)   | The `html` site and sitemap, plus one page per section of a module.  |
 
 ### JSON ([`@doc-kit/core`](./packages/core.md))
 

packages/core/src/__tests__/generators.test.mjs

@@ -72,6 +72,33 @@ const syntheticGenerators = {
       return { all: input };
     },
   },
+  // A generator delivered through another pipeline: it reads `metadata` and
+  // declares `gen-d-all` as its dependent, so `gen-d` reads from it instead.
+  'gen-splice': {
+    name: 'gen-splice',
+    dependsOn: 'metadata',
+    dependent: 'gen-d-all',
+    generate: async input => {
+      record('gen-splice');
+      return [...input, { spliced: true }];
+    },
+  },
+  'gen-d': {
+    name: 'gen-d',
+    dependsOn: 'metadata',
+    generate: async input => {
+      record('gen-d');
+      return { d: input };
+    },
+  },
+  'gen-d-all': {
+    name: 'gen-d-all',
+    dependsOn: 'gen-d',
+    generate: async input => {
+      record('gen-d-all');
+      return { all: input };
+    },
+  },
 };
 
 mock.module('../generators/loader.mjs', {
@@ -92,8 +119,10 @@ mock.module('../generators/loader.mjs', {
         const generator = syntheticGenerators[specifier];
         generators.set(specifier, generator);
 
-        if (generator.dependsOn) {
-          queue.push(generator.dependsOn);
+        for (const related of [generator.dependsOn, generator.dependent]) {
+          if (related) {
+            queue.push(related);
+          }
         }
       }
 
@@ -162,4 +191,24 @@ describe('createGenerator orchestration', () => {
 
     assert.deepStrictEqual(results, [[{ c: true }], { all: [{ c: true }] }]);
   });
+
+  it('delivers a generator through its dependent', async () => {
+    const { runGenerators } = createGenerator();
+
+    const results = await runGenerators({
+      target: ['gen-splice'],
+      threads: 1,
+    });
+
+    // Requesting only `gen-splice` runs its dependent's whole pipeline, with
+    // `gen-d` reading the spliced output instead of `metadata` directly.
+    assert.equal(runs.metadata, 1);
+    assert.equal(runs['gen-splice'], 1);
+    assert.equal(runs['gen-d'], 1);
+    assert.equal(runs['gen-d-all'], 1);
+
+    assert.deepStrictEqual(results, [
+      { all: { d: [{ meta: 1 }, { spliced: true }] } },
+    ]);
+  });
 });

packages/core/src/generators.mjs

@@ -5,6 +5,7 @@ import {
   loadGenerators,
   resolveGeneratorSpecifier,
 } from './generators/loader.mjs';
+import { resolvePipeline } from './generators/pipeline.mjs';
 import logger from './logger/index.mjs';
 import createWorkerPool from './threading/index.mjs';
 import createParallelWorker from './threading/parallel.mjs';
@@ -44,22 +45,24 @@ const createGenerator = () => {
    *
    * @param {string} specifier - Resolved generator specifier to schedule
    * @param {Map<string, GeneratorMetadata>} generators - Loaded generators
+   * @param {Map<string, string | undefined>} inputOf - Each generator's
+   * effective input generator (its dependency, unless another generator was
+   * spliced in front of it via `dependent`)
    * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options
    */
-  const scheduleGenerator = (specifier, generators, configuration) => {
+  const scheduleGenerator = (specifier, generators, inputOf, configuration) => {
     if (cache.has(specifier)) {
       return;
     }
 
     const generator = generators.get(specifier);
     const { name, generate, hasParallelProcessor } = generator;
 
-    const dependsOn =
-      generator.dependsOn && resolveGeneratorSpecifier(generator.dependsOn);
+    const dependsOn = inputOf.get(specifier);
 
     // Schedule dependency first
     if (dependsOn && !cache.has(dependsOn)) {
-      scheduleGenerator(dependsOn, generators, configuration);
+      scheduleGenerator(dependsOn, generators, inputOf, configuration);
     }
 
     generatorsLogger.debug(`Scheduling "${name}"`, {
@@ -104,8 +107,16 @@ const createGenerator = () => {
 
     // Resolve shorthand names and load the full dependency closure up front,
     // so scheduling below is fully synchronous.
-    const targets = target.map(resolveGeneratorSpecifier);
-    const generators = await loadGenerators(targets);
+    const generators = await loadGenerators(
+      target.map(resolveGeneratorSpecifier)
+    );
+
+    // Work out who reads from whom once generators declaring a `dependent`
+    // have been spliced in, and which generators the run finally collects.
+    const { targets, inputOf } = resolvePipeline(
+      target.map(resolveGeneratorSpecifier),
+      generators
+    );
 
     generatorsLogger.debug(`Starting pipeline`, {
       generators: targets.join(', '),
@@ -114,18 +125,14 @@ const createGenerator = () => {
 
     // Compute consumer counts up front so dependencies can be evicted as soon
     // as their last consumer runs (must be ready before any generator starts).
-    cache.populateConsumerCounts(targets, specifier => {
-      const { dependsOn } = generators.get(specifier);
-
-      return dependsOn && resolveGeneratorSpecifier(dependsOn);
-    });
+    cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier));
 
     // Create worker pool
     pool = createWorkerPool(threads);
 
     // Schedule all generators
     for (const specifier of targets) {
-      scheduleGenerator(specifier, generators, configuration);
+      scheduleGenerator(specifier, generators, inputOf, configuration);
     }
 
     // Start all collections in parallel (don't await sequentially). Consuming

packages/core/src/generators/__tests__/index.test.mjs

@@ -1,6 +1,8 @@
 import assert from 'node:assert/strict';
 import { describe, it } from 'node:test';
 
+import { enforceArray } from '#utils/array.mjs';
+
 import {
   allGenerators,
   deprecatedGenerators,
@@ -62,6 +64,17 @@ describe('All Generators', () => {
     });
   });
 
+  it('should have valid dependent references', () => {
+    loadedGenerators.forEach(([name, , generator]) => {
+      for (const dependent of enforceArray(generator.dependent ?? [])) {
+        assert.ok(
+          validDependencies.includes(dependent),
+          `Generator "${name}" declares dependent "${dependent}" which is not a valid generator specifier`
+        );
+      }
+    });
+  });
+
   it('should resolve deprecated aliases to loadable generators', async () => {
     for (const [name, specifier] of Object.entries(deprecatedGenerators)) {
       assert.equal(resolveGeneratorSpecifier(name), specifier);