diff --git a/.changeset/breezy-corners-push.md b/.changeset/breezy-corners-push.md new file mode 100644 index 000000000000..63afe1a9428c --- /dev/null +++ b/.changeset/breezy-corners-push.md @@ -0,0 +1,21 @@ +--- +'astro': minor +--- + +Adds a new `deferRender` option to the `glob()` content loader + +When set to `true`, renderable entries (such as Markdown) are not rendered during content sync. Instead, rendering is deferred until the entry is actually rendered in a page, using the same on-demand path that `.mdx` files already use. + +This reduces memory usage during `astro build` for large collections whose rendered output is much larger than the source — for example, Markdown that uses heavy rehype plugins like `rehype-katex`. Such builds could previously run out of memory while storing the eagerly-rendered HTML for every entry. + +```js +// src/content.config.ts +import { defineCollection } from 'astro:content'; +import { glob } from 'astro/loaders'; + +const docs = defineCollection({ + loader: glob({ pattern: '**/*.md', base: 'src/content/docs', deferRender: true }), +}); +``` + +By default `deferRender` is `false`, preserving the existing behavior of rendering entries eagerly during sync so their rendered HTML can be cached across builds. diff --git a/packages/astro/src/content/loaders/glob.ts b/packages/astro/src/content/loaders/glob.ts index 3c84c4a0f921..ab253578cca6 100644 --- a/packages/astro/src/content/loaders/glob.ts +++ b/packages/astro/src/content/loaders/glob.ts @@ -36,6 +36,13 @@ interface GlobOptions { * Defaults to `true`. */ retainBody?: boolean; + /** + * When `true`, renderable content entries (e.g. Markdown) will not be eagerly rendered + * during content sync. Instead, rendering is deferred until the entry is actually rendered + * in a page. This reduces memory usage for large collections with heavy rendered output. + * Defaults to `false`. + */ + deferRender?: boolean; } function generateIdDefault({ entry, base, data }: GenerateIdOptions, isLegacy?: boolean): string { @@ -182,7 +189,7 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean } } } - if (entryType.getRenderFunction) { + if (entryType.getRenderFunction && !globOptions.deferRender) { let render = renderFunctionByContentType.get(entryType); if (!render) { @@ -213,9 +220,10 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean } rendered, assetImports: rendered?.metadata?.imagePaths, }); - - // todo: add an explicit way to opt in to deferred rendering - } else if ('contentModuleTypes' in entryType) { + } else if ( + (entryType.getRenderFunction && globOptions.deferRender) || + 'contentModuleTypes' in entryType + ) { store.set({ id, data: parsedData, diff --git a/packages/astro/test/units/content-layer/glob-loader.test.ts b/packages/astro/test/units/content-layer/glob-loader.test.ts index a3114781c4a0..3a0966a6ad69 100644 --- a/packages/astro/test/units/content-layer/glob-loader.test.ts +++ b/packages/astro/test/units/content-layer/glob-loader.test.ts @@ -11,6 +11,20 @@ import { createMarkdownEntryType, } from './test-helpers.ts'; +/** + * A markdown entry type that also exposes a `getRenderFunction`, so it exercises + * the eager/deferred render branches of the glob loader (the base helper omits it). + */ +function createRenderableMarkdownEntryType() { + return { + ...createMarkdownEntryType(), + getRenderFunction: async () => async (entry: any) => ({ + html: `rendered: ${entry.body ?? ''}`, + metadata: {}, + }), + }; +} + describe('Glob Loader', () => { const root = new URL('../../fixtures/content-layer/', import.meta.url); @@ -151,6 +165,114 @@ describe('Glob Loader', () => { assert.equal(entry.body, undefined); }); + it('eagerly renders renderable entries during sync by default', async () => { + const store = new MutableDataStore(); + const settings = createMinimalSettings(root, { + contentEntryTypes: [createRenderableMarkdownEntryType()], + }); + const logger = new AstroLogger({ + destination: { write: () => true }, + level: 'silent', + }); + + const collections = { + spacecraft: defineCollection({ + loader: glob({ pattern: '*.md', base: 'src/content/space' }), + }), + }; + + const contentLayer = new ContentLayer({ + settings, + logger, + store, + contentConfigObserver: createTestConfigObserver(collections), + }); + + await contentLayer.sync(); + + const columbia = store.values('spacecraft').find((e) => e.id === 'columbia'); + assert.ok(columbia); + // Rendered HTML is stored eagerly, and the entry is not deferred + assert.ok(columbia.rendered); + assert.ok(columbia.rendered.html.includes('rendered:')); + assert.equal(columbia.deferredRender, undefined); + }); + + it('defers rendering of renderable entries when deferRender is true', async () => { + const store = new MutableDataStore(); + const settings = createMinimalSettings(root, { + contentEntryTypes: [createRenderableMarkdownEntryType()], + }); + const logger = new AstroLogger({ + destination: { write: () => true }, + level: 'silent', + }); + + const collections = { + spacecraft: defineCollection({ + loader: glob({ pattern: '*.md', base: 'src/content/space', deferRender: true }), + }), + }; + + const contentLayer = new ContentLayer({ + settings, + logger, + store, + contentConfigObserver: createTestConfigObserver(collections), + }); + + await contentLayer.sync(); + + const columbia = store.values('spacecraft').find((e) => e.id === 'columbia'); + assert.ok(columbia); + // Rendering is deferred: no rendered HTML is stored, entry is marked deferred + assert.equal(columbia.deferredRender, true); + assert.equal(columbia.rendered, undefined); + // The body is still retained so it can be rendered on demand + assert.ok(columbia.body); + }); + + it('does not defer non-renderable data entries when deferRender is true', async () => { + const store = new MutableDataStore(); + + const yamlEntryType = { + extensions: ['.yaml', '.yml'], + getEntryInfo: ({ contents }: any) => ({ data: { value: contents.trim() }, body: '' }), + }; + + const settings = createMinimalSettings(root, { + config: { + root, + srcDir: new URL('./src/', root), + }, + dataEntryTypes: [yamlEntryType], + }); + const logger = new AstroLogger({ + destination: { write: () => true }, + level: 'silent', + }); + + const collections = { + numbersYaml: defineCollection({ + loader: glob({ pattern: 'src/data/glob-yaml/*', base: '.', deferRender: true }), + }), + }; + + const contentLayer = new ContentLayer({ + settings, + logger, + store, + contentConfigObserver: createTestConfigObserver(collections), + }); + + await contentLayer.sync(); + + const entries = store.values('numbersYaml'); + assert.equal(entries.length, 3); + // Data entries have no render function, so deferRender must not mark them deferred + assert.ok(entries.every((e) => e.deferredRender === undefined)); + }); + it('loads YAML files with glob pattern', async () => { const store = new MutableDataStore();