From d57f86be419b82c1b1b87c2fcecf697859751e34 Mon Sep 17 00:00:00 2001 From: astrobot-houston Date: Sat, 4 Jul 2026 10:57:33 +0000 Subject: [PATCH 1/2] fix(content): add deferRender option to glob() loader to avoid OOM during content sync When glob() processes .md files, it eagerly renders them and stores the full HTML in the data store. For content with heavy rehype plugins (e.g. KaTeX), rendered HTML can be ~70x larger than source, exhausting memory for large collections. This adds a deferRender option to glob() that skips eager rendering and instead defers it until the entry is actually rendered in a page, matching the existing behavior for .mdx files. Usage: glob({ pattern: '**/*.md', base: 'src/content/docs', deferRender: true }) Fixes #17301 --- packages/astro/src/content/loaders/glob.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/astro/src/content/loaders/glob.ts b/packages/astro/src/content/loaders/glob.ts index 3c84c4a0f921..f58fdee6e237 100644 --- a/packages/astro/src/content/loaders/glob.ts +++ b/packages/astro/src/content/loaders/glob.ts @@ -36,6 +36,13 @@ interface GlobOptions { * Defaults to `true`. */ retainBody?: boolean; + /** + * When `true`, renderable content entries (e.g. Markdown) will not be eagerly rendered + * during content sync. Instead, rendering is deferred until the entry is actually rendered + * in a page. This reduces memory usage for large collections with heavy rendered output. + * Defaults to `false`. + */ + deferRender?: boolean; } function generateIdDefault({ entry, base, data }: GenerateIdOptions, isLegacy?: boolean): string { @@ -182,7 +189,7 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean } } } - if (entryType.getRenderFunction) { + if (entryType.getRenderFunction && !globOptions.deferRender) { let render = renderFunctionByContentType.get(entryType); if (!render) { @@ -213,9 +220,7 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean } rendered, assetImports: rendered?.metadata?.imagePaths, }); - - // todo: add an explicit way to opt in to deferred rendering - } else if ('contentModuleTypes' in entryType) { + } else if (globOptions.deferRender || 'contentModuleTypes' in entryType) { store.set({ id, data: parsedData, From fd53e18a91d1e680a8dea6d2de01aef848e731be Mon Sep 17 00:00:00 2001 From: Matthew Phillips Date: Sat, 4 Jul 2026 12:43:10 -0400 Subject: [PATCH 2/2] test(content): cover glob() deferRender; add changeset; only defer renderable entries --- .changeset/breezy-corners-push.md | 21 +++ packages/astro/src/content/loaders/glob.ts | 5 +- .../units/content-layer/glob-loader.test.ts | 122 ++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 .changeset/breezy-corners-push.md diff --git a/.changeset/breezy-corners-push.md b/.changeset/breezy-corners-push.md new file mode 100644 index 000000000000..63afe1a9428c --- /dev/null +++ b/.changeset/breezy-corners-push.md @@ -0,0 +1,21 @@ +--- +'astro': minor +--- + +Adds a new `deferRender` option to the `glob()` content loader + +When set to `true`, renderable entries (such as Markdown) are not rendered during content sync. Instead, rendering is deferred until the entry is actually rendered in a page, using the same on-demand path that `.mdx` files already use. + +This reduces memory usage during `astro build` for large collections whose rendered output is much larger than the source — for example, Markdown that uses heavy rehype plugins like `rehype-katex`. Such builds could previously run out of memory while storing the eagerly-rendered HTML for every entry. + +```js +// src/content.config.ts +import { defineCollection } from 'astro:content'; +import { glob } from 'astro/loaders'; + +const docs = defineCollection({ + loader: glob({ pattern: '**/*.md', base: 'src/content/docs', deferRender: true }), +}); +``` + +By default `deferRender` is `false`, preserving the existing behavior of rendering entries eagerly during sync so their rendered HTML can be cached across builds. diff --git a/packages/astro/src/content/loaders/glob.ts b/packages/astro/src/content/loaders/glob.ts index f58fdee6e237..ab253578cca6 100644 --- a/packages/astro/src/content/loaders/glob.ts +++ b/packages/astro/src/content/loaders/glob.ts @@ -220,7 +220,10 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean } rendered, assetImports: rendered?.metadata?.imagePaths, }); - } else if (globOptions.deferRender || 'contentModuleTypes' in entryType) { + } else if ( + (entryType.getRenderFunction && globOptions.deferRender) || + 'contentModuleTypes' in entryType + ) { store.set({ id, data: parsedData, diff --git a/packages/astro/test/units/content-layer/glob-loader.test.ts b/packages/astro/test/units/content-layer/glob-loader.test.ts index a3114781c4a0..3a0966a6ad69 100644 --- a/packages/astro/test/units/content-layer/glob-loader.test.ts +++ b/packages/astro/test/units/content-layer/glob-loader.test.ts @@ -11,6 +11,20 @@ import { createMarkdownEntryType, } from './test-helpers.ts'; +/** + * A markdown entry type that also exposes a `getRenderFunction`, so it exercises + * the eager/deferred render branches of the glob loader (the base helper omits it). + */ +function createRenderableMarkdownEntryType() { + return { + ...createMarkdownEntryType(), + getRenderFunction: async () => async (entry: any) => ({ + html: `rendered: ${entry.body ?? ''}`, + metadata: {}, + }), + }; +} + describe('Glob Loader', () => { const root = new URL('../../fixtures/content-layer/', import.meta.url); @@ -151,6 +165,114 @@ describe('Glob Loader', () => { assert.equal(entry.body, undefined); }); + it('eagerly renders renderable entries during sync by default', async () => { + const store = new MutableDataStore(); + const settings = createMinimalSettings(root, { + contentEntryTypes: [createRenderableMarkdownEntryType()], + }); + const logger = new AstroLogger({ + destination: { write: () => true }, + level: 'silent', + }); + + const collections = { + spacecraft: defineCollection({ + loader: glob({ pattern: '*.md', base: 'src/content/space' }), + }), + }; + + const contentLayer = new ContentLayer({ + settings, + logger, + store, + contentConfigObserver: createTestConfigObserver(collections), + }); + + await contentLayer.sync(); + + const columbia = store.values('spacecraft').find((e) => e.id === 'columbia'); + assert.ok(columbia); + // Rendered HTML is stored eagerly, and the entry is not deferred + assert.ok(columbia.rendered); + assert.ok(columbia.rendered.html.includes('rendered:')); + assert.equal(columbia.deferredRender, undefined); + }); + + it('defers rendering of renderable entries when deferRender is true', async () => { + const store = new MutableDataStore(); + const settings = createMinimalSettings(root, { + contentEntryTypes: [createRenderableMarkdownEntryType()], + }); + const logger = new AstroLogger({ + destination: { write: () => true }, + level: 'silent', + }); + + const collections = { + spacecraft: defineCollection({ + loader: glob({ pattern: '*.md', base: 'src/content/space', deferRender: true }), + }), + }; + + const contentLayer = new ContentLayer({ + settings, + logger, + store, + contentConfigObserver: createTestConfigObserver(collections), + }); + + await contentLayer.sync(); + + const columbia = store.values('spacecraft').find((e) => e.id === 'columbia'); + assert.ok(columbia); + // Rendering is deferred: no rendered HTML is stored, entry is marked deferred + assert.equal(columbia.deferredRender, true); + assert.equal(columbia.rendered, undefined); + // The body is still retained so it can be rendered on demand + assert.ok(columbia.body); + }); + + it('does not defer non-renderable data entries when deferRender is true', async () => { + const store = new MutableDataStore(); + + const yamlEntryType = { + extensions: ['.yaml', '.yml'], + getEntryInfo: ({ contents }: any) => ({ data: { value: contents.trim() }, body: '' }), + }; + + const settings = createMinimalSettings(root, { + config: { + root, + srcDir: new URL('./src/', root), + }, + dataEntryTypes: [yamlEntryType], + }); + const logger = new AstroLogger({ + destination: { write: () => true }, + level: 'silent', + }); + + const collections = { + numbersYaml: defineCollection({ + loader: glob({ pattern: 'src/data/glob-yaml/*', base: '.', deferRender: true }), + }), + }; + + const contentLayer = new ContentLayer({ + settings, + logger, + store, + contentConfigObserver: createTestConfigObserver(collections), + }); + + await contentLayer.sync(); + + const entries = store.values('numbersYaml'); + assert.equal(entries.length, 3); + // Data entries have no render function, so deferRender must not mark them deferred + assert.ok(entries.every((e) => e.deferredRender === undefined)); + }); + it('loads YAML files with glob pattern', async () => { const store = new MutableDataStore();