Add deferRender option to glob() loader to prevent OOM during content sync

.changeset/breezy-corners-push.md

@@ -0,0 +1,21 @@
+---
+'astro': minor
+---
+
+Adds a new `deferRender` option to the `glob()` content loader
+
+When set to `true`, renderable entries (such as Markdown) are not rendered during content sync. Instead, rendering is deferred until the entry is actually rendered in a page, using the same on-demand path that `.mdx` files already use.
+
+This reduces memory usage during `astro build` for large collections whose rendered output is much larger than the source — for example, Markdown that uses heavy rehype plugins like `rehype-katex`. Such builds could previously run out of memory while storing the eagerly-rendered HTML for every entry.
+
+```js
+// src/content.config.ts
+import { defineCollection } from 'astro:content';
+import { glob } from 'astro/loaders';
+
+const docs = defineCollection({
+  loader: glob({ pattern: '**/*.md', base: 'src/content/docs', deferRender: true }),
+});
+```
+
+By default `deferRender` is `false`, preserving the existing behavior of rendering entries eagerly during sync so their rendered HTML can be cached across builds.

packages/astro/src/content/loaders/glob.ts

@@ -36,6 +36,13 @@ interface GlobOptions {
 	 * Defaults to `true`.
 	 */
 	retainBody?: boolean;
+	/**
+	 * When `true`, renderable content entries (e.g. Markdown) will not be eagerly rendered
+	 * during content sync. Instead, rendering is deferred until the entry is actually rendered
+	 * in a page. This reduces memory usage for large collections with heavy rendered output.
+	 * Defaults to `false`.
+	 */
+	deferRender?: boolean;
 }
 
 function generateIdDefault({ entry, base, data }: GenerateIdOptions, isLegacy?: boolean): string {
@@ -182,7 +189,7 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean }
 					}
 				}
 
-				if (entryType.getRenderFunction) {
+				if (entryType.getRenderFunction && !globOptions.deferRender) {
 					let render = renderFunctionByContentType.get(entryType);
 
 					if (!render) {
@@ -213,9 +220,10 @@ export function glob(globOptions: GlobOptions & { [secretLegacyFlag]?: boolean }
 						rendered,
 						assetImports: rendered?.metadata?.imagePaths,
 					});
-
-					// todo: add an explicit way to opt in to deferred rendering
-				} else if ('contentModuleTypes' in entryType) {
+				} else if (
+					(entryType.getRenderFunction && globOptions.deferRender) ||
+					'contentModuleTypes' in entryType
+				) {
 					store.set({
 						id,
 						data: parsedData,

packages/astro/test/units/content-layer/glob-loader.test.ts

@@ -11,6 +11,20 @@ import {
 	createMarkdownEntryType,
 } from './test-helpers.ts';
 
+/**
+ * A markdown entry type that also exposes a `getRenderFunction`, so it exercises
+ * the eager/deferred render branches of the glob loader (the base helper omits it).
+ */
+function createRenderableMarkdownEntryType() {
+	return {
+		...createMarkdownEntryType(),
+		getRenderFunction: async () => async (entry: any) => ({
+			html: `rendered: ${entry.body ?? ''}`,
+			metadata: {},
+		}),
+	};
+}
+
 describe('Glob Loader', () => {
 	const root = new URL('../../fixtures/content-layer/', import.meta.url);
 
@@ -151,6 +165,114 @@ describe('Glob Loader', () => {
 		assert.equal(entry.body, undefined);
 	});
 
+	it('eagerly renders renderable entries during sync by default', async () => {
+		const store = new MutableDataStore();
+		const settings = createMinimalSettings(root, {
+			contentEntryTypes: [createRenderableMarkdownEntryType()],
+		});
+		const logger = new AstroLogger({
+			destination: { write: () => true },
+			level: 'silent',
+		});
+
+		const collections = {
+			spacecraft: defineCollection({
+				loader: glob({ pattern: '*.md', base: 'src/content/space' }),
+			}),
+		};
+
+		const contentLayer = new ContentLayer({
+			settings,
+			logger,
+			store,
+			contentConfigObserver: createTestConfigObserver(collections),
+		});
+
+		await contentLayer.sync();
+
+		const columbia = store.values('spacecraft').find((e) => e.id === 'columbia');
+		assert.ok(columbia);
+		// Rendered HTML is stored eagerly, and the entry is not deferred
+		assert.ok(columbia.rendered);
+		assert.ok(columbia.rendered.html.includes('rendered:'));
+		assert.equal(columbia.deferredRender, undefined);
+	});
+
+	it('defers rendering of renderable entries when deferRender is true', async () => {
+		const store = new MutableDataStore();
+		const settings = createMinimalSettings(root, {
+			contentEntryTypes: [createRenderableMarkdownEntryType()],
+		});
+		const logger = new AstroLogger({
+			destination: { write: () => true },
+			level: 'silent',
+		});
+
+		const collections = {
+			spacecraft: defineCollection({
+				loader: glob({ pattern: '*.md', base: 'src/content/space', deferRender: true }),
+			}),
+		};
+
+		const contentLayer = new ContentLayer({
+			settings,
+			logger,
+			store,
+			contentConfigObserver: createTestConfigObserver(collections),
+		});
+
+		await contentLayer.sync();
+
+		const columbia = store.values('spacecraft').find((e) => e.id === 'columbia');
+		assert.ok(columbia);
+		// Rendering is deferred: no rendered HTML is stored, entry is marked deferred
+		assert.equal(columbia.deferredRender, true);
+		assert.equal(columbia.rendered, undefined);
+		// The body is still retained so it can be rendered on demand
+		assert.ok(columbia.body);
+	});
+
+	it('does not defer non-renderable data entries when deferRender is true', async () => {
+		const store = new MutableDataStore();
+
+		const yamlEntryType = {
+			extensions: ['.yaml', '.yml'],
+			getEntryInfo: ({ contents }: any) => ({ data: { value: contents.trim() }, body: '' }),
+		};
+
+		const settings = createMinimalSettings(root, {
+			config: {
+				root,
+				srcDir: new URL('./src/', root),
+			},
+			dataEntryTypes: [yamlEntryType],
+		});
+		const logger = new AstroLogger({
+			destination: { write: () => true },
+			level: 'silent',
+		});
+
+		const collections = {
+			numbersYaml: defineCollection({
+				loader: glob({ pattern: 'src/data/glob-yaml/*', base: '.', deferRender: true }),
+			}),
+		};
+
+		const contentLayer = new ContentLayer({
+			settings,
+			logger,
+			store,
+			contentConfigObserver: createTestConfigObserver(collections),
+		});
+
+		await contentLayer.sync();
+
+		const entries = store.values('numbersYaml');
+		assert.equal(entries.length, 3);
+		// Data entries have no render function, so deferRender must not mark them deferred
+		assert.ok(entries.every((e) => e.deferredRender === undefined));
+	});
+
 	it('loads YAML files with glob pattern', async () => {
 		const store = new MutableDataStore();