diff --git a/content/docs/guide/meta.json b/content/docs/guide/meta.json index 8c15bc1..b486d3e 100644 --- a/content/docs/guide/meta.json +++ b/content/docs/guide/meta.json @@ -30,6 +30,9 @@ "remote/index", "remote/using-cli", "remote/providers", - "remote/http-spec" + "remote/http-spec", + "---AI---", + "external:[llms.txt](/llms.txt)", + "external:[llms-full.txt](/llms-full.txt)" ] } diff --git a/content/docs/guide/meta.ko.json b/content/docs/guide/meta.ko.json index 0035fc2..6902503 100644 --- a/content/docs/guide/meta.ko.json +++ b/content/docs/guide/meta.ko.json @@ -30,6 +30,9 @@ "remote/index", "remote/using-cli", "remote/providers", - "remote/http-spec" + "remote/http-spec", + "---AI---", + "external:[llms.txt](/ko/llms.txt)", + "external:[llms-full.txt](/ko/llms-full.txt)" ] } diff --git a/source.config.ts b/source.config.ts index 0b00e44..f67df0f 100644 --- a/source.config.ts +++ b/source.config.ts @@ -3,6 +3,15 @@ import lastModified from 'fumadocs-mdx/plugins/last-modified'; export const docs = defineDocs({ dir: 'content/docs', + docs: { + // Export each page's processed Markdown (as `_markdown`), which + // `page.data.getText('processed')` reads to build the `/llms.txt` routes. + // The `'raw'` alternative is not usable here: it reads the `.mdx` file from + // disk at request time, and the Worker has no filesystem. + postprocess: { + includeProcessedMarkdown: true, + }, + }, }); export default defineConfig({ diff --git a/src/lib/llms.ts b/src/lib/llms.ts new file mode 100644 index 0000000..27a6722 --- /dev/null +++ b/src/lib/llms.ts @@ -0,0 +1,51 @@ +import { llms } from 'fumadocs-core/source'; +import { docSource } from '../doc'; +import type { Locale } from './i18n'; + +// `llms.txt` support — plain-text views of the docs for LLMs to read. +// See https://llmstxt.org and https://fumadocs.dev/docs/integrations/llms. +// +// Every entry point takes an explicit locale. This is load-bearing: on an i18n +// source, both `llms().index()` and `getPages()` fall back to *every* language +// when the locale is omitted, so `/llms.txt` would serve English and Korean +// concatenated. Passing the locale keeps each file in one language, matching the +// URL it is served from (`/llms.txt` → `/docs/...`, `/ko/llms.txt` → `/ko/docs/...`). +// +// Note that a locale only has its own pages where a translation exists: fumadocs +// falls back to the default language per page, so the Korean files list the +// English text for pages with no `.ko.mdx` — the same content `/ko/docs/...` +// renders. + +type DocPage = (typeof docSource)['$inferPage']; + +function plainText(body: string): Response { + return new Response(body, { + headers: { 'content-type': 'text/plain; charset=utf-8' }, + }); +} + +/** + * Render one page for an LLM: a `# Title (url)` header, the description, then + * the page body as Markdown with the MDX components stripped out. + */ +export async function getLLMText(page: DocPage): Promise { + const { title, description } = page.data; + const parts = [`# ${title} (${page.url})`]; + if (description != null && description !== '') { + parts.push(description); + } + // Requires `includeProcessedMarkdown` (see source.config.ts). + parts.push(await page.data.getText('processed')); + return parts.join('\n\n'); +} + +/** `llms.txt` — an index of every page in `locale`, as a Markdown link list. */ +export function llmsIndexResponse(locale: Locale): Response { + return plainText(llms(docSource).index(locale)); +} + +/** `llms-full.txt` — the full text of every page in `locale`, concatenated. */ +export async function llmsFullResponse(locale: Locale): Promise { + const pages = await Promise.all(docSource.getPages(locale).map(getLLMText)); + return plainText(pages.join('\n\n')); +} diff --git a/src/routeTree.gen.ts b/src/routeTree.gen.ts index a88859b..f709415 100644 --- a/src/routeTree.gen.ts +++ b/src/routeTree.gen.ts @@ -9,9 +9,13 @@ // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. import { Route as rootRouteImport } from './routes/__root' +import { Route as LlmsDottxtRouteImport } from './routes/llms[.]txt' +import { Route as LlmsFullDottxtRouteImport } from './routes/llms-full[.]txt' import { Route as IndexRouteImport } from './routes/index' import { Route as KoIndexRouteImport } from './routes/ko/index' import { Route as DocsIndexRouteImport } from './routes/docs/index' +import { Route as KoLlmsDottxtRouteImport } from './routes/ko/llms[.]txt' +import { Route as KoLlmsFullDottxtRouteImport } from './routes/ko/llms-full[.]txt' import { Route as DocsChangelogRouteImport } from './routes/docs/changelog' import { Route as DocsSplatRouteImport } from './routes/docs/$' import { Route as ApiSearchRouteImport } from './routes/api/search' @@ -19,6 +23,16 @@ import { Route as KoDocsIndexRouteImport } from './routes/ko/docs/index' import { Route as KoDocsChangelogRouteImport } from './routes/ko/docs/changelog' import { Route as KoDocsSplatRouteImport } from './routes/ko/docs/$' +const LlmsDottxtRoute = LlmsDottxtRouteImport.update({ + id: '/llms.txt', + path: '/llms.txt', + getParentRoute: () => rootRouteImport, +} as any) +const LlmsFullDottxtRoute = LlmsFullDottxtRouteImport.update({ + id: '/llms-full.txt', + path: '/llms-full.txt', + getParentRoute: () => rootRouteImport, +} as any) const IndexRoute = IndexRouteImport.update({ id: '/', path: '/', @@ -34,6 +48,16 @@ const DocsIndexRoute = DocsIndexRouteImport.update({ path: '/docs/', getParentRoute: () => rootRouteImport, } as any) +const KoLlmsDottxtRoute = KoLlmsDottxtRouteImport.update({ + id: '/ko/llms.txt', + path: '/ko/llms.txt', + getParentRoute: () => rootRouteImport, +} as any) +const KoLlmsFullDottxtRoute = KoLlmsFullDottxtRouteImport.update({ + id: '/ko/llms-full.txt', + path: '/ko/llms-full.txt', + getParentRoute: () => rootRouteImport, +} as any) const DocsChangelogRoute = DocsChangelogRouteImport.update({ id: '/docs/changelog', path: '/docs/changelog', @@ -67,9 +91,13 @@ const KoDocsSplatRoute = KoDocsSplatRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof IndexRoute + '/llms-full.txt': typeof LlmsFullDottxtRoute + '/llms.txt': typeof LlmsDottxtRoute '/api/search': typeof ApiSearchRoute '/docs/$': typeof DocsSplatRoute '/docs/changelog': typeof DocsChangelogRoute + '/ko/llms-full.txt': typeof KoLlmsFullDottxtRoute + '/ko/llms.txt': typeof KoLlmsDottxtRoute '/docs/': typeof DocsIndexRoute '/ko/': typeof KoIndexRoute '/ko/docs/$': typeof KoDocsSplatRoute @@ -78,9 +106,13 @@ export interface FileRoutesByFullPath { } export interface FileRoutesByTo { '/': typeof IndexRoute + '/llms-full.txt': typeof LlmsFullDottxtRoute + '/llms.txt': typeof LlmsDottxtRoute '/api/search': typeof ApiSearchRoute '/docs/$': typeof DocsSplatRoute '/docs/changelog': typeof DocsChangelogRoute + '/ko/llms-full.txt': typeof KoLlmsFullDottxtRoute + '/ko/llms.txt': typeof KoLlmsDottxtRoute '/docs': typeof DocsIndexRoute '/ko': typeof KoIndexRoute '/ko/docs/$': typeof KoDocsSplatRoute @@ -90,9 +122,13 @@ export interface FileRoutesByTo { export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute + '/llms-full.txt': typeof LlmsFullDottxtRoute + '/llms.txt': typeof LlmsDottxtRoute '/api/search': typeof ApiSearchRoute '/docs/$': typeof DocsSplatRoute '/docs/changelog': typeof DocsChangelogRoute + '/ko/llms-full.txt': typeof KoLlmsFullDottxtRoute + '/ko/llms.txt': typeof KoLlmsDottxtRoute '/docs/': typeof DocsIndexRoute '/ko/': typeof KoIndexRoute '/ko/docs/$': typeof KoDocsSplatRoute @@ -103,9 +139,13 @@ export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '/' + | '/llms-full.txt' + | '/llms.txt' | '/api/search' | '/docs/$' | '/docs/changelog' + | '/ko/llms-full.txt' + | '/ko/llms.txt' | '/docs/' | '/ko/' | '/ko/docs/$' @@ -114,9 +154,13 @@ export interface FileRouteTypes { fileRoutesByTo: FileRoutesByTo to: | '/' + | '/llms-full.txt' + | '/llms.txt' | '/api/search' | '/docs/$' | '/docs/changelog' + | '/ko/llms-full.txt' + | '/ko/llms.txt' | '/docs' | '/ko' | '/ko/docs/$' @@ -125,9 +169,13 @@ export interface FileRouteTypes { id: | '__root__' | '/' + | '/llms-full.txt' + | '/llms.txt' | '/api/search' | '/docs/$' | '/docs/changelog' + | '/ko/llms-full.txt' + | '/ko/llms.txt' | '/docs/' | '/ko/' | '/ko/docs/$' @@ -137,9 +185,13 @@ export interface FileRouteTypes { } export interface RootRouteChildren { IndexRoute: typeof IndexRoute + LlmsFullDottxtRoute: typeof LlmsFullDottxtRoute + LlmsDottxtRoute: typeof LlmsDottxtRoute ApiSearchRoute: typeof ApiSearchRoute DocsSplatRoute: typeof DocsSplatRoute DocsChangelogRoute: typeof DocsChangelogRoute + KoLlmsFullDottxtRoute: typeof KoLlmsFullDottxtRoute + KoLlmsDottxtRoute: typeof KoLlmsDottxtRoute DocsIndexRoute: typeof DocsIndexRoute KoIndexRoute: typeof KoIndexRoute KoDocsSplatRoute: typeof KoDocsSplatRoute @@ -149,6 +201,20 @@ export interface RootRouteChildren { declare module '@tanstack/react-router' { interface FileRoutesByPath { + '/llms.txt': { + id: '/llms.txt' + path: '/llms.txt' + fullPath: '/llms.txt' + preLoaderRoute: typeof LlmsDottxtRouteImport + parentRoute: typeof rootRouteImport + } + '/llms-full.txt': { + id: '/llms-full.txt' + path: '/llms-full.txt' + fullPath: '/llms-full.txt' + preLoaderRoute: typeof LlmsFullDottxtRouteImport + parentRoute: typeof rootRouteImport + } '/': { id: '/' path: '/' @@ -170,6 +236,20 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DocsIndexRouteImport parentRoute: typeof rootRouteImport } + '/ko/llms.txt': { + id: '/ko/llms.txt' + path: '/ko/llms.txt' + fullPath: '/ko/llms.txt' + preLoaderRoute: typeof KoLlmsDottxtRouteImport + parentRoute: typeof rootRouteImport + } + '/ko/llms-full.txt': { + id: '/ko/llms-full.txt' + path: '/ko/llms-full.txt' + fullPath: '/ko/llms-full.txt' + preLoaderRoute: typeof KoLlmsFullDottxtRouteImport + parentRoute: typeof rootRouteImport + } '/docs/changelog': { id: '/docs/changelog' path: '/docs/changelog' @@ -217,9 +297,13 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, + LlmsFullDottxtRoute: LlmsFullDottxtRoute, + LlmsDottxtRoute: LlmsDottxtRoute, ApiSearchRoute: ApiSearchRoute, DocsSplatRoute: DocsSplatRoute, DocsChangelogRoute: DocsChangelogRoute, + KoLlmsFullDottxtRoute: KoLlmsFullDottxtRoute, + KoLlmsDottxtRoute: KoLlmsDottxtRoute, DocsIndexRoute: DocsIndexRoute, KoIndexRoute: KoIndexRoute, KoDocsSplatRoute: KoDocsSplatRoute, diff --git a/src/routes/ko/llms-full[.]txt.ts b/src/routes/ko/llms-full[.]txt.ts new file mode 100644 index 0000000..09e6068 --- /dev/null +++ b/src/routes/ko/llms-full[.]txt.ts @@ -0,0 +1,10 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { llmsFullResponse } from '../../lib/llms'; + +export const Route = createFileRoute('/ko/llms-full.txt')({ + server: { + handlers: { + GET: () => llmsFullResponse('ko'), + }, + }, +}); diff --git a/src/routes/ko/llms[.]txt.ts b/src/routes/ko/llms[.]txt.ts new file mode 100644 index 0000000..88108dd --- /dev/null +++ b/src/routes/ko/llms[.]txt.ts @@ -0,0 +1,10 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { llmsIndexResponse } from '../../lib/llms'; + +export const Route = createFileRoute('/ko/llms.txt')({ + server: { + handlers: { + GET: () => llmsIndexResponse('ko'), + }, + }, +}); diff --git a/src/routes/llms-full[.]txt.ts b/src/routes/llms-full[.]txt.ts new file mode 100644 index 0000000..d811e1a --- /dev/null +++ b/src/routes/llms-full[.]txt.ts @@ -0,0 +1,10 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { llmsFullResponse } from '../lib/llms'; + +export const Route = createFileRoute('/llms-full.txt')({ + server: { + handlers: { + GET: () => llmsFullResponse('en'), + }, + }, +}); diff --git a/src/routes/llms[.]txt.ts b/src/routes/llms[.]txt.ts new file mode 100644 index 0000000..8a50fad --- /dev/null +++ b/src/routes/llms[.]txt.ts @@ -0,0 +1,10 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { llmsIndexResponse } from '../lib/llms'; + +export const Route = createFileRoute('/llms.txt')({ + server: { + handlers: { + GET: () => llmsIndexResponse('en'), + }, + }, +});