diff --git a/.gitignore b/.gitignore index 2ddc373..87e329d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ dist/ .wrangler/ .dev.vars .idea +*.env # Local env files (e.g. VITE_SENTRY_DSN for production builds). Vite loads # `.env.production.local` automatically; keep the DSN out of this public repo. diff --git a/WORKS.md b/WORKS.md deleted file mode 100644 index 8b13789..0000000 --- a/WORKS.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/package.json b/package.json index 396ddab..e8c9057 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,6 @@ "fumadocs-core": "16.10.7", "fumadocs-mdx": "15.0.13", "fumadocs-ui": "16.10.7", - "garu-ko": "0.9.7", - "garu-orama-tokenizer": "0.4.1", "jose": "6.2.3", "react": "19.2.7", "react-dom": "19.2.7", @@ -55,8 +53,5 @@ "vite": "8.1.3", "wrangler": "4.107.0" }, - "resolutions": { - "garu-ko": "0.9.7" - }, "packageManager": "yarn@4.17.0" } diff --git a/public/apple-touch-icon.png b/public/apple-touch-icon.png new file mode 100644 index 0000000..8f1aba9 Binary files /dev/null and b/public/apple-touch-icon.png differ diff --git a/public/favicon-96x96.png b/public/favicon-96x96.png new file mode 100644 index 0000000..45dab07 Binary files /dev/null and b/public/favicon-96x96.png differ diff --git a/public/favicon.ico b/public/favicon.ico new file mode 100644 index 0000000..721f993 Binary files /dev/null and b/public/favicon.ico differ diff --git a/public/web-app-manifest-192x192.png b/public/web-app-manifest-192x192.png new file mode 100644 index 0000000..14d917b Binary files /dev/null and b/public/web-app-manifest-192x192.png differ diff --git a/public/web-app-manifest-512x512.png b/public/web-app-manifest-512x512.png new file mode 100644 index 0000000..bda9ac1 Binary files /dev/null and b/public/web-app-manifest-512x512.png differ diff --git a/src/garu-runtime.d.ts b/src/garu-runtime.d.ts deleted file mode 100644 index 3129675..0000000 --- a/src/garu-runtime.d.ts +++ /dev/null @@ -1,25 +0,0 @@ -// Type declarations for the garu-ko internals we import via Vite aliases -// (see `resolve.alias` in vite.config.ts). These bypass garu-ko's `exports` -// map so the Korean tokenizer can run in the Cloudflare Workers runtime. - -declare module 'garu-runtime/glue' { - /** wasm-bindgen synchronous init. Accepts a pre-compiled module. Idempotent. */ - export function initSync(input: { module: WebAssembly.Module } | WebAssembly.Module): unknown; - - /** The garu-ko WASM analyzer, constructed from the model bytes. */ - export class GaruWasm { - constructor(modelData: Uint8Array, normalizeJamo: boolean); - analyze(text: string): { tokens: { pos: string; text: string }[] }; - free(): void; - } -} - -declare module 'garu-runtime/wasm' { - const wasmModule: WebAssembly.Module; - export default wasmModule; -} - -declare module 'garu-runtime/model' { - const modelBytes: Uint8Array; - export default modelBytes; -} diff --git a/src/layouts/ErrorPage.tsx b/src/layouts/ErrorPage.tsx new file mode 100644 index 0000000..50ca675 --- /dev/null +++ b/src/layouts/ErrorPage.tsx @@ -0,0 +1,101 @@ +import * as Sentry from '@sentry/tanstackstart-react'; +import { type ErrorComponentProps, useRouter, useRouterState } from '@tanstack/react-router'; +import { useEffect, useRef, useState } from 'react'; +import { localeFromPathname, localizeHref, LocaleProvider, useLocale } from '../lib/locale'; +import { useUiStrings } from '../lib/ui-strings'; +import { SiteHeader } from './SiteHeader'; + +// Global error page. Rendered by the router's `defaultErrorComponent`, so it +// catches anything a route's render or loader throws — including the docs +// loaders — and, like `NotFound`, takes its locale from the path so a reader who +// broke on `/ko/...` stays in Korean. +export function ErrorPage({ error, info }: ErrorComponentProps) { + const pathname = useRouterState({ select: state => state.location.pathname }); + const locale = localeFromPathname(pathname); + + return ( + +
+ +
+ +
+
+ + ); +} + +function ErrorBody({ error, info }: Omit) { + const t = useUiStrings(); + const locale = useLocale(); + const router = useRouter(); + const [eventId, setEventId] = useState(); + const reported = useRef(undefined); + + // Report to Sentry once per error instance — the ref survives the re-renders + // and remounts a retry triggers, so a reader retrying a persistent failure + // doesn't send the same event repeatedly. `info.componentStack` is attached + // under the `react` context, the same key Sentry's React integration uses, so + // the stack shows up where the UI expects it. + // + // Effects never run during SSR, so this fires on the client only. That is the + // point: an error the router catches and renders here never propagates out of + // the server handler, so the Sentry request/function middleware (src/start.ts) + // never sees it. This is the report. + useEffect(() => { + if (reported.current === error) return; + reported.current = error; + setEventId( + Sentry.captureException(error, { + tags: { boundary: 'router', locale }, + contexts: { react: { componentStack: info?.componentStack } }, + }) + ); + }, [error, info, locale]); + + return ( + <> +

+ {t.error.eyebrow} +

+

+ {t.error.title} +

+

{t.error.message}

+ + {/* The raw message is useful while developing and noise (or a leak) in + production, where the error ID is what we'd actually ask a reader for. */} + {import.meta.env.DEV && ( + 
+          {error.message}
+
+ )} + +
+ {/* `reset` alone only clears the error boundary, which for a loader + failure just re-renders the same stored error. `invalidate` re-runs + the loaders and bumps the router's `loadedAt`, which is the key the + boundary resets on — so this actually retries. */} + + + {t.error.home} + +
+ + {eventId != null && ( +

+ {t.error.eventId}: {eventId} +

+ )} + + ); +} diff --git a/src/lib/korean-tokenizer.ts b/src/lib/korean-tokenizer.ts new file mode 100644 index 0000000..7da9369 --- /dev/null +++ b/src/lib/korean-tokenizer.ts @@ -0,0 +1,176 @@ +import type { Tokenizer } from '@orama/orama'; +import { tokenizer as oramaTokenizer } from '@orama/orama/components'; + +// A dependency-free Korean tokenizer for Orama. +// +// Orama has no `ko` language (its default map would throw "Language 'ko' is not +// supported"), and a real morphological analyzer (garu-ko) costs ~1MB of WASM + +// model in the Worker bundle to serve a docs-sized corpus. Two cheap properties +// replace it, because Korean glues its particles (조사) and endings (어미) onto +// the *end* of a word: +// +// 1. Orama's index is a radix tree and matches by prefix, so an indexed +// `설치하세요` is already found by the query `설치`, and `번들을` by `번들`. +// Inflection in the document therefore costs us nothing. +// 2. The reverse — a reader who *types* the inflected form, `번들을`, looking +// for a page that says `번들` — is what this tokenizer adds: it strips a +// trailing particle or ending, so both sides also index the bare stem. +// +// Tokens are additive (the raw word is always kept), so stripping can only add +// recall, never remove a match. +// +// Deliberately *not* done: character bigrams (the usual CJK trick). They buy +// mid-word matches, which Korean docs rarely need since they space their words, +// and they cost real precision — `오프라인` would rank `파이프라인` first, on the +// strength of the shared `프라` and `라인`. + +// Precomposed Hangul syllables (가-힣). Jamo-only text (ㄱ, ㅏ) is rare outside +// IME state and is left to the base tokenizer. +const HANGUL_RUN = /[가-힣]+/g; + +// Particles and endings, longest-first at build time so `번들에서는` strips +// `에서는` rather than the shorter `는`. Not exhaustive, and it doesn't need to +// be: this only has to catch what a reader is likely to type into a search box. +const SUFFIXES = [ + // 조사 (particles) + '으로부터', + '에서부터', + '로부터', + '에게서', + '한테서', + '에서는', + '에서도', + '에서의', + '으로서', + '으로써', + '으로는', + '으로도', + '이라는', + '이라고', + '에게는', + '에게도', + '에서', + '에게', + '한테', + '께서', + '으로', + '까지', + '부터', + '처럼', + '보다', + '마다', + '조차', + '밖에', + '이나', + '라는', + '라고', + '와는', + '과는', + '에는', + '에도', + '에만', + '들의', + '들을', + '들이', + '들은', + '은', + '는', + '이', + '가', + '을', + '를', + '에', + '의', + '와', + '과', + '도', + '만', + '로', + '들', + // 어미 (verb / adjective endings) + '하였습니다', + '했습니다', + '됐습니다', + '있습니다', + '없습니다', + '되었다', + '합니다', + '됩니다', + '입니다', + '습니다', + '해주세요', + '하십시오', + '하세요', + '하려면', + '하는', + '하고', + '하며', + '하면', + '하여', + '해서', + '했다', + '하지', + '하기', + '되는', + '되고', + '된다', + '되면', + '되어', + '있는', + '없는', + '한다', + '했던', + '었다', + '았다', + '였다', +].sort((a, b) => b.length - a.length); + +// Stems shorter than this are dropped: a 1-character stem (`참고` → `참`) is far +// more likely to be an over-strip than a useful term. +const MIN_STEM = 2; + +// The stem of a Hangul word, or undefined when no suffix applies. +function stem(word: string): string | undefined { + for (const suffix of SUFFIXES) { + if (word.length - suffix.length >= MIN_STEM && word.endsWith(suffix)) { + return word.slice(0, -suffix.length); + } + } + return undefined; +} + +function koreanTokens(raw: string): string[] { + const tokens: string[] = []; + for (const [word] of raw.normalize('NFC').matchAll(HANGUL_RUN)) { + tokens.push(word); + const root = stem(word); + if (root != null) { + tokens.push(root); + } + } + return tokens; +} + +// Korean pages here are dense with English API names, code identifiers, and +// numbers (`.wvb`, `createBundle`, `v1`). Orama's English tokenizer treats +// Hangul as a separator, so it yields exactly those Latin tokens — stopword +// filtered — and we union them with the Korean ones. +export function createKoreanTokenizer(): Tokenizer { + const base = oramaTokenizer.createTokenizer({ language: 'english' }); + + return { + language: 'korean', + normalizationCache: new Map(), + // The incoming `language` is deliberately dropped rather than forwarded: + // Orama's tokenizer throws LANGUAGE_NOT_SUPPORTED for any language that + // isn't its own, so handing our `korean` to an `english` base would blow up + // the moment a caller passes one (Orama leaves it undefined today, which is + // the only reason forwarding it appeared to work). `prop` and `withCache` + // still go through — they only drive the base's normalization cache. + tokenize(raw: string, _language?: string, prop?: string, withCache?: boolean): string[] { + return [ + ...new Set([...base.tokenize(raw, undefined, prop, withCache), ...koreanTokens(raw)]), + ]; + }, + }; +} diff --git a/src/lib/ui-strings.ts b/src/lib/ui-strings.ts index 787e269..b2529e1 100644 --- a/src/lib/ui-strings.ts +++ b/src/lib/ui-strings.ts @@ -46,6 +46,14 @@ export interface UiStrings { bundleFormatLink: string; }; notFound: { title: string; message: string; home: string; docs: string }; + error: { + eyebrow: string; + title: string; + message: string; + retry: string; + home: string; + eventId: string; + }; } const en: UiStrings = { @@ -147,6 +155,15 @@ const en: UiStrings = { home: 'Go home', docs: 'Browse docs', }, + error: { + eyebrow: 'Error', + title: 'Something went wrong', + message: + 'This page failed to load. The error has been reported to us — retrying often clears it.', + retry: 'Try again', + home: 'Go home', + eventId: 'Error ID', + }, }; const ko: UiStrings = { @@ -250,6 +267,15 @@ const ko: UiStrings = { home: '홈으로', docs: '문서 보기', }, + error: { + eyebrow: '오류', + title: '문제가 발생했습니다', + message: + '페이지를 불러오지 못했습니다. 오류는 자동으로 보고되었으며, 다시 시도하면 해결되는 경우가 많습니다.', + retry: '다시 시도', + home: '홈으로', + eventId: '오류 ID', + }, }; const DICT: Record = { en, ko }; diff --git a/src/router.tsx b/src/router.tsx index a8d9e35..11b4026 100644 --- a/src/router.tsx +++ b/src/router.tsx @@ -1,5 +1,6 @@ import * as Sentry from '@sentry/tanstackstart-react'; import { createRouter } from '@tanstack/react-router'; +import { ErrorPage } from './layouts/ErrorPage'; import { NotFound } from './layouts/NotFound'; import { routeTree } from './routeTree.gen'; @@ -8,6 +9,7 @@ export function getRouter() { routeTree, scrollRestoration: true, defaultNotFoundComponent: NotFound, + defaultErrorComponent: ErrorPage, }); if (!router.isServer) { diff --git a/src/routes/__root.tsx b/src/routes/__root.tsx index c7921f8..7408ec1 100644 --- a/src/routes/__root.tsx +++ b/src/routes/__root.tsx @@ -27,6 +27,22 @@ export const Route = createRootRoute({ rel: 'preconnect', href: 'https://static.wvb.dev', }, + { + rel: 'icon', + href: '/favicon.ico', + sizes: '32x32', + }, + { + rel: 'icon', + type: 'image/png', + href: '/favicon-96x96.png', + sizes: '96x96', + }, + { + rel: 'apple-touch-icon', + href: '/apple-touch-icon.png', + sizes: '180x180', + }, { rel: 'stylesheet', href: styles, diff --git a/src/routes/api/search.ts b/src/routes/api/search.ts index cd4f1d7..22c004b 100644 --- a/src/routes/api/search.ts +++ b/src/routes/api/search.ts @@ -1,87 +1,24 @@ -import { tokenizer as oramaTokenizer } from '@orama/orama/components'; import { createFileRoute } from '@tanstack/react-router'; import { createFromSource } from 'fumadocs-core/search/server'; -import type { Garu } from 'garu-ko'; import { docSource } from '../../doc'; +import { createKoreanTokenizer } from '../../lib/korean-tokenizer'; // Bilingual, per-locale search. English uses Orama's default tokenizer; Korean -// uses garu's real morphological analysis, so `먹다` matches `먹었다`/`먹지` and -// `학교` matches `학교에서` — particles and endings are stripped instead of being -// treated as word boundaries. The Korean tokenizer is composed with the default -// tokenizer (a union of both token sets) so English code and terms embedded in -// Korean pages still tokenize normally. -// -// (Without this, the default per-locale mapping sends `ko` to Orama language -// `ko`, which has no stemmer and throws "Language 'ko' is not supported".) -// -// garu-ko normally fetches/reads its WASM + model at runtime, which the workerd -// runtime forbids. Instead the WASM is imported as a pre-compiled module and the -// model is inlined as bytes (see vite.config.ts). Everything garu-related is -// dynamically imported so its ~1MB weight lands in a lazy chunk loaded only on -// the first search request, not at Worker startup. -let serverPromise: Promise> | undefined; - -async function buildSearchServer() { - const [glue, { default: garuWasmModule }, { default: garuModelBytes }, { createTokenizer }] = - await Promise.all([ - import('garu-runtime/glue'), - import('garu-runtime/wasm'), - import('garu-runtime/model'), - import('garu-orama-tokenizer'), - ]); - - // `initSync` only *instantiates* a pre-compiled module (allowed on workerd); - // compiling from bytes at runtime is not. Fail loudly if the build ever hands - // us something other than a compiled module (e.g. a plugin regression). - if (!(garuWasmModule instanceof WebAssembly.Module)) { - throw new Error('garu WASM import did not resolve to a WebAssembly.Module'); - } - - // Instantiate the pre-compiled WASM into garu-ko's wasm-bindgen singleton, - // then hand the model bytes to the analyzer. `GaruWasm` and `initSync` come - // from the same glue module, so they share that singleton. - glue.initSync({ module: garuWasmModule }); - const wasm = new glue.GaruWasm(garuModelBytes, false); - const garu = { analyze: (text: string) => wasm.analyze(text) } as unknown as Garu; - - const korean = await createTokenizer({ garu }); - const base = oramaTokenizer.createTokenizer({ language: 'english' }); - const koreanTokenizer = { - language: 'korean', - normalizationCache: new Map(), - tokenize(raw: string, language?: string, prop?: string, withCache?: boolean): string[] { - return [ - ...new Set([ - ...base.tokenize(raw, language, prop, withCache), - ...korean.tokenize(raw, language, prop), - ]), - ]; - }, - }; - - return createFromSource(docSource, { - // https://docs.orama.com/docs/orama-js/supported-languages - localeMap: { - en: 'english', - ko: { tokenizer: koreanTokenizer }, - }, - }); -} - -function getSearchServer() { - // Reset on failure so a transient first-request error (e.g. a cold-start CPU - // spike) doesn't permanently poison this isolate's cached search server. - serverPromise ??= buildSearchServer().catch(error => { - serverPromise = undefined; - throw error; - }); - return serverPromise; -} +// gets ours (see `korean-tokenizer.ts`), since Orama ships no `ko` language — +// the default per-locale mapping would send `ko` to Orama language `ko` and +// throw "Language 'ko' is not supported". +const server = createFromSource(docSource, { + // https://docs.orama.com/docs/orama-js/supported-languages + localeMap: { + en: 'english', + ko: { tokenizer: createKoreanTokenizer() }, + }, +}); export const Route = createFileRoute('/api/search')({ server: { handlers: { - GET: async ({ request }) => (await getSearchServer()).GET(request), + GET: ({ request }) => server.GET(request), }, }, }); diff --git a/vite.config.ts b/vite.config.ts index e176e23..e0a60df 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,60 +1,12 @@ -import { existsSync, readFileSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; import { cloudflare } from '@cloudflare/vite-plugin'; import { sentryVitePlugin } from '@sentry/vite-plugin'; import tailwindcss from '@tailwindcss/vite'; import { tanstackStart } from '@tanstack/react-start/plugin/vite'; import viteReact from '@vitejs/plugin-react'; import mdx from 'fumadocs-mdx/vite'; -import { createLogger, defineConfig, type Plugin } from 'vite'; +import { createLogger, defineConfig } from 'vite'; import * as MdxConfig from './source.config'; -// The Korean search tokenizer (garu-ko) ships a browser build that `fetch()`es -// its WASM/model and a node build that reads them from `fs` — neither works in -// the Cloudflare Workers (workerd) runtime where our search API runs. Instead we -// import the WASM as a module (compiled at startup, instantiated per isolate) and -// inline the ~1MB model as bytes. garu-ko's package `exports` map hides these -// internal files, so we alias directly to them. -const garuGlue = fileURLToPath(new URL('./node_modules/garu-ko/pkg/garu_wasm.js', import.meta.url)); -const garuWasm = fileURLToPath( - new URL('./node_modules/garu-ko/pkg/garu_wasm_bg.wasm', import.meta.url) -); -const garuModel = fileURLToPath( - new URL('./node_modules/garu-ko/models/base.gmdl', import.meta.url) -); - -// Load the garu model (`.gmdl`) as an inlined Uint8Array so it travels in the -// Worker bundle instead of being fetched/read at runtime, and assert the aliased -// garu-ko internals exist so a version bump that relocates them fails loudly. -function garuModelBytes(): Plugin { - return { - name: 'garu-model-bytes', - enforce: 'pre', - buildStart() { - const pkg = fileURLToPath(new URL('./node_modules/garu-ko/package.json', import.meta.url)); - const { version } = JSON.parse(readFileSync(pkg, 'utf8')) as { version: string }; - for (const file of [garuGlue, garuWasm, garuModel]) { - if (!existsSync(file)) { - this.error( - `[garu tokenizer] expected garu-ko internal file is missing: ${file} (garu-ko@${version}). ` + - 'garu-ko may have relocated its pkg/ or models/ files; pin garu-ko and update the aliases in vite.config.ts.' - ); - } - } - }, - load(id) { - if (id.replace(/\?.*$/, '') !== garuModel) return; - // Preallocated decode is faster than `Uint8Array.from(atob(...), cb)`, - // trimming the one-time first-search CPU cost on the Worker. - const base64 = readFileSync(garuModel).toString('base64'); - return `const binary = atob(${JSON.stringify(base64)}); -const bytes = new Uint8Array(binary.length); -for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); -export default bytes;`; - }, - }; -} - // Several dependencies (@tanstack/*, seroval) ship `//# sourceMappingURL` // comments without the referenced `.map` files. The Cloudflare SSR environment // bundles these packages (it can't externalize them for workerd), so Vite reads @@ -76,11 +28,6 @@ export default defineConfig(({ command }) => ({ }, resolve: { tsconfigPaths: true, - alias: [ - { find: /^garu-runtime\/glue$/, replacement: garuGlue }, - { find: /^garu-runtime\/wasm$/, replacement: garuWasm }, - { find: /^garu-runtime\/model$/, replacement: garuModel }, - ], }, build: { sourcemap: 'hidden', @@ -112,7 +59,6 @@ export default defineConfig(({ command }) => ({ }, }, plugins: [ - garuModelBytes(), mdx(MdxConfig), tailwindcss(), cloudflare({ diff --git a/yarn.lock b/yarn.lock index 9fcf8e9..768f33f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4289,24 +4289,6 @@ __metadata: languageName: node linkType: hard -"garu-ko@npm:0.9.7": - version: 0.9.7 - resolution: "garu-ko@npm:0.9.7" - checksum: 10c0/660f5e5d8bca42a2d4326c8add45dc25cbf1c066b9c0685fa285728665b80a31201768a37bb86dd744fad88540c49343cb5ff4474ee7677afc000e18417d365d - languageName: node - linkType: hard - -"garu-orama-tokenizer@npm:0.4.1": - version: 0.4.1 - resolution: "garu-orama-tokenizer@npm:0.4.1" - dependencies: - garu-ko: "npm:^0.9.0" - peerDependencies: - "@orama/orama": ">=2.0.0" - checksum: 10c0/19d1e885dfb32430a7869e2549de6d5ff8a4fb8363cafc8d6ecbe4b413983c8c4ef3c0e185b5f228688ddc9925533f58f390b8330e829614be0eb0ca9c045e60 - languageName: node - linkType: hard - "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -7272,8 +7254,6 @@ __metadata: fumadocs-core: "npm:16.10.7" fumadocs-mdx: "npm:15.0.13" fumadocs-ui: "npm:16.10.7" - garu-ko: "npm:0.9.7" - garu-orama-tokenizer: "npm:0.4.1" jose: "npm:6.2.3" oxfmt: "npm:0.57.0" oxlint: "npm:1.72.0"