From 06995e1e0dd3c6e13506124d226dd4541a2a17a1 Mon Sep 17 00:00:00 2001 From: Seokju Na Date: Wed, 1 Jul 2026 22:54:18 +0900 Subject: [PATCH] feat(search): Korean morphological tokenizer (garu) on the workerd search API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire garu-orama-tokenizer into the Orama search so Korean queries match by morpheme instead of whitespace: `먹다` matches `먹었다`/`먹지`, `학교` matches `학교에서`. It's composed with Orama's default tokenizer as a union, so English and other non-Korean search is unchanged. garu-ko normally fetches/reads its WASM+model at runtime, which the Cloudflare Workers (workerd) runtime forbids. Instead: - import the 341KB WASM as a pre-compiled module and instantiate it with `initSync({ module })` (workerd allows instantiating a compiled module; only compiling from bytes at runtime is banned); - inline the ~1MB model as bytes via a small Vite plugin; - alias garu-ko's non-exported internals and pin garu-ko + @orama/orama so the layout we depend on is version-locked, with a build-time existence guard. All garu code is dynamically imported so its weight lands in a lazy chunk loaded only on the first search. The failed-init cache is reset so a transient cold-start error can't permanently poison an isolate. Verified on the workerd runtime (dev Miniflare + prod build): Korean morphological matching works, English is unchanged, the client bundle stays clean (server-only), and the Worker is ~2.4MB gzipped (under the limits). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01HhaZ2zL7bAqT3c8mRWTSs9 --- package.json | 6 ++++ src/garu-runtime.d.ts | 25 ++++++++++++++ src/routes/api/search.ts | 73 +++++++++++++++++++++++++++++++++++++--- vite.config.ts | 56 +++++++++++++++++++++++++++++- yarn.lock | 23 ++++++++++++- 5 files changed, 176 insertions(+), 7 deletions(-) create mode 100644 src/garu-runtime.d.ts diff --git a/package.json b/package.json index 0ad55d0..a3c46e5 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@date-fns/tz": "1.5.0", "@fontsource-variable/inter": "5.2.8", "@fontsource-variable/jetbrains-mono": "5.2.8", + "@orama/orama": "3.1.18", "@sentry/cloudflare": "10.53.1", "@sentry/tanstackstart-react": "10.53.1", "@tanstack/react-router": "1.170.8", @@ -29,6 +30,8 @@ "fumadocs-core": "16.7.16", "fumadocs-mdx": "14.3.0", "fumadocs-ui": "16.7.16", + "garu-ko": "0.9.7", + "garu-orama-tokenizer": "0.4.1", "jose": "6.2.3", "react": "19.2.6", "react-dom": "19.2.6", @@ -50,5 +53,8 @@ "vite": "8.0.14", "wrangler": "4.94.0" }, + "resolutions": { + "garu-ko": "0.9.7" + }, "packageManager": "yarn@4.15.0" } diff --git a/src/garu-runtime.d.ts b/src/garu-runtime.d.ts new file mode 100644 index 0000000..3129675 --- /dev/null +++ b/src/garu-runtime.d.ts @@ -0,0 +1,25 @@ +// Type declarations for the garu-ko internals we import via Vite aliases +// (see `resolve.alias` in vite.config.ts). These bypass garu-ko's `exports` +// map so the Korean tokenizer can run in the Cloudflare Workers runtime. + +declare module 'garu-runtime/glue' { + /** wasm-bindgen synchronous init. Accepts a pre-compiled module. Idempotent. */ + export function initSync(input: { module: WebAssembly.Module } | WebAssembly.Module): unknown; + + /** The garu-ko WASM analyzer, constructed from the model bytes. */ + export class GaruWasm { + constructor(modelData: Uint8Array, normalizeJamo: boolean); + analyze(text: string): { tokens: { pos: string; text: string }[] }; + free(): void; + } +} + +declare module 'garu-runtime/wasm' { + const wasmModule: WebAssembly.Module; + export default wasmModule; +} + +declare module 'garu-runtime/model' { + const modelBytes: Uint8Array; + export default modelBytes; +} diff --git a/src/routes/api/search.ts b/src/routes/api/search.ts index 31ec783..8b856fb 100644 --- a/src/routes/api/search.ts +++ b/src/routes/api/search.ts @@ -1,16 +1,79 @@ +import { tokenizer as oramaTokenizer } from '@orama/orama/components'; import { createFileRoute } from '@tanstack/react-router'; import { createFromSource } from 'fumadocs-core/search/server'; +import type { Garu } from 'garu-ko'; import { docSource } from '../../doc'; -const server = createFromSource(docSource, { - // https://docs.orama.com/docs/orama-js/supported-languages - language: 'english', -}); +// Korean-aware search. The garu tokenizer runs real morphological analysis, so +// `먹다` matches `먹었다`/`먹지` and `학교` matches `학교에서` — particles and endings +// are stripped instead of being treated as word boundaries. We compose it with +// Orama's default tokenizer (a union of both token sets), so English and other +// non-Korean text keep their existing tokenization unchanged. +// +// garu-ko normally fetches/reads its WASM + model at runtime, which the workerd +// runtime forbids. Instead the WASM is imported as a pre-compiled module and the +// model is inlined as bytes (see vite.config.ts). Everything garu-related is +// dynamically imported so its ~1MB weight lands in a lazy chunk loaded only on +// the first search request, not at Worker startup. +let serverPromise: Promise> | undefined; + +async function buildSearchServer() { + const [glue, { default: garuWasmModule }, { default: garuModelBytes }, { createTokenizer }] = + await Promise.all([ + import('garu-runtime/glue'), + import('garu-runtime/wasm'), + import('garu-runtime/model'), + import('garu-orama-tokenizer'), + ]); + + // `initSync` only *instantiates* a pre-compiled module (allowed on workerd); + // compiling from bytes at runtime is not. Fail loudly if the build ever hands + // us something other than a compiled module (e.g. a plugin regression). + if (!(garuWasmModule instanceof WebAssembly.Module)) { + throw new Error('garu WASM import did not resolve to a WebAssembly.Module'); + } + + // Instantiate the pre-compiled WASM into garu-ko's wasm-bindgen singleton, + // then hand the model bytes to the analyzer. `GaruWasm` and `initSync` come + // from the same glue module, so they share that singleton. + glue.initSync({ module: garuWasmModule }); + const wasm = new glue.GaruWasm(garuModelBytes, false); + const garu = { analyze: (text: string) => wasm.analyze(text) } as unknown as Garu; + + const korean = await createTokenizer({ garu }); + const base = oramaTokenizer.createTokenizer({ language: 'english' }); + + return createFromSource(docSource, { + // https://docs.orama.com/docs/orama-js/supported-languages + tokenizer: { + language: 'english', + normalizationCache: new Map(), + tokenize(raw, language, prop, withCache) { + return [ + ...new Set([ + ...base.tokenize(raw, language, prop, withCache), + ...korean.tokenize(raw, language, prop), + ]), + ]; + }, + }, + }); +} + +function getSearchServer() { + // Reset on failure so a transient first-request error (e.g. a cold-start CPU + // spike) doesn't permanently poison this isolate's cached search server. + serverPromise ??= buildSearchServer().catch(error => { + serverPromise = undefined; + throw error; + }); + return serverPromise; +} export const Route = createFileRoute('/api/search')({ server: { handlers: { - GET: async ({ request }) => server.GET(request), + GET: async ({ request }) => (await getSearchServer()).GET(request), }, }, }); diff --git a/vite.config.ts b/vite.config.ts index e0a60df..e176e23 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,12 +1,60 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { cloudflare } from '@cloudflare/vite-plugin'; import { sentryVitePlugin } from '@sentry/vite-plugin'; import tailwindcss from '@tailwindcss/vite'; import { tanstackStart } from '@tanstack/react-start/plugin/vite'; import viteReact from '@vitejs/plugin-react'; import mdx from 'fumadocs-mdx/vite'; -import { createLogger, defineConfig } from 'vite'; +import { createLogger, defineConfig, type Plugin } from 'vite'; import * as MdxConfig from './source.config'; +// The Korean search tokenizer (garu-ko) ships a browser build that `fetch()`es +// its WASM/model and a node build that reads them from `fs` — neither works in +// the Cloudflare Workers (workerd) runtime where our search API runs. Instead we +// import the WASM as a module (compiled at startup, instantiated per isolate) and +// inline the ~1MB model as bytes. garu-ko's package `exports` map hides these +// internal files, so we alias directly to them. +const garuGlue = fileURLToPath(new URL('./node_modules/garu-ko/pkg/garu_wasm.js', import.meta.url)); +const garuWasm = fileURLToPath( + new URL('./node_modules/garu-ko/pkg/garu_wasm_bg.wasm', import.meta.url) +); +const garuModel = fileURLToPath( + new URL('./node_modules/garu-ko/models/base.gmdl', import.meta.url) +); + +// Load the garu model (`.gmdl`) as an inlined Uint8Array so it travels in the +// Worker bundle instead of being fetched/read at runtime, and assert the aliased +// garu-ko internals exist so a version bump that relocates them fails loudly. +function garuModelBytes(): Plugin { + return { + name: 'garu-model-bytes', + enforce: 'pre', + buildStart() { + const pkg = fileURLToPath(new URL('./node_modules/garu-ko/package.json', import.meta.url)); + const { version } = JSON.parse(readFileSync(pkg, 'utf8')) as { version: string }; + for (const file of [garuGlue, garuWasm, garuModel]) { + if (!existsSync(file)) { + this.error( + `[garu tokenizer] expected garu-ko internal file is missing: ${file} (garu-ko@${version}). ` + + 'garu-ko may have relocated its pkg/ or models/ files; pin garu-ko and update the aliases in vite.config.ts.' + ); + } + } + }, + load(id) { + if (id.replace(/\?.*$/, '') !== garuModel) return; + // Preallocated decode is faster than `Uint8Array.from(atob(...), cb)`, + // trimming the one-time first-search CPU cost on the Worker. + const base64 = readFileSync(garuModel).toString('base64'); + return `const binary = atob(${JSON.stringify(base64)}); +const bytes = new Uint8Array(binary.length); +for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); +export default bytes;`; + }, + }; +} + // Several dependencies (@tanstack/*, seroval) ship `//# sourceMappingURL` // comments without the referenced `.map` files. The Cloudflare SSR environment // bundles these packages (it can't externalize them for workerd), so Vite reads @@ -28,6 +76,11 @@ export default defineConfig(({ command }) => ({ }, resolve: { tsconfigPaths: true, + alias: [ + { find: /^garu-runtime\/glue$/, replacement: garuGlue }, + { find: /^garu-runtime\/wasm$/, replacement: garuWasm }, + { find: /^garu-runtime\/model$/, replacement: garuModel }, + ], }, build: { sourcemap: 'hidden', @@ -59,6 +112,7 @@ export default defineConfig(({ command }) => ({ }, }, plugins: [ + garuModelBytes(), mdx(MdxConfig), tailwindcss(), cloudflare({ diff --git a/yarn.lock b/yarn.lock index 1607949..182b2e3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1605,7 +1605,7 @@ __metadata: languageName: node linkType: hard -"@orama/orama@npm:^3.1.18": +"@orama/orama@npm:3.1.18, @orama/orama@npm:^3.1.18": version: 3.1.18 resolution: "@orama/orama@npm:3.1.18" checksum: 10c0/868be5143eb4a27e700d9cc866763d1cc04405ef0af6a6b6b6987dcf33ee5d4ba90d3a32cd134006d46d34d64e32500e13cd31436c718eadc396a940e793e7cd @@ -4944,6 +4944,24 @@ __metadata: languageName: node linkType: hard +"garu-ko@npm:0.9.7": + version: 0.9.7 + resolution: "garu-ko@npm:0.9.7" + checksum: 10c0/660f5e5d8bca42a2d4326c8add45dc25cbf1c066b9c0685fa285728665b80a31201768a37bb86dd744fad88540c49343cb5ff4474ee7677afc000e18417d365d + languageName: node + linkType: hard + +"garu-orama-tokenizer@npm:0.4.1": + version: 0.4.1 + resolution: "garu-orama-tokenizer@npm:0.4.1" + dependencies: + garu-ko: "npm:^0.9.0" + peerDependencies: + "@orama/orama": ">=2.0.0" + checksum: 10c0/19d1e885dfb32430a7869e2549de6d5ff8a4fb8363cafc8d6ecbe4b413983c8c4ef3c0e185b5f228688ddc9925533f58f390b8330e829614be0eb0ca9c045e60 + languageName: node + linkType: hard + "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -7956,6 +7974,7 @@ __metadata: "@date-fns/tz": "npm:1.5.0" "@fontsource-variable/inter": "npm:5.2.8" "@fontsource-variable/jetbrains-mono": "npm:5.2.8" + "@orama/orama": "npm:3.1.18" "@sentry/cloudflare": "npm:10.53.1" "@sentry/tanstackstart-react": "npm:10.53.1" "@sentry/vite-plugin": "npm:5.3.0" @@ -7972,6 +7991,8 @@ __metadata: fumadocs-core: "npm:16.7.16" fumadocs-mdx: "npm:14.3.0" fumadocs-ui: "npm:16.7.16" + garu-ko: "npm:0.9.7" + garu-orama-tokenizer: "npm:0.4.1" jose: "npm:6.2.3" oxfmt: "npm:0.51.0" oxlint: "npm:1.66.0"