feat(search): Korean morphological tokenizer on the workerd search API

package.json

@@ -19,6 +19,7 @@
     "@date-fns/tz": "1.5.0",
     "@fontsource-variable/inter": "5.2.8",
     "@fontsource-variable/jetbrains-mono": "5.2.8",
+    "@orama/orama": "3.1.18",
     "@sentry/cloudflare": "10.53.1",
     "@sentry/tanstackstart-react": "10.53.1",
     "@tanstack/react-router": "1.170.8",
@@ -29,6 +30,8 @@
     "fumadocs-core": "16.7.16",
     "fumadocs-mdx": "14.3.0",
     "fumadocs-ui": "16.7.16",
+    "garu-ko": "0.9.7",
+    "garu-orama-tokenizer": "0.4.1",
     "jose": "6.2.3",
     "react": "19.2.6",
     "react-dom": "19.2.6",
@@ -50,5 +53,8 @@
     "vite": "8.0.14",
     "wrangler": "4.94.0"
   },
+  "resolutions": {
+    "garu-ko": "0.9.7"
+  },
   "packageManager": "yarn@4.15.0"
 }

src/garu-runtime.d.ts

@@ -0,0 +1,25 @@
+// Type declarations for the garu-ko internals we import via Vite aliases
+// (see `resolve.alias` in vite.config.ts). These bypass garu-ko's `exports`
+// map so the Korean tokenizer can run in the Cloudflare Workers runtime.
+
+declare module 'garu-runtime/glue' {
+  /** wasm-bindgen synchronous init. Accepts a pre-compiled module. Idempotent. */
+  export function initSync(input: { module: WebAssembly.Module } | WebAssembly.Module): unknown;
+
+  /** The garu-ko WASM analyzer, constructed from the model bytes. */
+  export class GaruWasm {
+    constructor(modelData: Uint8Array, normalizeJamo: boolean);
+    analyze(text: string): { tokens: { pos: string; text: string }[] };
+    free(): void;
+  }
+}
+
+declare module 'garu-runtime/wasm' {
+  const wasmModule: WebAssembly.Module;
+  export default wasmModule;
+}
+
+declare module 'garu-runtime/model' {
+  const modelBytes: Uint8Array;
+  export default modelBytes;
+}

src/routes/api/search.ts

@@ -1,16 +1,79 @@
+import { tokenizer as oramaTokenizer } from '@orama/orama/components';
 import { createFileRoute } from '@tanstack/react-router';
 import { createFromSource } from 'fumadocs-core/search/server';
+import type { Garu } from 'garu-ko';
 import { docSource } from '../../doc';
 
-const server = createFromSource(docSource, {
-  // https://docs.orama.com/docs/orama-js/supported-languages
-  language: 'english',
-});
+// Korean-aware search. The garu tokenizer runs real morphological analysis, so
+// `먹다` matches `먹었다`/`먹지` and `학교` matches `학교에서` — particles and endings
+// are stripped instead of being treated as word boundaries. We compose it with
+// Orama's default tokenizer (a union of both token sets), so English and other
+// non-Korean text keep their existing tokenization unchanged.
+//
+// garu-ko normally fetches/reads its WASM + model at runtime, which the workerd
+// runtime forbids. Instead the WASM is imported as a pre-compiled module and the
+// model is inlined as bytes (see vite.config.ts). Everything garu-related is
+// dynamically imported so its ~1MB weight lands in a lazy chunk loaded only on
+// the first search request, not at Worker startup.
+let serverPromise: Promise<ReturnType<typeof createFromSource>> | undefined;
+
+async function buildSearchServer() {
+  const [glue, { default: garuWasmModule }, { default: garuModelBytes }, { createTokenizer }] =
+    await Promise.all([
+      import('garu-runtime/glue'),
+      import('garu-runtime/wasm'),
+      import('garu-runtime/model'),
+      import('garu-orama-tokenizer'),
+    ]);
+
+  // `initSync` only *instantiates* a pre-compiled module (allowed on workerd);
+  // compiling from bytes at runtime is not. Fail loudly if the build ever hands
+  // us something other than a compiled module (e.g. a plugin regression).
+  if (!(garuWasmModule instanceof WebAssembly.Module)) {
+    throw new Error('garu WASM import did not resolve to a WebAssembly.Module');
+  }
+
+  // Instantiate the pre-compiled WASM into garu-ko's wasm-bindgen singleton,
+  // then hand the model bytes to the analyzer. `GaruWasm` and `initSync` come
+  // from the same glue module, so they share that singleton.
+  glue.initSync({ module: garuWasmModule });
+  const wasm = new glue.GaruWasm(garuModelBytes, false);
+  const garu = { analyze: (text: string) => wasm.analyze(text) } as unknown as Garu;
+
+  const korean = await createTokenizer({ garu });
+  const base = oramaTokenizer.createTokenizer({ language: 'english' });
+
+  return createFromSource(docSource, {
+    // https://docs.orama.com/docs/orama-js/supported-languages
+    tokenizer: {
+      language: 'english',
+      normalizationCache: new Map(),
+      tokenize(raw, language, prop, withCache) {
+        return [
+          ...new Set([
+            ...base.tokenize(raw, language, prop, withCache),
+            ...korean.tokenize(raw, language, prop),
+          ]),
+        ];
+      },
+    },
+  });
+}
+
+function getSearchServer() {
+  // Reset on failure so a transient first-request error (e.g. a cold-start CPU
+  // spike) doesn't permanently poison this isolate's cached search server.
+  serverPromise ??= buildSearchServer().catch(error => {
+    serverPromise = undefined;
+    throw error;
+  });
+  return serverPromise;
+}
 
 export const Route = createFileRoute('/api/search')({
   server: {
     handlers: {
-      GET: async ({ request }) => server.GET(request),
+      GET: async ({ request }) => (await getSearchServer()).GET(request),
     },
   },
 });

vite.config.ts

@@ -1,12 +1,60 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
 import { cloudflare } from '@cloudflare/vite-plugin';
 import { sentryVitePlugin } from '@sentry/vite-plugin';
 import tailwindcss from '@tailwindcss/vite';
 import { tanstackStart } from '@tanstack/react-start/plugin/vite';
 import viteReact from '@vitejs/plugin-react';
 import mdx from 'fumadocs-mdx/vite';
-import { createLogger, defineConfig } from 'vite';
+import { createLogger, defineConfig, type Plugin } from 'vite';
 import * as MdxConfig from './source.config';
 
+// The Korean search tokenizer (garu-ko) ships a browser build that `fetch()`es
+// its WASM/model and a node build that reads them from `fs` — neither works in
+// the Cloudflare Workers (workerd) runtime where our search API runs. Instead we
+// import the WASM as a module (compiled at startup, instantiated per isolate) and
+// inline the ~1MB model as bytes. garu-ko's package `exports` map hides these
+// internal files, so we alias directly to them.
+const garuGlue = fileURLToPath(new URL('./node_modules/garu-ko/pkg/garu_wasm.js', import.meta.url));
+const garuWasm = fileURLToPath(
+  new URL('./node_modules/garu-ko/pkg/garu_wasm_bg.wasm', import.meta.url)
+);
+const garuModel = fileURLToPath(
+  new URL('./node_modules/garu-ko/models/base.gmdl', import.meta.url)
+);
+
+// Load the garu model (`.gmdl`) as an inlined Uint8Array so it travels in the
+// Worker bundle instead of being fetched/read at runtime, and assert the aliased
+// garu-ko internals exist so a version bump that relocates them fails loudly.
+function garuModelBytes(): Plugin {
+  return {
+    name: 'garu-model-bytes',
+    enforce: 'pre',
+    buildStart() {
+      const pkg = fileURLToPath(new URL('./node_modules/garu-ko/package.json', import.meta.url));
+      const { version } = JSON.parse(readFileSync(pkg, 'utf8')) as { version: string };
+      for (const file of [garuGlue, garuWasm, garuModel]) {
+        if (!existsSync(file)) {
+          this.error(
+            `[garu tokenizer] expected garu-ko internal file is missing: ${file} (garu-ko@${version}). ` +
+              'garu-ko may have relocated its pkg/ or models/ files; pin garu-ko and update the aliases in vite.config.ts.'
+          );
+        }
+      }
+    },
+    load(id) {
+      if (id.replace(/\?.*$/, '') !== garuModel) return;
+      // Preallocated decode is faster than `Uint8Array.from(atob(...), cb)`,
+      // trimming the one-time first-search CPU cost on the Worker.
+      const base64 = readFileSync(garuModel).toString('base64');
+      return `const binary = atob(${JSON.stringify(base64)});
+const bytes = new Uint8Array(binary.length);
+for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+export default bytes;`;
+    },
+  };
+}
+
 // Several dependencies (@tanstack/*, seroval) ship `//# sourceMappingURL`
 // comments without the referenced `.map` files. The Cloudflare SSR environment
 // bundles these packages (it can't externalize them for workerd), so Vite reads
@@ -28,6 +76,11 @@ export default defineConfig(({ command }) => ({
   },
   resolve: {
     tsconfigPaths: true,
+    alias: [
+      { find: /^garu-runtime\/glue$/, replacement: garuGlue },
+      { find: /^garu-runtime\/wasm$/, replacement: garuWasm },
+      { find: /^garu-runtime\/model$/, replacement: garuModel },
+    ],
   },
   build: {
     sourcemap: 'hidden',
@@ -59,6 +112,7 @@ export default defineConfig(({ command }) => ({
     },
   },
   plugins: [
+    garuModelBytes(),
     mdx(MdxConfig),
     tailwindcss(),
     cloudflare({

yarn.lock

@@ -1605,7 +1605,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@orama/orama@npm:^3.1.18":
+"@orama/orama@npm:3.1.18, @orama/orama@npm:^3.1.18":
   version: 3.1.18
   resolution: "@orama/orama@npm:3.1.18"
   checksum: 10c0/868be5143eb4a27e700d9cc866763d1cc04405ef0af6a6b6b6987dcf33ee5d4ba90d3a32cd134006d46d34d64e32500e13cd31436c718eadc396a940e793e7cd
@@ -4944,6 +4944,24 @@ __metadata:
   languageName: node
   linkType: hard
 
+"garu-ko@npm:0.9.7":
+  version: 0.9.7
+  resolution: "garu-ko@npm:0.9.7"
+  checksum: 10c0/660f5e5d8bca42a2d4326c8add45dc25cbf1c066b9c0685fa285728665b80a31201768a37bb86dd744fad88540c49343cb5ff4474ee7677afc000e18417d365d
+  languageName: node
+  linkType: hard
+
+"garu-orama-tokenizer@npm:0.4.1":
+  version: 0.4.1
+  resolution: "garu-orama-tokenizer@npm:0.4.1"
+  dependencies:
+    garu-ko: "npm:^0.9.0"
+  peerDependencies:
+    "@orama/orama": ">=2.0.0"
+  checksum: 10c0/19d1e885dfb32430a7869e2549de6d5ff8a4fb8363cafc8d6ecbe4b413983c8c4ef3c0e185b5f228688ddc9925533f58f390b8330e829614be0eb0ca9c045e60
+  languageName: node
+  linkType: hard
+
 "gensync@npm:^1.0.0-beta.2":
   version: 1.0.0-beta.2
   resolution: "gensync@npm:1.0.0-beta.2"
@@ -7956,6 +7974,7 @@ __metadata:
     "@date-fns/tz": "npm:1.5.0"
     "@fontsource-variable/inter": "npm:5.2.8"
     "@fontsource-variable/jetbrains-mono": "npm:5.2.8"
+    "@orama/orama": "npm:3.1.18"
     "@sentry/cloudflare": "npm:10.53.1"
     "@sentry/tanstackstart-react": "npm:10.53.1"
     "@sentry/vite-plugin": "npm:5.3.0"
@@ -7972,6 +7991,8 @@ __metadata:
     fumadocs-core: "npm:16.7.16"
     fumadocs-mdx: "npm:14.3.0"
     fumadocs-ui: "npm:16.7.16"
+    garu-ko: "npm:0.9.7"
+    garu-orama-tokenizer: "npm:0.4.1"
     jose: "npm:6.2.3"
     oxfmt: "npm:0.51.0"
     oxlint: "npm:1.66.0"