fix(processing): let Whitebox Dissolve take a grouping field

apps/geolibre-desktop/src/components/processing/ProcessingDialog.tsx

@@ -190,8 +190,9 @@ function isSubsetUrlParameter(tool: WhiteboxTool, param: WhiteboxToolParameter):
 // vector inputs (points_to_line's `line_field`/`sort_field`, and ~170 other
 // tools), so the dialog can offer the selected layer's attribute names instead
 // of asking the user to recall a column name (GeoLibre#1459). The kind check is
-// what keeps a same-named *dataset* param out (join_tables' `primary_key_field`
-// is a vector input): only a scalar string names a column.
+// what keeps a same-named *dataset* param out (the catalog types
+// classify_objects_svm's `class_field` as a LiDAR input): only a scalar string
+// names a column.
 function isFieldParameter(param: WhiteboxToolParameter): boolean {
   return parameterKind(param) === "string" && isFieldParameterName(param.name);
 }

apps/geolibre-desktop/src/lib/whitebox-field-params.ts

@@ -17,9 +17,12 @@ export const FIELD_PARAM_SUFFIX = /(^|_)(fields?|attributes?)$/i;
 /**
  * Whether a parameter name reads as an attribute-column name.
  *
- * Callers must also check the parameter is a scalar string: `join_tables`
- * exposes `primary_key_field` as a *dataset* input, and a dataset parameter
- * names a file, not a column.
+ * Callers must also check the parameter is a scalar string: the sidecar's
+ * catalog exposes `classify_objects_svm`'s `class_field` as a *dataset* input,
+ * and a dataset parameter names a file, not a column. (The WASM manifests used
+ * to mistype ~40 of these the same way, `dissolve_field` among them, until
+ * opengeos/whitebox-wasm#19 taught the manifest inference that a `*_field` name
+ * is a column; the sidecar catalog still carries a few.)
  *
  * @param name - The tool parameter's name.
  * @returns `true` when the name ends in a field/attribute suffix.

packages/processing/package.json

@@ -34,7 +34,7 @@
     "@turf/voronoi": "^7.4.0",
     "dggal": "^0.0.6",
     "fflate": "^0.8.3",
-    "geolibre-wasm": "^1.5.1",
+    "geolibre-wasm": "^1.5.2",
     "geotiff": "^3.0.5",
     "onnxruntime-web": "1.27.0",
     "s2js": "^1.44.0"

packages/processing/src/wasm-client.ts

@@ -8,6 +8,7 @@
 import type { FeatureCollection } from "geojson";
 import { convertGeoTiffToCog } from "./cog-convert";
 import { normalizeVectorOutputFormat } from "./sidecar-client";
+import { runWasmToolInBackground } from "./wasm-tool-runner";
 import type {
   RunWhiteboxToolRequest,
   VectorOutputFormat,
@@ -651,7 +652,6 @@ export async function ensureWhiteboxRasterCog(bytes: Uint8Array): Promise<Uint8A
  * (Cloud Optimized GeoTIFF) for `raster_out` - never a server path.
  */
 export async function runWhiteboxToolWasm(request: RunWhiteboxToolRequest): Promise<WhiteboxJob> {
-  const { runTool } = await loadToolsModule();
   const encoder = new TextEncoder();
   const input: Record<string, Uint8Array> = {};
   const args: string[] = [];
@@ -770,7 +770,14 @@ export async function runWhiteboxToolWasm(request: RunWhiteboxToolRequest): Prom
     }
   }
 
-  const { exitCode, stdout, files } = await runTool(request.tool_id, { args, input });
+  // Off the main thread: the WASI runner is one synchronous call with no yield
+  // points, so running it here would freeze the UI for the tool's whole
+  // duration (~60s for the 290-polygon dissolve in GeoLibre#1977).
+  const { exitCode, stdout, files } = await runWasmToolInBackground({
+    tool: request.tool_id,
+    args,
+    input,
+  });
   if (exitCode !== 0) {
     return job(
       request.tool_id,

packages/processing/src/wasm-convert.ts

@@ -14,7 +14,7 @@
 // All three run entirely client-side, so the web build needs no Python sidecar
 // for them.
 import type { RunToolOptions, ToolResult } from "geolibre-wasm/tools";
-import type { WasmToolRequest, WasmToolResponse } from "./wasm-convert.worker";
+import { runWasmToolInBackground } from "./wasm-tool-runner";
 
 /** The subset of `geolibre-wasm/tools` these converters use. */
 interface ConvertToolsModule {
@@ -59,61 +59,6 @@ export async function initConvertTools(
   await initTools(source);
 }
 
-/**
- * Run a tool on a one-shot Web Worker and resolve with its result.
- *
- * No timeout: how long a tool runs is bounded by the data, not the clock (a
- * country-scale tile pyramid is minutes), and cutting off work that would have
- * finished is worse than waiting. `error`/`messageerror` still reject, so the
- * promise settles on every failure the worker can report.
- */
-function runToolOnWorker(request: WasmToolRequest): Promise<ToolResult> {
-  return new Promise((resolve, reject) => {
-    const worker = new Worker(new URL("./wasm-convert.worker.ts", import.meta.url), {
-      type: "module",
-    });
-    worker.addEventListener("message", (event: MessageEvent<WasmToolResponse>) => {
-      worker.terminate();
-      if (event.data.ok) resolve(event.data.result);
-      else reject(new Error(event.data.error || `${request.tool} failed.`));
-    });
-    worker.addEventListener("error", (event) => {
-      worker.terminate();
-      reject(new Error(event.message || `The ${request.tool} worker failed.`));
-    });
-    // `error` does not fire when a posted message cannot be deserialized, which
-    // would otherwise leave this promise pending forever.
-    worker.addEventListener("messageerror", () => {
-      worker.terminate();
-      reject(new Error(`The ${request.tool} worker posted an undeserializable message.`));
-    });
-    // The input files are structured-cloned rather than transferred: these
-    // wrappers do not otherwise take ownership of the caller's bytes, and a
-    // neutered input array would be a trap the sibling converters don't set.
-    try {
-      worker.postMessage(request);
-    } catch (error) {
-      // A throw here (e.g. DataCloneError) rejects the promise on its own, but
-      // the worker is already spawned and would leak without this.
-      worker.terminate();
-      reject(error instanceof Error ? error : new Error(String(error)));
-    }
-  });
-}
-
-/**
- * Run a tool off the main thread where Workers exist, inline where they do not
- * (node, tests). The inline path is why {@link initConvertTools} still takes an
- * explicit wasm source: a worker resolves its own bundled copy instead.
- */
-async function runToolInBackground(request: WasmToolRequest): Promise<ToolResult> {
-  if (typeof Worker === "undefined") {
-    const { runTool } = await loadToolsModule();
-    return runTool(request.tool, { args: request.args, input: request.input });
-  }
-  return runToolOnWorker(request);
-}
-
 /** An input file for a WASM conversion: its name (the extension drives format
  * detection) and its raw bytes. */
 export interface WasmConvertFile {
@@ -295,7 +240,7 @@ export interface VectorToPmtilesOptions {
  * `siblings`, exactly as in {@link convertVectorWithWasm}.
  *
  * Unlike its siblings here this runs on a Web Worker (see
- * {@link runToolInBackground}). Tiling is by far the heaviest of these tools —
+ * {@link runWasmToolInBackground}). Tiling is by far the heaviest of these tools —
  * a US-wide layer to the default zoom 14 is millions of tiles and minutes of
  * uninterrupted WASM — so running it on the main thread would freeze the UI for
  * the whole conversion. The others finish quickly enough not to warrant the
@@ -322,7 +267,7 @@ export async function tileVectorToPmtiles(
   ]);
   const files: Record<string, Uint8Array> = { [input.name]: input.data };
   for (const sibling of siblings) files[sibling.name] = sibling.data;
-  const result = await runToolInBackground({
+  const result = await runWasmToolInBackground({
     tool: "vector_to_pmtiles",
     args,
     input: files,

packages/processing/src/wasm-tool-runner.ts

@@ -0,0 +1,134 @@
+// Shared plumbing for running a `geolibre-wasm/tools` WASI tool off the main
+// thread.
+//
+// The runner has no yield points once a tool starts: `wasi.start()` is one
+// synchronous call that returns only when the tool is done. On the main thread
+// that freezes the whole UI — no repaint, no input — for the tool's entire
+// duration, which is bounded by the data rather than the clock. Dissolving the
+// 290-polygon layer from GeoLibre#1977 blocks it for ~60s.
+//
+// wasm-convert.ts routed its tiling calls through a worker for exactly this
+// reason. This module is that machinery, lifted out so the Whitebox toolbox
+// (wasm-client.ts) shares one implementation with it instead of growing a
+// second copy that could drift.
+import type { ToolResult } from "geolibre-wasm/tools";
+import type { WasmToolRequest, WasmToolResponse } from "./wasm-tool.worker";
+
+export type { WasmToolRequest, WasmToolResponse };
+
+// Idle workers, kept alive to be reused. A worker compiles the ~23 MB
+// `geolibre-cli.wasm` in its *own* module scope, and the main thread's
+// already-compiled copy is not shared with it, so a worker discarded after
+// every run makes each run pay that fetch and compile again. That is invisible
+// next to a minutes-long tiling job — the only thing that used this path
+// before — but not next to the many Whitebox tools that finish in well under a
+// second, where it would dominate the run.
+//
+// Reuse rather than a single shared worker: a WASI run is synchronous inside
+// its worker, so one worker would serialize concurrent runs that used to
+// overlap. Taking an idle worker when there is one and spawning otherwise keeps
+// that parallelism and still pays the compile once per worker.
+const idleWorkers: Worker[] = [];
+
+// How many idle workers to keep warm. Each holds its compiled module (tens of
+// MB) for the rest of the session, and real usage is one tool at a time, so
+// one warm worker captures nearly all of the benefit; extras are terminated
+// rather than parked.
+const MAX_IDLE_WORKERS = 1;
+
+/**
+ * Terminate every parked worker and forget them, freeing the compiled module
+ * each one holds. Runs in flight are unaffected — they own their worker until
+ * it answers. Call it to reclaim that memory, and in tests, so a worker parked
+ * by one case is not handed to the next.
+ */
+export function releaseIdleWasmToolWorkers(): void {
+  for (const worker of idleWorkers.splice(0)) worker.terminate();
+}
+
+function acquireWorker(): Worker {
+  return (
+    idleWorkers.pop() ??
+    new Worker(new URL("./wasm-tool.worker.ts", import.meta.url), { type: "module" })
+  );
+}
+
+/** Park a still-healthy worker for reuse, or terminate it if enough are warm. */
+function releaseWorker(worker: Worker): void {
+  if (idleWorkers.length < MAX_IDLE_WORKERS) idleWorkers.push(worker);
+  else worker.terminate();
+}
+
+/**
+ * Run a tool on a Web Worker and resolve with its result.
+ *
+ * No timeout: how long a tool runs is bounded by the data, not the clock (a
+ * country-scale tile pyramid is minutes), and cutting off work that would have
+ * finished is worse than waiting. `error`/`messageerror` still reject, so the
+ * promise settles on every failure the worker can report.
+ *
+ * A worker that answers is parked for reuse; one that fails at the worker level
+ * is terminated, since its state after that is not something to hand the next
+ * caller. Listeners are removed on the way out so a reused worker does not
+ * accumulate them.
+ */
+function runToolOnWorker(request: WasmToolRequest): Promise<ToolResult> {
+  return new Promise((resolve, reject) => {
+    const worker = acquireWorker();
+    const onMessage = (event: MessageEvent<WasmToolResponse>) => {
+      cleanup();
+      releaseWorker(worker);
+      if (event.data.ok) resolve(event.data.result);
+      else reject(new Error(event.data.error || `${request.tool} failed.`));
+    };
+    const onError = (event: ErrorEvent) => {
+      cleanup();
+      worker.terminate();
+      reject(new Error(event.message || `The ${request.tool} worker failed.`));
+    };
+    // `error` does not fire when a posted message cannot be deserialized, which
+    // would otherwise leave this promise pending forever.
+    const onMessageError = () => {
+      cleanup();
+      worker.terminate();
+      reject(new Error(`The ${request.tool} worker posted an undeserializable message.`));
+    };
+    const cleanup = () => {
+      worker.removeEventListener("message", onMessage);
+      worker.removeEventListener("error", onError);
+      worker.removeEventListener("messageerror", onMessageError);
+    };
+    worker.addEventListener("message", onMessage);
+    worker.addEventListener("error", onError);
+    worker.addEventListener("messageerror", onMessageError);
+    // The input files are structured-cloned rather than transferred: these
+    // wrappers do not otherwise take ownership of the caller's bytes, and a
+    // neutered input array would be a trap the callers don't set.
+    try {
+      worker.postMessage(request);
+    } catch (error) {
+      // A throw here (e.g. DataCloneError) rejects the promise on its own, but
+      // the worker is already spawned and would leak without this.
+      cleanup();
+      worker.terminate();
+      reject(error instanceof Error ? error : new Error(String(error)));
+    }
+  });
+}
+
+/**
+ * Run a tool off the main thread where Workers exist, inline where they do not
+ * (node, tests). The inline path is why the callers still expose an explicit
+ * wasm-source init (`initConvertTools`): a worker resolves its own bundled copy
+ * instead, in its own module scope.
+ *
+ * @param request - The tool id, CLI args, and files to place under `/work`.
+ * @returns The tool's exit code, captured output, and the files it wrote.
+ */
+export async function runWasmToolInBackground(request: WasmToolRequest): Promise<ToolResult> {
+  if (typeof Worker === "undefined") {
+    const { runTool } = await import("geolibre-wasm/tools");
+    return runTool(request.tool, { args: request.args, input: request.input });
+  }
+  return runToolOnWorker(request);
+}

packages/processing/src/wasm-convert.worker.ts → packages/processing/src/wasm-tool.worker.ts

@@ -2,13 +2,17 @@
 import { runTool } from "geolibre-wasm/tools";
 import type { ToolResult } from "geolibre-wasm/tools";
 
-// Runs one `geolibre-wasm/tools` WASI tool off the main thread. The runner has
-// no yield points once a tool starts, so on the main thread a long job freezes
-// the whole UI for its duration — tiling a country-scale vector layer to zoom 14
-// is minutes of that, which is why wasm-convert.ts routes it here.
+// Runs `geolibre-wasm/tools` WASI tools off the main thread. The runner has no
+// yield points once a tool starts, so on the main thread a long job freezes the
+// whole UI for its duration — tiling a country-scale vector layer to zoom 14 is
+// minutes of that, and dissolving 290 polygons by an attribute is ~60s, which is
+// why wasm-convert.ts and wasm-client.ts both route through wasm-tool-runner.ts.
 //
-// One tool per worker: the caller terminates this worker as soon as the terminal
-// message arrives, so nothing here has to be reusable across runs.
+// One run at a time, but reusable across runs: the caller parks this worker for
+// its next call instead of terminating it, so the ~23 MB `geolibre-cli.wasm`
+// compile in this worker's module scope is paid once rather than per run. Each
+// run still gets a fresh WASI instance and a fresh /work from `runTool`, so
+// nothing carries over between them.
 const worker = self as unknown as DedicatedWorkerGlobalScope;
 
 /** A tool to run: its id, CLI args, and the files to place under /work. */