Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,9 @@ function isSubsetUrlParameter(tool: WhiteboxTool, param: WhiteboxToolParameter):
// vector inputs (points_to_line's `line_field`/`sort_field`, and ~170 other
// tools), so the dialog can offer the selected layer's attribute names instead
// of asking the user to recall a column name (GeoLibre#1459). The kind check is
// what keeps a same-named *dataset* param out (join_tables' `primary_key_field`
// is a vector input): only a scalar string names a column.
// what keeps a same-named *dataset* param out (the catalog types
// classify_objects_svm's `class_field` as a LiDAR input): only a scalar string
// names a column.
function isFieldParameter(param: WhiteboxToolParameter): boolean {
return parameterKind(param) === "string" && isFieldParameterName(param.name);
}
Expand Down
9 changes: 6 additions & 3 deletions apps/geolibre-desktop/src/lib/whitebox-field-params.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@ export const FIELD_PARAM_SUFFIX = /(^|_)(fields?|attributes?)$/i;
/**
* Whether a parameter name reads as an attribute-column name.
*
* Callers must also check the parameter is a scalar string: `join_tables`
* exposes `primary_key_field` as a *dataset* input, and a dataset parameter
* names a file, not a column.
* Callers must also check the parameter is a scalar string: the sidecar's
* catalog exposes `classify_objects_svm`'s `class_field` as a *dataset* input,
* and a dataset parameter names a file, not a column. (The WASM manifests used
* to mistype ~40 of these the same way, `dissolve_field` among them, until
* opengeos/whitebox-wasm#19 taught the manifest inference that a `*_field` name
* is a column; the sidecar catalog still carries a few.)
*
* @param name - The tool parameter's name.
* @returns `true` when the name ends in a field/attribute suffix.
Expand Down
8 changes: 4 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/processing/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
"@turf/voronoi": "^7.4.0",
"dggal": "^0.0.6",
"fflate": "^0.8.3",
"geolibre-wasm": "^1.5.1",
"geolibre-wasm": "^1.5.2",
Comment thread
coderabbitai[bot] marked this conversation as resolved.
"geotiff": "^3.0.5",
"onnxruntime-web": "1.27.0",
"s2js": "^1.44.0"
Expand Down
11 changes: 9 additions & 2 deletions packages/processing/src/wasm-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import type { FeatureCollection } from "geojson";
import { convertGeoTiffToCog } from "./cog-convert";
import { normalizeVectorOutputFormat } from "./sidecar-client";
import { runWasmToolInBackground } from "./wasm-tool-runner";
import type {
RunWhiteboxToolRequest,
VectorOutputFormat,
Expand Down Expand Up @@ -651,7 +652,6 @@ export async function ensureWhiteboxRasterCog(bytes: Uint8Array): Promise<Uint8A
* (Cloud Optimized GeoTIFF) for `raster_out` - never a server path.
*/
export async function runWhiteboxToolWasm(request: RunWhiteboxToolRequest): Promise<WhiteboxJob> {
const { runTool } = await loadToolsModule();
const encoder = new TextEncoder();
const input: Record<string, Uint8Array> = {};
const args: string[] = [];
Expand Down Expand Up @@ -770,7 +770,14 @@ export async function runWhiteboxToolWasm(request: RunWhiteboxToolRequest): Prom
}
}

const { exitCode, stdout, files } = await runTool(request.tool_id, { args, input });
// Off the main thread: the WASI runner is one synchronous call with no yield
// points, so running it here would freeze the UI for the tool's whole
// duration (~60s for the 290-polygon dissolve in GeoLibre#1977).
const { exitCode, stdout, files } = await runWasmToolInBackground({
Comment thread
giswqs marked this conversation as resolved.
tool: request.tool_id,
args,
input,
});
if (exitCode !== 0) {
return job(
request.tool_id,
Expand Down
61 changes: 3 additions & 58 deletions packages/processing/src/wasm-convert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
// All three run entirely client-side, so the web build needs no Python sidecar
// for them.
import type { RunToolOptions, ToolResult } from "geolibre-wasm/tools";
import type { WasmToolRequest, WasmToolResponse } from "./wasm-convert.worker";
import { runWasmToolInBackground } from "./wasm-tool-runner";

/** The subset of `geolibre-wasm/tools` these converters use. */
interface ConvertToolsModule {
Expand Down Expand Up @@ -59,61 +59,6 @@ export async function initConvertTools(
await initTools(source);
}

/**
* Run a tool on a one-shot Web Worker and resolve with its result.
*
* No timeout: how long a tool runs is bounded by the data, not the clock (a
* country-scale tile pyramid is minutes), and cutting off work that would have
* finished is worse than waiting. `error`/`messageerror` still reject, so the
* promise settles on every failure the worker can report.
*/
function runToolOnWorker(request: WasmToolRequest): Promise<ToolResult> {
return new Promise((resolve, reject) => {
const worker = new Worker(new URL("./wasm-convert.worker.ts", import.meta.url), {
type: "module",
});
worker.addEventListener("message", (event: MessageEvent<WasmToolResponse>) => {
worker.terminate();
if (event.data.ok) resolve(event.data.result);
else reject(new Error(event.data.error || `${request.tool} failed.`));
});
worker.addEventListener("error", (event) => {
worker.terminate();
reject(new Error(event.message || `The ${request.tool} worker failed.`));
});
// `error` does not fire when a posted message cannot be deserialized, which
// would otherwise leave this promise pending forever.
worker.addEventListener("messageerror", () => {
worker.terminate();
reject(new Error(`The ${request.tool} worker posted an undeserializable message.`));
});
// The input files are structured-cloned rather than transferred: these
// wrappers do not otherwise take ownership of the caller's bytes, and a
// neutered input array would be a trap the sibling converters don't set.
try {
worker.postMessage(request);
} catch (error) {
// A throw here (e.g. DataCloneError) rejects the promise on its own, but
// the worker is already spawned and would leak without this.
worker.terminate();
reject(error instanceof Error ? error : new Error(String(error)));
}
});
}

/**
* Run a tool off the main thread where Workers exist, inline where they do not
* (node, tests). The inline path is why {@link initConvertTools} still takes an
* explicit wasm source: a worker resolves its own bundled copy instead.
*/
async function runToolInBackground(request: WasmToolRequest): Promise<ToolResult> {
if (typeof Worker === "undefined") {
const { runTool } = await loadToolsModule();
return runTool(request.tool, { args: request.args, input: request.input });
}
return runToolOnWorker(request);
}

/** An input file for a WASM conversion: its name (the extension drives format
* detection) and its raw bytes. */
export interface WasmConvertFile {
Expand Down Expand Up @@ -295,7 +240,7 @@ export interface VectorToPmtilesOptions {
* `siblings`, exactly as in {@link convertVectorWithWasm}.
*
* Unlike its siblings here this runs on a Web Worker (see
* {@link runToolInBackground}). Tiling is by far the heaviest of these tools —
* {@link runWasmToolInBackground}). Tiling is by far the heaviest of these tools —
* a US-wide layer to the default zoom 14 is millions of tiles and minutes of
* uninterrupted WASM — so running it on the main thread would freeze the UI for
* the whole conversion. The others finish quickly enough not to warrant the
Expand All @@ -322,7 +267,7 @@ export async function tileVectorToPmtiles(
]);
const files: Record<string, Uint8Array> = { [input.name]: input.data };
for (const sibling of siblings) files[sibling.name] = sibling.data;
const result = await runToolInBackground({
const result = await runWasmToolInBackground({
tool: "vector_to_pmtiles",
args,
input: files,
Expand Down
40 changes: 0 additions & 40 deletions packages/processing/src/wasm-convert.worker.ts

This file was deleted.

Loading
Loading