Predicate-aware HNSW traversal: filtered vector search with user-function and RBAC participation

benchmarks/hnsw-search.js

@@ -155,3 +155,96 @@ console.log(`  Latency:            ${fmtNs(avgNs)}/query avg`);
 console.log(`  Throughput:         ${qps.toFixed(0)} qps`);
 console.log(`  Nodes visited/query: ${(hnsw.nodesVisitedCount / N_QUERIES).toFixed(1)}`);
 console.log(`  Recall@${TOP_K}:          ${(recall * 100).toFixed(1)}%\n`);
+
+// ---------------------------------------------------------------------------
+// Filtered-recall scenario (#1241): recall@K vs. selectivity for three strategies —
+//   post-filter    : plain search, then drop non-matching from the fixed top-ef set (today's behavior)
+//   predicate      : predicate-aware traversal (filter passed into search, ACORN-style)
+//   brute-force    : exact distance over the matching set only (the selective-filter fallback)
+// Predicate traversal should dominate post-filter on recall at low selectivity without pathological
+// latency; brute-force is the crossover once the matching set is tiny.
+// ---------------------------------------------------------------------------
+
+const idxOf = (key) => parseInt(key.slice(1));
+const SELECTIVITIES = [0.5, 0.1, 0.01, 0.001];
+
+console.log(`Filtered recall@${TOP_K} vs. selectivity (${N_QUERIES} queries):\n`);
+console.log('  selectivity   matches   post-filter   predicate   brute-force   pred.lat    bf.lat');
+console.log('  ' + '-'.repeat(88));
+
+for (const p of SELECTIVITIES) {
+	// Deterministic-ish membership: mark a p-fraction of vectors as matching the companion predicate.
+	const matches = new Array(N_VECTORS);
+	let matchCount = 0;
+	for (let i = 0; i < N_VECTORS; i++) {
+		matches[i] = Math.random() < p;
+		if (matches[i]) matchCount++;
+	}
+	if (matchCount === 0) {
+		console.log(
+			`  ${(p * 100).toFixed(1).padStart(9)}%   ${String(matchCount).padStart(7)}   (no matching vectors — skipped)`
+		);
+		continue;
+	}
+	const filter = (key) => matches[idxOf(key)];
+	const k = Math.min(TOP_K, matchCount);
+
+	// Ground truth: the k nearest among the matching vectors only.
+	const filteredGT = queries.map((query) => {
+		const scored = [];
+		for (let i = 0; i < N_VECTORS; i++) if (matches[i]) scored.push({ i, d: cosineDistance(query, vectors[i]) });
+		scored.sort((a, b) => a.d - b.d);
+		return new Set(scored.slice(0, k).map((x) => x.i));
+	});
+
+	let postRecall = 0;
+	let predRecall = 0;
+	let bfRecall = 0;
+	let predNs = 0;
+	let bfNs = 0;
+
+	for (let q = 0; q < N_QUERIES; q++) {
+		const query = queries[q];
+		const gt = filteredGT[q];
+
+		// post-filter: unfiltered search, then keep matching, take k
+		const plain = hnsw.search({ target: query, comparator: 'sort', descending: false }, {});
+		const postIdx = new Set(
+			plain
+				.filter((r) => matches[idxOf(r.key)])
+				.slice(0, k)
+				.map((r) => idxOf(r.key))
+		);
+		postRecall += overlap(gt, postIdx) / k;
+
+		// predicate-aware traversal
+		const t0 = performance.now();
+		const pred = hnsw.search({ target: query, comparator: 'sort', descending: false }, {}, filter);
+		predNs += (performance.now() - t0) * 1e6;
+		const predIdx = new Set(pred.slice(0, k).map((r) => idxOf(r.key)));
+		predRecall += overlap(gt, predIdx) / k;
+
+		// brute-force over the matching set
+		const b0 = performance.now();
+		const scored = [];
+		for (let i = 0; i < N_VECTORS; i++) if (matches[i]) scored.push({ i, d: cosineDistance(query, vectors[i]) });
+		scored.sort((a, b) => a.d - b.d);
+		bfNs += (performance.now() - b0) * 1e6;
+		const bfIdx = new Set(scored.slice(0, k).map((x) => x.i));
+		bfRecall += overlap(gt, bfIdx) / k;
+	}
+
+	const pct = (x) => `${((x / N_QUERIES) * 100).toFixed(1)}%`.padStart(9);
+	console.log(
+		`  ${(p * 100).toFixed(1).padStart(9)}%   ${String(matchCount).padStart(7)}   ` +
+			`${pct(postRecall).padStart(11)}   ${pct(predRecall).padStart(9)}   ${pct(bfRecall).padStart(11)}   ` +
+			`${fmtNs(predNs / N_QUERIES).padStart(8)}   ${fmtNs(bfNs / N_QUERIES).padStart(8)}`
+	);
+}
+console.log('');
+
+function overlap(a, b) {
+	let n = 0;
+	for (const x of a) if (b.has(x)) n++;
+	return n;
+}

resources/DESIGN.md

@@ -89,6 +89,7 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource`
 | How does a query opt out of a read snapshot?        | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`). |
 | How does a URL path map to a Resource?              | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below                                                                                                                                                                                                                       |
 | How does HNSW keep the graph connected on delete?   | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712)                                                                                                                                 |
+| How is a filter applied *during* a vector search?   | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a static `allowReadRecord` RBAC hook into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective *condition* filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering. |
 
 ---
 

resources/RequestTarget.ts

@@ -66,6 +66,15 @@ export class RequestTarget extends URLSearchParams {
 	declare allowFullScan?: boolean;
 	declare allowConditionsOnDynamicAttributes?: boolean;
 
+	/**
+	 * Predicate-aware vector search (#1241). A `(record) => boolean` filter evaluated during HNSW
+	 * traversal (and as a post-filter for other paths), so a vector sort keeps exploring until it has
+	 * enough MATCHING nearest neighbors instead of post-filtering an under-filled candidate set. The
+	 * function must be synchronous and side-effect free; the record it receives is frozen. JS-API only —
+	 * never parsed from a REST query string (no eval of user-supplied code over HTTP).
+	 */
+	declare vectorFilter?: (record: any) => boolean;
+
 	/**
 	 * When `false`, the query reads against the latest committed data without holding a consistent
 	 * read snapshot open for the duration of the iteration. This trades read consistency (rows

resources/ResourceInterface.ts

@@ -181,6 +181,12 @@ interface TypedDirectCondition<Record extends object, Property extends keyof Rec
 	 * full scan unless paired with another indexed condition.
 	 */
 	negated?: boolean;
+	/**
+	 * Internal (#1241): a `(primaryKey) => boolean` predicate composed by the query executor from
+	 * companion conditions, a caller `vectorFilter`, and record-level RBAC, pushed into a filterable
+	 * custom index (HNSW) so filtering happens during traversal. Not part of the public query surface.
+	 */
+	recordFilter?: (primaryKey: Id) => boolean;
 }
 
 interface ConditionGroup<Record extends object = any> {

resources/Table.ts

@@ -2854,6 +2854,17 @@ export function makeTable(options) {
 			// scans), the read transaction reads against the latest committed data without pinning a
 			// consistent snapshot, so the scan doesn't hold a snapshot that blocks compaction.
 			const readTxn = txn.useReadTxn(target.snapshot === false);
+			// Record-level read guard (#1241): a resource may define a static allowReadRecord(user, record)
+			// that participates in query execution (pushed into HNSW traversal for vector sorts, applied as a
+			// post-filter otherwise). Resolve off the actual (possibly subclassed) constructor so overrides win.
+			// SCOPE: this is a QUERY-result filter, not a general read-authorization boundary — direct
+			// single-record get(id) does not consult it (use allowRead/instance hooks for that). It must be
+			// synchronous, side-effect free, and fast; it can run once per candidate record during traversal.
+			const allowReadRecord = (this.constructor as any).allowReadRecord;
+			const recordAccess =
+				typeof allowReadRecord === 'function' || typeof target.vectorFilter === 'function'
+					? { allowReadRecord, user: (context as any)?.user, vectorFilter: target.vectorFilter }
+					: undefined;
 			const entries = executeConditions(
 				conditions,
 				operator,
@@ -2862,16 +2873,28 @@ export function makeTable(options) {
 				target,
 				context,
 				(results: any[], filters: Function[]) => transformToEntries(results, select, context, readTxn, filters),
-				filtered
+				filtered,
+				recordAccess
 			);
 			const ensure_loaded = (target as any).ensureLoaded !== false;
+			// Authoritative RBAC enforcement (#1241): the guards inside executeConditions evaluate the LOCAL
+			// record, but on a caching table transformEntryForSelect may then revalidate an expired/invalidated
+			// row from source and return a DIFFERENT record. An authorization check must hold on the record
+			// actually returned, so allowReadRecord is re-checked there, after materialization (the earlier
+			// evaluation stays as a prune that also bounds HNSW traversal). vectorFilter and condition filters
+			// intentionally keep the local-record semantics all query filters have on caching tables.
+			const recordGuard =
+				typeof allowReadRecord === 'function'
+					? (record: any) => allowReadRecord((context as any)?.user, record)
+					: undefined;
 			const transformToRecord = TableResource.transformEntryForSelect(
 				select,
 				context,
 				readTxn,
 				filtered,
 				ensure_loaded,
-				true
+				true,
+				recordGuard
 			);
 			let results = TableResource.transformToOrderedSelect(
 				entries,
@@ -3082,9 +3105,12 @@ export function makeTable(options) {
 		 * @param filtered
 		 * @param ensure_loaded
 		 * @param canSkip
+		 * @param recordGuard record-level read check (#1241) applied to the record actually being
+		 * returned — i.e. AFTER any caching-source revalidation replaces a stale local copy — so an
+		 * authorization verdict can't be made on bytes that differ from what the caller receives.
 		 * @returns
 		 */
-		static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded?, canSkip?) {
+		static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded?, canSkip?, recordGuard?) {
 			let checkLoaded;
 			if (
 				ensure_loaded &&
@@ -3164,6 +3190,10 @@ export function makeTable(options) {
 					}
 				}
 				if (record == null) return canSkip ? SKIP : record;
+				// Record-level RBAC (#1241): enforced here because `record` is now the final, materialized
+				// record — a caching table's source revalidation (above) may have replaced the local copy the
+				// query filters evaluated. Fail closed on the record actually being returned.
+				if (recordGuard && !recordGuard(record)) return canSkip ? SKIP : undefined;
 				if (select && !(select[0] === '*' && select.length === 1)) {
 					let promises: Promise<any>[];
 					const selectAttribute = (attribute, callback) => {