From b2f473b89a3ef34a1546c7d21e9b2b57dd9d9404 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 12 Jul 2026 08:38:07 -0600 Subject: [PATCH 1/3] Predicate-aware HNSW traversal: filtered vector search with RBAC (#1241) Make HNSW beam search predicate-aware so a vector sort combined with a filter keeps exploring until it has `ef` MATCHING results instead of post-filtering a fixed top-`ef` candidate set (which under-fills under selective filters and leaks the presence of nearby restricted records). - HierarchicalNavigableSmallWorld.search() accepts an optional (primaryKey) => boolean predicate. It gates result admission at layer 0 only; upper layers and the candidate/visited sets ignore it, so non-matching nodes still route (ACORN). A visit budget (ef * filterExpansion) bounds the under-filled/selective case and returns partial results; nodesVisited / filterEvaluations exposed for tuning. - search.ts composes companion AND conditions + a request `vectorFilter` + a static `allowReadRecord` RBAC hook into one predicate and pushes it into a leading filterable custom index; kept in the post-filter chain as a deterministic re-check. Very selective condition filters still go to the exact brute-force path via the planner's existing ordering. - vectorFilter (JS-API only) and record-level allowReadRecord wired through Table.search. - Unit tests (index-level mechanism + end-to-end vectorFilter/RBAC/int8), filtered-recall benchmark scenario, DESIGN.md note. Co-Authored-By: Claude Fable 5 --- benchmarks/hnsw-search.js | 93 +++++ resources/DESIGN.md | 1 + resources/RequestTarget.ts | 9 + resources/ResourceInterface.ts | 6 + resources/Table.ts | 36 +- .../HierarchicalNavigableSmallWorld.ts | 174 ++++++++- resources/search.ts | 118 +++++- unitTests/resources/vectorIndex.test.js | 346 ++++++++++++++++++ 8 files changed, 755 insertions(+), 28 deletions(-) diff --git a/benchmarks/hnsw-search.js b/benchmarks/hnsw-search.js index 8fac85b446..ca734ce7a6 100644 --- a/benchmarks/hnsw-search.js +++ b/benchmarks/hnsw-search.js @@ -155,3 +155,96 @@ console.log(` Latency: ${fmtNs(avgNs)}/query avg`); console.log(` Throughput: ${qps.toFixed(0)} qps`); console.log(` Nodes visited/query: ${(hnsw.nodesVisitedCount / N_QUERIES).toFixed(1)}`); console.log(` Recall@${TOP_K}: ${(recall * 100).toFixed(1)}%\n`); + +// --------------------------------------------------------------------------- +// Filtered-recall scenario (#1241): recall@K vs. selectivity for three strategies — +// post-filter : plain search, then drop non-matching from the fixed top-ef set (today's behavior) +// predicate : predicate-aware traversal (filter passed into search, ACORN-style) +// brute-force : exact distance over the matching set only (the selective-filter fallback) +// Predicate traversal should dominate post-filter on recall at low selectivity without pathological +// latency; brute-force is the crossover once the matching set is tiny. +// --------------------------------------------------------------------------- + +const idxOf = (key) => parseInt(key.slice(1)); +const SELECTIVITIES = [0.5, 0.1, 0.01, 0.001]; + +console.log(`Filtered recall@${TOP_K} vs. selectivity (${N_QUERIES} queries):\n`); +console.log(' selectivity matches post-filter predicate brute-force pred.lat bf.lat'); +console.log(' ' + '-'.repeat(88)); + +for (const p of SELECTIVITIES) { + // Deterministic-ish membership: mark a p-fraction of vectors as matching the companion predicate. + const matches = new Array(N_VECTORS); + let matchCount = 0; + for (let i = 0; i < N_VECTORS; i++) { + matches[i] = Math.random() < p; + if (matches[i]) matchCount++; + } + if (matchCount === 0) { + console.log( + ` ${(p * 100).toFixed(1).padStart(9)}% ${String(matchCount).padStart(7)} (no matching vectors — skipped)` + ); + continue; + } + const filter = (key) => matches[idxOf(key)]; + const k = Math.min(TOP_K, matchCount); + + // Ground truth: the k nearest among the matching vectors only. + const filteredGT = queries.map((query) => { + const scored = []; + for (let i = 0; i < N_VECTORS; i++) if (matches[i]) scored.push({ i, d: cosineDistance(query, vectors[i]) }); + scored.sort((a, b) => a.d - b.d); + return new Set(scored.slice(0, k).map((x) => x.i)); + }); + + let postRecall = 0; + let predRecall = 0; + let bfRecall = 0; + let predNs = 0; + let bfNs = 0; + + for (let q = 0; q < N_QUERIES; q++) { + const query = queries[q]; + const gt = filteredGT[q]; + + // post-filter: unfiltered search, then keep matching, take k + const plain = hnsw.search({ target: query, comparator: 'sort', descending: false }, {}); + const postIdx = new Set( + plain + .filter((r) => matches[idxOf(r.key)]) + .slice(0, k) + .map((r) => idxOf(r.key)) + ); + postRecall += overlap(gt, postIdx) / k; + + // predicate-aware traversal + const t0 = performance.now(); + const pred = hnsw.search({ target: query, comparator: 'sort', descending: false }, {}, filter); + predNs += (performance.now() - t0) * 1e6; + const predIdx = new Set(pred.slice(0, k).map((r) => idxOf(r.key))); + predRecall += overlap(gt, predIdx) / k; + + // brute-force over the matching set + const b0 = performance.now(); + const scored = []; + for (let i = 0; i < N_VECTORS; i++) if (matches[i]) scored.push({ i, d: cosineDistance(query, vectors[i]) }); + scored.sort((a, b) => a.d - b.d); + bfNs += (performance.now() - b0) * 1e6; + const bfIdx = new Set(scored.slice(0, k).map((x) => x.i)); + bfRecall += overlap(gt, bfIdx) / k; + } + + const pct = (x) => `${((x / N_QUERIES) * 100).toFixed(1)}%`.padStart(9); + console.log( + ` ${(p * 100).toFixed(1).padStart(9)}% ${String(matchCount).padStart(7)} ` + + `${pct(postRecall).padStart(11)} ${pct(predRecall).padStart(9)} ${pct(bfRecall).padStart(11)} ` + + `${fmtNs(predNs / N_QUERIES).padStart(8)} ${fmtNs(bfNs / N_QUERIES).padStart(8)}` + ); +} +console.log(''); + +function overlap(a, b) { + let n = 0; + for (const x of a) if (b.has(x)) n++; + return n; +} diff --git a/resources/DESIGN.md b/resources/DESIGN.md index 1164d26801..b3d3f7a091 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -89,6 +89,7 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource` | How does a query opt out of a read snapshot? | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`). | | How does a URL path map to a Resource? | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below | | How does HNSW keep the graph connected on delete? | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712) | +| How is a filter applied *during* a vector search? | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a static `allowReadRecord` RBAC hook into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective *condition* filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering. | --- diff --git a/resources/RequestTarget.ts b/resources/RequestTarget.ts index cfc2904b74..160585ad22 100644 --- a/resources/RequestTarget.ts +++ b/resources/RequestTarget.ts @@ -66,6 +66,15 @@ export class RequestTarget extends URLSearchParams { declare allowFullScan?: boolean; declare allowConditionsOnDynamicAttributes?: boolean; + /** + * Predicate-aware vector search (#1241). A `(record) => boolean` filter evaluated during HNSW + * traversal (and as a post-filter for other paths), so a vector sort keeps exploring until it has + * enough MATCHING nearest neighbors instead of post-filtering an under-filled candidate set. The + * function must be synchronous and side-effect free; the record it receives is frozen. JS-API only — + * never parsed from a REST query string (no eval of user-supplied code over HTTP). + */ + declare vectorFilter?: (record: any) => boolean; + /** * When `false`, the query reads against the latest committed data without holding a consistent * read snapshot open for the duration of the iteration. This trades read consistency (rows diff --git a/resources/ResourceInterface.ts b/resources/ResourceInterface.ts index 1c3abdd76e..ed44dff8d1 100644 --- a/resources/ResourceInterface.ts +++ b/resources/ResourceInterface.ts @@ -181,6 +181,12 @@ interface TypedDirectCondition boolean` predicate composed by the query executor from + * companion conditions, a caller `vectorFilter`, and record-level RBAC, pushed into a filterable + * custom index (HNSW) so filtering happens during traversal. Not part of the public query surface. + */ + recordFilter?: (primaryKey: Id) => boolean; } interface ConditionGroup { diff --git a/resources/Table.ts b/resources/Table.ts index 66d1e9f875..2240f5ec99 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2854,6 +2854,17 @@ export function makeTable(options) { // scans), the read transaction reads against the latest committed data without pinning a // consistent snapshot, so the scan doesn't hold a snapshot that blocks compaction. const readTxn = txn.useReadTxn(target.snapshot === false); + // Record-level read guard (#1241): a resource may define a static allowReadRecord(user, record) + // that participates in query execution (pushed into HNSW traversal for vector sorts, applied as a + // post-filter otherwise). Resolve off the actual (possibly subclassed) constructor so overrides win. + // SCOPE: this is a QUERY-result filter, not a general read-authorization boundary — direct + // single-record get(id) does not consult it (use allowRead/instance hooks for that). It must be + // synchronous, side-effect free, and fast; it can run once per candidate record during traversal. + const allowReadRecord = (this.constructor as any).allowReadRecord; + const recordAccess = + typeof allowReadRecord === 'function' || typeof target.vectorFilter === 'function' + ? { allowReadRecord, user: (context as any)?.user, vectorFilter: target.vectorFilter } + : undefined; const entries = executeConditions( conditions, operator, @@ -2862,16 +2873,28 @@ export function makeTable(options) { target, context, (results: any[], filters: Function[]) => transformToEntries(results, select, context, readTxn, filters), - filtered + filtered, + recordAccess ); const ensure_loaded = (target as any).ensureLoaded !== false; + // Authoritative RBAC enforcement (#1241): the guards inside executeConditions evaluate the LOCAL + // record, but on a caching table transformEntryForSelect may then revalidate an expired/invalidated + // row from source and return a DIFFERENT record. An authorization check must hold on the record + // actually returned, so allowReadRecord is re-checked there, after materialization (the earlier + // evaluation stays as a prune that also bounds HNSW traversal). vectorFilter and condition filters + // intentionally keep the local-record semantics all query filters have on caching tables. + const recordGuard = + typeof allowReadRecord === 'function' + ? (record: any) => allowReadRecord((context as any)?.user, record) + : undefined; const transformToRecord = TableResource.transformEntryForSelect( select, context, readTxn, filtered, ensure_loaded, - true + true, + recordGuard ); let results = TableResource.transformToOrderedSelect( entries, @@ -3082,9 +3105,12 @@ export function makeTable(options) { * @param filtered * @param ensure_loaded * @param canSkip + * @param recordGuard record-level read check (#1241) applied to the record actually being + * returned — i.e. AFTER any caching-source revalidation replaces a stale local copy — so an + * authorization verdict can't be made on bytes that differ from what the caller receives. * @returns */ - static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded?, canSkip?) { + static transformEntryForSelect(select, context, readTxn, filtered, ensure_loaded?, canSkip?, recordGuard?) { let checkLoaded; if ( ensure_loaded && @@ -3164,6 +3190,10 @@ export function makeTable(options) { } } if (record == null) return canSkip ? SKIP : record; + // Record-level RBAC (#1241): enforced here because `record` is now the final, materialized + // record — a caching table's source revalidation (above) may have replaced the local copy the + // query filters evaluated. Fail closed on the record actually being returned. + if (recordGuard && !recordGuard(record)) return canSkip ? SKIP : undefined; if (select && !(select[0] === '*' && select.length === 1)) { let promises: Promise[]; const selectAttribute = (attribute, callback) => { diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index a079a96110..342db9175a 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -146,7 +146,12 @@ export class HierarchicalNavigableSmallWorld { // Index options that only affect search, not the stored graph — changing them must not trigger a // reindex (databases.ts persists the new value but skips rebuilding). efConstructionSearch is the // search-time candidate-list size; the build uses efConstruction/M/distance, which are structural. - static searchOnlyOptions = ['efConstructionSearch']; + // filterExpansion is the visit-budget multiplier for predicate-aware (filtered) traversal. + static searchOnlyOptions = ['efConstructionSearch', 'filterExpansion']; + // Signals to search.ts that this index accepts a per-record predicate in search() and applies it + // during traversal (predicate-aware / ACORN-style filtering), so companion conditions and RBAC can + // be pushed down instead of post-filtering an under-filled candidate set (#1241). + filteredSearch = true; indexStore: any; M: number = 16; // max number of connections per layer efConstruction: number = 100; // size of dynamic candidate list @@ -156,6 +161,17 @@ export class HierarchicalNavigableSmallWorld { // a value of 1 is extremely aggressive. optimizeRouting = 0.5; nodesVisitedCount = 0; + // Visit-budget multiplier for predicate-aware traversal (#1241). When a filter is selective enough + // that layer-0 results never fill `ef`, the "closest candidate worse than worst result" stop rule + // never triggers, so maxVisits = ef * filterExpansion caps how many nodes an under-filled filtered + // search visits before returning what it has (approximate, like all ANN). It does NOT bound a filter + // that fills `ef` — that terminates naturally. Default 24 (not the 8 the design sketch assumed): + // this HNSW visits a large fraction of the graph per query, so filling `ef` at selectivity `s` needs + // ~ef/s visits; a multiplier of 24 fills down to ~4% selectivity before the budget bites, keeping + // recall at or above post-filtering across the range the query planner routes to traversal. Raising + // it is nearly free for condition-derived filters (they fill and self-terminate); it mainly trades + // latency for recall on selective *function* predicates. Per-query override via the search options. + filterExpansion = 24; idIncrementer: BigInt64Array | undefined; distance: (a: number[], b: number[]) => number; @@ -192,6 +208,7 @@ export class HierarchicalNavigableSmallWorld { if (options.efConstructionSearch !== undefined) this.efConstructionSearch = options.efConstructionSearch; if (options.mL !== undefined) this.mL = options.mL; if (options.optimizeRouting !== undefined) this.optimizeRouting = options.optimizeRouting; + if (options.filterExpansion !== undefined) this.filterExpansion = options.filterExpansion; } } index(primaryKey: Id, vector: number[], existingVector?: number[], options: any = {}) { @@ -735,7 +752,13 @@ export class HierarchicalNavigableSmallWorld { ef: number, level: number, options: { transaction?: any } = {}, - distanceFunction = this.distance + distanceFunction = this.distance, + // Predicate-aware traversal (#1241, layer 0 only). `filter` decides result admission by primary + // key; non-matching nodes still route (they stay in the candidate/visited sets) so the graph + // remains navigable under selective filters (ACORN). `filterState` carries the visit budget and + // per-query counters. Both are undefined for routing layers and for unfiltered searches. + filter?: (primaryKey: Id) => boolean, + filterState?: FilterState ): SearchResults { // Pre-compute query magnitude for cosine; use cached invMag on stored nodes to skip sqrt per neighbor. // Asymmetric distance: the query stays full-precision float; a stored neighbor may be int8 @@ -789,13 +812,60 @@ export class HierarchicalNavigableSmallWorld { const candidates = new MinHeap(); candidates.push(initialCandidate); - const results = [initialCandidate] as SearchResults; + if (!filter) { + // Unfiltered path (unchanged): results are seeded with the entry point and every visited + // node is admitted; the stop rule is "closest remaining candidate worse than worst result". + const results = [initialCandidate] as SearchResults; + while (candidates.size > 0) { + const current = candidates.pop()!; + const furthestDistance = results[results.length - 1].distance; + + if (current.distance > furthestDistance) break; + + for (const { id: neighborId } of current.node[level] || []) { + if (visited.has(neighborId) || neighborId === undefined) continue; + visited.add(neighborId); + + const neighbor = this.safeGetSync(neighborId, options); + if (!neighbor) continue; + this.nodesVisitedCount++; + const distance = computeDistance(neighbor.vector, neighbor.invMag, neighbor.scale); + + if (distance < furthestDistance || results.length < ef) { + const candidate: Candidate = { id: neighborId, distance, node: neighbor }; + candidates.push(candidate); + results.splice(bisectInsert(results, distance), 0, candidate); + if (results.length > ef) results.pop(); + } + } + } + results.visited = visited.size; + return results; + } + + // Predicate-aware path (#1241). `results` holds only nodes the filter admits, but every visited + // node still routes: it enters the candidate heap and can be expanded, preserving connectivity + // through non-matching regions. Because matches accrue slower than visits, the distance stop rule + // only applies once `ef` matches are in hand; until then we keep expanding. + // + // maxVisits = ef * filterExpansion is a hard ceiling on layer-0 node visits. It has to be + // generous enough NOT to truncate a non-selective filter before it fills `ef` (filling at + // selectivity `s` costs ~ef/s visits) — that is why the default filterExpansion is larger than a + // standard-HNSW rule of thumb, since this index visits a large fraction of the graph per query. + // It matters as a genuine bound in two cases: a selective filter that can never fill `ef` (where + // the alternative is crawling the whole graph — the same regime the planner diverts to the exact + // brute-force path for condition filters, leaving function predicates the real beneficiary), and + // a filter that fills `ef` with distant matches (a loose worst-match bound would otherwise let + // the distance rule explore almost everything). + const results = [] as unknown as SearchResults; + if (this.admit(filter, filterState, entryPoint.primaryKey)) results.push(initialCandidate); + let budgetExhausted = false; while (candidates.size > 0) { const current = candidates.pop()!; - const furthestDistance = results[results.length - 1].distance; - - if (current.distance > furthestDistance) break; + // Once we have ef matches, the worst of them bounds useful exploration; before that, keep going. + const furthestDistance = results.length >= ef ? results[results.length - 1].distance : Infinity; + if (results.length >= ef && current.distance > furthestDistance) break; for (const { id: neighborId } of current.node[level] || []) { if (visited.has(neighborId) || neighborId === undefined) continue; @@ -804,20 +874,41 @@ export class HierarchicalNavigableSmallWorld { const neighbor = this.safeGetSync(neighborId, options); if (!neighbor) continue; this.nodesVisitedCount++; + filterState!.nodesVisited++; const distance = computeDistance(neighbor.vector, neighbor.invMag, neighbor.scale); + // Route through any node that could still improve the result set (under-filled or nearer + // than the current worst match) — filtering does not prune the graph, only admission. if (distance < furthestDistance || results.length < ef) { const candidate: Candidate = { id: neighborId, distance, node: neighbor }; candidates.push(candidate); - results.splice(bisectInsert(results, distance), 0, candidate); - if (results.length > ef) results.pop(); + if (this.admit(filter, filterState, neighbor.primaryKey)) { + results.splice(bisectInsert(results, distance), 0, candidate); + if (results.length > ef) results.pop(); + } + } + if (filterState!.nodesVisited >= filterState!.maxVisits) { + budgetExhausted = true; + break; } } + if (budgetExhausted) break; } results.visited = visited.size; return results; } + /** + * Evaluate the traversal predicate for one node, counting the evaluation for `explain`. Kept as a + * tiny helper so both the entry-point seed and the neighbor loop share the counting path. The filter + * itself memoizes verdicts per query (a node can be reached from multiple neighbors), so this stays + * cheap even when the predicate loads a record. + */ + private admit(filter: (primaryKey: Id) => boolean, filterState: FilterState | undefined, primaryKey: Id): boolean { + if (filterState) filterState.filterEvaluations++; + return filter(primaryKey); + } + /** * This the main entry from Harper's query functionality, where we actually search for an ordered list of nearest * neighbors, using the provided sort/order definition object and performing the multi-layer skip-list search. @@ -837,6 +928,7 @@ export class HierarchicalNavigableSmallWorld { distance, comparator, ef, + filterExpansion, }: { target: number[]; value: number; @@ -844,8 +936,14 @@ export class HierarchicalNavigableSmallWorld { distance: string; comparator: string; ef?: number; + filterExpansion?: number; }, - context: any + context: any, + // Predicate-aware traversal (#1241). When provided, only nodes for which `filter(primaryKey)` + // returns true are admitted to the result list at layer 0; routing is unaffected. Composed by + // search.ts from companion AND conditions, a caller-supplied `vectorFilter`, and record-level + // RBAC. Must be synchronous and side-effect free. JS-API only (never from a REST query string). + filter?: (primaryKey: Id) => boolean ) { let limit: number | undefined; // only set for threshold comparators; 0 is a valid threshold (e.g. dotProduct) let limitInclusive = false; // true for `le`, false for `lt` @@ -885,14 +983,34 @@ export class HierarchicalNavigableSmallWorld { : (this.indexStore.getStats?.()?.entryCount ?? 0); effectiveEf = autoScaleEf(nodeCount); } + // Predicate-aware traversal budget (#1241): matches accrue slower than visits under a selective + // filter, so bound layer-0 work at ef * filterExpansion nodes. Only built when a filter is active. + const filterState: FilterState | undefined = filter + ? { + maxVisits: effectiveEf * (filterExpansion && filterExpansion > 0 ? filterExpansion : this.filterExpansion), + nodesVisited: 0, + filterEvaluations: 0, + } + : undefined; let entryPoint = this.getEntryPoint(options); - if (!entryPoint) return []; + if (!entryPoint) return withStats([], filterState); let entryPointId = entryPoint.id; let results: Candidate[] = []; - // For each level from top to bottom + // For each level from top to bottom. The filter applies only at layer 0 (result admission); + // upper layers route unfiltered so non-matching hubs still guide the descent. for (let l = entryPoint.level; l >= 0; l--) { // Search for closest neighbors at current level - results = this.searchLayer(target, entryPointId, entryPoint, effectiveEf, l, options, distanceFunction); + results = this.searchLayer( + target, + entryPointId, + entryPoint, + effectiveEf, + l, + options, + distanceFunction, + l === 0 ? filter : undefined, + l === 0 ? filterState : undefined + ); if (results.length > 0) { const neighbor = results[0]; // closest neighbor becomes new entry point @@ -904,11 +1022,14 @@ export class HierarchicalNavigableSmallWorld { results = results.filter((candidate) => limitInclusive ? candidate.distance <= limit : candidate.distance < limit ); - return results.map((candidate) => ({ - // we return the result as an entry so we can provide distance as metadata - key: candidate.node.primaryKey, // return value - distance: candidate.distance, - })); + return withStats( + results.map((candidate) => ({ + // we return the result as an entry so we can provide distance as metadata + key: candidate.node.primaryKey, // return value + distance: candidate.distance, + })), + filterState + ); } /** * Exact distance between a query and a record's FULL-precision vector, mirroring search()'s metric @@ -1144,3 +1265,22 @@ type Candidate = { node: Node; }; type SearchResults = Candidate[] & { visited: number }; +// Per-query state for predicate-aware traversal (#1241): the visit budget plus counters surfaced for +// tuning (nodesVisited / filterEvaluations). One instance per search(), threaded to the layer-0 searchLayer. +type FilterState = { + maxVisits: number; + nodesVisited: number; + filterEvaluations: number; +}; +/** + * Attach filtered-traversal counters to the returned entries array so callers (search.ts / explain) can + * report how much of the graph a filtered query touched. No-op for unfiltered searches. Non-enumerable so + * the stats don't leak into iteration/serialization of the result entries. + */ +function withStats(entries: T, filterState: FilterState | undefined): T { + if (filterState) { + Object.defineProperty(entries, 'nodesVisited', { value: filterState.nodesVisited, enumerable: false }); + Object.defineProperty(entries, 'filterEvaluations', { value: filterState.filterEvaluations, enumerable: false }); + } + return entries; +} diff --git a/resources/search.ts b/resources/search.ts index ad2d1630e7..9c701c2bba 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -44,8 +44,24 @@ export const COERCIBLE_OPERATORS = { ne: true, eq: true, }; -export function executeConditions(conditions, operator, table, txn, request, context, transformToEntries, filtered) { +export function executeConditions( + conditions, + operator, + table, + txn, + request, + context, + transformToEntries, + filtered, + recordAccess? +) { const firstSearch = conditions[0]; + // Record-level guards (a caller-supplied vectorFilter + RBAC allowReadRecord) apply to every record + // the query returns, independent of which condition leads (#1241). `recordAccess` is supplied only on + // the top-level executeConditions call (Table.search) and deliberately NOT threaded into the recursive + // calls below, so the guards run exactly once — on the final result set — rather than redundantly at + // each nested group. + const recordGuards = buildRecordGuards(recordAccess); // both AND and OR start by getting an iterator for the ids for first condition // and then things diverge... if (operator === 'or') { @@ -58,7 +74,7 @@ export function executeConditions(conditions, operator, table, txn, request, con results = results.concat(nextResults); } const returnedIds = new Set(); - return results.filter((entry) => { + const deduped = results.filter((entry) => { const id = entry.key ?? entry; if (returnedIds.has(id)) // skip duplicate ids @@ -66,14 +82,35 @@ export function executeConditions(conditions, operator, table, txn, request, con returnedIds.add(id); return true; }); + // Record guards must hold on the OR union too — loading each surviving record and dropping the + // ones the guards reject (transformToEntries loads the record to evaluate a filter). Without this + // a record-level RBAC check would be bypassed by any OR query. + return recordGuards ? transformToEntries(deduped, recordGuards) : deduped; } else { - // AND + // AND: use the indexed query for the first condition and filter by all subsequent conditions. + if (isFilterablePushdown(firstSearch, table)) { + // The lead is a filterable custom index (HNSW). It doesn't populate the join map (`filtered`) + // that sibling filters read, so — unlike the general case below — we can build the sibling + // filters BEFORE executing the lead and push them into the traversal, so filtering happens + // during the graph walk instead of post-filtering an under-filled candidate set (#1241). They + // also stay in the post-filter chain as a deterministic, harmless re-check. + const filters = mapConditionsToFilters(conditions.slice(1), true, firstSearch.estimated_count); + const recordFilters = recordGuards ? filters.concat(recordGuards) : filters; + // Execute a COPY carrying the pushed-down predicate, never mutating the caller's condition + // object (which may be reused across requests with different users / guards). + const lead = + recordFilters.length > 0 + ? { ...firstSearch, recordFilter: composeRecordFilter(recordFilters, table, context) } + : firstSearch; + const results = executeCondition(lead); + return recordFilters.length > 0 ? transformToEntries(results, recordFilters) : results; + } + // General path: execute the lead FIRST — a leading join populates `filtered`, which the sibling + // filters read as they are built — then apply the sibling filters plus any record guards. const results = executeCondition(firstSearch); - // get the intersection of condition searches by using the indexed query for the first condition - // and then filtering by all subsequent conditions. - // now apply filters that require looking up records const filters = mapConditionsToFilters(conditions.slice(1), true, firstSearch.estimated_count); - return filters.length > 0 ? transformToEntries(results, filters) : results; + const recordFilters = recordGuards ? filters.concat(recordGuards) : filters; + return recordFilters.length > 0 ? transformToEntries(results, recordFilters) : results; } function executeCondition(condition) { if (condition.conditions) @@ -86,6 +123,7 @@ export function executeConditions(conditions, operator, table, txn, request, con context, transformToEntries, filtered + // recordAccess intentionally omitted: guards run once, at the top level (see above). ); return searchByIndex( condition, @@ -122,6 +160,65 @@ export function executeConditions(conditions, operator, table, txn, request, con } } +/** + * Build the record-level guards that apply to a query independent of its conditions (#1241): + * - `vectorFilter`: a caller-supplied `(record) => boolean` predicate (JS-API only). + * - `allowReadRecord`: a resource's static record-level RBAC check `(user, record) => boolean`. + * Both arrive already resolved on `recordAccess` (assembled once in Table.search). Returns an array of + * `(record) => boolean` predicates, or undefined when neither is defined (the common case — zero + * overhead). Predicates must be synchronous and side-effect free; records passed in are frozen. + */ +function buildRecordGuards(recordAccess): ((record: any) => boolean)[] | undefined { + if (!recordAccess) return undefined; + const guards: ((record: any) => boolean)[] = []; + const vectorFilter = recordAccess.vectorFilter; + if (typeof vectorFilter === 'function') guards.push((record) => vectorFilter(record)); + const allowReadRecord = recordAccess.allowReadRecord; + if (typeof allowReadRecord === 'function') { + const user = recordAccess.user; + guards.push((record) => allowReadRecord(user, record)); + } + return guards.length > 0 ? guards : undefined; +} + +/** True when a condition's index is a custom index that participates in predicate-aware traversal (HNSW). */ +function isFilterablePushdown(condition, table): boolean { + const attributeName = condition?.attribute ?? condition?.[0]; + if (attributeName == null) return false; + const index = attributeName === table.primaryKey ? table.primaryStore : table.indices?.[attributeName]; + return Boolean(index?.customIndex?.filteredSearch); +} + +/** + * Compose a set of record predicates into the single `(primaryKey) => boolean` the custom index calls + * during traversal (#1241). The record is loaded lazily (only when the index reaches a node) via + * `primaryStore.getEntry` and frozen before the predicates see it. Verdicts are memoized per query since + * the graph can reach a node from multiple neighbors. A missing/deleted record fails the predicate. + */ +function composeRecordFilter(recordFilters, table, context): (primaryKey: Id) => boolean { + const memo = new Map(); + const transaction = context && table._readTxnForContext ? table._readTxnForContext(context) : undefined; + return (primaryKey: Id) => { + const cached = memo.get(primaryKey); + if (cached !== undefined) return cached; + let verdict = false; + const entry = table.primaryStore.getEntry(primaryKey, { transaction }); + const record = entry?.value; + if (record != null) { + freezeRecord(record); + verdict = true; + for (const filter of recordFilters) { + if (!filter(record, entry)) { + verdict = false; + break; + } + } + } + memo.set(primaryKey, verdict); + return verdict; + }; +} + /** * Search for records or keys, based on the search condition, using an index if available * @param searchCondition @@ -387,7 +484,12 @@ export function searchByIndex( return results; } else if (index && !skipIndex) { if (index.customIndex) { - const loaded = index.customIndex.search(searchCondition, context).map((entry) => { + // Predicate-aware traversal (#1241): a record filter composed from companion AND conditions, + // a caller-supplied vectorFilter, and record-level RBAC is pushed into the index so it keeps + // exploring until it has enough MATCHING results, rather than post-filtering an under-filled + // candidate set. Only indexes that opt in (filteredSearch) receive it; others post-filter as before. + const recordFilter = index.customIndex.filteredSearch ? searchCondition.recordFilter : undefined; + const loaded = index.customIndex.search(searchCondition, context, recordFilter).map((entry) => { // if the custom index returns an entry with metadata, merge it with the loaded entry if (typeof entry === 'object' && entry) { const { key, ...otherProps } = entry; diff --git a/unitTests/resources/vectorIndex.test.js b/unitTests/resources/vectorIndex.test.js index 69f4de90bb..527e842850 100644 --- a/unitTests/resources/vectorIndex.test.js +++ b/unitTests/resources/vectorIndex.test.js @@ -1274,6 +1274,352 @@ describeUnlessLmdb('HNSW int8 threshold queries (lt/le) — exact boundary after }); }); +// A self-contained in-memory index store, mirroring the one used by the 5.1 GA fixes above, so the +// predicate-aware traversal mechanism (#1241) can be exercised at the index level without a real DB. +function newMockIndexStore() { + const nodes = new Map(); + let ep; + return { + encoder: { useFloat32: false }, + getSync(key) { + if (key === Symbol.for('entryPoint')) return ep; + const k = typeof key === 'number' ? key : JSON.stringify(key); + return nodes.get(k); + }, + put(key, value) { + if (key === Symbol.for('entryPoint')) return void (ep = value); + const k = typeof key === 'number' ? key : JSON.stringify(key); + nodes.set(k, value); + }, + remove(key) { + if (key === Symbol.for('entryPoint')) return void (ep = undefined); + const k = typeof key === 'number' ? key : JSON.stringify(key); + nodes.delete(k); + }, + *getRange({ start = 0, end = Infinity } = {}) { + for (const [k, v] of nodes) if (typeof k === 'number' && k >= start && k <= end) yield { key: k, value: v }; + }, + getKeys() { + return []; + }, + getUserSharedBuffer(_name, buffer) { + return buffer; + }, + }; +} + +const describeUnlessLmdbFilter = process.env.HARPER_STORAGE_ENGINE === 'lmdb' ? describe.skip : describe; + +describeUnlessLmdbFilter('HNSW predicate-aware traversal (#1241)', () => { + // 1D euclidean vectors [i] put euclidean distance from [0] at i^2 — nearest neighbor is the smallest key. + function buildLine(count, options = { distance: 'euclidean', quantization: 'none', optimizeRouting: 0 }) { + const hnsw = new HierarchicalNavigableSmallWorld(newMockIndexStore(), options); + for (let i = 0; i < count; i++) hnsw.index(i, [i], null, {}); + return hnsw; + } + + it('admits only matching nodes at layer 0, while non-matching nodes still route', () => { + const hnsw = buildLine(100); + const even = (pk) => Number(pk) % 2 === 0; + const results = hnsw.search( + { target: [0], comparator: 'sort', descending: false }, + { transaction: undefined }, + even + ); + assert(results.length > 0, 'expected matching results'); + assert( + results.every((r) => Number(r.key) % 2 === 0), + 'every admitted node must satisfy the filter' + ); + // key 1 is the 2nd-nearest overall but odd: it must route (help reach 0/2/…) yet never be admitted. + assert(!results.some((r) => r.key === 1), 'a non-matching nearest node must not be admitted'); + assert.strictEqual(results[0].key, 0, 'nearest matching node first'); + assert.strictEqual(results[1].key, 2, 'then the next matching node'); + }); + + it('keeps traversing until it has the k nearest MATCHING nodes (no under-fill)', () => { + const hnsw = buildLine(100); + const everyTenth = (pk) => Number(pk) % 10 === 0; // matches 0,10,20,…,90 — sparse near the target + const results = hnsw.search( + { target: [0], comparator: 'sort', descending: false }, + { transaction: undefined }, + everyTenth + ); + assert( + results.every((r) => Number(r.key) % 10 === 0), + 'all results match' + ); + // The five nearest matching nodes, in order — a fixed-candidate post-filter would miss most of these. + assert.deepStrictEqual( + results.slice(0, 5).map((r) => r.key), + [0, 10, 20, 30, 40] + ); + }); + + it('bounds work with a visit budget and returns partial results, not an error', () => { + const hnsw = buildLine(200); + const matchesNothing = () => false; + // ef 10 * filterExpansion 2 = 20-node budget; 200 nodes, none matching → budget must stop it. + const results = hnsw.search( + { target: [0], comparator: 'sort', descending: false, ef: 10, filterExpansion: 2 }, + { transaction: undefined }, + matchesNothing + ); + assert.strictEqual(results.length, 0, 'no matches yields an empty result, not an error'); + assert(results.nodesVisited > 0, 'some nodes were visited'); + assert(results.nodesVisited <= 20, `visit budget must bound traversal, got ${results.nodesVisited}`); + }); + + it('exposes nodesVisited and filterEvaluations for tuning', () => { + const hnsw = buildLine(50); + const even = (pk) => Number(pk) % 2 === 0; + const results = hnsw.search( + { target: [0], comparator: 'sort', descending: false }, + { transaction: undefined }, + even + ); + assert.strictEqual(typeof results.nodesVisited, 'number'); + assert.strictEqual(typeof results.filterEvaluations, 'number'); + assert(results.nodesVisited > 0 && results.filterEvaluations > 0, 'counters populated'); + assert(results.filterEvaluations >= results.length, 'at least one evaluation per admitted result'); + }); + + it('excludes deleted nodes from filtered results', () => { + const hnsw = buildLine(50); + hnsw.index(10, null, null, {}); // delete two matching nodes + hnsw.index(20, null, null, {}); + const even = (pk) => Number(pk) % 2 === 0; + const results = hnsw.search( + { target: [0], comparator: 'sort', descending: false }, + { transaction: undefined }, + even + ); + assert(!results.some((r) => r.key === 10 || r.key === 20), 'deleted nodes must not be returned'); + assert( + results.some((r) => r.key === 0) && results.some((r) => r.key === 2), + 'surviving matching nodes remain findable' + ); + }); + + it('works on an int8 (quantized) index', () => { + const hnsw = buildLine(50, { distance: 'euclidean' }); // int8 quantization on by default + const even = (pk) => Number(pk) % 2 === 0; + const results = hnsw.search( + { target: [0], comparator: 'sort', descending: false }, + { transaction: undefined }, + even + ); + assert( + results.every((r) => Number(r.key) % 2 === 0), + 'quantized graph still honors the filter' + ); + assert.strictEqual(results[0].key, 0, 'nearest matching node first on the int8 graph'); + }); +}); + +describeUnlessLmdbFilter('HNSW filtered search via Table.search (#1241)', () => { + let T, Tq; + before(async () => { + T = table({ + table: 'HNSWFilter', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'group', indexed: true }, + { name: 'ownerId', indexed: true }, + { name: 'active', indexed: true, type: 'Boolean' }, + { name: 'vector', indexed: { type: 'HNSW', distance: 'euclidean', quantization: 'none' }, type: 'Array' }, + ], + }); + for (let i = 0; i < 60; i++) { + await T.put(i, { + group: i % 2 === 0 ? 'blue' : 'red', + ownerId: i % 3, + active: i % 5 !== 0, + vector: [i, 0], + }); + } + Tq = table({ + table: 'HNSWFilterInt8', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'group', indexed: true }, + { name: 'vector', indexed: { type: 'HNSW', distance: 'euclidean' }, type: 'Array' }, // int8 default + ], + }); + for (let i = 0; i < 40; i++) { + await Tq.put(i, { group: i % 2 === 0 ? 'blue' : 'red', vector: [i, 0.5] }); + } + }); + after(() => { + T.dropTable(); + Tq.dropTable(); + }); + + it('vectorFilter returns the k nearest MATCHING records', async () => { + const results = await fromAsync( + T.search( + { + sort: { attribute: 'vector', target: [0, 0], distance: 'euclidean' }, + vectorFilter: (record) => record.group === 'blue', + select: ['id', 'group'], + limit: 5, + }, + {} + ) + ); + assert.strictEqual(results.length, 5, 'limit is filled with matching records, not under-filled'); + assert( + results.every((r) => r.group === 'blue'), + 'every returned record matches the vectorFilter' + ); + assert.deepStrictEqual( + results.map((r) => r.id), + [0, 2, 4, 6, 8], + 'the five nearest blue records' + ); + }); + + it('allowReadRecord restricts a vector search to records the user may see', async () => { + T.allowReadRecord = (user, record) => record.ownerId === user.id; + try { + const results = await fromAsync( + T.search( + { + sort: { attribute: 'vector', target: [0, 0], distance: 'euclidean' }, + select: ['id', 'ownerId'], + limit: 5, + }, + { user: { id: 1 } } + ) + ); + assert( + results.every((r) => r.ownerId === 1), + 'a restricted user only sees their own records' + ); + // ownerId === 1 for ids 1,4,7,10,13,… — the k nearest VISIBLE, not k-minus-redacted. + assert.deepStrictEqual( + results.map((r) => r.id), + [1, 4, 7, 10, 13] + ); + } finally { + delete T.allowReadRecord; + } + }); + + it('re-checks allowReadRecord on the source-revalidated record, not the stale cached copy', async () => { + // A caching table: the query filters evaluate the LOCAL (possibly stale) copy, but the + // returned record may be revalidated from source. The authorization verdict must hold on + // the record actually returned (transformEntryForSelect recordGuard), or ownership changes + // at the source would leak through the stale-copy verdict. + const sourceRecords = new Map(); + const C = table({ + table: 'RBACCachingTest', + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'kind', indexed: true }, { name: 'ownerId' }], + }); + C.sourcedFrom({ + get(id) { + return sourceRecords.get(id); + }, + }); + C.allowReadRecord = (user, record) => record.ownerId === user.id; + try { + sourceRecords.set(1, { kind: 'doc', ownerId: 1 }); + C.setTTLExpiration(0.01); // 10ms — expiry retains the stale value (unlike invalidate), which is the vulnerable path + const cached = await C.get(1, { user: { id: 1 } }); + assert.strictEqual(cached.ownerId, 1, 'cache populated with the original owner'); + // Ownership changes at the source; let the cached copy expire so the next query revalidates + // while the STALE value (ownerId 1) is still what the query filters evaluate. + sourceRecords.set(1, { kind: 'doc', ownerId: 2 }); + await new Promise((resolve) => setTimeout(resolve, 20)); + const results = await fromAsync( + C.search({ conditions: [{ attribute: 'kind', value: 'doc' }] }, { user: { id: 1 } }) + ); + assert.strictEqual(results.length, 0, 'a record now owned by another user must not be returned'); + } finally { + delete C.allowReadRecord; + C.dropTable(); + } + }); + + it('enforces allowReadRecord on OR queries (no RBAC bypass)', async () => { + T.allowReadRecord = (user, record) => record.ownerId === user.id; + try { + // active=true OR active=false spans every record; the record guard must still filter the union. + const results = await fromAsync( + T.search( + { + conditions: [ + { attribute: 'active', comparator: 'equals', value: true }, + { attribute: 'active', comparator: 'equals', value: false }, + ], + operator: 'or', + select: ['id', 'ownerId'], + }, + { user: { id: 2 } } + ) + ); + assert(results.length > 0, 'expected some visible records'); + assert( + results.every((r) => r.ownerId === 2), + 'an OR union must still be filtered by allowReadRecord' + ); + } finally { + delete T.allowReadRecord; + } + }); + + it('honors a companion AND condition during the vector query', async () => { + const results = await fromAsync( + T.search( + { + conditions: [{ attribute: 'active', comparator: 'equals', value: true }], + sort: { attribute: 'vector', target: [0, 0], distance: 'euclidean' }, + select: ['id', 'active'], + limit: 5, + }, + {} + ) + ); + assert.strictEqual(results.length, 5); + assert( + results.every((r) => r.active === true), + 'every result satisfies the companion condition' + ); + // active === (i % 5 !== 0) → inactive at 0,5,10,… so nearest active are 1,2,3,4,6. + assert.deepStrictEqual( + results.map((r) => r.id), + [1, 2, 3, 4, 6] + ); + }); + + it('combines int8 quantization + vectorFilter + exact rerank', async () => { + const results = await fromAsync( + Tq.search( + { + sort: { attribute: 'vector', target: [0, 0.5], distance: 'euclidean' }, + vectorFilter: (record) => record.group === 'blue', + select: ['id', 'group', 'vector', '$distance'], + limit: 5, + }, + {} + ) + ); + assert.strictEqual(results.length, 5); + assert( + results.every((r) => r.group === 'blue'), + 'filter honored on the quantized graph' + ); + for (const r of results) { + // $distance is recomputed from the full-precision record vector (exact), not the int8 approximation. + const exact = (r.vector[0] - 0) ** 2 + (r.vector[1] - 0.5) ** 2; + assert(Math.abs(r.$distance - exact) < 1e-9, `$distance ${r.$distance} should equal exact ${exact}`); + } + }); +}); + async function fromAsync(iterable) { let results = []; for await (let entry of iterable) { From b8f36c7bd817084c86f453cdac490fcfaba1519c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 12 Jul 2026 09:13:34 -0600 Subject: [PATCH 2/3] fix: prettier reflow of DESIGN.md table Co-Authored-By: Claude Fable 5 --- resources/DESIGN.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/resources/DESIGN.md b/resources/DESIGN.md index b3d3f7a091..b5a2380ae5 100644 --- a/resources/DESIGN.md +++ b/resources/DESIGN.md @@ -75,21 +75,21 @@ One giant `makeTable()` factory that returns a `TableResource extends Resource` ## "Where is X" cheat sheet -| Question | Where | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| How is a CRUD request authorized? | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.) | -| Where does versioning / conflict resolution happen? | `Table.ts → _writeUpdate` (`#section: write-path-internals`) | -| How does `search()` choose an index? | `Table.ts → search` (`#section: search-query`) | -| How are subscriptions replayed? | `Table.ts → subscribe` (`#section: pub-sub`) | -| How is the response body shaped (select clause)? | `Table.ts → transformEntryForSelect` (`#section: search-query`) | -| Where is record-level TTL evaluated? | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`) | -| How are residencies enforced (replication)? | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`) | -| How is the RecordObject prototype applied? | `RecordEncoder.ts` (see `../DESIGN.md`) | -| Where is the per-request transaction stored? | `transaction.ts` + `contextStorage` (AsyncLocalStorage) | -| How does a query opt out of a read snapshot? | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`). | -| How does a URL path map to a Resource? | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below | -| How does HNSW keep the graph connected on delete? | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712) | -| How is a filter applied *during* a vector search? | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a static `allowReadRecord` RBAC hook into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective *condition* filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering. | +| Question | Where | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| How is a CRUD request authorized? | `Table.ts → #section: authz-hooks`; defaults in `Resource.ts` (`allowRead` etc.) | +| Where does versioning / conflict resolution happen? | `Table.ts → _writeUpdate` (`#section: write-path-internals`) | +| How does `search()` choose an index? | `Table.ts → search` (`#section: search-query`) | +| How are subscriptions replayed? | `Table.ts → subscribe` (`#section: pub-sub`) | +| How is the response body shaped (select clause)? | `Table.ts → transformEntryForSelect` (`#section: search-query`) | +| Where is record-level TTL evaluated? | `Table.ts → setTTLExpiration` (`#section: lifecycle-admin`); `Updatable.getExpiresAt` (`#section: setup-and-factory`) | +| How are residencies enforced (replication)? | `Table.ts → #section: lifecycle-admin` (residency block: `getResidencyRecord`, `setResidency`, `setResidencyById`, `getResidency`) | +| How is the RecordObject prototype applied? | `RecordEncoder.ts` (see `../DESIGN.md`) | +| Where is the per-request transaction stored? | `transaction.ts` + `contextStorage` (AsyncLocalStorage) | +| How does a query opt out of a read snapshot? | Pass `snapshot: false` on the search request (e.g. `get_analytics`). `Table.ts → search` calls `txn.useReadTxn(snapshot === false)`; on RocksDB `DatabaseTransaction.getReadTxn` then builds the read txn with `{ disableSnapshot: true }` so a long scan reads latest without pinning a snapshot. No-op on LMDB (`LMDBTransaction.useReadTxn`). | +| How does a URL path map to a Resource? | `Resources.ts → getMatch` (exact/prefix fast path) then `matchParamRoute` (parameterised routes); see "Path routing" below | +| How does HNSW keep the graph connected on delete? | `indexes/HierarchicalNavigableSmallWorld.ts → index()` delete path: zero-degree orphans reindexed via `needsReindexing`; severed multi-node islands detected and reconnected by `repairSeveredNeighbors` (#1712) | +| How is a filter applied _during_ a vector search? | Predicate-aware traversal (#1241): `search.ts → executeConditions` composes companion AND conditions + a request `vectorFilter` + a static `allowReadRecord` RBAC hook into one `(primaryKey) => boolean` (`composeRecordFilter`) and passes it to `HierarchicalNavigableSmallWorld.search(cond, ctx, filter)`. The filter gates result admission at layer 0 only (routing ignores it, ACORN-style); a visit budget (`filterExpansion`) bounds the under-filled/selective case. Very selective _condition_ filters are instead diverted to the exact brute-force path by the query planner's `estimateCountAsSort` ordering. | --- From dfc4a6ecc0c7df825ca52d92bd97df20240227da Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 12 Jul 2026 09:15:25 -0600 Subject: [PATCH 3/3] review: default filterState for direct searchLayer callers, null-guard table in isFilterablePushdown Co-Authored-By: Claude Fable 5 --- resources/indexes/HierarchicalNavigableSmallWorld.ts | 6 ++++-- resources/search.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index 342db9175a..5d09d120b2 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -858,6 +858,8 @@ export class HierarchicalNavigableSmallWorld { // brute-force path for condition filters, leaving function predicates the real beneficiary), and // a filter that fills `ef` with distant matches (a loose worst-match bound would otherwise let // the distance rule explore almost everything). + // search() always supplies filterState alongside filter; default one for any direct caller that doesn't. + if (!filterState) filterState = { maxVisits: Infinity, nodesVisited: 0, filterEvaluations: 0 }; const results = [] as unknown as SearchResults; if (this.admit(filter, filterState, entryPoint.primaryKey)) results.push(initialCandidate); let budgetExhausted = false; @@ -874,7 +876,7 @@ export class HierarchicalNavigableSmallWorld { const neighbor = this.safeGetSync(neighborId, options); if (!neighbor) continue; this.nodesVisitedCount++; - filterState!.nodesVisited++; + filterState.nodesVisited++; const distance = computeDistance(neighbor.vector, neighbor.invMag, neighbor.scale); // Route through any node that could still improve the result set (under-filled or nearer @@ -887,7 +889,7 @@ export class HierarchicalNavigableSmallWorld { if (results.length > ef) results.pop(); } } - if (filterState!.nodesVisited >= filterState!.maxVisits) { + if (filterState.nodesVisited >= filterState.maxVisits) { budgetExhausted = true; break; } diff --git a/resources/search.ts b/resources/search.ts index 9c701c2bba..5a062c9a81 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -184,7 +184,7 @@ function buildRecordGuards(recordAccess): ((record: any) => boolean)[] | undefin /** True when a condition's index is a custom index that participates in predicate-aware traversal (HNSW). */ function isFilterablePushdown(condition, table): boolean { const attributeName = condition?.attribute ?? condition?.[0]; - if (attributeName == null) return false; + if (attributeName == null || table == null) return false; const index = attributeName === table.primaryKey ? table.primaryStore : table.indices?.[attributeName]; return Boolean(index?.customIndex?.filteredSearch); }