Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions packages/agent/src/dkg-agent-base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ import {
pickNetworkTunables,
isSparqlUpdateOperation,
} from '@origintrail-official/dkg-core';
import { GraphManager, PrivateContentStore, SystemRecordLaneForwarderV1, createTripleStore, isExternalBackend, structuredMutationMightMutate, structuredMutationTouchedGraphs, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig, type QueryOptions } from '@origintrail-official/dkg-storage';
import { GraphManager, PrivateContentStore, SystemRecordLaneForwarderV1, captureStructuredMutationEffects, createTripleStore, isExternalBackend, type TripleStore, type TripleStoreConfig, type Quad, type LargeLiteralStorageConfig, type QueryOptions } from '@origintrail-official/dkg-storage';
import { emptyRpcUsageWindow, EVMChainAdapter, NoChainAdapter, enrichEvmError, buildKnowledgeAssetUal, type EVMAdapterConfig, type ChainAdapter, type CreateContextGraphParams, type CreateOnChainContextGraphParams, type CreateOnChainContextGraphResult, type TxResult, type V10PublishingConvictionAccountInfo, type RpcUsageWindow } from '@origintrail-official/dkg-chain';
import {
DKGPublisher, PublishHandler, SharedMemoryHandler, UpdateHandler, ChainEventPoller, AccessHandler, AccessClient,
Expand Down Expand Up @@ -578,14 +578,13 @@ export function createListContextGraphsCacheInvalidatingStore(
: undefined,
structuredMutation: innerStore.structuredMutation
? (mutation, options) => {
// Capture scope before the first await so caller-side mutation cannot
// redirect cache invalidation after the backend has committed.
const targetGraphs = [...structuredMutationTouchedGraphs(mutation)];
const mightMutate = structuredMutationMightMutate(mutation);
const effects = captureStructuredMutationEffects(mutation);
return invalidateAfterMutation(
() => innerStore.structuredMutation!(mutation, options),
() => mightMutate,
() => targetGraphs.forEach((graph) => markProjectionDirty?.(undefined, graph)),
() => effects !== undefined,
() => effects?.touchedGraphs.forEach(
(graph) => markProjectionDirty?.(undefined, graph),
),
);
}
: undefined,
Expand Down
74 changes: 73 additions & 1 deletion packages/agent/test/replace-subject-agent-wrapper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
CHANGELOG_GRAPH,
OxigraphStore,
Expand Down Expand Up @@ -150,4 +150,76 @@ describe('#1863 replaceSubject through the agent store wrapper', () => {
proj.markDirtyForGraph('urn:dkg:publisher:control-plane');
expect(entries.has('urn:dkg:publisher:control-plane')).toBe(false);
});

it('invalidates structured mutation effects after success without decoding them in Agent', async () => {
let release!: () => void;
const inFlight = new Promise<void>((resolve) => { release = resolve; });
const options = { source: 'agent.test.structured-mutation-effects' };
let inner!: TripleStore;
const structuredMutation = vi.fn(function (
this: TripleStore,
_mutation: unknown,
receivedOptions: unknown,
) {
expect(this).toBe(inner);
expect(receivedOptions).toBe(options);
return inFlight;
});
inner = { structuredMutation } as unknown as TripleStore;
const invalidate = vi.fn();
const markProjectionDirty = vi.fn();
const store = createListContextGraphsCacheInvalidatingStore(
inner,
invalidate,
markProjectionDirty,
);
const mutation = {
kind: 'copy-subject-projection' as const,
input: {
sourceGraphUris: ['urn:test:source'],
targetGraphUri: 'urn:test:target',
roots: ['urn:test:root'],
descendantSuffix: '/',
excludedPredicates: [],
},
};

const pending = store.structuredMutation!(mutation, options);
mutation.input.targetGraphUri = 'urn:test:redirected';
expect(invalidate).not.toHaveBeenCalled();
release();
await pending;

expect(invalidate).toHaveBeenCalledOnce();
expect(markProjectionDirty).toHaveBeenCalledOnce();
expect(markProjectionDirty).toHaveBeenCalledWith(undefined, 'urn:test:target');
});

it('does not invalidate structured mutation failures or structural no-ops', async () => {
const invalidate = vi.fn();
const markProjectionDirty = vi.fn();
const inner = {
structuredMutation: vi.fn(async () => undefined),
} as unknown as TripleStore;
const store = createListContextGraphsCacheInvalidatingStore(
inner,
invalidate,
markProjectionDirty,
);

await store.structuredMutation!({
kind: 'delete-subjects',
input: { graphUri: 'urn:test:target', subjects: [] },
});
expect(invalidate).not.toHaveBeenCalled();
expect(markProjectionDirty).not.toHaveBeenCalled();

inner.structuredMutation = vi.fn(async () => { throw new Error('commit failed'); });
await expect(store.structuredMutation!({
kind: 'delete-subjects',
input: { graphUri: 'urn:test:target', subjects: ['urn:test:subject'] },
})).rejects.toThrow('commit failed');
expect(invalidate).not.toHaveBeenCalled();
expect(markProjectionDirty).not.toHaveBeenCalled();
});
});
5 changes: 3 additions & 2 deletions packages/storage/src/adapters/blazegraph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
} from '../atomic-graph-replace.js';
import {
buildStructuredMutationUpdate,
captureStructuredMutationEffects,
normalizeStructuredMutation,
structuredMutationMightMutate,
} from '../bounded-structured-mutation.js';
import { quadToNQuad } from '../bounded-rdf.js';
import { readResponseTextBounded } from '../http-response-limit.js';
Expand Down Expand Up @@ -417,14 +417,15 @@
options?: QueryOptions,
): Promise<void> {
const normalized = normalizeStructuredMutation(mutation);
const effects = captureStructuredMutationEffects(normalized);
if (normalized.kind === 'replace-subject-predicates') {
assertQuadLiteralsMutf8Safe([...normalized.input.replacementQuads], {
maxBytes: JAVA_WRITE_UTF_MAX_BYTES,
label: 'BlazegraphStore.structuredMutation',
});
}
const update = buildStructuredMutationUpdate(normalized);
if (!update || !structuredMutationMightMutate(normalized)) return;
if (!update || !effects) return;
await this.sparqlUpdate(
update,
{ ...options, source: options?.source ?? 'blazegraph.structuredMutation' },
Expand Down Expand Up @@ -544,7 +545,7 @@

async listGraphs(options?: TripleStoreQueryOptions): Promise<string[]> {
const r = await this.query(
'SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o } }',

Check notice on line 548 in packages/storage/src/adapters/blazegraph.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R2 graph-var-scan

All-variable triple inside GRAPH ?var enumerates every graph × every triple (the #1597 listGraphs-storm shape). Bind the graph (VALUES/exact IRI), bind a term, or use a FILTER EXISTS existence probe. [pre-existing (grandfathered; fix when touched)] To acknowledge: "sparql-scan-allow: R2 -- <why this is bounded>"
options,
);
if (r.type !== 'bindings') return [];
Expand All @@ -560,7 +561,7 @@
async countQuads(graphUri?: string, options?: QueryOptions): Promise<number> {
const sparql = graphUri
? `SELECT (COUNT(*) AS ?c) WHERE { GRAPH <${escapeUri(graphUri)}> { ?s ?p ?o } }`
: `SELECT (COUNT(*) AS ?c) WHERE { { ?s ?p ?o } UNION { GRAPH ?g { ?s ?p ?o } } }`;

Check notice on line 564 in packages/storage/src/adapters/blazegraph.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R2 graph-var-scan

All-variable triple inside GRAPH ?var enumerates every graph × every triple (the #1597 listGraphs-storm shape). Bind the graph (VALUES/exact IRI), bind a term, or use a FILTER EXISTS existence probe. [pre-existing (grandfathered; fix when touched)] To acknowledge: "sparql-scan-allow: R2 -- <why this is bounded>"

Check notice on line 564 in packages/storage/src/adapters/blazegraph.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R1 unscoped-all-var-scan

All-variable triple pattern with no graph scope scans the ENTIRE store. Scope it to an exact named graph, bind at least one term, or add LIMIT (without ORDER BY). [pre-existing (grandfathered; fix when touched)] To acknowledge: "sparql-scan-allow: R1 -- <why this is bounded>"
const r = await this.query(sparql, {
...options,
source: options?.source ?? 'blazegraph.countQuads',
Expand Down
8 changes: 3 additions & 5 deletions packages/storage/src/adapters/oxigraph-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ import type {
import { registerTripleStoreAdapter } from '../triple-store.js';
import { GraphWriteGenTracker } from '../graph-write-gen.js';
import {
captureStructuredMutationEffects,
normalizeStructuredMutation,
structuredMutationMightMutate,
structuredMutationTouchedGraphs,
} from '../bounded-structured-mutation.js';

/**
Expand Down Expand Up @@ -698,10 +697,9 @@ export class OxigraphWorkerStore implements TripleStore {
_options?: TripleStoreQueryOptions,
): Promise<void> {
const normalized = normalizeStructuredMutation(mutation);
const effects = captureStructuredMutationEffects(normalized);
await this.call('structuredMutation', normalized);
if (structuredMutationMightMutate(normalized)) {
this.writeGen.recordGraphWrites(structuredMutationTouchedGraphs(normalized));
}
if (effects) this.writeGen.recordGraphWrites(effects.touchedGraphs);
}
async query(sparql: string, options?: TripleStoreQueryOptions): Promise<QueryResult> {
return this.callWithTimeout<QueryResult>(this.operationTimeoutMs, options?.signal, 'query', sparql);
Expand Down
8 changes: 4 additions & 4 deletions packages/storage/src/adapters/oxigraph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ import {
} from '../atomic-graph-replace.js';
import {
buildStructuredMutationUpdate,
captureStructuredMutationEffects,
normalizeStructuredMutation,
structuredMutationMightMutate,
structuredMutationTouchedGraphs,
} from '../bounded-structured-mutation.js';
import { quadsToNQuads } from '../bounded-rdf.js';
import { assertQuadLiteralsMutf8Safe, JAVA_WRITE_UTF_MAX_BYTES } from '@origintrail-official/dkg-core';
Expand Down Expand Up @@ -421,17 +420,18 @@ export class OxigraphStore implements TripleStore {
_options?: TripleStoreQueryOptions,
): Promise<void> {
const normalized = normalizeStructuredMutation(mutation);
const effects = captureStructuredMutationEffects(normalized);
if (normalized.kind === 'replace-subject-predicates') {
assertQuadLiteralsMutf8Safe([...normalized.input.replacementQuads], {
maxBytes: JAVA_WRITE_UTF_MAX_BYTES,
label: 'OxigraphStore.structuredMutation',
});
}
const update = buildStructuredMutationUpdate(normalized);
if (!update || !structuredMutationMightMutate(normalized)) return;
if (!update || !effects) return;
this.store.update(update);
this.scheduleFlush();
this.writeGen.recordGraphWrites(structuredMutationTouchedGraphs(normalized));
this.writeGen.recordGraphWrites(effects.touchedGraphs);
}

async listGraphs(options?: TripleStoreQueryOptions): Promise<string[]> {
Expand Down
13 changes: 6 additions & 7 deletions packages/storage/src/adapters/sparql-http.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,9 @@
} from '../atomic-graph-replace.js';
import {
buildStructuredMutationUpdate,
captureStructuredMutationEffects,
normalizeStructuredMutation,
structuredMutationGuardedGraphs,
structuredMutationMightMutate,
structuredMutationTouchedGraphs,
} from '../bounded-structured-mutation.js';
import {
assertNotReservedInternalGraphV1,
Expand Down Expand Up @@ -1299,7 +1298,7 @@
options?: QueryOptions,
): Promise<void> {
const normalized = normalizeStructuredMutation(mutation);
const touchedGraphs = structuredMutationTouchedGraphs(normalized);
const effects = captureStructuredMutationEffects(normalized);
this.assertGenericMutationScope(structuredMutationGuardedGraphs(normalized), 'structuredMutation');
if (normalized.kind === 'replace-subject-predicates') {
assertQuadLiteralsMutf8Safe([...normalized.input.replacementQuads], {
Expand All @@ -1308,24 +1307,24 @@
});
}
const update = buildStructuredMutationUpdate(normalized);
if (!update || !structuredMutationMightMutate(normalized)) return;
if (!update || !effects) return;
try {
await this.postUpdate(
update,
{ ...options, source: options?.source ?? 'sparql-http.structuredMutation' },
'structuredMutation',
touchedGraphs,
effects.touchedGraphs,
);
} catch (error) {
// A remote endpoint may commit before its response is lost. Fail open for
// cache coherence: invalidate graph enumeration and advance each affected
// write generation even though the caller still receives the failure.
this.invalidateListGraphsCache();
this.writeGen.recordGraphWrites(touchedGraphs);
this.writeGen.recordGraphWrites(effects.touchedGraphs);
throw error;
}
this.invalidateListGraphsCache();
this.writeGen.recordGraphWrites(touchedGraphs);
this.writeGen.recordGraphWrites(effects.touchedGraphs);
}

async query(sparql: string, options?: SparqlHttpQueryOptions): Promise<QueryResult> {
Expand Down Expand Up @@ -1522,7 +1521,7 @@
async countQuads(graphUri?: string, options?: QueryOptions): Promise<number> {
const sparql = graphUri
? `SELECT (COUNT(*) AS ?c) WHERE { GRAPH <${escapeUri(graphUri)}> { ?s ?p ?o } }`
: `SELECT (COUNT(*) AS ?c) WHERE { { ?s ?p ?o } UNION { GRAPH ?g { ?s ?p ?o } } }`;

Check notice on line 1524 in packages/storage/src/adapters/sparql-http.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R2 graph-var-scan

All-variable triple inside GRAPH ?var enumerates every graph × every triple (the #1597 listGraphs-storm shape). Bind the graph (VALUES/exact IRI), bind a term, or use a FILTER EXISTS existence probe. [pre-existing (grandfathered; fix when touched)] To acknowledge: "sparql-scan-allow: R2 -- <why this is bounded>"

Check notice on line 1524 in packages/storage/src/adapters/sparql-http.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R1 unscoped-all-var-scan

All-variable triple pattern with no graph scope scans the ENTIRE store. Scope it to an exact named graph, bind at least one term, or add LIMIT (without ORDER BY). [pre-existing (grandfathered; fix when touched)] To acknowledge: "sparql-scan-allow: R1 -- <why this is bounded>"
const r = await this.query(sparql, {
...options,
source: options?.source ?? 'sparql-http.countQuads',
Expand Down
14 changes: 14 additions & 0 deletions packages/storage/src/bounded-structured-mutation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@
# sparql-scan-allow: R3 -- one bounded retention prune; OFFSET <= 100000 and LIMIT <= 10000; never page-walked
GROUP BY ?subject
ORDER BY DESC(COALESCE(
xsd:integer(STR(?latestPrimaryRank)),

Check notice on line 227 in packages/storage/src/bounded-structured-mutation.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R3 offset-pagination

OFFSET pagination re-scans O(offset) rows per page (O(n²) for a full walk) and tears on mutable data. Use a keyset/seek cursor, a retained snapshot, or acknowledge with a pragma. [acknowledged by pragma] To acknowledge: "sparql-scan-allow: R3 -- <why this is bounded>"
xsd:integer(STR(?latestSecondaryRank)),
0
)) DESC(STR(?subject))
Expand Down Expand Up @@ -547,7 +547,7 @@
WHERE {
VALUES ?sourceGraph { ${sources} }
VALUES ?root { ${roots} }
# sparql-scan-allow: R2 -- ?sourceGraph is VALUES-bound to at most 8 validated exact graph IRIs

Check notice on line 550 in packages/storage/src/bounded-structured-mutation.ts

View workflow job for this annotation

GitHub Actions / SPARQL scalability lint

sparql-scale-lint R2 graph-var-scan

All-variable triple inside GRAPH ?var enumerates every graph × every triple (the #1597 listGraphs-storm shape). Bind the graph (VALUES/exact IRI), bind a term, or use a FILTER EXISTS existence probe. [acknowledged by pragma] To acknowledge: "sparql-scan-allow: R2 -- <why this is bounded>"
GRAPH ?sourceGraph { ?subject ?predicate ?object }
FILTER(?subject = ?root || STRSTARTS(STR(?subject), CONCAT(STR(?root), ${sparqlString(normalized.descendantSuffix)})))
${excluded}
Expand Down Expand Up @@ -680,3 +680,17 @@
export function structuredMutationMightMutate(mutation: StructuredMutation): boolean {
return mutation.kind !== 'delete-subjects' || mutation.input.subjects.length > 0;
}

/** Immutable graph-scoped effects captured before a structured mutation is dispatched. */
export interface StructuredMutationEffects {
readonly touchedGraphs: readonly string[];
}

/** Capture canonical effects without executing or probing a store capability. */
export function captureStructuredMutationEffects(
mutation: StructuredMutation,
): StructuredMutationEffects | undefined {
if (!structuredMutationMightMutate(mutation)) return undefined;
const touchedGraphs = Object.freeze([...structuredMutationTouchedGraphs(mutation)]);
return Object.freeze({ touchedGraphs });
}
8 changes: 4 additions & 4 deletions packages/storage/src/changelog-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,9 @@ import {
} from './store-chain-capability.js';
import type { SystemRecordLaneControllerV1 } from './system-record-materializer-v1.js';
import {
captureStructuredMutationEffects,
normalizeStructuredMutation,
structuredMutationGuardedGraphs,
structuredMutationMightMutate,
structuredMutationTouchedGraphs,
} from './bounded-structured-mutation.js';

/**
Expand Down Expand Up @@ -460,6 +459,7 @@ export class ChangelogStore implements TripleStore, ChangelogReader {

async structuredMutation(mutation: StructuredMutation, options?: QueryOptions): Promise<void> {
const normalized = normalizeStructuredMutation(mutation);
const effects = captureStructuredMutationEffects(normalized);
const operation = this.inner.structuredMutation;
if (!operation) {
throw new UnsupportedTripleStoreCapabilityError('structuredMutation', 'ChangelogStore');
Expand All @@ -477,8 +477,8 @@ export class ChangelogStore implements TripleStore, ChangelogReader {
}
throw error;
}
if (structuredMutationMightMutate(normalized)) {
await this.markPostMutation(structuredMutationTouchedGraphs(normalized), options);
if (effects) {
await this.markPostMutation(effects.touchedGraphs, options);
}
});
}
Expand Down
8 changes: 4 additions & 4 deletions packages/storage/src/graph-set-index-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@ import {
import { isAtomicGraphReplaceStagingGraph } from './atomic-graph-replace.js';
import { ManagedOxigraphBackendUnownedError } from './managed-oxigraph-ownership-v1-internal.js';
import {
captureStructuredMutationEffects,
normalizeStructuredMutation,
structuredMutationMightMutate,
structuredMutationTouchedGraphs,
} from './bounded-structured-mutation.js';
import {
CACHED_READ_GATE_V1,
Expand Down Expand Up @@ -559,6 +558,7 @@ export class GraphSetIndexStore implements TripleStore {

async structuredMutation(mutation: StructuredMutation, options?: QueryOptions): Promise<void> {
const normalized = normalizeStructuredMutation(mutation);
const effects = captureStructuredMutationEffects(normalized);
const operation = this.inner.structuredMutation;
if (!operation) {
throw new UnsupportedTripleStoreCapabilityError('structuredMutation', 'GraphSetIndexStore');
Expand All @@ -571,10 +571,10 @@ export class GraphSetIndexStore implements TripleStore {
}
throw error;
}
if (!this.enabled || !structuredMutationMightMutate(normalized)) return;
if (!this.enabled || !effects) return;
this.bumpMutation();
await this.maintainTouchedGraphs(
[...structuredMutationTouchedGraphs(normalized)],
[...effects.touchedGraphs],
'structuredMutation',
options,
);
Expand Down
2 changes: 2 additions & 0 deletions packages/storage/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,11 @@ export {
} from './atomic-graph-replace.js';
export {
BOUNDED_MUTATION_MAX_PRUNE_DELETE,
captureStructuredMutationEffects,
chunkCopySubjectProjectionInput,
structuredMutationMightMutate,
structuredMutationTouchedGraphs,
type StructuredMutationEffects,
} from './bounded-structured-mutation.js';
// System-record V1 (#2052 Stack B2). Default-unused: these modules perform no
// I/O, scheduling, timer, or per-store lane work until the daemon supervisor
Expand Down
Loading
Loading