diff --git a/apps/api/migrations/20261028120000_file_events.sql b/apps/api/migrations/20261028120000_file_events.sql new file mode 100644 index 000000000..fdc225bf5 --- /dev/null +++ b/apps/api/migrations/20261028120000_file_events.sql @@ -0,0 +1,47 @@ +-- Append-only log of classifier attempts and member hold overrides +-- (issue #1136, storage redesign step 1). +-- +-- Rows are inserted and never updated. An overwrite clears the current +-- `ai.*` rows and then appends another event; it does not delete the +-- earlier ones. Object delete and retention leave the log in place, so a +-- release cannot be erased by deleting the file. Workspace teardown +-- removes the workspace's rows. +-- +-- `content_sha256` is the bytes this event describes. It is the future +-- blob id for content-addressed storage and is not a foreign key. An +-- override's hash is never null: it is the object's `content-sha256`, or +-- the hash of the body restored from the held bucket. An override applies +-- to a key only while that key's current bytes hash to `content_sha256`. +-- A different hash is a different set of bytes and the override is ignored. +-- +-- `kind` is classified, reused_exact, reused_perceptual, skipped, failed, +-- or override. `action` is release or hold, and only on an override. +-- Safety columns are the answers in force for that attempt. They are a +-- copy. Nothing here writes `file_metadata`. +CREATE TABLE file_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + workspace TEXT NOT NULL, + object_key TEXT NOT NULL, + content_sha256 TEXT, + kind TEXT NOT NULL, + classifier_version TEXT, + model TEXT, + reason TEXT, + action TEXT, + actor TEXT, + rating TEXT, + rating_confidence TEXT, + content_flag TEXT, + content_confidence TEXT, + secret TEXT, + secret_confidence TEXT, + pii TEXT, + pii_confidence TEXT, + created_at TEXT NOT NULL +); + +-- History of one key, oldest first. Also serves the latest-override read. +CREATE INDEX file_events_key_idx ON file_events (workspace, object_key, id); + +-- Future blob lookup: every event that described these bytes. +CREATE INDEX file_events_hash_idx ON file_events (workspace, content_sha256, id); diff --git a/apps/api/src/classifier.ts b/apps/api/src/classifier.ts index 9211ba3c9..d14893e55 100644 --- a/apps/api/src/classifier.ts +++ b/apps/api/src/classifier.ts @@ -21,7 +21,9 @@ * A call that times out, errors, or answers below the confidence floor * writes `ai.classifier=failed` and `ai.error`, but only when these bytes * have no earlier result. New bytes clear every classifier row first - * (`CLASSIFIER_ROW_KEYS`). + * (`CLASSIFIER_ROW_KEYS`). Each attempt also appends a `file_events` row + * (`classified`, `reused_exact`, `reused_perceptual`, `skipped`, `failed`). + * The log keeps the earlier attempt after those rows are cleared. * * Before calling, a finished classification in the same workspace can stand * in: identical bytes reuse all rows (including safety flags and holds); an @@ -40,6 +42,7 @@ */ import { contentHashDonorKeys, contentHashForKey } from "./content-hash"; +import { appendClassificationEvent, type ClassificationEventKind } from "./file-events"; import { PHASH_META_KEY, SAFETY_FLAG_KEYS, @@ -1159,6 +1162,31 @@ function classifierVersionMarker(provider: ClassifierProvider, resized: boolean) return resized ? CLASSIFIER_RESIZED_VERSION : CLASSIFIER_VERSION; } +function classifierModel(provider: ClassifierProvider): string { + return provider === "luna" ? CLASSIFIER_LUNA_MODEL : CLASSIFIER_JEV_MODEL; +} + +async function recordClassificationEvent( + env: Env, + workspaceName: string, + key: string, + bytes: Uint8Array, + kind: ClassificationEventKind, + meta: Record | undefined, + extra: { model?: string | null; reason?: string | null } = {}, +): Promise { + await appendClassificationEvent(env, { + workspace: workspaceName, + objectKey: key, + bytes, + kind, + classifierVersion: meta?.["ai.classifier"] ?? null, + model: extra.model ?? null, + reason: extra.reason ?? null, + safety: meta, + }); +} + /** Display labels a skip must not leave behind from an earlier classification. */ const CLASSIFIER_DISPLAY_KEYS = ["ai.kind", "ai.surface", "ai.screen", "ai.ready"] as const; @@ -1178,6 +1206,7 @@ async function storeSkip( key: string, reason: string, provider: ClassifierProvider, + bytes: Uint8Array, ): Promise> { const db = dbFor(env); await deleteServerFileMetadataKeys(db, workspaceName, key, [ @@ -1186,6 +1215,7 @@ async function storeSkip( ]); const meta = skipMeta(reason, provider); await setServerFileMetadata(db, workspaceName, key, meta); + await recordClassificationEvent(env, workspaceName, key, bytes, "skipped", meta, { reason }); return meta; } @@ -1212,14 +1242,31 @@ async function storeFailure( key: string, reason: ClassifierErrorReason, provider: ClassifierProvider, + bytes: Uint8Array, ): Promise | undefined> { console.log({ event: "classifier_call_failed", reason, provider }); + const model = classifierModel(provider); const db = dbFor(env); const existing = (await getFileMetadata(db, workspaceName, key))["ai.classifier"]; - if (existing !== undefined && existing !== CLASSIFIER_FAILED) return undefined; + if (existing !== undefined && existing !== CLASSIFIER_FAILED) { + await recordClassificationEvent( + env, + workspaceName, + key, + bytes, + "failed", + { "ai.classifier": CLASSIFIER_FAILED }, + { model, reason }, + ); + return undefined; + } await deleteServerFileMetadataKeys(db, workspaceName, key, ["ai.skip"]); const meta = { "ai.classifier": CLASSIFIER_FAILED, "ai.error": reason }; await setServerFileMetadata(db, workspaceName, key, meta); + await recordClassificationEvent(env, workspaceName, key, bytes, "failed", meta, { + model, + reason, + }); return meta; } @@ -1421,6 +1468,7 @@ async function matchPerceptualReuse( workspaceName: string, key: string, phash: string, + bytes: Uint8Array, ): Promise | undefined> { const hits = await findObjectsByMetadata( dbFor(env), @@ -1433,7 +1481,9 @@ async function matchPerceptualReuse( const meta = perceptualReuseMeta(hit.metadata, phash); if (!meta) continue; console.log({ event: "classifier_reused", match: "phash", workspace: workspaceName }); - return storeDecision(env, workspaceName, key, meta); + const stored = await storeDecision(env, workspaceName, key, meta); + await recordClassificationEvent(env, workspaceName, key, bytes, "reused_perceptual", stored); + return stored; } return undefined; } @@ -1478,7 +1528,9 @@ async function tryReuseClassification( ]); if (recorded === sha && hasReuseBody(own)) { console.log({ event: "classifier_reused", match: "sha256", workspace: workspaceName }); - return { meta: classifierRowsOf(own), hold: true }; + const meta = classifierRowsOf(own); + await recordClassificationEvent(env, workspaceName, key, bytes, "reused_exact", meta); + return { meta, hold: true }; } const phash = imagePhash(bytes, contentType); @@ -1486,11 +1538,13 @@ async function tryReuseClassification( const meta = exactReuseMeta(await getFileMetadata(db, workspaceName, donorKey), phash); if (!meta) continue; console.log({ event: "classifier_reused", match: "sha256", workspace: workspaceName }); - return { meta: await storeDecision(env, workspaceName, key, meta), hold: true }; + const stored = await storeDecision(env, workspaceName, key, meta); + await recordClassificationEvent(env, workspaceName, key, bytes, "reused_exact", stored); + return { meta: stored, hold: true }; } if (phash) { - const meta = await matchPerceptualReuse(env, workspaceName, key, phash); + const meta = await matchPerceptualReuse(env, workspaceName, key, phash, bytes); if (meta) return { meta, hold: false }; } return { hold: false, phash }; @@ -1521,7 +1575,7 @@ export async function classifyAndStore( if (!(await classificationAllowed(env, ws, workspaceName))) return undefined; const provider = await resolveClassifierProvider(env, ws, workspaceName); const trivial = trivialImageSkip(contentType, bytes); - if (trivial) return await storeSkip(env, workspaceName, key, trivial, provider); + if (trivial) return await storeSkip(env, workspaceName, key, trivial, provider, bytes); const reused = await tryReuseClassification(env, workspaceName, key, bytes, contentType); if (reused.meta) { @@ -1555,7 +1609,7 @@ export async function classifyAndStore( if (!resizeController.signal.aborted) { console.error({ event: "classifier_image_transform_failed", err }); } - return await storeSkip(env, workspaceName, key, reason, provider); + return await storeSkip(env, workspaceName, key, reason, provider, bytes); } if (!prepared.image) { @@ -1565,6 +1619,7 @@ export async function classifyAndStore( key, prepared.skip ?? CLASSIFIER_SKIP_UNSUPPORTED, provider, + bytes, ); } @@ -1573,7 +1628,7 @@ export async function classifyAndStore( phash = imagePhash(prepared.hashable.bytes, prepared.hashable.contentType); if (phash) { try { - const meta = await matchPerceptualReuse(env, workspaceName, key, phash); + const meta = await matchPerceptualReuse(env, workspaceName, key, phash, bytes); if (meta) { return finishReused( env, @@ -1611,13 +1666,23 @@ export async function classifyAndStore( ? CLASSIFIER_ERROR_TIMEOUT : CLASSIFIER_ERROR_PROVIDER; console.error({ event: "classifier_failed", workspace: workspaceName, key, reason, err }); - return await storeFailure(env, workspaceName, key, reason, provider); + return await storeFailure(env, workspaceName, key, reason, provider, bytes); } if (!meta) { - return await storeFailure(env, workspaceName, key, CLASSIFIER_ERROR_NO_ANSWER, provider); + return await storeFailure( + env, + workspaceName, + key, + CLASSIFIER_ERROR_NO_ANSWER, + provider, + bytes, + ); } if (phash) meta[PHASH_META_KEY] = phash; const stored = await storeDecision(env, workspaceName, key, meta); + await recordClassificationEvent(env, workspaceName, key, bytes, "classified", stored, { + model: classifierModel(provider), + }); // Loaded lazily: the hold re-syncs the PR comment, and the comment // service imports files-core, which imports this module. const { applyUploadHold, resyncCommentAfterClassification } = diff --git a/apps/api/src/file-events.ts b/apps/api/src/file-events.ts new file mode 100644 index 000000000..26543732a --- /dev/null +++ b/apps/api/src/file-events.ts @@ -0,0 +1,220 @@ +import { ValidationError } from "@uploads/errors"; +import { dbFor, type D1Queryable } from "./db-session"; +import { contentSha256Hex } from "./provenance"; + +const INSERT_SQL = `INSERT INTO file_events ( + workspace, object_key, content_sha256, kind, classifier_version, model, + reason, action, actor, + rating, rating_confidence, content_flag, content_confidence, + secret, secret_confidence, pii, pii_confidence, created_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; + +export type ClassificationEventKind = + | "classified" + | "reused_exact" + | "reused_perceptual" + | "skipped" + | "failed"; + +function ignorable(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return message.includes("no such table"); +} + +const SHA256_RE = /^[a-f0-9]{64}$/; + +async function insertEvent( + env: Env, + row: { + workspace: string; + objectKey: string; + contentSha256: string | null; + kind: string; + classifierVersion?: string | null; + model?: string | null; + reason?: string | null; + action?: string | null; + actor?: string | null; + safety?: Record; + at: string; + }, +): Promise { + const meta = row.safety; + const read = (key: string) => meta?.[key] ?? null; + try { + await dbFor(env) + .prepare(INSERT_SQL) + .bind( + row.workspace, + row.objectKey, + row.contentSha256, + row.kind, + row.classifierVersion ?? null, + row.model ?? null, + row.reason ?? null, + row.action ?? null, + row.actor ?? null, + read("ai.rating"), + read("ai.rating.confidence"), + read("ai.content"), + read("ai.content.confidence"), + read("ai.secret"), + read("ai.secret.confidence"), + read("ai.pii"), + read("ai.pii.confidence"), + row.at, + ) + .run(); + } catch (err) { + if (ignorable(err)) return; + console.error({ event: "file_event_append_failed", workspace: row.workspace, err }); + } +} + +export async function appendClassificationEvent( + env: Env, + input: { + workspace: string; + objectKey: string; + bytes: Uint8Array; + kind: ClassificationEventKind; + classifierVersion?: string | null; + model?: string | null; + reason?: string | null; + safety?: Record; + }, +): Promise { + let contentSha256: string | null = null; + try { + contentSha256 = await contentSha256Hex(input.bytes); + } catch { + contentSha256 = null; + } + await insertEvent(env, { + workspace: input.workspace, + objectKey: input.objectKey, + contentSha256, + kind: input.kind, + classifierVersion: input.classifierVersion, + model: input.model, + reason: input.reason, + safety: input.safety, + at: new Date().toISOString(), + }); +} + +export async function appendOverrideEvent( + env: Env, + input: { + workspace: string; + objectKey: string; + contentSha256: string; + action: "release" | "hold"; + actor?: string | null; + reason?: string | null; + safety?: Record; + at: string; + }, +): Promise { + if (!SHA256_RE.test(input.contentSha256)) { + throw new ValidationError( + "this file has no content hash, so the hold change cannot be recorded", + { + code: "content_hash_unknown", + }, + ); + } + await insertEvent(env, { + workspace: input.workspace, + objectKey: input.objectKey, + contentSha256: input.contentSha256, + kind: "override", + action: input.action, + actor: input.actor, + reason: input.reason, + safety: input.safety, + at: input.at, + }); +} + +export interface OverrideEvent { + action: string; + actor: string | null; + contentSha256: string | null; + createdAt: string; +} + +/** Latest override per key, newest id first. One key is `latestOverridesForKeys(db, ws, [key])`. */ +export async function latestOverridesForKeys( + db: D1Queryable, + workspace: string, + keys: string[], +): Promise> { + const found = new Map(); + if (keys.length === 0) return found; + try { + const chunkSize = 80; + for (let i = 0; i < keys.length; i += chunkSize) { + const chunk = keys.slice(i, i + chunkSize); + const placeholders = chunk.map(() => "?").join(", "); + const result = await db + .prepare( + `SELECT object_key, action, actor, content_sha256, created_at + FROM file_events + WHERE workspace = ? AND kind = 'override' AND object_key IN (${placeholders}) + ORDER BY id DESC`, + ) + .bind(workspace, ...chunk) + .all<{ + object_key: string; + action: string; + actor: string | null; + content_sha256: string | null; + created_at: string; + }>(); + for (const row of result.results ?? []) { + if (found.has(row.object_key)) continue; + found.set(row.object_key, { + action: row.action, + actor: row.actor, + contentSha256: row.content_sha256, + createdAt: row.created_at, + }); + } + } + } catch (err) { + if (ignorable(err)) return found; + console.error({ event: "file_event_read_failed", workspace, err }); + return found; + } + return found; +} + +export function overrideApplies(eventSha: string | null, currentSha: string | null): boolean { + if (eventSha === null) return currentSha === null; + return eventSha === currentSha; +} + +async function runDelete( + db: D1Queryable, + workspace: string, + sql: string, + binds: unknown[], +): Promise { + try { + await db + .prepare(sql) + .bind(...binds) + .run(); + } catch (err) { + if (ignorable(err)) return; + console.error({ event: "file_event_delete_failed", workspace, err }); + } +} + +export async function deleteFileEventsForWorkspace( + db: D1Queryable, + workspace: string, +): Promise { + await runDelete(db, workspace, `DELETE FROM file_events WHERE workspace = ?`, [workspace]); +} diff --git a/apps/api/src/file-metadata.ts b/apps/api/src/file-metadata.ts index 1230e3fea..1a9fc9caf 100644 --- a/apps/api/src/file-metadata.ts +++ b/apps/api/src/file-metadata.ts @@ -56,7 +56,8 @@ export const SAFETY_FLAG_FIELDS = CLASSIFIER_SAFETY_FIELDS; * classification is scheduled. A classification that writes a value * (including a confident `none`) replaces that key; one that omits a key * leaves the old value. Object delete removes them. They don't bump - * `ai.classifier`. + * `ai.classifier`. The append-only history is `file_events`. An overwrite, + * a delete, and retention leave it in place. Workspace teardown removes it. */ export const SAFETY_FLAG_KEYS: ReadonlySet = new Set( SAFETY_FLAG_FIELDS.flatMap((field) => [`ai.${field}`, `ai.${field}.confidence`]), diff --git a/apps/api/src/held-storage.ts b/apps/api/src/held-storage.ts index 1ed622021..f7e56660c 100644 --- a/apps/api/src/held-storage.ts +++ b/apps/api/src/held-storage.ts @@ -17,10 +17,16 @@ import { createStorage, type Files } from "@uploads/storage"; import { dbFor } from "./db-session"; import { setServerFileMetadata } from "./file-metadata"; import { UPLOAD_CACHE_CONTROL } from "./files-core"; +import { contentSha256Hex } from "./provenance"; import { HOLD_MOVED_KEY } from "./upload-hold"; import { isKeyHeld } from "./upload-hold-read"; import { posterKeyFor } from "./poster"; -import { isSharedLane, resolveObjectLane, storageConfigs } from "./storage"; +import { + isSharedLane, + resolveObjectLane, + resolveObjectLaneWithHead, + storageConfigs, +} from "./storage"; import type { WorkspaceRecord } from "./workspace"; /** The held bucket, or `undefined` where it is not bound (tests, older deploys). */ @@ -67,24 +73,38 @@ export async function moveToHeld( /** * Copy a held object (and its poster) back to its key on the hosted lane, - * then drop the held copy. `false` when the held bucket has no copy. + * then drop the held copy. The return is the SHA-256 of the restored body, + * or `null` when the held bucket has no copy. */ export async function restoreFromHeld( env: Env, ws: WorkspaceRecord, workspaceName: string, key: string, -): Promise { +): Promise { const held = heldBucket(env); - if (!held) return false; + if (!held) return null; const store = await sharedLaneStore(env, ws); - if (!store) return false; - const restored = await copyIn(held, store, heldObjectKey(workspaceName, key), key); - if (!restored) return false; + if (!store) return null; + const sha = await copyIn(held, store, heldObjectKey(workspaceName, key), key); + if (!sha) return null; const posterKey = posterKeyFor(key); await copyIn(held, store, heldObjectKey(workspaceName, posterKey), posterKey); await held.delete(heldKeysFor(workspaceName, key)); - return true; + return sha; +} + +const SHA256_RE = /^[a-f0-9]{64}$/; + +/** `content-sha256` on the object, for a hold that never left the public bucket. */ +export async function objectContentSha256( + env: Env, + ws: WorkspaceRecord, + key: string, +): Promise { + const lane = await resolveObjectLaneWithHead(env, ws, key); + const value = lane?.head.metadata?.["content-sha256"]; + return typeof value === "string" && SHA256_RE.test(value) ? value : undefined; } /** @@ -154,14 +174,14 @@ async function copyIn( store: Files, heldKey: string, key: string, -): Promise { +): Promise { const source = await held.get(heldKey); - if (!source) return false; + if (!source) return null; const bytes = new Uint8Array(await source.arrayBuffer()); await store.upload(key, bytes, { contentType: source.httpMetadata?.contentType ?? "application/octet-stream", cacheControl: UPLOAD_CACHE_CONTROL, metadata: { ...source.customMetadata }, }); - return true; + return contentSha256Hex(bytes); } diff --git a/apps/api/src/routes/workspace-files.ts b/apps/api/src/routes/workspace-files.ts index 822465449..6b306af4d 100644 --- a/apps/api/src/routes/workspace-files.ts +++ b/apps/api/src/routes/workspace-files.ts @@ -48,18 +48,8 @@ import { setObjectVisibility, } from "../files-core"; import { approveUpload, approverLabel, holdAgain } from "../upload-hold-service"; -import { listHeldKeys } from "../upload-hold-read"; -import { - HOLD_APPROVED, - HOLD_AREA_KEY, - HOLD_AT_KEY, - HOLD_BY_KEY, - HOLD_META_KEY, - HOLD_MOVED_KEY, - HOLD_PENDING, - parseHoldArea, - storedHoldReason, -} from "../upload-hold"; +import { effectiveHold, listHeldKeys } from "../upload-hold-read"; +import { HOLD_AREA_KEY, HOLD_MOVED_KEY, parseHoldArea, storedHoldReason } from "../upload-hold"; import { readHeld } from "../held-storage"; import { getFileMetadata, @@ -524,7 +514,8 @@ export const workspaceFiles = new Hono() async (c) => { const key = c.req.query("key") ?? ""; if (badKey(key)) throw new NotFoundError(); - const result = await holdAgain(c.env, c.get("workspace"), c.req.param("workspace"), key); + const by = await approverLabel(c.env, c.get("uploaderIdentity")); + const result = await holdAgain(c.env, c.get("workspace"), c.req.param("workspace"), key, by); return c.json({ key, ...result }); }, ) @@ -535,14 +526,15 @@ export const workspaceFiles = new Hono() .get("/:workspace/files/review", dualWorkspaceAuth(), scoped("files:read"), async (c) => { const key = c.req.query("key") ?? ""; if (badKey(key)) throw new NotFoundError(); - const meta = await getFileMetadata(dbFor(c.env), c.get("workspaceName"), key); - const hold = meta[HOLD_META_KEY]; + const workspaceName = c.get("workspaceName"); + const meta = await getFileMetadata(dbFor(c.env), workspaceName, key); + const view = await effectiveHold(c.env, workspaceName, key, meta); return c.json({ key, - hold: hold === HOLD_PENDING || hold === HOLD_APPROVED ? hold : null, - reason: hold ? storedHoldReason(meta) : null, - by: hold === HOLD_APPROVED ? (meta[HOLD_BY_KEY] ?? null) : null, - at: hold === HOLD_APPROVED ? (meta[HOLD_AT_KEY] ?? null) : null, + hold: view.hold, + reason: view.hold ? storedHoldReason(meta) : null, + by: view.by, + at: view.at, }); }) diff --git a/apps/api/src/upload-hold-read.ts b/apps/api/src/upload-hold-read.ts index ba5ecb55a..989cc6652 100644 --- a/apps/api/src/upload-hold-read.ts +++ b/apps/api/src/upload-hold-read.ts @@ -6,9 +6,18 @@ */ import { dbFor } from "./db-session"; +import { contentHashForKey } from "./content-hash"; import { getMetadataForKeys } from "./file-metadata"; +import { latestOverridesForKeys, overrideApplies } from "./file-events"; import { aiRows } from "./classifier-admin"; -import { HOLD_META_KEY, HOLD_PENDING, isHeld } from "./upload-hold"; +import { + HOLD_APPROVED, + HOLD_AT_KEY, + HOLD_BY_KEY, + HOLD_META_KEY, + HOLD_PENDING, + isHeld, +} from "./upload-hold"; /** The subset of `keys` held for approval. One batched D1 read. */ export async function heldKeysAmong( @@ -30,6 +39,52 @@ export async function isKeyHeld(env: Env, workspaceName: string, key: string): P return (await heldKeysAmong(env, workspaceName, [key])).has(key); } +export interface EffectiveHold { + hold: "pending" | "approved" | null; + by: string | null; + at: string | null; +} + +/** + * Hold state for a member view. `ai.hold` is the effective state for the + * current bytes (pending, approved, or absent). A release override for + * those same bytes supplies who released it and when. An override for a + * different hash is an earlier set of bytes and is ignored. A pending row + * stays pending: that is what comments and the public gate already read. + */ +export async function effectiveHold( + env: Env, + workspace: string, + objectKey: string, + meta: Record, +): Promise { + const row = meta[HOLD_META_KEY]; + const fromRow = (): EffectiveHold => { + if (row !== HOLD_PENDING && row !== HOLD_APPROVED) return { hold: null, by: null, at: null }; + const released = row === HOLD_APPROVED; + return { + hold: row, + by: released ? (meta[HOLD_BY_KEY] ?? null) : null, + at: released ? (meta[HOLD_AT_KEY] ?? null) : null, + }; + }; + if (row !== HOLD_APPROVED) return fromRow(); + const db = dbFor(env); + const [overrides, currentSha] = await Promise.all([ + latestOverridesForKeys(db, workspace, [objectKey]), + contentHashForKey(db, workspace, objectKey), + ]); + const override = overrides.get(objectKey) ?? null; + if ( + !override || + override.action !== "release" || + !overrideApplies(override.contentSha256, currentSha ?? null) + ) { + return fromRow(); + } + return { hold: "approved", by: override.actor, at: override.createdAt }; +} + /** * Every key in the workspace held for approval. A moved hold has no public * object to list, so the comment and `uploads approve` find held files diff --git a/apps/api/src/upload-hold-service.ts b/apps/api/src/upload-hold-service.ts index 11451508f..91bf467dd 100644 --- a/apps/api/src/upload-hold-service.ts +++ b/apps/api/src/upload-hold-service.ts @@ -9,16 +9,17 @@ * (`hold-email-notify.ts`). */ -import { ServiceUnavailableError } from "@uploads/errors"; +import { ServiceUnavailableError, ValidationError } from "@uploads/errors"; import { classifierEvaluationContext } from "./classifier"; import { dbFor } from "./db-session"; +import { appendOverrideEvent } from "./file-events"; import { deleteServerFileMetadataKeys, getFileMetadata, setServerFileMetadata, } from "./file-metadata"; import { parseAttachmentKey } from "./github-attachment-index"; -import { moveToHeld, restoreFromHeld } from "./held-storage"; +import { moveToHeld, objectContentSha256, restoreFromHeld } from "./held-storage"; import { emailHold } from "./hold-email-notify"; import { notifySlackHoldResolved, notifySlackOfHold } from "./slack-hold-notify"; import { postManagedComment } from "./github-comment-service"; @@ -124,7 +125,8 @@ export type ApproveUploadResult = { approved: true } | { approved: false; reason /** * Release a held file. A file that is not held (never flagged, already * approved, or no row for this key) is `not_held`, which the route reports - * as a 200 so a retried approve is harmless. + * as a 200 so a retried approve is harmless. Safety flags stay as + * classified. The release also appends an override event. */ export async function approveUpload( env: Env, @@ -136,21 +138,43 @@ export async function approveUpload( const db = dbFor(env); const existing = await getFileMetadata(db, workspaceName, key); if (existing[HOLD_META_KEY] !== HOLD_PENDING) return { approved: false, reason: "not_held" }; + let contentSha256: string | undefined; if (existing[HOLD_MOVED_KEY] === "1") { // Bytes first: approving a file whose bytes cannot come back would - // leave its links broken with no hold left to explain why. - if (!(await restoreFromHeld(env, ws, workspaceName, key))) { + // leave its links broken with no hold left to explain why. The hash is + // of that body, not the best-effort D1 index. + const restored = await restoreFromHeld(env, ws, workspaceName, key); + if (!restored) { throw new ServiceUnavailableError("the held copy of this file could not be restored", { code: "held_restore_failed", }); } + contentSha256 = restored; await deleteServerFileMetadataKeys(db, workspaceName, key, [HOLD_MOVED_KEY]); + } else { + contentSha256 = await objectContentSha256(env, ws, key); } + if (!contentSha256) { + throw new ValidationError( + "this file has no content hash, so the hold change cannot be recorded", + { code: "content_hash_unknown" }, + ); + } + const at = new Date().toISOString(); await setServerFileMetadata(db, workspaceName, key, { [HOLD_META_KEY]: HOLD_APPROVED, - [HOLD_AT_KEY]: new Date().toISOString(), + [HOLD_AT_KEY]: at, ...(by ? { [HOLD_BY_KEY]: by } : {}), }); + await appendOverrideEvent(env, { + workspace: workspaceName, + objectKey: key, + contentSha256, + action: "release", + actor: by ?? null, + safety: existing, + at, + }); console.log({ event: "upload_approved", workspace: workspaceName }); await Promise.all([ resyncAttachmentComment(env, ws, workspaceName, key, existing), @@ -174,13 +198,30 @@ export async function holdAgain( ws: WorkspaceRecord, workspaceName: string, key: string, + by?: string | null, ): Promise { const db = dbFor(env); const existing = await getFileMetadata(db, workspaceName, key); if (existing[HOLD_META_KEY] !== HOLD_APPROVED) return { held: false, reason: "not_approved" }; + const contentSha256 = await objectContentSha256(env, ws, key); + if (!contentSha256) { + throw new ValidationError( + "this file has no content hash, so the hold change cannot be recorded", + { code: "content_hash_unknown" }, + ); + } // The row lands first, as in `applyUploadHold`. await setServerFileMetadata(db, workspaceName, key, { [HOLD_META_KEY]: HOLD_PENDING }); await deleteServerFileMetadataKeys(db, workspaceName, key, [HOLD_BY_KEY, HOLD_AT_KEY]); + await appendOverrideEvent(env, { + workspace: workspaceName, + objectKey: key, + contentSha256, + action: "hold", + actor: by ?? null, + safety: existing, + at: new Date().toISOString(), + }); const moved = await moveToHeld(env, ws, workspaceName, key); console.log({ event: "upload_held_again", workspace: workspaceName, moved }); await resyncAttachmentComment(env, ws, workspaceName, key, existing); diff --git a/apps/api/src/upload-hold.ts b/apps/api/src/upload-hold.ts index 2edd047d7..82aeabecf 100644 --- a/apps/api/src/upload-hold.ts +++ b/apps/api/src/upload-hold.ts @@ -8,9 +8,13 @@ * with `POST /v1/workspaces/:workspace/files/approve`, which moves the * bytes back to the same key. * - * State is one server-owned row, `ai.hold`: `pending` or `approved`. It is - * in `CLASSIFIER_ROW_KEYS`, so it shares the classifier's lifetime: - * identical bytes keep it (an approval sticks), and new bytes clear it. + * State for the current bytes is one server-owned row, `ai.hold`: + * `pending` or `approved`. It is in `CLASSIFIER_ROW_KEYS`, so it shares + * the classifier's lifetime: identical bytes keep it (an approval sticks), + * and new bytes clear it. A release also appends an override event + * (`file-events.ts`) with who released it and the safety flags in force. + * That event does not change `ai.secret`, `ai.pii`, `ai.rating`, or + * `ai.content`, and it stays after new bytes clear the row. * * Env-free, so the CLI's local-gh comment path can share the predicate. */ diff --git a/apps/api/src/workspace-teardown.ts b/apps/api/src/workspace-teardown.ts index 037be96be..dfb6c8768 100644 --- a/apps/api/src/workspace-teardown.ts +++ b/apps/api/src/workspace-teardown.ts @@ -17,6 +17,7 @@ */ import { dbFor } from "./db-session"; import { deleteFileMetadataForWorkspace } from "./file-metadata"; +import { deleteFileEventsForWorkspace } from "./file-events"; import { deleteGalleriesForWorkspace } from "./galleries"; import { deleteFeedsForWorkspace } from "./feeds"; import { deleteAttachmentsForWorkspaceSafe } from "./github-attachment-index"; @@ -105,6 +106,7 @@ export async function teardownWorkspace( const { galleries } = await deleteGalleriesForWorkspace(dbFor(env), name); await deleteFeedsForWorkspace(dbFor(env), name); await deleteFileMetadataForWorkspace(dbFor(env), name); + await deleteFileEventsForWorkspace(dbFor(env), name); // The attachment index is workspace-scoped like file_metadata (issue // #934) — a torn-down workspace must not leave rows behind. await deleteAttachmentsForWorkspaceSafe(dbFor(env), name); diff --git a/apps/api/test/classifier-upload.test.ts b/apps/api/test/classifier-upload.test.ts index 4fc2556ae..cab097d63 100644 --- a/apps/api/test/classifier-upload.test.ts +++ b/apps/api/test/classifier-upload.test.ts @@ -1768,3 +1768,54 @@ describe("held uploads move out of the public bucket", () => { expect(await rows(harness.env)).toBeUndefined(); }); }); + +describe("classifier event log", () => { + it("appends skip, classify, and reuse events across an overwrite", async () => { + const harness = makeClassifierEnv(() => FLAGGED); + const events: Array<{ kind: string; reason: string | null; secret: string | null }> = []; + const db = harness.env.DB as unknown as { prepare: (sql: string) => unknown }; + const prepare = db.prepare.bind(db); + db.prepare = (sql: string) => { + const normalized = sql.replace(/\s+/g, " ").trim(); + if (!normalized.includes("file_events")) return prepare(sql); + let values: unknown[] = []; + return { + bind(...next: unknown[]) { + values = next; + return this; + }, + async run() { + if (normalized.startsWith("INSERT INTO file_events")) { + events.push({ + kind: String(values[3]), + reason: (values[6] as string | null) ?? null, + secret: (values[13] as string | null) ?? null, + }); + } + return { success: true, meta: { changes: 0 }, results: [] }; + }, + async all() { + return { success: true, results: [], meta: {} }; + }, + async first() { + return null; + }, + }; + }; + + await putFlagged(harness, pngOf(8, 8)); + await putFlagged(harness, FITTING_PNG, { replace: true }); + await putFlagged(harness, FITTING_PNG, { replace: true }); + + expect(events).toEqual([ + { kind: "skipped", reason: "small", secret: null }, + { kind: "classified", reason: null, secret: "api_key" }, + { kind: "reused_exact", reason: null, secret: "api_key" }, + ]); + const stored = ( + await getMetadataForKeys(harness.env.DB, WORKSPACE, ["images/flagged.png"]) + ).get("images/flagged.png"); + expect(stored?.["ai.secret"]).toBe("api_key"); + expect(stored?.["ai.skip"]).toBeUndefined(); + }); +}); diff --git a/apps/api/test/file-events.test.ts b/apps/api/test/file-events.test.ts new file mode 100644 index 000000000..ba95a6a9e --- /dev/null +++ b/apps/api/test/file-events.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { getFileMetadata, setServerFileMetadata } from "../src/file-metadata"; +import { approveUpload } from "../src/upload-hold-service"; +import type { WorkspaceRecord } from "../src/workspace"; +import { FakeR2Bucket } from "./fake-r2"; +import { SqliteD1, database } from "./helpers/sqlite-d1"; + +const SHA = "ab".repeat(32); + +const ws: WorkspaceRecord = { + provider: "r2", + bucket: "uploads-default", + binding: "UPLOADS_DEFAULT", + prefix: "acme/", + publicBaseUrl: "https://storage.uploads.sh", +}; + +describe("file_events", () => { + it("records a release override and leaves the safety flags", async () => { + const sqlite = new SqliteD1([ + "migrations/20260713210559_file_metadata.sql", + "migrations/20260724140000_file_content_hash.sql", + "migrations/20261028120000_file_events.sql", + ]); + const db = database(sqlite); + const bucket = new FakeR2Bucket(); + await bucket.put("acme/shots/a.png", new Uint8Array([1, 2, 3]), { + customMetadata: { "content-sha256": SHA }, + }); + const env = { DB: db, UPLOADS_DEFAULT: bucket } as unknown as Env; + await setServerFileMetadata(db, "acme", "shots/a.png", { + "ai.hold": "pending", + "ai.secret": "api_key", + "ai.pii": "financial", + "ai.rating": "explicit", + "ai.content": "sexual", + }); + expect(await approveUpload(env, ws, "acme", "shots/a.png", "Member")).toEqual({ + approved: true, + }); + + const stored = await getFileMetadata(db, "acme", "shots/a.png"); + expect(stored["ai.secret"]).toBe("api_key"); + expect(stored["ai.pii"]).toBe("financial"); + expect(stored["ai.rating"]).toBe("explicit"); + expect(stored["ai.content"]).toBe("sexual"); + const event = await db + .prepare( + `SELECT kind, action, actor, secret, content_sha256 FROM file_events + WHERE workspace = ? AND object_key = ?`, + ) + .bind("acme", "shots/a.png") + .first<{ + kind: string; + action: string; + actor: string; + secret: string; + content_sha256: string; + }>(); + expect(event).toEqual({ + kind: "override", + action: "release", + actor: "Member", + secret: "api_key", + content_sha256: SHA, + }); + sqlite.close(); + }); + + it("still approves when the event table is missing", async () => { + const sqlite = new SqliteD1(["migrations/20260713210559_file_metadata.sql"]); + const db = database(sqlite); + const bucket = new FakeR2Bucket(); + await bucket.put("acme/a.png", new Uint8Array([1]), { + customMetadata: { "content-sha256": SHA }, + }); + const env = { DB: db, UPLOADS_DEFAULT: bucket } as unknown as Env; + await setServerFileMetadata(db, "acme", "a.png", { + "ai.hold": "pending", + "ai.secret": "api_key", + }); + expect(await approveUpload(env, ws, "acme", "a.png", "Member")).toEqual({ approved: true }); + const stored = await getFileMetadata(db, "acme", "a.png"); + expect(stored["ai.secret"]).toBe("api_key"); + expect(stored["ai.hold"]).toBe("approved"); + sqlite.close(); + }); +}); diff --git a/apps/api/test/routes-workspace-files.test.ts b/apps/api/test/routes-workspace-files.test.ts index 815d9d274..763f68fca 100644 --- a/apps/api/test/routes-workspace-files.test.ts +++ b/apps/api/test/routes-workspace-files.test.ts @@ -151,6 +151,7 @@ async function makeEnv( async function seed(bucket: FakeR2Bucket) { await bucket.put("acme/shots/a.png", new Uint8Array([1, 2, 3]).buffer, { httpMetadata: { contentType: "image/png" }, + customMetadata: { "content-sha256": "ab".repeat(32) }, }); } diff --git a/apps/api/test/upload-hold-release.test.ts b/apps/api/test/upload-hold-release.test.ts index bae4e388d..3b09f9a6e 100644 --- a/apps/api/test/upload-hold-release.test.ts +++ b/apps/api/test/upload-hold-release.test.ts @@ -28,7 +28,7 @@ function makeEnv() { REGISTRY: registry, UPLOADS_DEFAULT: bucket, } as unknown as Env; - return { env, registry, record }; + return { env, registry, record, bucket }; } async function hold(env: Env, workspace: string, key: string) { @@ -53,9 +53,12 @@ describe("operator hold release", () => { }); it("approves every pending hold, or one workspace's", async () => { - const { env, registry, record } = makeEnv(); + const { env, registry, record, bucket } = makeEnv(); await registry.put("ws:acme", JSON.stringify(record("acme"))); await registry.put("ws:beta", JSON.stringify(record("beta"))); + const sha = { "content-sha256": "ab".repeat(32) }; + await bucket.put("acme/a.png", new Uint8Array([1]), { customMetadata: sha }); + await bucket.put("beta/c.png", new Uint8Array([2]), { customMetadata: sha }); await hold(env, "acme", "a.png"); await hold(env, "beta", "c.png"); diff --git a/apps/mcp/test/mcp.test.ts b/apps/mcp/test/mcp.test.ts index 995aa7392..a95fa4128 100644 --- a/apps/mcp/test/mcp.test.ts +++ b/apps/mcp/test/mcp.test.ts @@ -1492,9 +1492,12 @@ describe("mcp worker", () => { }); it("list_held, get_metadata, and approve cover a held file", async () => { - const { env, metadata } = await makeEnv(); + const { env, bucket, metadata } = await makeEnv(); const key = "gh/acme/web/pull/12/env.png"; - // A moved hold: rows only, no object in the bucket. + // Bytes stay in the bucket and carry the hash the release records. + await bucket.put(key, new Uint8Array([1]), { + customMetadata: { "content-sha256": "ab".repeat(32) }, + }); metadata.set( `test-ws ${key}`, new Map([ diff --git a/apps/mcp/test/staged-media-tools.test.ts b/apps/mcp/test/staged-media-tools.test.ts index 340a72823..feae772b3 100644 --- a/apps/mcp/test/staged-media-tools.test.ts +++ b/apps/mcp/test/staged-media-tools.test.ts @@ -14,12 +14,10 @@ class PreviewBucket extends FakeR2Bucket { privateKeys = new Set(); override async head(key: string) { const head = await super.head(key); - return ( - head && { - ...head, - customMetadata: this.privateKeys.has(key) ? { visibility: "private" } : {}, - } - ); + const customMetadata: Record = this.privateKeys.has(key) + ? { visibility: "private" } + : {}; + return head && { ...head, customMetadata }; } } diff --git a/apps/web/src/content/docs/ai-labels.mdx b/apps/web/src/content/docs/ai-labels.mdx index 4d5636a95..6f6f674cd 100644 --- a/apps/web/src/content/docs/ai-labels.mdx +++ b/apps/web/src/content/docs/ai-labels.mdx @@ -65,6 +65,8 @@ A stored flag stays until a later classification writes that same key. A confide The classifier version does not change for these flags. `v5`, `v6`, `v4`, and `skipped` still name how the image was prepared. A file labeled before the flags existed simply has none of these keys. +You cannot set, change, or delete these flags from the CLI, the API, or MCP. A member release does not change them either. It records a separate override: who released the hold, and when. +
@@ -94,7 +96,7 @@ curl -X POST -H "Authorization: Bearer $UPLOADS_TOKEN" \ With the [Slack app](/docs/slack#held-uploads) connected, the uploader (or, without one, the workspace admins) gets a Slack message for each new hold, with an **Approve** button. To get an email for each new hold instead, or as well, turn on **An upload is held for review** under Email notifications in your [account settings](/account/profile). It's off by default. -Approval writes `ai.hold=approved` and re-syncs the comment. Re-uploading identical bytes keeps the approval. New bytes are classified again and can be held again. `ai.pii=contact` and `ai.pii=other` never hold a file, and the `ai.rating` and `ai.content` flags never hold one. +Approval records that override, writes `ai.hold=approved`, and re-syncs the comment. The safety flags stay as the classifier wrote them. Re-uploading identical bytes keeps the approval. New bytes are classified again and can be held again. `ai.pii=contact` and `ai.pii=other` never hold a file, and the `ai.rating` and `ai.content` flags never hold one.
diff --git a/docs/ops.md b/docs/ops.md index 7ce565f0f..579b5fb7c 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -1422,7 +1422,18 @@ logs `classifier_hold_area` or `classifier_hold_area_failed` and leaves the hold as it is. `POST /v1/workspaces/:workspace/files/approve?key=` writes -`ai.hold=approved` and re-syncs the comment (`upload_approved`). `ai.hold` +`ai.hold=approved` and re-syncs the comment (`upload_approved`). It does +not change `ai.secret`, `ai.pii`, `ai.rating`, or `ai.content`. The same +call appends a `file_events` row: who released the hold, when, the content +hash, and the flags in force. The hash is the object's `content-sha256`, +or the hash of the body copied back from the held bucket. A file with +neither fails the approve. Classification appends `classified`, +`reused_exact`, `reused_perceptual`, `skipped`, or `failed` the same way. +New bytes still clear the current `ai.*` rows. The log keeps the earlier +row. Deleting the file, or the retention sweep, does not delete the log. +Workspace teardown does. `content_sha256` is those bytes, for a later blob +id. The table is on the existing D1 database. A missing migration skips +the append. `ai.hold` is in `CLASSIFIER_ROW_KEYS`, so identical bytes keep an approval and new bytes start over. Promote and attach copy every classifier row onto the new key, since the copy is not classified again. `ai.rating`, diff --git a/packages/storage/test/fake-r2.ts b/packages/storage/test/fake-r2.ts index 9b1f3ba08..501b9e41f 100644 --- a/packages/storage/test/fake-r2.ts +++ b/packages/storage/test/fake-r2.ts @@ -6,6 +6,7 @@ interface StoredObject { data: Uint8Array; contentType?: string; + customMetadata?: Record; } export class FakeR2Bucket { @@ -22,7 +23,7 @@ export class FakeR2Bucket { storageClass: "Standard", checksums: {}, httpMetadata: { contentType: obj.contentType }, - customMetadata: {}, + customMetadata: obj.customMetadata ? { ...obj.customMetadata } : {}, writeHttpMetadata() {}, }; } @@ -30,7 +31,10 @@ export class FakeR2Bucket { async put( key: string, value: ArrayBuffer | ArrayBufferView | string | ReadableStream | null, - opts?: { httpMetadata?: { contentType?: string } | Headers }, + opts?: { + httpMetadata?: { contentType?: string } | Headers; + customMetadata?: Record; + }, ) { let data: Uint8Array; if (typeof value === "string") data = new TextEncoder().encode(value); @@ -58,7 +62,11 @@ export class FakeR2Bucket { httpMetadata instanceof Headers ? (httpMetadata.get("content-type") ?? undefined) : httpMetadata?.contentType; - const obj = { data, contentType }; + const obj = { + data, + contentType, + customMetadata: opts?.customMetadata ? { ...opts.customMetadata } : undefined, + }; this.store.set(key, obj); return this.meta(key, obj); }