From bc1a7fc2278191bfaf008c1df546c987e9e5b469 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 23 Jul 2026 14:39:01 +0200 Subject: [PATCH 1/5] feat: implment bulk radius api Signed-off-by: Umberto Sgueglia --- .../api/public/v1/akrites-external/index.ts | 17 ++ .../public/v1/akrites-external/openapi.yaml | 197 +++++++++++++++++- .../v1/packages/blastRadiusBatch.test.ts | 93 +++++++++ .../public/v1/packages/blastRadiusBatch.ts | 83 ++++++++ .../v1/packages/getBlastRadiusJobBatch.ts | 57 +++++ .../v1/packages/submitBlastRadiusJobBatch.ts | 102 +++++++++ 6 files changed, 541 insertions(+), 8 deletions(-) create mode 100644 backend/src/api/public/v1/packages/blastRadiusBatch.test.ts create mode 100644 backend/src/api/public/v1/packages/blastRadiusBatch.ts create mode 100644 backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts create mode 100644 backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts diff --git a/backend/src/api/public/v1/akrites-external/index.ts b/backend/src/api/public/v1/akrites-external/index.ts index 07bed2dd1b..7f1df0af7c 100644 --- a/backend/src/api/public/v1/akrites-external/index.ts +++ b/backend/src/api/public/v1/akrites-external/index.ts @@ -12,7 +12,9 @@ import { getAkritesExternalContactDetailBatch } from '../packages/getAkritesExte import { getAkritesExternalPackageDetail } from '../packages/getAkritesExternalPackageDetail' import { getAkritesExternalPackageDetailBatch } from '../packages/getAkritesExternalPackageDetailBatch' import { getBlastRadiusJob } from '../packages/getBlastRadiusJob' +import { getBlastRadiusJobBatch } from '../packages/getBlastRadiusJobBatch' import { submitBlastRadiusJob } from '../packages/submitBlastRadiusJob' +import { submitBlastRadiusJobBatch } from '../packages/submitBlastRadiusJobBatch' const rateLimiter = createRateLimiter({ max: 60, windowMs: 60 * 1000 }) @@ -75,7 +77,22 @@ export function akritesExternalRouter(): Router { const blastRadiusSubRouter = Router() blastRadiusSubRouter.use(requireScopes([SCOPES.READ_PACKAGES])) blastRadiusSubRouter.post('/jobs', blastRadiusRateLimiter, safeWrap(submitBlastRadiusJob)) + // Bulk submit multiplies Temporal workflow starts per request (up to + // MAX_BLAST_RADIUS_JOBS_PER_BATCH), so it sits behind the same strict + // blastRadiusRateLimiter as the single-job route, not the regular one. + blastRadiusSubRouter.post( + /^\/jobs:batch\/?$/, + blastRadiusRateLimiter, + safeWrap(submitBlastRadiusJobBatch), + ) blastRadiusSubRouter.get('/jobs/:analysisId', rateLimiter, safeWrap(getBlastRadiusJob)) + // Bulk poll is read-only, same cost profile as the other batch endpoints, so + // it uses the regular rateLimiter. + blastRadiusSubRouter.post( + /^\/jobs:batch\/poll\/?$/, + rateLimiter, + safeWrap(getBlastRadiusJobBatch), + ) router.use('/blast-radius', blastRadiusSubRouter) return router diff --git a/backend/src/api/public/v1/akrites-external/openapi.yaml b/backend/src/api/public/v1/akrites-external/openapi.yaml index 08327892fa..12209f0fdb 100644 --- a/backend/src/api/public/v1/akrites-external/openapi.yaml +++ b/backend/src/api/public/v1/akrites-external/openapi.yaml @@ -51,14 +51,16 @@ tags: - name: Blast Radius description: > Advisory reachability analysis — submit (2a) and poll (2b) are both - implemented. Submitting kicks off a Temporal workflow that runs the - npm reachability pipeline (other ecosystems fail fast with - ECOSYSTEM_NOT_SUPPORTED); poll returns job status and, once done, - results. Rate-limited independently of the other akrites-external - endpoints (default 5 requests/hour, configurable via - AKRITES_BLAST_RADIUS_RATE_LIMIT_MAX / _WINDOW_MS). Same interim scope - note as Advisories applies (read:packages, pending a dedicated - read:advisories scope). + implemented, each with a bulk counterpart. Submitting kicks off a + Temporal workflow that runs the npm reachability pipeline (other + ecosystems fail fast with ECOSYSTEM_NOT_SUPPORTED); poll returns job + status and, once done, results. Bulk submit (jobs:batch) is capped at + 20 jobs per request (10 recommended as the default batch size) — each + entry starts its own workflow — and stays behind the same strict rate + limiter as the single-job route; bulk poll + (jobs:batch/poll) is read-only and capped at 100 like the other batch + endpoints. Same interim scope note as Advisories applies (read:packages, + pending a dedicated read:advisories scope). components: securitySchemes: @@ -429,6 +431,69 @@ components: enum: [pending, running, done, failed] description: Always pending — the response is returned before the Temporal workflow runs. + BlastRadiusJobBatchRequest: + type: object + required: [jobs] + properties: + jobs: + type: array + minItems: 1 + maxItems: 20 + description: > + Capped much lower than the 100-item read batches — each entry + starts its own Temporal workflow, so the batch multiplies + workflow starts (and reachability-analysis cost) per request. + 10 is the recommended default batch size; 20 is the hard limit. + items: + $ref: '#/components/schemas/BlastRadiusJobRequest' + + BlastRadiusJobBatchResponse: + type: object + required: [results] + description: > + Plain array in request order, one entry per submitted job — unlike the + read batches there is no found/not-found case, every job is submitted. + properties: + results: + type: array + items: + $ref: '#/components/schemas/BlastRadiusJobEntry' + + BlastRadiusJobPollBatchRequest: + type: object + required: [analysisIds] + properties: + analysisIds: + type: array + minItems: 1 + maxItems: 100 + items: + type: string + format: uuid + page: + type: integer + minimum: 1 + default: 1 + pageSize: + type: integer + minimum: 1 + maximum: 100 + default: 20 + + BlastRadiusAnalysisBulkEntry: + type: object + required: [requestedAnalysisId, found, analysis] + properties: + requestedAnalysisId: + type: string + found: + type: boolean + analysis: + type: object + nullable: true + allOf: + - $ref: '#/components/schemas/BlastRadiusAnalysis' + BlastRadiusResultConfidence: type: string enum: [high, medium, low] @@ -1045,6 +1110,122 @@ paths: schema: $ref: '#/components/schemas/Error' + /akrites-external/blast-radius/jobs:batch: + post: + operationId: submitBlastRadiusJobBatch + summary: 2a bulk — Submit multiple blast-radius analysis jobs + description: > + One job per array entry, same semantics as the single-job submit — + omit package for an advisory-wide analysis, provide it to narrow to a + single package. Each entry starts its own Temporal workflow, so the + batch is capped at 20 jobs (10 recommended as the default batch + size), much lower than the 100-item read batches, and stays behind + the same strict rate limiter as the single-job route. + + + A per-job failure does not fail the whole batch — that entry comes + back with status: 'failed' and the rest still submit. + tags: [Blast Radius] + security: + - M2MBearer: + - read:packages + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BlastRadiusJobBatchRequest' + responses: + '202': + description: Jobs accepted, in request order. + content: + application/json: + schema: + $ref: '#/components/schemas/BlastRadiusJobBatchResponse' + '400': + description: Validation error (empty array, >20 items, or an invalid job entry). + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Missing or invalid bearer token. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Token missing read:packages scope. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '429': + description: Too many requests — rate-limited independently of the other endpoints. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /akrites-external/blast-radius/jobs:batch/poll: + post: + operationId: getBlastRadiusJobBatch + summary: 2b bulk — Poll multiple blast-radius analysis jobs + description: > + Same found/not-found echo shape as the other batch endpoints: an + unknown analysisId comes back { found: false, analysis: null } + instead of 404ing the whole request. Read-only, so it is rate-limited + the same as the other non-blast-radius endpoints, not the strict + submit limiter. + tags: [Blast Radius] + security: + - M2MBearer: + - read:packages + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BlastRadiusJobPollBatchRequest' + responses: + '200': + description: One page of results, in request order. + content: + application/json: + schema: + type: object + required: [page, pageSize, total, results] + properties: + page: + type: integer + pageSize: + type: integer + total: + type: integer + description: Total number of requested analysisIds, across all pages. + results: + type: array + items: + $ref: '#/components/schemas/BlastRadiusAnalysisBulkEntry' + '400': + description: Validation error (empty array, >100 items, or a non-uuid analysisId). + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '401': + description: Missing or invalid bearer token. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '403': + description: Token missing read:packages scope. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + /akrites-external/blast-radius/jobs/{analysisId}: get: operationId: getBlastRadiusJob diff --git a/backend/src/api/public/v1/packages/blastRadiusBatch.test.ts b/backend/src/api/public/v1/packages/blastRadiusBatch.test.ts new file mode 100644 index 0000000000..b7201336fa --- /dev/null +++ b/backend/src/api/public/v1/packages/blastRadiusBatch.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest' + +import { + MAX_BLAST_RADIUS_JOBS_PER_BATCH, + MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH, + blastRadiusJobBatchRequestSchema, + blastRadiusJobPollBatchRequestSchema, + paginateAnalysisIds, +} from './blastRadiusBatch' + +describe('blastRadiusJobBatchRequestSchema', () => { + it('accepts a batch of valid job requests', () => { + const result = blastRadiusJobBatchRequestSchema.safeParse({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'CVE-2024-12345', ecosystem: 'npm', package: 'pkg:npm/lodash' }, + ], + }) + expect(result.success).toBe(true) + }) + + it('rejects an empty jobs array', () => { + const result = blastRadiusJobBatchRequestSchema.safeParse({ jobs: [] }) + expect(result.success).toBe(false) + }) + + it('rejects more than MAX_BLAST_RADIUS_JOBS_PER_BATCH jobs', () => { + const jobs = Array.from({ length: MAX_BLAST_RADIUS_JOBS_PER_BATCH + 1 }, () => ({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'npm', + })) + const result = blastRadiusJobBatchRequestSchema.safeParse({ jobs }) + expect(result.success).toBe(false) + }) + + it('rejects a batch containing one invalid job', () => { + const result = blastRadiusJobBatchRequestSchema.safeParse({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'not-an-advisory-id', ecosystem: 'npm' }, + ], + }) + expect(result.success).toBe(false) + }) +}) + +describe('blastRadiusJobPollBatchRequestSchema', () => { + const validId = '3fa85f64-5717-4562-b3fc-2c963f66afa6' + + it('accepts a batch of valid analysisIds and defaults page/pageSize', () => { + const result = blastRadiusJobPollBatchRequestSchema.parse({ analysisIds: [validId] }) + expect(result.page).toBe(1) + expect(result.pageSize).toBe(20) + }) + + it('rejects an empty analysisIds array', () => { + const result = blastRadiusJobPollBatchRequestSchema.safeParse({ analysisIds: [] }) + expect(result.success).toBe(false) + }) + + it('rejects a non-uuid analysisId', () => { + const result = blastRadiusJobPollBatchRequestSchema.safeParse({ analysisIds: ['not-a-uuid'] }) + expect(result.success).toBe(false) + }) + + it('rejects more than MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH analysisIds', () => { + const analysisIds = Array.from( + { length: MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH + 1 }, + () => validId, + ) + const result = blastRadiusJobPollBatchRequestSchema.safeParse({ analysisIds }) + expect(result.success).toBe(false) + }) +}) + +describe('paginateAnalysisIds', () => { + it('slices the requested page out of the full analysisIds array', () => { + const analysisIds = ['a', 'b', 'c', 'd', 'e'] + const result = paginateAnalysisIds({ analysisIds, page: 2, pageSize: 2 }) + expect(result).toEqual({ + page: 2, + pageSize: 2, + total: 5, + pagedAnalysisIds: ['c', 'd'], + }) + }) + + it('returns an empty page past the end of the array', () => { + const result = paginateAnalysisIds({ analysisIds: ['a'], page: 2, pageSize: 20 }) + expect(result.pagedAnalysisIds).toEqual([]) + expect(result.total).toBe(1) + }) +}) diff --git a/backend/src/api/public/v1/packages/blastRadiusBatch.ts b/backend/src/api/public/v1/packages/blastRadiusBatch.ts new file mode 100644 index 0000000000..7989d722d7 --- /dev/null +++ b/backend/src/api/public/v1/packages/blastRadiusBatch.ts @@ -0,0 +1,83 @@ +import { z } from 'zod' + +import { blastRadiusJobRequestSchema } from './blastRadius' +import type { BlastRadiusAnalysis } from './blastRadiusAnalysis' +import { DEFAULT_BATCH_PAGE_SIZE } from './purl' + +// Read-only batches (packages/advisories/contacts) cap at 100 — cheap indexed +// lookups. Batch submit multiplies Temporal workflow starts (and their LLM +// reachability cost) per request, so it gets a much lower cap, independent of +// that constant. 20 is the hard limit agreed with Joana after the cost test +// (2026-07-23) — 10 is the recommended/default batch size for callers, not a +// separate enforced value, since `jobs` is an explicit client-provided array. +export const MAX_BLAST_RADIUS_JOBS_PER_BATCH = 20 + +// Polling is read-only, same cost profile as the other batches, so it reuses +// their 100 cap. +export const MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH = 100 + +export const blastRadiusJobBatchRequestSchema = z.object({ + jobs: z + .array(blastRadiusJobRequestSchema) + .min(1) + .max( + MAX_BLAST_RADIUS_JOBS_PER_BATCH, + `Maximum ${MAX_BLAST_RADIUS_JOBS_PER_BATCH} jobs per request`, + ), +}) + +export type BlastRadiusJobBatchRequest = z.infer + +// Unlike the read batches (purls that may or may not resolve to a package), every +// job in a submit batch is genuinely submitted — there is no "not found" case, so +// the response is a plain array in request order, not a found/not-found wrapper. +const analysisIdSchema = z.uuid() + +export const blastRadiusJobPollBatchRequestSchema = z.object({ + analysisIds: z + .array(analysisIdSchema) + .min(1) + .max( + MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH, + `Maximum ${MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH} analysisIds per request`, + ), + page: z.coerce.number().int().min(1).default(1), + pageSize: z.coerce + .number() + .int() + .min(1) + .max(MAX_BLAST_RADIUS_POLL_IDS_PER_BATCH) + .default(DEFAULT_BATCH_PAGE_SIZE), +}) + +export type BlastRadiusJobPollBatchRequest = z.infer + +export interface BlastRadiusAnalysisBulkEntry { + requestedAnalysisId: string + found: boolean + analysis: BlastRadiusAnalysis | null +} + +export interface PaginatedAnalysisIds { + page: number + pageSize: number + total: number + pagedAnalysisIds: string[] +} + +// Mirrors paginatePurls (purl.ts): slice the requested page out of the full +// analysisIds array. No normalization step — analysisIds are UUIDs, unlike purls. +export function paginateAnalysisIds(body: { + analysisIds: string[] + page: number + pageSize: number +}): PaginatedAnalysisIds { + const { analysisIds, page, pageSize } = body + const start = (page - 1) * pageSize + return { + page, + pageSize, + total: analysisIds.length, + pagedAnalysisIds: analysisIds.slice(start, start + pageSize), + } +} diff --git a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts new file mode 100644 index 0000000000..2c7d532064 --- /dev/null +++ b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts @@ -0,0 +1,57 @@ +import type { Request, Response } from 'express' + +import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' + +import { getPackagesQx } from '@/db/packagesDb' +import { ok } from '@/utils/api' +import { validateOrThrow } from '@/utils/validation' + +import { toBlastRadiusAnalysis } from './blastRadiusAnalysis' +import { + type BlastRadiusAnalysisBulkEntry, + blastRadiusJobPollBatchRequestSchema, + paginateAnalysisIds, +} from './blastRadiusBatch' + +// 2b bulk — poll multiple blast-radius analyses in one request. Same +// found/not-found echo shape as the other batch endpoints (packages, advisories, +// contacts): an unknown analysisId comes back { found: false, analysis: null } +// instead of 404ing the whole request. Read-only, so it stays behind the +// regular rateLimiter, not the strict blastRadiusRateLimiter. +export async function getBlastRadiusJobBatch(req: Request, res: Response): Promise { + const { page, pageSize, total, pagedAnalysisIds } = paginateAnalysisIds( + validateOrThrow(blastRadiusJobPollBatchRequestSchema, req.body), + ) + + const qx = await getPackagesQx() + + const results: BlastRadiusAnalysisBulkEntry[] = await Promise.all( + pagedAnalysisIds.map((requestedAnalysisId) => pollOneAnalysis(qx, requestedAnalysisId)), + ) + + ok(res, { page, pageSize, total, results }) +} + +async function pollOneAnalysis( + qx: Awaited>, + requestedAnalysisId: string, +): Promise { + const analysis = await blastRadiusDal.getAnalysisDetail(qx, requestedAnalysisId) + if (!analysis) { + return { requestedAnalysisId, found: false, analysis: null } + } + + const done = analysis.status === 'done' + const [verdictRows, excludedByRangeCount] = done + ? await Promise.all([ + blastRadiusDal.getVerdictResults(qx, requestedAnalysisId), + blastRadiusDal.getDependentsExcludedByRangeCount(qx, requestedAnalysisId), + ]) + : [[], 0] + + return { + requestedAnalysisId, + found: true, + analysis: toBlastRadiusAnalysis(analysis, verdictRows, excludedByRangeCount), + } +} diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts new file mode 100644 index 0000000000..fa0d7b6f89 --- /dev/null +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts @@ -0,0 +1,102 @@ +import type { Request, Response } from 'express' + +import { generateUUIDv4 } from '@crowd/common' +import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { Client } from '@crowd/temporal' +import { ITriggerBlastRadiusAnalysis, TemporalWorkflowId } from '@crowd/types' + +import { getPackagesQx } from '@/db/packagesDb' +import { getPackagesTemporalClient } from '@/db/packagesTemporal' +import { validateOrThrow } from '@/utils/validation' + +import { + type BlastRadiusJobEntry, + type BlastRadiusJobRequest, + toBlastRadiusJobEntry, +} from './blastRadius' +import { blastRadiusJobBatchRequestSchema } from './blastRadiusBatch' + +// 2a bulk — submit multiple blast-radius analysis jobs in one request, one per +// array entry. Same lifecycle as the single-job submit, just looped: each entry +// gets its own analysisId, its own pending row, and its own Temporal workflow +// start. Unlike the read-only batch endpoints (packages/advisories/contacts), +// this multiplies workflow starts per request, so the batch size is capped much +// lower (see MAX_BLAST_RADIUS_JOBS_PER_BATCH) and the route stays behind the same +// strict blastRadiusRateLimiter as the single-job route. +// +// A per-job failure (e.g. workflow.start throwing) does not fail the whole +// batch — that job's entry comes back status: 'failed' and the rest still +// submit, matching the partial-result shape of the other batch endpoints. +export async function submitBlastRadiusJobBatch(req: Request, res: Response): Promise { + const { jobs } = validateOrThrow(blastRadiusJobBatchRequestSchema, req.body) + + const qx = await getPackagesQx() + const packagesTemporal = await getPackagesTemporalClient() + + const results: BlastRadiusJobEntry[] = await Promise.all( + jobs.map((body) => submitOneJob(qx, packagesTemporal, body)), + ) + + res.status(202).json({ results }) +} + +async function submitOneJob( + qx: QueryExecutor, + packagesTemporal: Client, + body: BlastRadiusJobRequest, +): Promise { + const jobPackage = body.package ?? null + const jobEcosystem = body.ecosystem + const analysisId = generateUUIDv4() + const analysisInput = { + id: analysisId, + advisoryOsvId: body.advisoryId, + packageName: jobPackage, + ecosystem: jobEcosystem, + force: body.force, + } + + try { + // Create the pending row synchronously, before starting the workflow — see the + // same comment on submitBlastRadiusJob for why (avoids a poll-race 404). This is + // inside the try too — unlike the single-job submit, a createAnalysis failure + // must not reject the whole batch's Promise.all, only this job's entry. + await blastRadiusDal.createAnalysis(qx, analysisInput) + + await packagesTemporal.workflow.start('analyzeBlastRadius', { + taskQueue: 'blast-radius-worker', + workflowId: `${TemporalWorkflowId.BLAST_RADIUS_ANALYSIS}/${analysisId}`, + retry: { maximumAttempts: 1 }, + args: [ + { + analysisId, + advisoryId: body.advisoryId, + package: jobPackage, + ecosystem: jobEcosystem, + force: body.force, + } satisfies ITriggerBlastRadiusAnalysis, + ], + }) + + return toBlastRadiusJobEntry({ + analysisId, + advisoryId: body.advisoryId, + package: jobPackage, + ecosystem: jobEcosystem, + }) + } catch (err) { + // Unlike the single-job submit, this does not rethrow — one job's workflow + // failing to start must not take the rest of the batch down with it. + const errorMessage = err instanceof Error ? err.message : String(err) + await blastRadiusDal.failAnalysis(qx, analysisInput, errorMessage) + + return { + analysisId, + advisoryId: body.advisoryId, + package: jobPackage, + ecosystem: jobEcosystem, + status: 'failed', + } + } +} From 8f416826ec5e68eb8259f89b542edd327ddd2cdd Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 23 Jul 2026 15:14:18 +0200 Subject: [PATCH 2/5] feat: add tests Signed-off-by: Umberto Sgueglia --- .../public/v1/akrites-external/openapi.yaml | 11 +- .../packages/getBlastRadiusJobBatch.test.ts | 157 ++++++++++++++++++ .../v1/packages/getBlastRadiusJobBatch.ts | 61 ++++--- .../submitBlastRadiusJobBatch.test.ts | 124 ++++++++++++++ .../src/packages/blastRadius.ts | 75 +++++++++ 5 files changed, 401 insertions(+), 27 deletions(-) create mode 100644 backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts create mode 100644 backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts diff --git a/backend/src/api/public/v1/akrites-external/openapi.yaml b/backend/src/api/public/v1/akrites-external/openapi.yaml index 12209f0fdb..a1d1ef1540 100644 --- a/backend/src/api/public/v1/akrites-external/openapi.yaml +++ b/backend/src/api/public/v1/akrites-external/openapi.yaml @@ -411,7 +411,10 @@ components: BlastRadiusJobEntry: type: object required: [analysisId, advisoryId, package, ecosystem, status] - description: Response body of 2a — one job per request, never wrapped in an array. + description: > + Response body of 2a for a single submitted job. Returned directly (not + wrapped in an array) by the single-job submit; the batch submit (2a bulk) + returns an array of these under `results` — see BlastRadiusJobBatchResponse. properties: analysisId: type: string @@ -429,7 +432,11 @@ components: status: type: string enum: [pending, running, done, failed] - description: Always pending — the response is returned before the Temporal workflow runs. + description: > + Pending in the single-job submit response, always returned before the + Temporal workflow runs. In the batch submit response, a job whose + workflow failed to start comes back as failed instead — the rest of + the batch is unaffected. BlastRadiusJobBatchRequest: type: object diff --git a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts new file mode 100644 index 0000000000..ff5107b4bd --- /dev/null +++ b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts @@ -0,0 +1,157 @@ +import type { Request, Response } from 'express' +import { describe, expect, it, vi } from 'vitest' + +import { getBlastRadiusJobBatch } from './getBlastRadiusJobBatch' + +const { getAnalysisDetailsByIds, getVerdictResultsBatch, getDependentsExcludedByRangeCountBatch } = + vi.hoisted(() => ({ + getAnalysisDetailsByIds: vi.fn(), + getVerdictResultsBatch: vi.fn(), + getDependentsExcludedByRangeCountBatch: vi.fn(), + })) + +vi.mock('@crowd/data-access-layer/src/packages/blastRadius', () => ({ + getAnalysisDetailsByIds, + getVerdictResultsBatch, + getDependentsExcludedByRangeCountBatch, +})) + +vi.mock('@/db/packagesDb', () => ({ + getPackagesQx: vi.fn().mockResolvedValue({}), +})) + +const PENDING_ID = '11111111-1111-4111-8111-111111111111' +const DONE_ID = '22222222-2222-4222-8222-222222222222' +const MISSING_ID = '33333333-3333-4333-8333-333333333333' + +function mockReqRes(body: unknown) { + getAnalysisDetailsByIds.mockClear() + getVerdictResultsBatch.mockClear() + getDependentsExcludedByRangeCountBatch.mockClear() + getVerdictResultsBatch.mockResolvedValue([]) + getDependentsExcludedByRangeCountBatch.mockResolvedValue([]) + + const req = { body } as unknown as Request + + const json = vi.fn() + const status = vi.fn().mockReturnValue({ json }) + const res = { status, json } as unknown as Response + + return { req, res, json } +} + +describe('getBlastRadiusJobBatch', () => { + it('returns found/not-found entries in request order with page metadata', async () => { + getAnalysisDetailsByIds.mockResolvedValue([ + { + id: PENDING_ID, + advisory_osv_id: 'GHSA-jf85-cpcp-j695', + package_name: null, + ecosystem: 'npm', + status: 'pending', + error: null, + candidates_considered: null, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: null, + }, + ]) + + const { req, res, json } = mockReqRes({ + analysisIds: [PENDING_ID, MISSING_ID], + }) + + await getBlastRadiusJobBatch(req, res) + + expect(json).toHaveBeenCalledWith( + expect.objectContaining({ + page: 1, + pageSize: expect.any(Number), + total: 2, + results: [ + expect.objectContaining({ + requestedAnalysisId: PENDING_ID, + found: true, + analysis: expect.objectContaining({ analysisId: PENDING_ID, status: 'pending' }), + }), + { requestedAnalysisId: MISSING_ID, found: false, analysis: null }, + ], + }), + ) + }) + + it('only fetches verdicts/excluded counts for done analyses', async () => { + getAnalysisDetailsByIds.mockResolvedValue([ + { + id: PENDING_ID, + advisory_osv_id: 'GHSA-jf85-cpcp-j695', + package_name: null, + ecosystem: 'npm', + status: 'pending', + error: null, + candidates_considered: null, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: null, + }, + { + id: DONE_ID, + advisory_osv_id: 'GHSA-652q-gvq3-74qv', + package_name: 'lodash', + ecosystem: 'npm', + status: 'done', + error: null, + candidates_considered: 10, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: '2026-07-01T01:00:00.000Z', + }, + ]) + getVerdictResultsBatch.mockResolvedValue([ + { + analysisId: DONE_ID, + name: 'benchmark.js', + version: '2.1.4', + downloads: 500000, + reachable_verdict: 'affected', + confidence: 0.9, + evidence: null, + reasoning: 'uses merge', + }, + ]) + getDependentsExcludedByRangeCountBatch.mockResolvedValue([{ analysisId: DONE_ID, count: 8 }]) + + const { req, res, json } = mockReqRes({ analysisIds: [PENDING_ID, DONE_ID] }) + + await getBlastRadiusJobBatch(req, res) + + expect(getVerdictResultsBatch).toHaveBeenCalledWith(expect.anything(), [DONE_ID]) + expect(getDependentsExcludedByRangeCountBatch).toHaveBeenCalledWith(expect.anything(), [ + DONE_ID, + ]) + + const [{ results }] = json.mock.calls[0] + expect(results[0]).toMatchObject({ requestedAnalysisId: PENDING_ID, found: true }) + expect(results[0].analysis).toMatchObject({ status: 'pending', summary: null, results: null }) + expect(results[1]).toMatchObject({ requestedAnalysisId: DONE_ID, found: true }) + expect(results[1].analysis).toMatchObject({ + status: 'done', + summary: expect.objectContaining({ dependentsExcludedUpfront: 8 }), + }) + }) + + it('rejects a batch with a malformed uuid without querying the database', async () => { + const { req, res } = mockReqRes({ analysisIds: [PENDING_ID, 'not-a-uuid'] }) + + await expect(getBlastRadiusJobBatch(req, res)).rejects.toThrow() + expect(getAnalysisDetailsByIds).not.toHaveBeenCalled() + }) + + it('rejects a batch with more than 100 analysisIds', async () => { + const analysisIds = Array.from( + { length: 101 }, + (_, i) => `44444444-4444-4444-8444-${String(i).padStart(12, '0')}`, + ) + const { req, res } = mockReqRes({ analysisIds }) + + await expect(getBlastRadiusJobBatch(req, res)).rejects.toThrow() + expect(getAnalysisDetailsByIds).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts index 2c7d532064..2dd397be19 100644 --- a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts +++ b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts @@ -25,33 +25,44 @@ export async function getBlastRadiusJobBatch(req: Request, res: Response): Promi const qx = await getPackagesQx() - const results: BlastRadiusAnalysisBulkEntry[] = await Promise.all( - pagedAnalysisIds.map((requestedAnalysisId) => pollOneAnalysis(qx, requestedAnalysisId)), - ) + const analysisRows = await blastRadiusDal.getAnalysisDetailsByIds(qx, pagedAnalysisIds) + const analysisById = new Map(analysisRows.map((row) => [row.id, row])) - ok(res, { page, pageSize, total, results }) -} + const doneIds = analysisRows.filter((row) => row.status === 'done').map((row) => row.id) + const [verdictRows, excludedByRangeCounts] = await Promise.all([ + blastRadiusDal.getVerdictResultsBatch(qx, doneIds), + blastRadiusDal.getDependentsExcludedByRangeCountBatch(qx, doneIds), + ]) -async function pollOneAnalysis( - qx: Awaited>, - requestedAnalysisId: string, -): Promise { - const analysis = await blastRadiusDal.getAnalysisDetail(qx, requestedAnalysisId) - if (!analysis) { - return { requestedAnalysisId, found: false, analysis: null } + const verdictsByAnalysisId = new Map() + for (const row of verdictRows) { + const bucket = verdictsByAnalysisId.get(row.analysisId) + if (bucket) { + bucket.push(row) + } else { + verdictsByAnalysisId.set(row.analysisId, [row]) + } } + const excludedByRangeCountByAnalysisId = new Map( + excludedByRangeCounts.map(({ analysisId, count }) => [analysisId, count]), + ) - const done = analysis.status === 'done' - const [verdictRows, excludedByRangeCount] = done - ? await Promise.all([ - blastRadiusDal.getVerdictResults(qx, requestedAnalysisId), - blastRadiusDal.getDependentsExcludedByRangeCount(qx, requestedAnalysisId), - ]) - : [[], 0] - - return { - requestedAnalysisId, - found: true, - analysis: toBlastRadiusAnalysis(analysis, verdictRows, excludedByRangeCount), - } + const results: BlastRadiusAnalysisBulkEntry[] = pagedAnalysisIds.map((requestedAnalysisId) => { + const analysis = analysisById.get(requestedAnalysisId) + if (!analysis) { + return { requestedAnalysisId, found: false, analysis: null } + } + + return { + requestedAnalysisId, + found: true, + analysis: toBlastRadiusAnalysis( + analysis, + verdictsByAnalysisId.get(requestedAnalysisId) ?? [], + excludedByRangeCountByAnalysisId.get(requestedAnalysisId) ?? 0, + ), + } + }) + + ok(res, { page, pageSize, total, results }) } diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts new file mode 100644 index 0000000000..e3f37ae73d --- /dev/null +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts @@ -0,0 +1,124 @@ +import type { Request, Response } from 'express' +import { describe, expect, it, vi } from 'vitest' + +import { submitBlastRadiusJobBatch } from './submitBlastRadiusJobBatch' + +const { start, createAnalysis, failAnalysis } = vi.hoisted(() => ({ + start: vi.fn().mockResolvedValue(undefined), + createAnalysis: vi.fn().mockResolvedValue(undefined), + failAnalysis: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock('@/db/packagesTemporal', () => ({ + getPackagesTemporalClient: vi.fn().mockResolvedValue({ workflow: { start } }), +})) + +vi.mock('@/db/packagesDb', () => ({ + getPackagesQx: vi.fn().mockResolvedValue({}), +})) + +vi.mock('@crowd/data-access-layer/src/packages/blastRadius', () => ({ + createAnalysis, + failAnalysis, +})) + +function mockReqRes(body: unknown) { + start.mockClear() + createAnalysis.mockClear() + failAnalysis.mockClear() + + const req = { body } as unknown as Request + + const json = vi.fn() + const status = vi.fn().mockReturnValue({ json }) + const res = { status } as unknown as Response + + return { req, res, start, status, json } +} + +describe('submitBlastRadiusJobBatch', () => { + it('starts one workflow per job and responds 202 with results in request order', async () => { + const { req, res, status, json } = mockReqRes({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'GHSA-652q-gvq3-74qv', package: 'pkg:npm/lodash', ecosystem: 'npm' }, + ], + }) + + await submitBlastRadiusJobBatch(req, res) + + expect(createAnalysis).toHaveBeenCalledTimes(2) + expect(start).toHaveBeenCalledTimes(2) + expect(status).toHaveBeenCalledWith(202) + + const [{ results }] = json.mock.calls[0] + expect(results).toHaveLength(2) + expect(results[0]).toMatchObject({ + advisoryId: 'GHSA-jf85-cpcp-j695', + package: null, + ecosystem: 'npm', + status: 'pending', + }) + expect(results[1]).toMatchObject({ + advisoryId: 'GHSA-652q-gvq3-74qv', + package: 'pkg:npm/lodash', + ecosystem: 'npm', + status: 'pending', + }) + expect(typeof results[0].analysisId).toBe('string') + expect(typeof results[1].analysisId).toBe('string') + }) + + it('isolates a per-job workflow.start failure to that job only', async () => { + const { req, res, json } = mockReqRes({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'npm' }, + ], + }) + start.mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('temporal unreachable')) + + await submitBlastRadiusJobBatch(req, res) + + const [{ results }] = json.mock.calls[0] + expect(results).toHaveLength(2) + expect(results[0]).toMatchObject({ advisoryId: 'GHSA-jf85-cpcp-j695', status: 'pending' }) + expect(results[1]).toMatchObject({ advisoryId: 'GHSA-652q-gvq3-74qv', status: 'failed' }) + + expect(failAnalysis).toHaveBeenCalledTimes(1) + const [, , errorMessage] = failAnalysis.mock.calls[0] + expect(errorMessage).toBe('temporal unreachable') + }) + + it('rejects a batch containing an unsupported ecosystem without submitting any job', async () => { + const { req, res, start } = mockReqRes({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'maven' }, + ], + }) + + await expect(submitBlastRadiusJobBatch(req, res)).rejects.toThrow() + expect(start).not.toHaveBeenCalled() + expect(createAnalysis).not.toHaveBeenCalled() + }) + + it('rejects a batch with more than 20 jobs without submitting any job', async () => { + const jobs = Array.from({ length: 21 }, () => ({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'npm', + })) + const { req, res, start } = mockReqRes({ jobs }) + + await expect(submitBlastRadiusJobBatch(req, res)).rejects.toThrow() + expect(start).not.toHaveBeenCalled() + expect(createAnalysis).not.toHaveBeenCalled() + }) + + it('rejects an empty jobs array', async () => { + const { req, res, start } = mockReqRes({ jobs: [] }) + + await expect(submitBlastRadiusJobBatch(req, res)).rejects.toThrow() + expect(start).not.toHaveBeenCalled() + }) +}) diff --git a/services/libs/data-access-layer/src/packages/blastRadius.ts b/services/libs/data-access-layer/src/packages/blastRadius.ts index f71cf36f92..030657f9ce 100644 --- a/services/libs/data-access-layer/src/packages/blastRadius.ts +++ b/services/libs/data-access-layer/src/packages/blastRadius.ts @@ -177,6 +177,28 @@ export async function getAnalysisDetail( ) } +// Bulk counterpart of getAnalysisDetail for batch polling — one query for the whole +// page instead of one per id. Order is not guaranteed to match analysisIds; callers +// key the result by row.id. +export async function getAnalysisDetailsByIds( + qx: QueryExecutor, + analysisIds: string[], +): Promise { + if (analysisIds.length === 0) { + return [] + } + return qx.select( + ` + SELECT + id, advisory_osv_id, package_name, ecosystem, status, error, + candidates_considered, started_at, completed_at + FROM blast_radius_analyses + WHERE id = ANY($(analysisIds)::uuid[]) + `, + { analysisIds }, + ) +} + export async function setDependentsMeta( qx: QueryExecutor, analysisId: string, @@ -374,6 +396,31 @@ export async function getDependentsExcludedByRangeCount( return Number(row.count) || 0 } +// Bulk counterpart of getDependentsExcludedByRangeCount — one grouped query for the +// whole page instead of one per id. Ids with no excluded rows are simply absent from +// the result; callers should default to 0 for those. +export async function getDependentsExcludedByRangeCountBatch( + qx: QueryExecutor, + analysisIds: string[], +): Promise<{ analysisId: string; count: number }[]> { + if (analysisIds.length === 0) { + return [] + } + const rows = await qx.select( + ` + SELECT analysis_id, COUNT(*) as count + FROM blast_radius_dependents + WHERE analysis_id = ANY($(analysisIds)::uuid[]) AND excluded_by_range = TRUE + GROUP BY analysis_id + `, + { analysisIds }, + ) + return rows.map((row: Record) => ({ + analysisId: row.analysis_id as string, + count: Number(row.count) || 0, + })) +} + // Clears prior dependents for this analysis before a fresh scan re-inserts. Needed // because a retry (stage failed after insertDependents but before completeStageRun) // would otherwise leave stale rows from the earlier attempt's scan around — upserting @@ -491,6 +538,34 @@ export async function getVerdictResults( ) } +// Bulk counterpart of getVerdictResults — one query for the whole page instead of one +// per id. Each row carries its analysisId so callers can group results back per id; +// order within a group still follows d.downloads DESC NULLS LAST. +export async function getVerdictResultsBatch( + qx: QueryExecutor, + analysisIds: string[], +): Promise<(VerdictResultRow & { analysisId: string })[]> { + if (analysisIds.length === 0) { + return [] + } + const rows = await qx.select( + ` + SELECT + v.analysis_id, d.name, d.version, d.downloads, + v.reachable_verdict, v.confidence, v.evidence, v.reasoning + FROM blast_radius_verdicts v + JOIN blast_radius_dependents d ON d.id = v.dependent_id + WHERE v.analysis_id = ANY($(analysisIds)::uuid[]) + ORDER BY d.downloads DESC NULLS LAST + `, + { analysisIds }, + ) + return rows.map((row: Record) => ({ + ...(row as unknown as VerdictResultRow), + analysisId: row.analysis_id as string, + })) +} + // ---- stage runs (monitoring) ---- export async function startStageRun(qx: QueryExecutor, input: StageRunInput): Promise { From aadcaaf6e5b54f892129f5ddea639d3f82bb28f3 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 23 Jul 2026 15:16:16 +0200 Subject: [PATCH 3/5] feat: high rate limit Signed-off-by: Umberto Sgueglia --- backend/src/api/public/v1/akrites-external/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/public/v1/akrites-external/index.ts b/backend/src/api/public/v1/akrites-external/index.ts index 7f1df0af7c..c99fb41497 100644 --- a/backend/src/api/public/v1/akrites-external/index.ts +++ b/backend/src/api/public/v1/akrites-external/index.ts @@ -20,7 +20,7 @@ const rateLimiter = createRateLimiter({ max: 60, windowMs: 60 * 1000 }) // Blast-radius jobs kick off a Temporal workflow per request, so they get their own, // much stricter limiter — configurable via env so it can be tuned without a redeploy. -// Defaults to 5 requests/hour. +// Defaults to 50 requests/hour. const blastRadiusRateLimitMax = Number(process.env.AKRITES_BLAST_RADIUS_RATE_LIMIT_MAX) const blastRadiusRateLimitWindowMs = Number(process.env.AKRITES_BLAST_RADIUS_RATE_LIMIT_WINDOW_MS) @@ -28,7 +28,7 @@ const blastRadiusRateLimiter = createRateLimiter({ max: Number.isSafeInteger(blastRadiusRateLimitMax) && blastRadiusRateLimitMax > 0 ? blastRadiusRateLimitMax - : 5, + : 50, windowMs: Number.isSafeInteger(blastRadiusRateLimitWindowMs) && blastRadiusRateLimitWindowMs > 0 ? blastRadiusRateLimitWindowMs From a93f850c77dfdb969deab423802e7d6012bae9e2 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 23 Jul 2026 15:26:06 +0200 Subject: [PATCH 4/5] fix: test Signed-off-by: Umberto Sgueglia --- .../src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts index ff5107b4bd..f1953cea41 100644 --- a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts +++ b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.test.ts @@ -80,6 +80,8 @@ describe('getBlastRadiusJobBatch', () => { }) it('only fetches verdicts/excluded counts for done analyses', async () => { + const { req, res, json } = mockReqRes({ analysisIds: [PENDING_ID, DONE_ID] }) + getAnalysisDetailsByIds.mockResolvedValue([ { id: PENDING_ID, @@ -118,8 +120,6 @@ describe('getBlastRadiusJobBatch', () => { ]) getDependentsExcludedByRangeCountBatch.mockResolvedValue([{ analysisId: DONE_ID, count: 8 }]) - const { req, res, json } = mockReqRes({ analysisIds: [PENDING_ID, DONE_ID] }) - await getBlastRadiusJobBatch(req, res) expect(getVerdictResultsBatch).toHaveBeenCalledWith(expect.anything(), [DONE_ID]) From bfd8a2bf21314fed700d98f766ec224266112f55 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 23 Jul 2026 16:52:52 +0200 Subject: [PATCH 5/5] fix: add advisory cache Signed-off-by: Umberto Sgueglia --- .../public/v1/akrites-external/openapi.yaml | 33 +++++--- .../src/api/public/v1/packages/blastRadius.ts | 57 ++++++++++++- .../v1/packages/getBlastRadiusJobBatch.ts | 11 +-- .../v1/packages/submitBlastRadiusJob.test.ts | 64 ++++++++++++++- .../v1/packages/submitBlastRadiusJob.ts | 25 +++++- .../submitBlastRadiusJobBatch.test.ts | 80 ++++++++++++++++++- .../v1/packages/submitBlastRadiusJobBatch.ts | 38 +++++++-- .../src/packages/blastRadius.ts | 28 +++++++ 8 files changed, 298 insertions(+), 38 deletions(-) diff --git a/backend/src/api/public/v1/akrites-external/openapi.yaml b/backend/src/api/public/v1/akrites-external/openapi.yaml index a1d1ef1540..e1b2ca4d7f 100644 --- a/backend/src/api/public/v1/akrites-external/openapi.yaml +++ b/backend/src/api/public/v1/akrites-external/openapi.yaml @@ -12,8 +12,10 @@ info: Packages, Advisories and Contacts endpoints are implemented. Blast Radius submit (2a) and poll (2b) are both implemented, backed by a 4-stage Temporal pipeline (intel, dependents, reachability, report) for npm - packages; other ecosystems fail fast with ECOSYSTEM_NOT_SUPPORTED. The - 7-day result cache is specced separately and not yet built. + packages; other ecosystems fail fast with ECOSYSTEM_NOT_SUPPORTED. Submit + reuses a 'done' analysis for the same advisory/package/ecosystem if it + completed within the last day (configurable, see the `force` field below) + instead of starting a new Temporal workflow. TODO: scopes below (read:packages, read:stewardships) are the existing @@ -57,10 +59,11 @@ tags: status and, once done, results. Bulk submit (jobs:batch) is capped at 20 jobs per request (10 recommended as the default batch size) — each entry starts its own workflow — and stays behind the same strict rate - limiter as the single-job route; bulk poll - (jobs:batch/poll) is read-only and capped at 100 like the other batch - endpoints. Same interim scope note as Advisories applies (read:packages, - pending a dedicated read:advisories scope). + limiter as the single-job route, defaulting to 50 requests/hour + (configurable via AKRITES_BLAST_RADIUS_RATE_LIMIT_MAX / _WINDOW_MS); + bulk poll (jobs:batch/poll) is read-only and capped at 100 like the + other batch endpoints. Same interim scope note as Advisories applies + (read:packages, pending a dedicated read:advisories scope). components: securitySchemes: @@ -406,7 +409,10 @@ components: force: type: boolean default: false - description: Bypasses the 7-day cache and always triggers a new run. Use sparingly. + description: > + Bypasses the advisory cache (a 'done' analysis for the same + advisory/package/ecosystem completed within the last day, by + default) and always triggers a new run. Use sparingly. BlastRadiusJobEntry: type: object @@ -433,8 +439,10 @@ components: type: string enum: [pending, running, done, failed] description: > - Pending in the single-job submit response, always returned before the - Temporal workflow runs. In the batch submit response, a job whose + Pending when a job is freshly submitted, returned before the Temporal + workflow runs. Done when the advisory cache is reused instead — see + force above — in which case analysisId is the cached analysis's own + id, already completed. In the batch submit response, a job whose workflow failed to start comes back as failed instead — the rest of the batch is unaffected. @@ -1067,10 +1075,9 @@ paths: Starts a Temporal workflow running the 4-stage reachability pipeline (intel, dependents, reachability, report) for npm; other ecosystems fail fast with ECOSYSTEM_NOT_SUPPORTED. Poll status/results via - GET /jobs/{analysisId}. - - - Not yet implemented: the 7-day result cache and force-bypass semantics. + GET /jobs/{analysisId}. Reuses a 'done' analysis for the same + advisory/package/ecosystem completed within the last day (by + default) instead of starting a new workflow, unless force is true. tags: [Blast Radius] security: - M2MBearer: diff --git a/backend/src/api/public/v1/packages/blastRadius.ts b/backend/src/api/public/v1/packages/blastRadius.ts index 69c7c5b21a..72c535b661 100644 --- a/backend/src/api/public/v1/packages/blastRadius.ts +++ b/backend/src/api/public/v1/packages/blastRadius.ts @@ -1,5 +1,8 @@ import { z } from 'zod' +import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + // The reachability pipeline is npm-only for now — every other ecosystem (including // a missing one) is rejected by the schema below before the Temporal workflow is // triggered. @@ -10,6 +13,16 @@ export const SUPPORTED_BLAST_RADIUS_ECOSYSTEMS = ['npm'] as const // so it is NOT run through purlFieldSchema/normalizePurl like the other endpoints. const ADVISORY_ID_PATTERN = /^(GHSA-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}|CVE-\d{4}-\d{4,})$/ +// How recent a 'done' analysis for the same (advisoryId, package, ecosystem) has to +// be for submit to reuse it instead of starting a new Temporal workflow — see +// getRecentDoneAnalysis. Configurable via env so it can be tuned without a redeploy; +// defaults to 1 day. force=true on the request always bypasses this cache. +const blastRadiusCacheMaxAgeDaysEnv = Number(process.env.AKRITES_BLAST_RADIUS_CACHE_MAX_AGE_DAYS) +export const BLAST_RADIUS_CACHE_MAX_AGE_DAYS = + Number.isSafeInteger(blastRadiusCacheMaxAgeDaysEnv) && blastRadiusCacheMaxAgeDaysEnv > 0 + ? blastRadiusCacheMaxAgeDaysEnv + : 1 + export const blastRadiusJobRequestSchema = z.object({ advisoryId: z .string() @@ -36,19 +49,57 @@ export interface BlastRadiusJobEntry { status: BlastRadiusJobStatus } -// Builds the 2a response body. The pipeline isn't implemented yet, so every freshly -// submitted job comes back pending — see analyzeBlastRadius in packages_worker. +// Builds the 2a response body. status defaults to 'pending' (a freshly submitted +// job — see analyzeBlastRadius in packages_worker) but a cache hit passes the +// cached analysis's own status (always 'done' — see getRecentDoneAnalysis) so the +// caller doesn't need to poll a job that's already finished. export function toBlastRadiusJobEntry(params: { analysisId: string advisoryId: string package: string | null ecosystem: BlastRadiusJobEcosystem + status?: BlastRadiusJobStatus }): BlastRadiusJobEntry { return { analysisId: params.analysisId, advisoryId: params.advisoryId, package: params.package, ecosystem: params.ecosystem, - status: 'pending', + status: params.status ?? 'pending', + } +} + +// Shared by submitBlastRadiusJob and submitBlastRadiusJobBatch — looks up a +// recent 'done' analysis for the same (advisoryId, package, ecosystem) and, if +// found, builds the job entry for it. Returns null on a cache miss or when +// force=true (which bypasses the cache entirely). +export async function getCachedJobEntry( + qx: QueryExecutor, + params: { + advisoryId: string + package: string | null + ecosystem: BlastRadiusJobEcosystem + force: boolean + }, +): Promise { + if (params.force) { + return null } + + const cached = await blastRadiusDal.getRecentDoneAnalysis( + qx, + { advisoryOsvId: params.advisoryId, packageName: params.package, ecosystem: params.ecosystem }, + BLAST_RADIUS_CACHE_MAX_AGE_DAYS, + ) + if (!cached) { + return null + } + + return toBlastRadiusJobEntry({ + analysisId: cached.id, + advisoryId: params.advisoryId, + package: params.package, + ecosystem: params.ecosystem, + status: 'done', + }) } diff --git a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts index 2dd397be19..8f52e81b09 100644 --- a/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts +++ b/backend/src/api/public/v1/packages/getBlastRadiusJobBatch.ts @@ -1,5 +1,6 @@ import type { Request, Response } from 'express' +import { groupBy } from '@crowd/common' import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' import { getPackagesQx } from '@/db/packagesDb' @@ -34,15 +35,7 @@ export async function getBlastRadiusJobBatch(req: Request, res: Response): Promi blastRadiusDal.getDependentsExcludedByRangeCountBatch(qx, doneIds), ]) - const verdictsByAnalysisId = new Map() - for (const row of verdictRows) { - const bucket = verdictsByAnalysisId.get(row.analysisId) - if (bucket) { - bucket.push(row) - } else { - verdictsByAnalysisId.set(row.analysisId, [row]) - } - } + const verdictsByAnalysisId = groupBy(verdictRows, (row) => row.analysisId) const excludedByRangeCountByAnalysisId = new Map( excludedByRangeCounts.map(({ analysisId, count }) => [analysisId, count]), ) diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts index eb70ad1bd9..1fab59b719 100644 --- a/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts @@ -3,10 +3,11 @@ import { describe, expect, it, vi } from 'vitest' import { submitBlastRadiusJob } from './submitBlastRadiusJob' -const { start, createAnalysis, failAnalysis } = vi.hoisted(() => ({ +const { start, createAnalysis, failAnalysis, getRecentDoneAnalysis } = vi.hoisted(() => ({ start: vi.fn().mockResolvedValue(undefined), createAnalysis: vi.fn().mockResolvedValue(undefined), failAnalysis: vi.fn().mockResolvedValue(undefined), + getRecentDoneAnalysis: vi.fn(), })) vi.mock('@/db/packagesTemporal', () => ({ @@ -20,12 +21,15 @@ vi.mock('@/db/packagesDb', () => ({ vi.mock('@crowd/data-access-layer/src/packages/blastRadius', () => ({ createAnalysis, failAnalysis, + getRecentDoneAnalysis, })) function mockReqRes(body: unknown) { start.mockClear() createAnalysis.mockClear() failAnalysis.mockClear() + getRecentDoneAnalysis.mockClear() + getRecentDoneAnalysis.mockResolvedValue(null) const req = { body } as unknown as Request @@ -152,4 +156,62 @@ describe('submitBlastRadiusJob', () => { }) expect(errorMessage).toBe('temporal unreachable') }) + + it('reuses a recent done analysis instead of starting a workflow', async () => { + const { req, res, start, status, json } = mockReqRes({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'npm', + }) + getRecentDoneAnalysis.mockResolvedValue({ + id: 'cached-analysis-id', + advisory_osv_id: 'GHSA-jf85-cpcp-j695', + package_name: null, + ecosystem: 'npm', + status: 'done', + error: null, + candidates_considered: 5, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: '2026-07-01T01:00:00.000Z', + }) + + await submitBlastRadiusJob(req, res) + + expect(createAnalysis).not.toHaveBeenCalled() + expect(start).not.toHaveBeenCalled() + expect(status).toHaveBeenCalledWith(202) + expect(json).toHaveBeenCalledWith( + expect.objectContaining({ + analysisId: 'cached-analysis-id', + advisoryId: 'GHSA-jf85-cpcp-j695', + package: null, + ecosystem: 'npm', + status: 'done', + }), + ) + }) + + it('bypasses the cache and starts a new workflow when force is true, even with a recent done analysis', async () => { + const { req, res, start } = mockReqRes({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'npm', + force: true, + }) + getRecentDoneAnalysis.mockResolvedValue({ + id: 'cached-analysis-id', + advisory_osv_id: 'GHSA-jf85-cpcp-j695', + package_name: null, + ecosystem: 'npm', + status: 'done', + error: null, + candidates_considered: 5, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: '2026-07-01T01:00:00.000Z', + }) + + await submitBlastRadiusJob(req, res) + + expect(getRecentDoneAnalysis).not.toHaveBeenCalled() + expect(createAnalysis).toHaveBeenCalledTimes(1) + expect(start).toHaveBeenCalledTimes(1) + }) }) diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJob.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJob.ts index d92ace7b75..f60eebbdcb 100644 --- a/backend/src/api/public/v1/packages/submitBlastRadiusJob.ts +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJob.ts @@ -8,16 +8,36 @@ import { getPackagesQx } from '@/db/packagesDb' import { getPackagesTemporalClient } from '@/db/packagesTemporal' import { validateOrThrow } from '@/utils/validation' -import { blastRadiusJobRequestSchema, toBlastRadiusJobEntry } from './blastRadius' +import { + blastRadiusJobRequestSchema, + getCachedJobEntry, + toBlastRadiusJobEntry, +} from './blastRadius' // 2a — submit a blast-radius analysis job. Always exactly one job per request. -// Every submission gets a fresh analysisId and status pending. +// Every submission gets a fresh analysisId and status pending, unless a 'done' +// analysis for the same (advisoryId, package, ecosystem) is still within the +// advisory cache window — see BLAST_RADIUS_CACHE_MAX_AGE_DAYS — in which case that +// cached analysis is returned instead, with no new row and no workflow start. +// force=true on the request skips this cache entirely. export async function submitBlastRadiusJob(req: Request, res: Response): Promise { const body = validateOrThrow(blastRadiusJobRequestSchema, req.body) const jobPackage = body.package ?? null const jobEcosystem = body.ecosystem + const qx = await getPackagesQx() + const cached = await getCachedJobEntry(qx, { + advisoryId: body.advisoryId, + package: jobPackage, + ecosystem: jobEcosystem, + force: body.force, + }) + if (cached) { + res.status(202).json(cached) + return + } + const analysisId = generateUUIDv4() // Create the pending row synchronously, before starting the workflow — otherwise a @@ -25,7 +45,6 @@ export async function submitBlastRadiusJob(req: Request, res: Response): Promise // blastRadiusStart's own createAnalysis call and get a 404 for a job that was, in // fact, accepted. blastRadiusStart's createAnalysis upserts the same row, so this // is safe to run again from the workflow. - const qx = await getPackagesQx() await blastRadiusDal.createAnalysis(qx, { id: analysisId, advisoryOsvId: body.advisoryId, diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts index e3f37ae73d..d0be4e37b2 100644 --- a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts @@ -3,10 +3,11 @@ import { describe, expect, it, vi } from 'vitest' import { submitBlastRadiusJobBatch } from './submitBlastRadiusJobBatch' -const { start, createAnalysis, failAnalysis } = vi.hoisted(() => ({ +const { start, createAnalysis, failAnalysis, getRecentDoneAnalysis } = vi.hoisted(() => ({ start: vi.fn().mockResolvedValue(undefined), createAnalysis: vi.fn().mockResolvedValue(undefined), failAnalysis: vi.fn().mockResolvedValue(undefined), + getRecentDoneAnalysis: vi.fn(), })) vi.mock('@/db/packagesTemporal', () => ({ @@ -20,12 +21,15 @@ vi.mock('@/db/packagesDb', () => ({ vi.mock('@crowd/data-access-layer/src/packages/blastRadius', () => ({ createAnalysis, failAnalysis, + getRecentDoneAnalysis, })) function mockReqRes(body: unknown) { start.mockClear() createAnalysis.mockClear() failAnalysis.mockClear() + getRecentDoneAnalysis.mockClear() + getRecentDoneAnalysis.mockResolvedValue(null) const req = { body } as unknown as Request @@ -90,6 +94,24 @@ describe('submitBlastRadiusJobBatch', () => { expect(errorMessage).toBe('temporal unreachable') }) + it('still resolves the batch when failAnalysis itself throws after a workflow.start failure', async () => { + const { req, res, json } = mockReqRes({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'npm' }, + ], + }) + start.mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('temporal unreachable')) + failAnalysis.mockRejectedValueOnce(new Error('db unreachable')) + + await expect(submitBlastRadiusJobBatch(req, res)).resolves.toBeUndefined() + + const [{ results }] = json.mock.calls[0] + expect(results).toHaveLength(2) + expect(results[0]).toMatchObject({ advisoryId: 'GHSA-jf85-cpcp-j695', status: 'pending' }) + expect(results[1]).toMatchObject({ advisoryId: 'GHSA-652q-gvq3-74qv', status: 'failed' }) + }) + it('rejects a batch containing an unsupported ecosystem without submitting any job', async () => { const { req, res, start } = mockReqRes({ jobs: [ @@ -121,4 +143,60 @@ describe('submitBlastRadiusJobBatch', () => { await expect(submitBlastRadiusJobBatch(req, res)).rejects.toThrow() expect(start).not.toHaveBeenCalled() }) + + it('reuses a recent done analysis for one job while starting a fresh workflow for the other', async () => { + const { req, res, start, json } = mockReqRes({ + jobs: [ + { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, + { advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'npm' }, + ], + }) + getRecentDoneAnalysis.mockResolvedValueOnce({ + id: 'cached-analysis-id', + advisory_osv_id: 'GHSA-jf85-cpcp-j695', + package_name: null, + ecosystem: 'npm', + status: 'done', + error: null, + candidates_considered: 5, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: '2026-07-01T01:00:00.000Z', + }) + + await submitBlastRadiusJobBatch(req, res) + + expect(createAnalysis).toHaveBeenCalledTimes(1) + expect(start).toHaveBeenCalledTimes(1) + + const [{ results }] = json.mock.calls[0] + expect(results[0]).toMatchObject({ + analysisId: 'cached-analysis-id', + advisoryId: 'GHSA-jf85-cpcp-j695', + status: 'done', + }) + expect(results[1]).toMatchObject({ advisoryId: 'GHSA-652q-gvq3-74qv', status: 'pending' }) + }) + + it('bypasses the cache and starts a new workflow when force is true', async () => { + const { req, res, start } = mockReqRes({ + jobs: [{ advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm', force: true }], + }) + getRecentDoneAnalysis.mockResolvedValue({ + id: 'cached-analysis-id', + advisory_osv_id: 'GHSA-jf85-cpcp-j695', + package_name: null, + ecosystem: 'npm', + status: 'done', + error: null, + candidates_considered: 5, + started_at: '2026-07-01T00:00:00.000Z', + completed_at: '2026-07-01T01:00:00.000Z', + }) + + await submitBlastRadiusJobBatch(req, res) + + expect(getRecentDoneAnalysis).not.toHaveBeenCalled() + expect(createAnalysis).toHaveBeenCalledTimes(1) + expect(start).toHaveBeenCalledTimes(1) + }) }) diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts index fa0d7b6f89..8c0d97c2ee 100644 --- a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.ts @@ -13,6 +13,7 @@ import { validateOrThrow } from '@/utils/validation' import { type BlastRadiusJobEntry, type BlastRadiusJobRequest, + getCachedJobEntry, toBlastRadiusJobEntry, } from './blastRadius' import { blastRadiusJobBatchRequestSchema } from './blastRadiusBatch' @@ -20,7 +21,10 @@ import { blastRadiusJobBatchRequestSchema } from './blastRadiusBatch' // 2a bulk — submit multiple blast-radius analysis jobs in one request, one per // array entry. Same lifecycle as the single-job submit, just looped: each entry // gets its own analysisId, its own pending row, and its own Temporal workflow -// start. Unlike the read-only batch endpoints (packages/advisories/contacts), +// start — unless a 'done' analysis for the same (advisoryId, package, ecosystem) +// is still within the advisory cache window (see BLAST_RADIUS_CACHE_MAX_AGE_DAYS), +// in which case that entry reuses the cached analysis instead. Unlike the +// read-only batch endpoints (packages/advisories/contacts), // this multiplies workflow starts per request, so the batch size is capped much // lower (see MAX_BLAST_RADIUS_JOBS_PER_BATCH) and the route stays behind the same // strict blastRadiusRateLimiter as the single-job route. @@ -58,10 +62,21 @@ async function submitOneJob( } try { + // Cache lookup is inside the try too — like createAnalysis/workflow.start below, + // a DB error here must resolve this job's entry as 'failed', not reject the whole + // batch's Promise.all and 500 every other job in it. + const cached = await getCachedJobEntry(qx, { + advisoryId: body.advisoryId, + package: jobPackage, + ecosystem: jobEcosystem, + force: body.force, + }) + if (cached) { + return cached + } + // Create the pending row synchronously, before starting the workflow — see the - // same comment on submitBlastRadiusJob for why (avoids a poll-race 404). This is - // inside the try too — unlike the single-job submit, a createAnalysis failure - // must not reject the whole batch's Promise.all, only this job's entry. + // same comment on submitBlastRadiusJob for why (avoids a poll-race 404). await blastRadiusDal.createAnalysis(qx, analysisInput) await packagesTemporal.workflow.start('analyzeBlastRadius', { @@ -87,16 +102,23 @@ async function submitOneJob( }) } catch (err) { // Unlike the single-job submit, this does not rethrow — one job's workflow - // failing to start must not take the rest of the batch down with it. + // failing to start must not take the rest of the batch down with it. The + // failAnalysis call below is deliberately its own try/catch too — if marking + // the row failed also fails (e.g. DB unreachable), that must still resolve + // this job's entry rather than reject the whole Promise.all and 500 the batch. const errorMessage = err instanceof Error ? err.message : String(err) - await blastRadiusDal.failAnalysis(qx, analysisInput, errorMessage) + try { + await blastRadiusDal.failAnalysis(qx, analysisInput, errorMessage) + } catch { + // best-effort — the job is already being reported as failed below + } - return { + return toBlastRadiusJobEntry({ analysisId, advisoryId: body.advisoryId, package: jobPackage, ecosystem: jobEcosystem, status: 'failed', - } + }) } } diff --git a/services/libs/data-access-layer/src/packages/blastRadius.ts b/services/libs/data-access-layer/src/packages/blastRadius.ts index 030657f9ce..fc3530d4c1 100644 --- a/services/libs/data-access-layer/src/packages/blastRadius.ts +++ b/services/libs/data-access-layer/src/packages/blastRadius.ts @@ -177,6 +177,34 @@ export async function getAnalysisDetail( ) } +// Advisory-cache lookup for submit: finds the most recent 'done' analysis for the +// same (advisoryOsvId, packageName, ecosystem) triple completed within maxAgeDays, +// so a submit can reuse it instead of starting a new Temporal workflow. packageName +// uses IS NOT DISTINCT FROM since it's nullable (advisory-wide analyses have no +// package) and NULL = NULL is never true in plain SQL equality. +export async function getRecentDoneAnalysis( + qx: QueryExecutor, + input: { advisoryOsvId: string; packageName: string | null; ecosystem: string }, + maxAgeDays: number, +): Promise { + return qx.selectOneOrNone( + ` + SELECT + id, advisory_osv_id, package_name, ecosystem, status, error, + candidates_considered, started_at, completed_at + FROM blast_radius_analyses + WHERE advisory_osv_id = $(advisoryOsvId) + AND package_name IS NOT DISTINCT FROM $(packageName) + AND ecosystem = $(ecosystem) + AND status = 'done' + AND completed_at >= NOW() - make_interval(days => $(maxAgeDays)) + ORDER BY completed_at DESC + LIMIT 1 + `, + { ...input, maxAgeDays }, + ) +} + // Bulk counterpart of getAnalysisDetail for batch polling — one query for the whole // page instead of one per id. Order is not guaranteed to match analysisIds; callers // key the result by row.id.