From e2da239d94c2c9f00065764f32749a087630a392 Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 30 Sep 2026 15:54:14 +0100 Subject: [PATCH 1/2] Fall back to default release versions when none specified Bring the surveys and prevalences endpoints in line with the metadata endpoint behaviour of defaulting to the latest model release (or, in the case of the surveys endpoint, the data release that the latest model release is pinned to). --- README.md | 4 +++ src/app.ts | 18 +++++++------ src/utils/endpoints.ts | 4 +-- src/utils/releases.ts | 27 +++++++++++++++++++ src/utils/validators.ts | 7 ++--- .../prevalences.integration.test.ts | 24 ++++++++++++++++- tests/integration/surveys.integration.test.ts | 13 +++++++++ tests/unit/validators.test.ts | 18 ++++++++++--- 8 files changed, 96 insertions(+), 19 deletions(-) create mode 100644 src/utils/releases.ts diff --git a/README.md b/README.md index 485e6b1..1b95bc8 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,8 @@ response: An endpoint for querying survey data, as stored in `/data/stave//survey_data.parquet`. +The optional `data_release` parameter defaults to the data release that the latest model release (as configured via `config.ts`) depends on. + Note that this endpoint actually returns multiple entries per STAVE survey - that is, we have one entry per variant per STAVE survey. Thus these objects match the STAVE concept of a '[count](https://mrc-ide.github.io/STAVE/articles/howto_counts_table.html)' (which counts a particular mutation) a bit more closely than the concept of a '[survey](https://mrc-ide.github.io/STAVE/articles/howto_surveys_table.html)' (which would collect multiple genetic variants). Example: @@ -124,6 +126,8 @@ response: An endpoint for querying model outputs, as stored in `/data/model//admin.parquet`. +The optional `model_release` parameter defaults to the latest model release, as configured via `config.ts`. + The `admin_level` query parameter determines the granularity of the model outputs, while the query parameters `admin0`, `admin1` and `admin2` scope the results to a particular region. Thus for example, to request results within the `admin0` region of Mali (`MLI`), at the finest level of granularity: request: diff --git a/src/app.ts b/src/app.ts index 7ecfb5e..e522330 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,11 +1,13 @@ +// TODO: JSON schema for telling consumers what shape response to expect? + import express, { type Express, type Request, type Response } from 'express'; -import { join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { join } from 'node:path'; import config from './config/config.ts'; import { errorHandler } from './middlewares/errorHandler.ts'; import { globalBounds, modelVersions } from './constants.ts'; import type { QueryParams } from './types.ts'; import { validateModelRelease } from './utils/validators.ts'; +import { getDataRelease, getModelRelease, readModelMetadata } from './utils/releases.ts'; import { validateSurveysRequest, validatePrevalencesRequest } from './utils/endpoints.ts'; import { executeParquetQuery } from './utils/data.ts'; import { getMutationsByGene } from './utils/metadata.ts'; @@ -14,12 +16,11 @@ export const createApp = (): Express => { const app: Express = express(); app.get('/metadata', async (req: Request, res: Response) => { - const modelVersion = (req.query['model_release'] ?? config.latestModelVersion) as string; - if (!validateModelRelease(modelVersion, res)) return; + if (!validateModelRelease(req, res)) return; + const modelVersion = getModelRelease(req); const mutationsByGene = await getMutationsByGene(modelVersion); - const metadataPath = pathToFileURL(resolve(config.dataDir, "model", modelVersion, "metadata.json")).href; - const { default: modelMetadata } = await import(metadataPath, { with: { type: "json" } }); + const modelMetadata = await readModelMetadata(modelVersion); const dataVersion = modelMetadata.data_release; res.send({ @@ -39,7 +40,7 @@ export const createApp = (): Express => { app.get('/surveys', async (req: Request, res: Response) => { if (!validateSurveysRequest(req, res)) return; - const dataVersion = req.query['data_release'] as string; + const dataVersion = getDataRelease(req); const surveyDataParquet = join(config.dataDir, "stave", dataVersion, "survey_data.parquet"); const result = await executeParquetQuery(req.query as QueryParams, "/surveys", surveyDataParquet, res); @@ -51,11 +52,12 @@ export const createApp = (): Express => { app.get('/prevalences', async (req: Request, res: Response) => { if (!validatePrevalencesRequest(req, res)) return; + const modelVersion = getModelRelease(req); const queryParams = req.query as QueryParams; // Client may request results at any of the available levels of granularity. const adminLevel = queryParams.admin_level as string; - const prevalencesParquet = join(config.dataDir, "model", queryParams.model_release!, `admin${adminLevel}.parquet`); + const prevalencesParquet = join(config.dataDir, "model", modelVersion, `admin${adminLevel}.parquet`); const result = await executeParquetQuery(queryParams, "/prevalences", prevalencesParquet, res); if (!result) return; diff --git a/src/utils/endpoints.ts b/src/utils/endpoints.ts index 8cc9cbd..845e76f 100644 --- a/src/utils/endpoints.ts +++ b/src/utils/endpoints.ts @@ -11,7 +11,7 @@ export const validateSurveysRequest = (req: Request, res: Response) => { export const validatePrevalencesRequest = (req: Request, res: Response) => { return validateRequiredQueryParams(req, res) - && validateModelRelease(req.query['model_release'] as string, res) + && validateModelRelease(req, res) && validateDateParams(req, res) && validateAdminLevel(req, res) }; @@ -43,7 +43,6 @@ export interface EndpointConfig { export const endpointConfigs: Record = { "/surveys": { requiredParams: [ - "data_release", "properties", SURVEY_COLUMNS.GENE, SURVEY_COLUMNS.MUTATION, @@ -56,7 +55,6 @@ export const endpointConfigs: Record = { }, "/prevalences": { requiredParams: [ - "model_release", "admin_level", "properties", PREVALENCE_COLUMNS.GENE, diff --git a/src/utils/releases.ts b/src/utils/releases.ts new file mode 100644 index 0000000..d8b1158 --- /dev/null +++ b/src/utils/releases.ts @@ -0,0 +1,27 @@ +import type { Request } from "express"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import config from "../config/config.ts"; + +export interface ModelMetadata { + version: string; + data_release: string; + shapefile_source: string; +} + +export const readModelMetadata = async (modelVersion: string): Promise => { + const metadataPath = join(config.dataDir, "model", modelVersion, "metadata.json"); + return JSON.parse(await readFile(metadataPath, "utf8")) as ModelMetadata; +}; + +export const defaultModelRelease = config.latestModelVersion; + +// Each model release depends on a specific data release, so the default data release +// is the one that the default model release depends on. +export const defaultDataRelease = (await readModelMetadata(defaultModelRelease)).data_release; + +export const getModelRelease = (req: Request): string => + (req.query['model_release'] ?? defaultModelRelease) as string; + +export const getDataRelease = (req: Request): string => + (req.query['data_release'] ?? defaultDataRelease) as string; diff --git a/src/utils/validators.ts b/src/utils/validators.ts index 5f675f5..867f147 100644 --- a/src/utils/validators.ts +++ b/src/utils/validators.ts @@ -1,6 +1,7 @@ import { type Request, type Response } from 'express'; import { modelVersions, dataVersions, adminLevels } from '../constants.ts'; import type { Column } from '../types.ts'; +import { getDataRelease, getModelRelease } from './releases.ts'; import { endpointConfigs, type DateFormat, type Endpoint } from './endpoints.ts'; const dateRegexes: Record = { @@ -45,7 +46,8 @@ export const validateRequestedProperties = ( // The release-version validators below are intended to guard against SQL injection // by checking the requested version is a filepath within the relevant data directory. -export const validateModelRelease = (modelVersion: string, res: Response): boolean => { +export const validateModelRelease = (req: Request, res: Response): boolean => { + const modelVersion = getModelRelease(req); if (!modelVersions.includes(modelVersion)) { res.status(404).send({ error: `Unknown model release: ${modelVersion}` }); return false; @@ -54,8 +56,7 @@ export const validateModelRelease = (modelVersion: string, res: Response): boole }; export const validateDataRelease = (req: Request, res: Response): boolean => { - const dataVersion = req.query['data_release'] as string; - + const dataVersion = getDataRelease(req); if (!dataVersions.includes(dataVersion)) { res.status(404).send({ error: `Unknown data release requested: ${dataVersion}` }); return false; diff --git a/tests/integration/prevalences.integration.test.ts b/tests/integration/prevalences.integration.test.ts index cb508d9..583ce3e 100644 --- a/tests/integration/prevalences.integration.test.ts +++ b/tests/integration/prevalences.integration.test.ts @@ -164,10 +164,32 @@ describe('GET /prevalences', () => { expect(response.status).toBe(400); expect(response.body.error).toBe( - 'Missing required query parameters: model_release, properties, gene, mutation', + 'Missing required query parameters: properties, gene, mutation', ); }); + it('uses the latest model release when none is specified', async () => { + const query = { gene: baseQuery.gene, mutation: baseQuery.mutation, admin_level: '0', properties: 'admin0,date,median' }; + + const defaultResponse = await request(app).get('/prevalences').query(query); + const explicitResponse = await request(app) + .get('/prevalences') + .query({ ...query, model_release: fixtureConfig.modelRelease }); + + expect(defaultResponse.status).toBe(200); + expect(defaultResponse.body.median.length).toBeGreaterThan(0); + expect(defaultResponse.body).toEqual(explicitResponse.body); + }); + + it('rejects an unknown model release', async () => { + const response = await request(app) + .get('/prevalences') + .query({ ...baseQuery, model_release: '../private', admin_level: '0', properties: 'median' }); + + expect(response.status).toBe(404); + expect(response.body).toEqual({ error: 'Unknown model release: ../private' }); + }); + it('rejects a containing region more granular than the requested results', async () => { const response = await request(app) .get('/prevalences') diff --git a/tests/integration/surveys.integration.test.ts b/tests/integration/surveys.integration.test.ts index 887b0a0..16e1fcb 100644 --- a/tests/integration/surveys.integration.test.ts +++ b/tests/integration/surveys.integration.test.ts @@ -131,6 +131,19 @@ describe('GET /surveys', () => { expect(response.body).toEqual({ error: 'ISO code not found: ZZZ' }); }); + it('uses the data release of the latest model release when none is specified', async () => { + const query = { gene: baseQuery.gene, mutation: baseQuery.mutation, properties: pointProperties }; + + const defaultResponse = await request(app).get('/surveys').query(query); + const explicitResponse = await request(app) + .get('/surveys') + .query({ ...query, data_release: fixtureConfig.dataRelease }); + + expect(defaultResponse.status).toBe(200); + expect(defaultResponse.body.length).toBeGreaterThan(0); + expect(defaultResponse.body).toEqual(explicitResponse.body); + }); + it('rejects an unknown data release', async () => { const response = await request(app) .get('/surveys') diff --git a/tests/unit/validators.test.ts b/tests/unit/validators.test.ts index b089468..1573676 100644 --- a/tests/unit/validators.test.ts +++ b/tests/unit/validators.test.ts @@ -105,14 +105,19 @@ describe('validateRequestedProperties', () => { describe('release validators', () => { it('accepts a known model release', () => { - const { res } = mockReqRes({}); - expect(validateModelRelease(fixtureConfig.modelRelease, res)).toBe(true); + const { req, res } = mockReqRes({ model_release: fixtureConfig.modelRelease }); + expect(validateModelRelease(req, res)).toBe(true); + }); + + it('accepts a missing model release by falling back to the default', () => { + const { req, res } = mockReqRes({}); + expect(validateModelRelease(req, res)).toBe(true); }); it('rejects an unknown model release', () => { - const { res } = mockReqRes({ model_release: '../private' }); + const { req, res } = mockReqRes({ model_release: '../private' }); - expect(validateModelRelease("../private", res)).toBe(false); + expect(validateModelRelease(req, res)).toBe(false); expect(res.status).toHaveBeenCalledWith(404); }); @@ -121,6 +126,11 @@ describe('release validators', () => { expect(validateDataRelease(req, res)).toBe(true); }); + it('accepts a missing data release by falling back to the default', () => { + const { req, res } = mockReqRes({}); + expect(validateDataRelease(req, res)).toBe(true); + }); + it('rejects an unknown data release', () => { const { req, res } = mockReqRes({ data_release: '../private' }); From 63bf2c214840e1b694a07a85824212f05b394d89 Mon Sep 17 00:00:00 2001 From: David Mears Date: Wed, 30 Sep 2026 16:53:17 +0100 Subject: [PATCH 2/2] Remove todo comment --- src/app.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index e522330..63fa62a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,5 +1,3 @@ -// TODO: JSON schema for telling consumers what shape response to expect? - import express, { type Express, type Request, type Response } from 'express'; import { join } from 'node:path'; import config from './config/config.ts';