Use YYYY-MM strings for prevalence dates

README.md

@@ -131,7 +131,7 @@ GET /prevalences?
   &admin0=MLI
   &gene=k13
   &mutation=469Y
-  &date=2024-05-01
+  &date=2024-05
   &properties=median,admin2
 ```
 

scripts/create_example_model_outputs.R

@@ -196,6 +196,7 @@ build_level_chunk <- function(level, regions_tbl) {
 
   output_table <- base |>
     mutate(
+      date = format(date, "%Y-%m"),
       mean = mean_prevalence,
       median = median_prevalence,
       SD = prevalence_sd,

src/utils/endpoints.ts

@@ -1,5 +1,5 @@
 import type { Request, Response } from 'express';
-import { validateRequiredQueryParams, validateDataRelease, validateDateParams, validateModelRelease, validateDateIsFirstOfMonth, validateAdminLevel } from './validators.ts';
+import { validateRequiredQueryParams, validateDataRelease, validateDateParams, validateModelRelease, validateAdminLevel } from './validators.ts';
 import type { Column } from '../types.ts';
 import { PREVALENCE_COLUMNS, SURVEY_COLUMNS } from '../constants.ts';
 
@@ -13,7 +13,6 @@ export const validatePrevalencesRequest = (req: Request, res: Response) => {
   return validateRequiredQueryParams(req, res)
     && validateModelRelease(req.query['model_release'] as string, res)
     && validateDateParams(req, res)
-    && validateDateIsFirstOfMonth(req, res)
     && validateAdminLevel(req, res)
 };
 
@@ -27,6 +26,7 @@ const Admin0Mode = {
 type Admin0Mode = typeof Admin0Mode[keyof typeof Admin0Mode];
 
 export type Endpoint = "/surveys" | "/prevalences";
+export type DateFormat = "YYYY-MM" | "YYYY-MM-DD";
 export interface EndpointConfig<T extends Column = Column> {
   // Query parameters that must be present in the request.
   requiredParams: string[];
@@ -36,6 +36,7 @@ export interface EndpointConfig<T extends Column = Column> {
   filterableParams: string[];
   // The column to filter on for requests that scope by date_from/date_to.
   dateColumn: T;
+  dateFormat: DateFormat;
   admin0Mode: Admin0Mode;
 }
 
@@ -50,6 +51,7 @@ export const endpointConfigs: Record<Endpoint, EndpointConfig> = {
     requestableProperties: Object.values(SURVEY_COLUMNS),
     filterableParams: ["admin0", "survey_id", "date_from", "date_to", "gene", "mutation"],
     dateColumn: "collection_day",
+    dateFormat: "YYYY-MM-DD",
     admin0Mode: "bounds",
   },
   "/prevalences": {
@@ -63,6 +65,7 @@ export const endpointConfigs: Record<Endpoint, EndpointConfig> = {
     requestableProperties: Object.values(PREVALENCE_COLUMNS),
     filterableParams: ["admin0", "admin1", "admin2", "gene", "mutation", "date", "date_from", "date_to"],
     dateColumn: "date",
+    dateFormat: "YYYY-MM",
     admin0Mode: "column",
   },
 } as const;

src/utils/metadata.ts

@@ -20,8 +20,8 @@ export const getMutationsByGene = async (
       ANY_VALUE(gene) AS gene,
       ANY_VALUE(mutation) AS mutation,
       variant,
-      STRFTIME(MIN("date"), '%Y-%m-%d') AS min_date,
-      STRFTIME(MAX("date"), '%Y-%m-%d') AS max_date
+      MIN("date") AS min_date,
+      MAX("date") AS max_date
     FROM '${join(config.dataDir, "model", modelVersion, "admin0.parquet")}'
     GROUP BY variant
   `);

src/utils/validators.ts

@@ -1,9 +1,12 @@
 import { type Request, type Response } from 'express';
 import { modelVersions, dataVersions, adminLevels } from '../constants.ts';
 import type { Column } from '../types.ts';
-import { endpointConfigs, type Endpoint } from './endpoints.ts';
+import { endpointConfigs, type DateFormat, type Endpoint } from './endpoints.ts';
 
-const dateRegex = /^\d{4}-\d{2}-\d{2}$/;
+const dateRegexes: Record<DateFormat, RegExp> = {
+  "YYYY-MM": /^\d{4}-(0[1-9]|1[0-2])$/,
+  "YYYY-MM-DD": /^\d{4}-\d{2}-\d{2}$/,
+};
 
 export const validateRequiredQueryParams = (
   req: Request,
@@ -61,13 +64,18 @@ export const validateDataRelease = (req: Request, res: Response): boolean => {
 };
 
 export const validateDateParams = (req: Request, res: Response): boolean => {
+  console.log("path:", req.path);
+  const path = req.path as Endpoint;
+  const dateFormat = endpointConfigs[path].dateFormat;
   const queryParams = req.query as Record<string, string | undefined>;
 
   for (const param of ["date", "date_from", "date_to"]) {
     const value = queryParams[param];
     if (!value) continue;
-    if (!dateRegex.test(value) || Number.isNaN(Date.parse(value))) {
-      res.status(400).send({ error: `Invalid date for parameter '${param}'. Expected YYYY-MM-DD.` });
+    const isValid = dateRegexes[dateFormat].test(value)
+      && (dateFormat !== "YYYY-MM-DD" || !Number.isNaN(Date.parse(value)));
+    if (!isValid) {
+      res.status(400).send({ error: `Invalid date for parameter '${param}'. Expected ${dateFormat}.` });
       return false;
     }
   }
@@ -80,22 +88,13 @@ export const validateDateParams = (req: Request, res: Response): boolean => {
     return false;
   }
 
-  if (date_from && date_to && new Date(date_from) > new Date(date_to)) {
+  if (date_from && date_to && date_from > date_to) {
     res.status(400).send({ error: "'date_from' cannot be later than 'date_to'." });
     return false;
   }
   return true;
 };
 
-export const validateDateIsFirstOfMonth = (req: Request, res: Response): boolean => {
-  const date = req.query['date'] as string | undefined;
-  if (date && new Date(date).getDate() !== 1) {
-    res.status(400).send({ error: "Invalid `date` parameter. The date must be the first of a month." });
-    return false;
-  }
-  return true;
-};
-
 export const validateAdminLevel = (req: Request, res: Response): boolean => {
   const adminLevel = req.query['admin_level'] as string | undefined;
   if (!adminLevel || !adminLevels.includes(adminLevel)) {
@@ -113,4 +112,3 @@ export const validateAdminLevel = (req: Request, res: Response): boolean => {
   }
   return true;
 };
-

tests/fixtures/fixture-config.json

@@ -10,15 +10,15 @@
       "variant": "crt:76:K",
       "gene": "crt",
       "mutation": "76K",
-      "start": "2023-05-01",
-      "end": "2025-05-01"
+      "start": "2023-05",
+      "end": "2025-05"
     },
     {
       "variant": "k13:469:Y",
       "gene": "k13",
       "mutation": "469Y",
-      "start": "2024-05-01",
-      "end": "2025-05-01"
+      "start": "2024-05",
+      "end": "2025-05"
     }
   ],
   "surveyIds": [

tests/fixtures/generate-test-fixtures.ts

@@ -41,7 +41,7 @@ for (const path of prevalencePaths) {
       WHERE
         admin0 IN (${fixtureConfig.countries.map(c => `'${c}'`).join(', ')})
         AND (${fixtureConfig.variantWindows.map(w =>
-          `(variant = '${w.variant}' AND date BETWEEN DATE '${w.start}' AND DATE '${w.end}')`,
+          `(variant = '${w.variant}' AND date BETWEEN '${w.start}' AND '${w.end}')`,
         ).join(' OR ')})
     )
     TO '${prevalenceDestination}' (FORMAT PARQUET)

tests/integration/metadata.integration.test.ts

@@ -27,14 +27,14 @@ describe('GET /metadata', () => {
         gene: 'crt',
         mutations: [{
           mutation: '76K',
-          date_range: { start: '2023-05-01', end: '2025-05-01' },
+          date_range: { start: '2023-05', end: '2025-05' },
         }],
       },
       {
         gene: 'k13',
         mutations: [{
           mutation: '469Y',
-          date_range: { start: '2024-05-01', end: '2025-05-01' },
+          date_range: { start: '2024-05', end: '2025-05' },
         }],
       },
     ];

tests/integration/prevalences.integration.test.ts

@@ -17,7 +17,7 @@ describe('GET /prevalences', () => {
       .query({
         ...baseQuery,
         admin_level: '0',
-        date: '2024-05-01',
+        date: '2024-05',
         properties: 'admin0,median',
       });
 
@@ -35,8 +35,8 @@ describe('GET /prevalences', () => {
       .query({
         ...baseQuery,
         admin_level: '0',
-        date_from: '2024-06-01',
-        date_to: '2025-01-01',
+        date_from: '2024-06',
+        date_to: '2025-01',
         properties: 'admin0,median,date',
       });
 
@@ -45,10 +45,10 @@ describe('GET /prevalences', () => {
     expect(response.body.admin0).toHaveLength(expectedLength);
     expect(response.body.median).toHaveLength(expectedLength);
     expect(response.body.date).toHaveLength(expectedLength);
-    expect(response.body.date).toContain('2024-06-01');
-    expect(response.body.date).toContain('2025-01-01');
-    expect(response.body.date).not.toContain('2023-01-01');
-    expect(response.body.date).not.toContain('2026-01-01');
+    expect(response.body.date).toContain('2024-06');
+    expect(response.body.date).toContain('2025-01');
+    expect(response.body.date).not.toContain('2023-01');
+    expect(response.body.date).not.toContain('2026-01');
   });
 
   it('returns details for one region', async () => {
@@ -58,7 +58,7 @@ describe('GET /prevalences', () => {
         ...baseQuery,
         admin_level: '1',
         admin1: 'MLI.1_1',
-        date: '2024-05-01',
+        date: '2024-05',
         properties: [
           'median',
           'mean',
@@ -95,7 +95,7 @@ describe('GET /prevalences', () => {
         ...baseQuery,
         admin_level: '1',
         admin0: 'MLI',
-        date: '2024-05-01',
+        date: '2024-05',
         properties: 'median,admin1',
       });
 
@@ -118,7 +118,7 @@ describe('GET /prevalences', () => {
         ...baseQuery,
         admin_level: '2',
         admin0: 'MLI',
-        date: '2024-05-01',
+        date: '2024-05',
         properties: 'median,admin2',
       });
 
@@ -151,9 +151,9 @@ describe('GET /prevalences', () => {
     }));
     expect(rows).toHaveLength(25);
     expect(rows).toEqual(expect.arrayContaining([
-      { date: '2023-05-01', median: 0.476 },
-      { date: '2024-05-01', median: 0.287 },
-      { date: '2025-05-01', median: 0.214 },
+      { date: '2023-05', median: 0.476 },
+      { date: '2024-05', median: 0.287 },
+      { date: '2025-05', median: 0.214 },
     ]));
   });
 
@@ -188,7 +188,7 @@ describe('GET /prevalences', () => {
       .query({
         ...baseQuery,
         admin_level: '1',
-        date: '2024-05-01',
+        date: '2024-05',
         properties: 'admin1,password',
       });
 

tests/unit/data.test.ts

@@ -22,7 +22,7 @@ const columnsTypes = {
   admin1: 'VARCHAR',
   gene: 'VARCHAR',
   mutation: 'VARCHAR',
-  date: 'DATE',
+  date: 'VARCHAR',
   collection_day: 'DATE',
   median: 'DOUBLE',
   no_of_informing_surveys: 'INTEGER',
@@ -58,7 +58,7 @@ describe('executeParquetQuery SQL generation', () => {
         properties: 'admin1,median',
         gene: 'crt',
         mutation: '76K',
-        date: '2024-05-01',
+        date: '2024-05',
       },
       '/prevalences',
       admin1ParquetPath,
@@ -72,7 +72,7 @@ describe('executeParquetQuery SQL generation', () => {
     expect(db.bind).toHaveBeenCalledWith({
       gene: 'crt',
       mutation: '76K',
-      date: '2024-05-01',
+      date: '2024-05',
     });
   });
 
@@ -82,8 +82,8 @@ describe('executeParquetQuery SQL generation', () => {
     await executeParquetQuery(
       {
         properties: 'admin1',
-        date_from: '2023-05-01',
-        date_to: '2025-05-01',
+        date_from: '2023-05',
+        date_to: '2025-05',
       },
       '/prevalences',
       admin1ParquetPath,
@@ -95,8 +95,8 @@ describe('executeParquetQuery SQL generation', () => {
       'WHERE p.date >= $date_from AND p.date <= $date_to',
     );
     expect(db.bind).toHaveBeenCalledWith({
-      date_from: '2023-05-01',
-      date_to: '2025-05-01',
+      date_from: '2023-05',
+      date_to: '2025-05',
     });
   });
 

tests/unit/metadata.test.ts

@@ -11,22 +11,22 @@ vi.mock('../../src/queryEngine.ts', () => {
         gene: 'crt',
         mutation: '76K',
         variant: 'crt:76:K',
-        min_date: '2003-05-01',
-        max_date: '2025-05-01',
+        min_date: '2003-05',
+        max_date: '2025-05',
       },
       {
         gene: 'k13',
         mutation: '469Y',
         variant: 'k13:469:Y',
-        min_date: '2020-05-01',
-        max_date: '2024-05-01',
+        min_date: '2020-05',
+        max_date: '2024-05',
       },
       {
         gene: 'k13',
         mutation: '469F',
         variant: 'k13:469:F',
-        min_date: '2021-05-01',
-        max_date: '2023-05-01',
+        min_date: '2021-05',
+        max_date: '2023-05',
       },
     ],
   }));
@@ -48,19 +48,19 @@ describe('getMutationsByGene', () => {
         gene: 'crt',
         mutations: [{
           mutation: '76K',
-          date_range: { start: '2003-05-01', end: '2025-05-01' },
+          date_range: { start: '2003-05', end: '2025-05' },
         }],
       },
       {
         gene: 'k13',
         mutations: [
           {
             mutation: '469Y',
-            date_range: { start: '2020-05-01', end: '2024-05-01' },
+            date_range: { start: '2020-05', end: '2024-05' },
           },
           {
             mutation: '469F',
-            date_range: { start: '2021-05-01', end: '2023-05-01' },
+            date_range: { start: '2021-05', end: '2023-05' },
           },
         ],
       },