From 36ae138238985f69dd5309e292dad563635b1a7e Mon Sep 17 00:00:00 2001 From: Andy Pols Date: Tue, 28 Jul 2026 14:51:41 +0000 Subject: [PATCH 1/4] feat(db): add backend-agnostic migration runner Signed-off-by: Andrew Pols --- src/db/migrations/index.ts | 70 +++++++++++++ src/db/types.ts | 3 + test/db/migrations/runner.test.ts | 165 ++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 src/db/migrations/index.ts create mode 100644 test/db/migrations/runner.test.ts diff --git a/src/db/migrations/index.ts b/src/db/migrations/index.ts new file mode 100644 index 000000000..7967e9872 --- /dev/null +++ b/src/db/migrations/index.ts @@ -0,0 +1,70 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Sink } from '../types'; + +export interface Migration { + id: string; + up: (sink: Sink) => Promise; + down?: (sink: Sink) => Promise; +} + +const byId = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); + +const assertUniqueIds = (migrations: Migration[]): void => { + const seen = new Set(); + for (const { id } of migrations) { + if (seen.has(id)) { + throw new Error(`Duplicate migration id: ${id}`); + } + seen.add(id); + } +}; + +export const runMigrations = async (sink: Sink, migrations: Migration[]): Promise => { + assertUniqueIds(migrations); + const applied = new Set(await sink.getAppliedMigrations()); + const pending = migrations.filter((m) => !applied.has(m.id)).sort((a, b) => byId(a.id, b.id)); + for (const migration of pending) { + await migration.up(sink); + // record per-migration, not in one batch, so an interrupted run resumes cleanly + await sink.recordMigration(migration.id); + } +}; + +export const rollbackMigrations = async ( + sink: Sink, + migrations: Migration[], + ids: string[], +): Promise => { + const applied = new Set(await sink.getAppliedMigrations()); + const byIdMap = new Map(migrations.map((m) => [m.id, m])); + const ordered = ids + .filter((id) => applied.has(id)) + .sort(byId) + .reverse(); + for (const id of ordered) { + const migration = byIdMap.get(id); + if (!migration) { + throw new Error(`Cannot roll back unknown migration: ${id}`); + } + if (!migration.down) { + throw new Error(`Migration ${id} is irreversible (no down())`); + } + await migration.down(sink); + await sink.unrecordMigration(id); + } +}; diff --git a/src/db/types.ts b/src/db/types.ts index 495dd98b2..476eb5f19 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -175,4 +175,7 @@ export interface Sink { addPublicKey: (username: string, publicKey: PublicKeyRecord) => Promise; removePublicKey: (username: string, fingerprint: string) => Promise; getPublicKeys: (username: string) => Promise; + getAppliedMigrations: () => Promise; + recordMigration: (id: string) => Promise; + unrecordMigration: (id: string) => Promise; } diff --git a/test/db/migrations/runner.test.ts b/test/db/migrations/runner.test.ts new file mode 100644 index 000000000..31ee2af32 --- /dev/null +++ b/test/db/migrations/runner.test.ts @@ -0,0 +1,165 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { runMigrations, rollbackMigrations } from '../../../src/db/migrations'; +import type { Migration } from '../../../src/db/migrations'; +import type { Sink } from '../../../src/db/types'; + +interface FakeSink { + sink: Sink; + applied: () => string[]; +} + +// The runner is backend-agnostic: it only ever calls getAppliedMigrations / +// recordMigration / unrecordMigration plus each migration's up()/down(). +const makeSink = (initiallyApplied: string[] = []): FakeSink => { + const applied = new Set(initiallyApplied); + const sink = { + getAppliedMigrations: async () => [...applied], + recordMigration: async (id: string) => { + applied.add(id); + }, + unrecordMigration: async (id: string) => { + applied.delete(id); + }, + } as unknown as Sink; + return { sink, applied: () => [...applied] }; +}; + +const makeMigration = (id: string, log: string[]): Migration => ({ + id, + up: async () => { + log.push(`up:${id}`); + }, + down: async () => { + log.push(`down:${id}`); + }, +}); + +const A = '20260701-alpha'; +const B = '20260702-bravo'; +const C = '20260703-charlie'; + +describe('runMigrations', () => { + let log: string[]; + + beforeEach(() => { + log = []; + }); + + it('applies every unapplied migration in id order on a fresh db', async () => { + const { sink, applied } = makeSink(); + const migrations = [C, A, B].map((id) => makeMigration(id, log)); // deliberately unsorted + + await runMigrations(sink, migrations); + + expect(log).toEqual([`up:${A}`, `up:${B}`, `up:${C}`]); + expect(applied()).toEqual([A, B, C]); + }); + + it('skips migrations already recorded as applied', async () => { + const { sink } = makeSink([A, B]); + const migrations = [A, B, C].map((id) => makeMigration(id, log)); + + await runMigrations(sink, migrations); + + expect(log).toEqual([`up:${C}`]); + }); + + it('is a no-op when everything is already applied', async () => { + const { sink } = makeSink([A, B, C]); + const migrations = [A, B, C].map((id) => makeMigration(id, log)); + + await runMigrations(sink, migrations); + + expect(log).toEqual([]); + }); + + // The reason this model exists: a migration authored on a parallel branch + // lands with an id that sorts *before* one already applied on main. A single + // "current version" integer would skip it; tracking applied ids does not. + it('applies a lower-id migration merged in after a higher-id one already ran', async () => { + const { sink, applied } = makeSink([C]); + const migrations = [A, B, C].map((id) => makeMigration(id, log)); + + await runMigrations(sink, migrations); + + expect(log).toEqual([`up:${A}`, `up:${B}`]); + expect(applied().sort()).toEqual([A, B, C]); + }); + + it('records each migration as it is applied, not in a single batch at the end', async () => { + const recorded: string[] = []; + const applied = new Set(); + const sink = { + getAppliedMigrations: async () => [...applied], + recordMigration: async (id: string) => { + recorded.push(id); + applied.add(id); + }, + } as unknown as Sink; + + await runMigrations( + sink, + [A, B, C].map((id) => makeMigration(id, log)), + ); + + expect(recorded).toEqual([A, B, C]); + }); + + it('throws when two migrations share an id', async () => { + const { sink } = makeSink(); + const migrations = [makeMigration(A, log), makeMigration(A, log)]; + + await expect(runMigrations(sink, migrations)).rejects.toThrow(/duplicate/i); + }); +}); + +describe('rollbackMigrations', () => { + let log: string[]; + + beforeEach(() => { + log = []; + }); + + it('rolls back the given applied migrations in reverse id order', async () => { + const { sink, applied } = makeSink([A, B, C]); + const migrations = [A, B, C].map((id) => makeMigration(id, log)); + + await rollbackMigrations(sink, migrations, [B, C]); + + expect(log).toEqual([`down:${C}`, `down:${B}`]); + expect(applied()).toEqual([A]); + }); + + it('ignores ids that were never applied', async () => { + const { sink, applied } = makeSink([A]); + const migrations = [A, B, C].map((id) => makeMigration(id, log)); + + await rollbackMigrations(sink, migrations, [B, C]); + + expect(log).toEqual([]); + expect(applied()).toEqual([A]); + }); + + it('throws when asked to roll back a migration that has no down()', async () => { + const { sink } = makeSink([A]); + const migrations: Migration[] = [{ id: A, up: async () => {} }]; + + await expect(rollbackMigrations(sink, migrations, [A])).rejects.toThrow(/irreversible|down/i); + }); +}); From cdc658d38950fc73df5d86f9df4703b00118bc4c Mon Sep 17 00:00:00 2001 From: Andy Pols Date: Tue, 28 Jul 2026 15:11:32 +0000 Subject: [PATCH 2/4] feat(db): add migration store to file and mongo backends Signed-off-by: Andrew Pols --- src/db/file/index.ts | 3 + src/db/file/migrations.ts | 68 ++++++++++++++++++++ src/db/mongo/index.ts | 3 + src/db/mongo/migrations.ts | 35 ++++++++++ test/db/file/migrations.test.ts | 68 ++++++++++++++++++++ test/db/mongo/migrations.integration.test.ts | 65 +++++++++++++++++++ test/setup-integration.ts | 2 +- 7 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 src/db/file/migrations.ts create mode 100644 src/db/mongo/migrations.ts create mode 100644 test/db/file/migrations.test.ts create mode 100644 test/db/mongo/migrations.integration.test.ts diff --git a/src/db/file/index.ts b/src/db/file/index.ts index 0693e860e..2d4d7def7 100644 --- a/src/db/file/index.ts +++ b/src/db/file/index.ts @@ -18,9 +18,12 @@ import * as users from './users'; import * as repo from './repo'; import * as pushes from './pushes'; import * as helper from './helper'; +import * as migrations from './migrations'; export const { getSessionStore } = helper; +export const { getAppliedMigrations, recordMigration, unrecordMigration } = migrations; + export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; export const { diff --git a/src/db/file/migrations.ts b/src/db/file/migrations.ts new file mode 100644 index 000000000..be5ce4b15 --- /dev/null +++ b/src/db/file/migrations.ts @@ -0,0 +1,68 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Datastore from '@seald-io/nedb'; + +const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day + +// export for testing purposes +export let db: Datastore; +if (process.env.NODE_ENV === 'test') { + db = new Datastore({ inMemoryOnly: true, autoload: true }); +} else { + db = new Datastore({ filename: './.data/db/migrations.db', autoload: true }); +} +db.ensureIndex({ fieldName: 'id', unique: true }); +db.setAutocompactionInterval(COMPACTION_INTERVAL); + +export const getAppliedMigrations = (): Promise => { + return new Promise((resolve, reject) => { + db.find({}, (err: Error | null, docs: { id: string }[]) => { + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(docs.map((doc) => doc.id)); + } + }); + }); +}; + +export const recordMigration = (id: string): Promise => { + return new Promise((resolve, reject) => { + db.update({ id }, { id }, { upsert: true }, (err) => { + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +}; + +export const unrecordMigration = (id: string): Promise => { + return new Promise((resolve, reject) => { + db.remove({ id }, {}, (err) => { + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +}; diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index e5b0b9f25..4ce1e8571 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -18,9 +18,12 @@ import * as helper from './helper'; import * as pushes from './pushes'; import * as repo from './repo'; import * as users from './users'; +import * as migrations from './migrations'; export const { getSessionStore } = helper; +export const { getAppliedMigrations, recordMigration, unrecordMigration } = migrations; + export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; export const { diff --git a/src/db/mongo/migrations.ts b/src/db/mongo/migrations.ts new file mode 100644 index 000000000..2bd27f822 --- /dev/null +++ b/src/db/mongo/migrations.ts @@ -0,0 +1,35 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { connect } from './helper'; + +const collectionName = 'migrations'; + +export const getAppliedMigrations = async (): Promise => { + const collection = await connect(collectionName); + const docs = await collection.find({}).toArray(); + return docs.map((doc) => doc.id as string); +}; + +export const recordMigration = async (id: string): Promise => { + const collection = await connect(collectionName); + await collection.updateOne({ id }, { $set: { id } }, { upsert: true }); +}; + +export const unrecordMigration = async (id: string): Promise => { + const collection = await connect(collectionName); + await collection.deleteOne({ id }); +}; diff --git a/test/db/file/migrations.test.ts b/test/db/file/migrations.test.ts new file mode 100644 index 000000000..f6a3b1cb8 --- /dev/null +++ b/test/db/file/migrations.test.ts @@ -0,0 +1,68 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import * as migrations from '../../../src/db/file/migrations'; + +describe('File DB migrations store', () => { + beforeEach(async () => { + await new Promise((resolve, reject) => { + migrations.db.remove({}, { multi: true }, (err) => (err ? reject(err) : resolve())); + }); + }); + + it('returns no applied migrations for a fresh store', async () => { + expect(await migrations.getAppliedMigrations()).toEqual([]); + }); + + it('records a migration so it is reported as applied', async () => { + await migrations.recordMigration('20260701-alpha'); + + expect(await migrations.getAppliedMigrations()).toEqual(['20260701-alpha']); + }); + + it('reports multiple applied migrations', async () => { + await migrations.recordMigration('20260701-alpha'); + await migrations.recordMigration('20260702-bravo'); + + expect((await migrations.getAppliedMigrations()).sort()).toEqual([ + '20260701-alpha', + '20260702-bravo', + ]); + }); + + it('records the same migration idempotently', async () => { + await migrations.recordMigration('20260701-alpha'); + await migrations.recordMigration('20260701-alpha'); + + expect(await migrations.getAppliedMigrations()).toEqual(['20260701-alpha']); + }); + + it('unrecords a migration so it is no longer applied', async () => { + await migrations.recordMigration('20260701-alpha'); + await migrations.recordMigration('20260702-bravo'); + + await migrations.unrecordMigration('20260701-alpha'); + + expect(await migrations.getAppliedMigrations()).toEqual(['20260702-bravo']); + }); + + it('unrecording an unknown migration is a no-op', async () => { + await migrations.unrecordMigration('does-not-exist'); + + expect(await migrations.getAppliedMigrations()).toEqual([]); + }); +}); diff --git a/test/db/mongo/migrations.integration.test.ts b/test/db/mongo/migrations.integration.test.ts new file mode 100644 index 000000000..48bb106ad --- /dev/null +++ b/test/db/mongo/migrations.integration.test.ts @@ -0,0 +1,65 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect } from 'vitest'; +import { + getAppliedMigrations, + recordMigration, + unrecordMigration, +} from '../../../src/db/mongo/migrations'; + +const shouldRunMongoTests = process.env.RUN_MONGO_TESTS === 'true'; + +describe.runIf(shouldRunMongoTests)('MongoDB migrations store', () => { + it('returns no applied migrations for a fresh store', async () => { + expect(await getAppliedMigrations()).toEqual([]); + }); + + it('records a migration so it is reported as applied', async () => { + await recordMigration('20260701-alpha'); + + expect(await getAppliedMigrations()).toEqual(['20260701-alpha']); + }); + + it('reports multiple applied migrations', async () => { + await recordMigration('20260701-alpha'); + await recordMigration('20260702-bravo'); + + expect((await getAppliedMigrations()).sort()).toEqual(['20260701-alpha', '20260702-bravo']); + }); + + it('records the same migration idempotently', async () => { + await recordMigration('20260701-alpha'); + await recordMigration('20260701-alpha'); + + expect(await getAppliedMigrations()).toEqual(['20260701-alpha']); + }); + + it('unrecords a migration so it is no longer applied', async () => { + await recordMigration('20260701-alpha'); + await recordMigration('20260702-bravo'); + + await unrecordMigration('20260701-alpha'); + + expect(await getAppliedMigrations()).toEqual(['20260702-bravo']); + }); + + it('unrecording an unknown migration is a no-op', async () => { + await unrecordMigration('does-not-exist'); + + expect(await getAppliedMigrations()).toEqual([]); + }); +}); diff --git a/test/setup-integration.ts b/test/setup-integration.ts index 4b61a5222..c220668b5 100644 --- a/test/setup-integration.ts +++ b/test/setup-integration.ts @@ -20,7 +20,7 @@ import { resetConnection } from '../src/db/mongo/helper'; import { invalidateCache } from '../src/config'; const DEFAULT_TEST_DB_NAME = 'git-proxy-test'; -const COLLECTIONS = ['repos', 'users', 'pushes', 'user_session']; +const COLLECTIONS = ['repos', 'users', 'pushes', 'user_session', 'migrations']; let client: MongoClient | null = null; From 446616b241d05dea8bea08ee790d89a905d72ddd Mon Sep 17 00:00:00 2001 From: Andy Pols Date: Tue, 28 Jul 2026 15:25:56 +0000 Subject: [PATCH 3/4] feat(db): run migrations on startup via registry Signed-off-by: Andrew Pols --- src/db/index.ts | 3 +++ src/db/migrations/registry.ts | 19 ++++++++++++++++++ src/proxy/index.ts | 4 +++- test/db/db.test.ts | 13 ++++++++++++ test/db/migrations/registry.test.ts | 31 +++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/db/migrations/registry.ts create mode 100644 test/db/migrations/registry.test.ts diff --git a/src/db/index.ts b/src/db/index.ts index 8c6a19c72..b02f410fa 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -25,6 +25,8 @@ import MongoDBStore from 'connect-mongo'; import { CompletedAttestation, Rejection } from '../proxy/processors/types'; import { processGitUrl } from '../proxy/routes/helper'; import { initializeFolders } from './file/helper'; +import { runMigrations as applyMigrations } from './migrations'; +import { migrations } from './migrations/registry'; let _sink: Sink | null = null; @@ -178,6 +180,7 @@ export const canUserCancelPush = async (id: string, user: string) => { } }; +export const runMigrations = (): Promise => applyMigrations(start(), migrations); export const getSessionStore = (): MongoDBStore | undefined => start().getSessionStore(); export const getPushes = (query: Partial): Promise => start().getPushes(query); export const writeAudit = (action: Action): Promise => start().writeAudit(action); diff --git a/src/db/migrations/registry.ts b/src/db/migrations/registry.ts new file mode 100644 index 000000000..6a774e1e1 --- /dev/null +++ b/src/db/migrations/registry.ts @@ -0,0 +1,19 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Migration } from './index'; + +export const migrations: Migration[] = []; diff --git a/src/proxy/index.ts b/src/proxy/index.ts index 041fab7ba..ae45b13de 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -32,7 +32,7 @@ import { getServerPort, getHttpsServerPort, } from '../config'; -import { addUserCanAuthorise, addUserCanPush, createRepo, getRepos } from '../db'; +import { addUserCanAuthorise, addUserCanPush, createRepo, getRepos, runMigrations } from '../db'; import { PluginLoader } from '../plugin'; import chain from './chain'; import { Repo } from '../db/types'; @@ -73,6 +73,8 @@ export class Proxy { console.log('Creating the .remote dir...'); fs.mkdirSync(remoteDir); + await runMigrations(); + const plugins = getPlugins(); const pluginLoader = new PluginLoader(plugins); await pluginLoader.load(); diff --git a/test/db/db.test.ts b/test/db/db.test.ts index b245be501..c13340a7b 100644 --- a/test/db/db.test.ts +++ b/test/db/db.test.ts @@ -19,10 +19,16 @@ import { SAMPLE_REPO } from '../../src/proxy/constants'; vi.mock('../../src/db/mongo', () => ({ getRepoByUrl: vi.fn(), + getAppliedMigrations: vi.fn().mockResolvedValue([]), + recordMigration: vi.fn(), + unrecordMigration: vi.fn(), })); vi.mock('../../src/db/file', () => ({ getRepoByUrl: vi.fn(), + getAppliedMigrations: vi.fn().mockResolvedValue([]), + recordMigration: vi.fn(), + unrecordMigration: vi.fn(), })); vi.mock('../../src/config', () => ({ @@ -41,6 +47,13 @@ describe('db', () => { vi.restoreAllMocks(); }); + describe('runMigrations', () => { + it('runs the registry against the configured backend sink', async () => { + await db.runMigrations(); + expect(mongo.getAppliedMigrations).toHaveBeenCalled(); + }); + }); + describe('isUserPushAllowed', () => { it('returns true if user is in canPush', async () => { vi.mocked(mongo.getRepoByUrl).mockResolvedValue({ diff --git a/test/db/migrations/registry.test.ts b/test/db/migrations/registry.test.ts new file mode 100644 index 000000000..ecc01a7f8 --- /dev/null +++ b/test/db/migrations/registry.test.ts @@ -0,0 +1,31 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect } from 'vitest'; +import { migrations } from '../../../src/db/migrations/registry'; + +describe('migration registry', () => { + it('has a unique id for every migration', () => { + const ids = migrations.map((m) => m.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + it('gives every migration an up()', () => { + for (const migration of migrations) { + expect(typeof migration.up).toBe('function'); + } + }); +}); From 3759c3138cb7bd652826ed5490a0b119ef96fcd2 Mon Sep 17 00:00:00 2001 From: Andy Pols Date: Tue, 28 Jul 2026 16:10:44 +0000 Subject: [PATCH 4/4] feat(db): add updateRepo, deriveCreatedAt and migration template Signed-off-by: Andrew Pols --- src/db/file/index.ts | 4 +- src/db/file/migrations.ts | 2 + src/db/file/repo.ts | 28 +++++++++ src/db/migrations/example.ts | 61 ++++++++++++++++++ src/db/mongo/index.ts | 4 +- src/db/mongo/migrations.ts | 9 +++ src/db/mongo/repo.ts | 17 +++++ src/db/types.ts | 2 + test/db/file/repo.test.ts | 39 ++++++++++++ test/db/migrations/example.test.ts | 86 ++++++++++++++++++++++++++ test/db/mongo/deriveCreatedAt.test.ts | 32 ++++++++++ test/db/mongo/repo.integration.test.ts | 32 ++++++++++ 12 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 src/db/migrations/example.ts create mode 100644 test/db/migrations/example.test.ts create mode 100644 test/db/mongo/deriveCreatedAt.test.ts diff --git a/src/db/file/index.ts b/src/db/file/index.ts index 2d4d7def7..25cb97c46 100644 --- a/src/db/file/index.ts +++ b/src/db/file/index.ts @@ -22,7 +22,8 @@ import * as migrations from './migrations'; export const { getSessionStore } = helper; -export const { getAppliedMigrations, recordMigration, unrecordMigration } = migrations; +export const { getAppliedMigrations, recordMigration, unrecordMigration, deriveCreatedAt } = + migrations; export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; @@ -32,6 +33,7 @@ export const { getRepoByUrl, getRepoById, createRepo, + updateRepo, addUserCanPush, addUserCanAuthorise, removeUserCanPush, diff --git a/src/db/file/migrations.ts b/src/db/file/migrations.ts index be5ce4b15..801f84d7c 100644 --- a/src/db/file/migrations.ts +++ b/src/db/file/migrations.ts @@ -28,6 +28,8 @@ if (process.env.NODE_ENV === 'test') { db.ensureIndex({ fieldName: 'id', unique: true }); db.setAutocompactionInterval(COMPACTION_INTERVAL); +export const deriveCreatedAt = (): string | undefined => undefined; + export const getAppliedMigrations = (): Promise => { return new Promise((resolve, reject) => { db.find({}, (err: Error | null, docs: { id: string }[]) => { diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 76cc8be9b..41873b0b6 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -119,6 +119,34 @@ export const createRepo = async (repo: Repo): Promise => { }); }; +export const updateRepo = async (repo: Partial): Promise => { + const { _id, ...fields } = repo; + const set: Record = {}; + const unset: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (value === undefined) unset[key] = true; + else set[key] = value; + } + const modifier: Record = {}; + if (Object.keys(set).length > 0) modifier.$set = set; + if (Object.keys(unset).length > 0) modifier.$unset = unset; + + return new Promise((resolve, reject) => { + if (Object.keys(modifier).length === 0) { + resolve(); + return; + } + db.update({ _id: _id }, modifier, { multi: false, upsert: false }, (err) => { + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); +}; + export const addUserCanPush = async (_id: string, user: string): Promise => { user = user.toLowerCase(); const repo = await getRepoById(_id); diff --git a/src/db/migrations/example.ts b/src/db/migrations/example.ts new file mode 100644 index 000000000..271f247ef --- /dev/null +++ b/src/db/migrations/example.ts @@ -0,0 +1,61 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Template for a real migration. Copy this file, give it a timestamped id, and +// add it to registry.ts to make it run. It is intentionally NOT registered. +// +// It shows the pattern: read through the sink, use a backend capability +// (deriveCreatedAt) that each backend answers as best it can, write through the +// sink. Backfilling dateCreated/lastModified on repos is the worked example. + +import type { Repo } from '../types'; +import type { Migration } from './index'; + +export const exampleMigration: Migration = { + id: '20260728-example-add-repo-timestamps', + + up: async (sink) => { + const repos = await sink.getRepos(); + for (const repo of repos) { + const record = repo as unknown as Record; + if (record.dateCreated || !repo._id) { + continue; + } + const created = sink.deriveCreatedAt(repo._id) ?? new Date().toISOString(); + // cast: dateCreated/lastModified are new fields not yet on the Repo type + await sink.updateRepo({ + _id: repo._id, + dateCreated: created, + lastModified: created, + } as Partial); + } + }, + + down: async (sink) => { + const repos = await sink.getRepos(); + for (const repo of repos) { + if (!repo._id) { + continue; + } + + await sink.updateRepo({ + _id: repo._id, + dateCreated: undefined, + lastModified: undefined, + } as Partial); + } + }, +}; diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index 4ce1e8571..2a85c8a20 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -22,7 +22,8 @@ import * as migrations from './migrations'; export const { getSessionStore } = helper; -export const { getAppliedMigrations, recordMigration, unrecordMigration } = migrations; +export const { getAppliedMigrations, recordMigration, unrecordMigration, deriveCreatedAt } = + migrations; export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; @@ -32,6 +33,7 @@ export const { getRepoByUrl, getRepoById, createRepo, + updateRepo, addUserCanPush, addUserCanAuthorise, removeUserCanPush, diff --git a/src/db/mongo/migrations.ts b/src/db/mongo/migrations.ts index 2bd27f822..4d7073357 100644 --- a/src/db/mongo/migrations.ts +++ b/src/db/mongo/migrations.ts @@ -14,10 +14,19 @@ * limitations under the License. */ +import { ObjectId } from 'mongodb'; import { connect } from './helper'; const collectionName = 'migrations'; +export const deriveCreatedAt = (id: string): string | undefined => { + try { + return new ObjectId(id).getTimestamp().toISOString(); + } catch { + return undefined; + } +}; + export const getAppliedMigrations = async (): Promise => { const collection = await connect(collectionName); const docs = await collection.find({}).toArray(); diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index b2ec4f4df..ade3d24cd 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -56,6 +56,23 @@ export const createRepo = async (repo: Repo): Promise => { return repo; }; +export const updateRepo = async (repo: Partial): Promise => { + const { _id, ...fields } = repo; + const collection = await connect(collectionName); + const set: Record = {}; + const unset: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (value === undefined) unset[key] = ''; + else set[key] = value; + } + const update: Record = {}; + if (Object.keys(set).length > 0) update.$set = set; + if (Object.keys(unset).length > 0) update.$unset = unset; + if (Object.keys(update).length > 0) { + await collection.updateOne({ _id: new ObjectId(_id as string) }, update); + } +}; + export const addUserCanPush = async (_id: string, user: string): Promise => { user = user.toLowerCase(); const collection = await connect(collectionName); diff --git a/src/db/types.ts b/src/db/types.ts index 476eb5f19..e8791ccd3 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -158,6 +158,7 @@ export interface Sink { getRepoByUrl: (url: string) => Promise; getRepoById: (_id: string) => Promise; createRepo: (repo: Repo) => Promise; + updateRepo: (repo: Partial) => Promise; addUserCanPush: (_id: string, user: string) => Promise; addUserCanAuthorise: (_id: string, user: string) => Promise; removeUserCanPush: (_id: string, user: string) => Promise; @@ -178,4 +179,5 @@ export interface Sink { getAppliedMigrations: () => Promise; recordMigration: (id: string) => Promise; unrecordMigration: (id: string) => Promise; + deriveCreatedAt: (id: string) => string | undefined; } diff --git a/test/db/file/repo.test.ts b/test/db/file/repo.test.ts index 8b802e61f..524264b88 100644 --- a/test/db/file/repo.test.ts +++ b/test/db/file/repo.test.ts @@ -86,4 +86,43 @@ describe('File DB', () => { ).rejects.toThrow('DB error'); }); }); + + describe('updateRepo', () => { + it('sets the given fields on the stored repo', async () => { + const created = await repoModule.createRepo( + new Repo('proj', 'update-me', 'https://example.com/update-me.git'), + ); + + await repoModule.updateRepo({ + _id: created._id, + dateCreated: '2020-01-01T00:00:00.000Z', + } as Partial); + + const fetched = (await repoModule.getRepoById(created._id!)) as Record< + string, + unknown + > | null; + expect(fetched?.dateCreated).toBe('2020-01-01T00:00:00.000Z'); + expect(fetched?.name).toBe('update-me'); + }); + + it('removes fields passed as undefined', async () => { + const created = await repoModule.createRepo( + new Repo('proj', 'unset-me', 'https://example.com/unset-me.git'), + ); + await repoModule.updateRepo({ + _id: created._id, + dateCreated: '2020-01-01T00:00:00.000Z', + } as Partial); + + await repoModule.updateRepo({ _id: created._id, dateCreated: undefined } as Partial); + + const fetched = (await repoModule.getRepoById(created._id!)) as Record< + string, + unknown + > | null; + expect(fetched?.dateCreated).toBeUndefined(); + expect(fetched?.name).toBe('unset-me'); + }); + }); }); diff --git a/test/db/migrations/example.test.ts b/test/db/migrations/example.test.ts new file mode 100644 index 000000000..0ca09d69b --- /dev/null +++ b/test/db/migrations/example.test.ts @@ -0,0 +1,86 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { exampleMigration } from '../../../src/db/migrations/example'; +import type { Sink } from '../../../src/db/types'; + +type RepoRecord = Record & { _id: string }; + +const makeSink = (repos: RepoRecord[], deriveCreatedAt: (id: string) => string | undefined) => { + const store = new Map(repos.map((r) => [r._id, { ...r }])); + const updateRepo = vi.fn(async (repo: Record) => { + const { _id, ...fields } = repo; + Object.assign(store.get(_id as string) as RepoRecord, fields); + }); + const sink = { + getRepos: async () => [...store.values()], + updateRepo, + deriveCreatedAt, + } as unknown as Sink; + return { sink, updateRepo, get: (id: string) => store.get(id) }; +}; + +describe('example migration', () => { + it('sets dateCreated from the backend-derived value and mirrors it to lastModified', async () => { + const { sink, get } = makeSink([{ _id: 'r1' }], () => '2020-01-01T00:00:00.000Z'); + + await exampleMigration.up(sink); + + expect(get('r1')?.dateCreated).toBe('2020-01-01T00:00:00.000Z'); + expect(get('r1')?.lastModified).toBe('2020-01-01T00:00:00.000Z'); + }); + + it('falls back to a valid timestamp when the backend cannot derive one', async () => { + const { sink, get } = makeSink([{ _id: 'r1' }], () => undefined); + + await exampleMigration.up(sink); + + const created = get('r1')?.dateCreated; + expect(created).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/); + expect(get('r1')?.lastModified).toBe(created); + }); + + it('leaves repos that already have a dateCreated untouched', async () => { + const { sink, updateRepo, get } = makeSink( + [{ _id: 'r1', dateCreated: '2019-05-05T00:00:00.000Z' }], + () => '2020-01-01T00:00:00.000Z', + ); + + await exampleMigration.up(sink); + + expect(updateRepo).not.toHaveBeenCalled(); + expect(get('r1')?.dateCreated).toBe('2019-05-05T00:00:00.000Z'); + }); + + it('down removes the timestamps it added', async () => { + const { sink, get } = makeSink( + [ + { + _id: 'r1', + dateCreated: '2020-01-01T00:00:00.000Z', + lastModified: '2020-01-01T00:00:00.000Z', + }, + ], + () => undefined, + ); + + await exampleMigration.down!(sink); + + expect(get('r1')?.dateCreated).toBeUndefined(); + expect(get('r1')?.lastModified).toBeUndefined(); + }); +}); diff --git a/test/db/mongo/deriveCreatedAt.test.ts b/test/db/mongo/deriveCreatedAt.test.ts new file mode 100644 index 000000000..cf74566e0 --- /dev/null +++ b/test/db/mongo/deriveCreatedAt.test.ts @@ -0,0 +1,32 @@ +/** + * Copyright 2026 GitProxy Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect } from 'vitest'; +import { ObjectId } from 'mongodb'; +import { deriveCreatedAt } from '../../../src/db/mongo/migrations'; + +describe('mongo deriveCreatedAt', () => { + it('derives the creation time embedded in an ObjectId', () => { + const when = new Date('2021-06-15T12:00:00.000Z'); + const id = ObjectId.createFromTime(when.getTime() / 1000).toHexString(); + + expect(deriveCreatedAt(id)).toBe(when.toISOString()); + }); + + it('returns undefined for an id that is not an ObjectId', () => { + expect(deriveCreatedAt('not-an-object-id')).toBeUndefined(); + }); +}); diff --git a/test/db/mongo/repo.integration.test.ts b/test/db/mongo/repo.integration.test.ts index 5652d2d38..f9008f172 100644 --- a/test/db/mongo/repo.integration.test.ts +++ b/test/db/mongo/repo.integration.test.ts @@ -26,6 +26,7 @@ import { removeUserCanPush, removeUserCanAuthorise, deleteRepo, + updateRepo, } from '../../../src/db/mongo/repo'; import { Repo } from '../../../src/db/types'; @@ -164,6 +165,37 @@ describe.runIf(shouldRunMongoTests)('MongoDB Repo Integration Tests', () => { }); }); + describe('updateRepo', () => { + it('sets the given fields on the stored repo', async () => { + const created = await createRepo(createTestRepo({ name: 'update-fields-repo' })); + + await updateRepo({ + _id: created._id, + dateCreated: '2020-01-01T00:00:00.000Z', + lastModified: '2020-01-01T00:00:00.000Z', + } as Partial); + + const fetched = (await getRepoById(created._id!)) as Record | null; + expect(fetched?.dateCreated).toBe('2020-01-01T00:00:00.000Z'); + expect(fetched?.lastModified).toBe('2020-01-01T00:00:00.000Z'); + expect(fetched?.name).toBe('update-fields-repo'); + }); + + it('removes fields passed as undefined', async () => { + const created = await createRepo(createTestRepo({ name: 'unset-fields-repo' })); + await updateRepo({ + _id: created._id, + dateCreated: '2020-01-01T00:00:00.000Z', + } as Partial); + + await updateRepo({ _id: created._id, dateCreated: undefined } as Partial); + + const fetched = (await getRepoById(created._id!)) as Record | null; + expect(fetched?.dateCreated).toBeUndefined(); + expect(fetched?.name).toBe('unset-fields-repo'); + }); + }); + describe('deleteRepo', () => { it('should delete a repo by id', async () => { const repo = createTestRepo({ name: 'delete-test-repo' });