From 6e6cfd6fe31ed188855ceecb9309a72e22106ece Mon Sep 17 00:00:00 2001 From: Omer Cohen <639682+omercnet@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:50:49 +0000 Subject: [PATCH 1/2] chore(dev): Compose stack with PostgreSQL 17 and SeaweedFS, database roles and Makefile --- .env.example | 30 +++++++++++++++++++++ .github/workflows/ci.yml | 56 +++++++++++++++++++++++++++++++++++++++ AGENTS.md | 24 +++++++++++------ Makefile | 38 ++++++++++++++++++++++++++ compose.yaml | 44 ++++++++++++++++++++++++++++++ db/init/01-roles.sh | 4 +++ db/init/02-extensions.sql | 6 +++++ db/init/roles.psql | 15 +++++++++++ 8 files changed, 209 insertions(+), 8 deletions(-) create mode 100644 .env.example create mode 100644 Makefile create mode 100644 compose.yaml create mode 100644 db/init/01-roles.sh create mode 100644 db/init/02-extensions.sql create mode 100644 db/init/roles.psql diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..83aac58 --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Local development only. Every value below is a dummy; never reuse in production. +# `make up` copies this file to .env (ignored by git), which Compose reads: the keys below are the one place the +# development file-store keys and database password are written down for Compose (the db/init SQL repeats the password). +COMPOSE_PROJECT_NAME=grimoire +GRIMOIRE_DEV_DB_PASSWORD=grimoire-dev +GRIMOIRE_DEV_S3_ACCESS_KEY=grimoire-dev +GRIMOIRE_DEV_S3_SECRET_KEY=grimoire-dev-secret +GRIMOIRE_DATABASE_URL=postgresql://grimoire_app:grimoire-dev@127.0.0.1:55432/grimoire +GRIMOIRE_DATABASE_MIGRATION_URL=postgresql://grimoire_migrator:grimoire-dev@127.0.0.1:55432/grimoire +GRIMOIRE_S3_ENDPOINT=http://127.0.0.1:8333 +GRIMOIRE_S3_BUCKET=grimoire +GRIMOIRE_S3_ACCESS_KEY=grimoire-dev +GRIMOIRE_S3_SECRET_KEY=grimoire-dev-secret +# GRIMOIRE_DEV_TOKEN_SECRET is not written here on purpose: `make up` generates a random one into .env on first use (a fixed value would be a shared password). +# Empty is safe for local development; a non-local server requires a comma-separated list of lowercase domains. +GRIMOIRE_ALLOWED_EMAIL_DOMAINS= +# Empty selects the default Descope API endpoint; use an HTTP(S) URL only for a custom or local key server. +# Maximum accepted issued-token lifetime: 24 hours; the loader accepts 1 to 86400 seconds. +GRIMOIRE_DESCOPE_MAX_TOKEN_LIFETIME_SECONDS=86400 +GRIMOIRE_DESCOPE_BASE_URL= +# Always empty in this example: this is a secret, and no real or realistic key belongs in the repository. +GRIMOIRE_DESCOPE_MANAGEMENT_KEY= +# Re-check collaboration access every 60 seconds; the loader accepts an integer from 1 to 600. +GRIMOIRE_COLLAB_RECHECK_SECONDS=60 +# Unauthenticated sockets are closed and released after 5 seconds; the loader accepts 1 to 60 seconds. +GRIMOIRE_COLLAB_PREAUTH_TIMEOUT_SECONDS=5 +# Authenticated resources per user: 20 sockets, 50 document sessions, and 2 MiB of updates per second across pages. +GRIMOIRE_COLLAB_MAX_SOCKETS_PER_USER=20 +GRIMOIRE_COLLAB_MAX_DOCUMENTS_PER_USER=50 +GRIMOIRE_COLLAB_BYTES_PER_SECOND_PER_USER=2097152 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2378abb..3b66ff7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,60 @@ jobs: if-no-files-found: ignore retention-days: 7 + compose-config: + name: Compose config + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Create .env and validate the Compose files + run: | + cp .env.example .env + docker compose -p grimoire config --quiet + + integration: + name: Integration tests + needs: changes + if: needs.changes.outputs.code == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ${{ format('runs-on={0}-{1}-integration/runner=4cpu-linux-x64', github.run_id, github.run_attempt) }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: jdx/mise-action@236a21c3b91f5db6d4e174b348674a0f0caac06d # v5.2.9 + with: + version: 2026.11.9 + sha256: f13f3e5ec88807e0161669b3aa07a91d84478cdf1e95b5b92b63f85a25d42395 + install_args: node bun + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.bun/install/cache + key: bun-${{ runner.os }}-${{ hashFiles('bun.lock', 'bunfig.toml') }} + restore-keys: bun-${{ runner.os }}- + - run: bun install --frozen-lockfile + - run: make db-up + - run: bun run build + - run: bun run test:integration + - name: Collect Compose logs + if: failure() + run: docker compose -p grimoire logs --no-color > compose.log + - name: Upload integration-logs + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: integration-logs + path: compose.log + retention-days: 7 + if-no-files-found: ignore + - name: Tear down + if: always() + run: make down + ci-required: name: CI Required runs-on: ubuntu-latest @@ -179,6 +233,8 @@ jobs: - static - unit - licence + - compose-config + - integration steps: - name: Verify every needed job succeeded env: diff --git a/AGENTS.md b/AGENTS.md index 7c873ad..8f1b4c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,9 @@ Run from the repository root. Each line works on `main` today. | Benchmarks | `bun run test:benchmark` | | Licence check (installed packages and the recorded rest) | `bun run licence` | | Licence check of the container base images | `bun run licence:check-images` | +| Start PostgreSQL and the file store (Compose project `grimoire`) | `make db-up` | +| Stop the stack, keep the data | `make down` | +| Stop the stack and delete its data volumes | `make down-volumes` | | Sleep check alone | `node tools/ci/check-test-sleeps.mjs` | | Secret scan, history | `gitleaks git --config .gitleaks.toml --redact .` | | Secret scan, files | `gitleaks dir --config .gitleaks.toml --redact .` | @@ -48,14 +51,16 @@ Run from the repository root. Each line works on `main` today. `mise.lock`. A tool missing from `mise.toml` is added there in a pull request. Today the integration, container and benchmark commands find no tests and exit -with success. They start to matter when the first such tests arrive. +with success. They start to matter when the first such tests arrive. The +integration tests need the database and the file store: run `make db-up` first. The list grows. Each later slice adds its line here: the -exit tests (`make` targets), the Compose stack. Do not document a command that -is not on `main`. +exit tests (`make` targets). Do not document a command that is not on `main`. -CI runs lint, typecheck, build, unit tests, the tool tests and the secret scan. -It does not run the integration, container or benchmark tests, and a pull +CI runs lint, typecheck, build, unit tests, the tool tests, the licence check, +the Compose file check, the integration tests (against the Compose database and +file store) and the secret scan. It does not run the container or benchmark +tests, and a pull request that changes only documentation skips the static and unit jobs. A green `CI Required` is therefore not proof that the other suites pass: run them yourself when your change can affect them. The required check is `CI Required`. @@ -102,7 +107,12 @@ stand in this document. There is no project OMP configuration. sample file (the samples are the `.ts.txt` files in `fixtures/`). - The core never reads `process.env`. Only a config adapter reads it. - Server-only libraries stay out of the browser client and shared libraries. -- Add a fake for every new port, next to the others in `src/testing`. +- Every port gets an in-memory fake next to the others in `src/testing`, no later + than the first code whose tests depend on that port. Today the clock, config, + logger, database, Markdown converter, user store and access store have one. The + auth verifier gets its fake with the auth adapters, and the collaboration host + and page store get theirs (an in-memory page store) with the collaboration + adapter. - A forbidden import is fixed by adding or using a port. Never loosen the rule. ## House rules @@ -172,8 +182,6 @@ Do not trust a claim, including your own. Check on the real thing. ## Docker safety -Applies once Compose files exist in the repository. - - Use only this project's Compose stack, through the `make` targets. Never run raw `docker` commands that touch other containers, images or volumes. - Never prune (`docker system prune`, `docker volume prune` and the like). diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..401fa15 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +# COMPOSE_FILE (for example compose.yaml:compose.offline.yaml) is honoured; the project name is fixed. +COMPOSE := docker compose -p grimoire + +.PHONY: db-up db-down down down-volumes test + +# Compose reads .env for the development passwords and keys; .env.example is the single place they are written down. +# Hand-written: Compose has no "copy a default env file" feature. +.env: + cp .env.example .env + chmod 600 .env + +# Development tokens carry a per-checkout secret (GRIMOIRE_DEV_TOKEN_SECRET): generated on first use, never a fixed default, kept in the ignored .env +# (mode 600). 32 bytes from the kernel's CSPRNG (/dev/urandom), hex encoded (64 characters). An existing value is never rewritten. +# Every target that starts the app or reads its environment depends on this one (compose-settings.test.ts checks `up` reaches it). +.PHONY: dev-secret +dev-secret: .env + @chmod 600 .env + @grep -q '^GRIMOIRE_DEV_TOKEN_SECRET=.' .env || { [ -z "$$(tail -c1 .env)" ] || echo >> .env; echo "GRIMOIRE_DEV_TOKEN_SECRET=$$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')" >> .env; echo "generated GRIMOIRE_DEV_TOKEN_SECRET in .env"; } + +# NOTE: database initialisation scripts (db/init) run only on an EMPTY volume. A database volume created before +# the roles or passwords changed must be reset with `make reset` (or `make down-volumes`, which deletes the data) before the next `make up` or `make db-up`. +db-up: .env dev-secret + $(COMPOSE) up -d --wait --wait-timeout 120 postgres seaweedfs + + +# Stops and removes the containers and the network; the data volumes stay, so `make up` brings the same data back. +down: + $(COMPOSE) --profile tools down --remove-orphans + +db-down: down + +# The only targets that delete data (database and file store volumes). +down-volumes: + $(COMPOSE) --profile tools down -v --remove-orphans + +test: + bun run test + diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..d290159 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,44 @@ +name: grimoire + +services: + postgres: + image: postgres:17@sha256:b60e90042aa6b8dbecd58e224b3e36114dbf8ad754465264c2476ad58a86e45c + environment: + POSTGRES_DB: grimoire + POSTGRES_USER: grimoire_admin + POSTGRES_PASSWORD: ${GRIMOIRE_DEV_DB_PASSWORD} # development only; value in .env (copied from .env.example) + GRIMOIRE_DEV_DB_PASSWORD: ${GRIMOIRE_DEV_DB_PASSWORD} # read by db/init/01-roles.sh + ports: + - "127.0.0.1:55432:5432" + volumes: + - grimoire-pgdata:/var/lib/postgresql/data + - ./db/init:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U grimoire_admin -d grimoire"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + + seaweedfs: + image: chrislusf/seaweedfs:4.54@sha256:4b42c212543a0acd4c0e675215ecdcaf62a186ef98def76c7e12701871222d15 + command: ["mini", "-dir=/data", "-s3", "-ip.bind=0.0.0.0"] + environment: + AWS_ACCESS_KEY_ID: ${GRIMOIRE_DEV_S3_ACCESS_KEY} # development only; value in .env + AWS_SECRET_ACCESS_KEY: ${GRIMOIRE_DEV_S3_SECRET_KEY} # development only; value in .env + ports: + - "127.0.0.1:8333:8333" + volumes: + - grimoire-seaweed:/data + healthcheck: + # Signed ListBuckets must return HTTP 200 with a ListAllMyBucketsResult body: + # proves the S3 gateway is serving and the filer/master behind it answer. + test: ["CMD-SHELL", "curl -sf --aws-sigv4 aws:amz:us-east-1:s3 --user \"$$AWS_ACCESS_KEY_ID:$$AWS_SECRET_ACCESS_KEY\" http://127.0.0.1:8333/ | grep -q ListAllMyBucketsResult"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 3s + +volumes: + grimoire-pgdata: + grimoire-seaweed: diff --git a/db/init/01-roles.sh b/db/init/01-roles.sh new file mode 100644 index 0000000..ee43a98 --- /dev/null +++ b/db/init/01-roles.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Hand-written because the postgres image runs *.sql init files without variables: this passes the shared dev password to psql. +set -eu +psql -v ON_ERROR_STOP=1 -v pw="$GRIMOIRE_DEV_DB_PASSWORD" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -f /docker-entrypoint-initdb.d/roles.psql diff --git a/db/init/02-extensions.sql b/db/init/02-extensions.sql new file mode 100644 index 0000000..d6445a8 --- /dev/null +++ b/db/init/02-extensions.sql @@ -0,0 +1,6 @@ +-- Local development only. Run once by the postgres image's init step (after 01-roles.sh), as the container superuser, against database grimoire. +-- grimoire_migrator has only CONNECT on the database and cannot create extensions; migration 0004 repeats `create extension if not exists citext`, which is a no-op once this ran. +-- Both are trusted extensions. On a managed service (Cloud SQL) the administrator is not a superuser: install them there with a role that may create extensions +-- (cloudsqlsuperuser) before the first `grimoire migrate`. +CREATE EXTENSION IF NOT EXISTS citext; +CREATE EXTENSION IF NOT EXISTS pg_trgm; diff --git a/db/init/roles.psql b/db/init/roles.psql new file mode 100644 index 0000000..dbd9662 --- /dev/null +++ b/db/init/roles.psql @@ -0,0 +1,15 @@ +-- Local development only. Run once by 01-roles.sh (psql -v pw=...), as the container superuser grimoire_admin, against database grimoire. +-- The password is the GRIMOIRE_DEV_DB_PASSWORD value, passed as the psql variable :pw. +CREATE ROLE grimoire_migrator LOGIN PASSWORD :'pw'; +CREATE ROLE grimoire_app LOGIN PASSWORD :'pw'; + +REVOKE ALL ON DATABASE grimoire FROM PUBLIC; +GRANT CONNECT ON DATABASE grimoire TO grimoire_migrator, grimoire_app; + +-- The migration role owns the schema and may change it. +ALTER SCHEMA public OWNER TO grimoire_migrator; +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT USAGE ON SCHEMA public TO grimoire_app; + +-- Runtime role: no default grants. Each migration grants the runtime role exactly what it needs on the tables it creates; +-- Kysely's own bookkeeping tables (kysely_migration, kysely_migration_lock) get no grant. From 9422c9880d40f21e1715f9a0624a29820737d27f Mon Sep 17 00:00:00 2001 From: Omer Cohen <639682+omercnet@users.noreply.github.com> Date: Sat, 21 Nov 2026 13:07:25 +0000 Subject: [PATCH 2/2] fix(dev): the database healthcheck waits for the final server; AGENTS: make rows are not pre-approved --- AGENTS.md | 8 +++++--- compose.yaml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8f1b4c6..bb5a8cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,9 +71,11 @@ yourself when your change can affect them. The required check is `CI Required`. key is explained here. The keys are documented in the Claude Code settings reference. -- `permissions.allow`: exactly the commands in the table above, bare or wrapped - in `mise exec --`, with no extra arguments. They run without asking. Nothing that writes outside - the repository or touches the network is listed. +- `permissions.allow`: the commands in the table above, bare or wrapped in + `mise exec --`, with no extra arguments, except the `make` rows: those pull + images and create or delete Docker volumes, so each use is approved. Allowed + commands run without asking. Nothing that writes outside the repository or + touches the network is listed. - `permissions.deny`: no edits to `bun.lock` (change `package.json` and run `bun install`) or to any `dist/` (generated), and no reading of `.env` files at any depth. - Skills are discovered from `.claude/skills/`; no setting is needed. diff --git a/compose.yaml b/compose.yaml index d290159..b20c140 100644 --- a/compose.yaml +++ b/compose.yaml @@ -14,7 +14,7 @@ services: - grimoire-pgdata:/var/lib/postgresql/data - ./db/init:/docker-entrypoint-initdb.d:ro healthcheck: - test: ["CMD-SHELL", "pg_isready -U grimoire_admin -d grimoire"] + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U grimoire_admin -d grimoire"] interval: 2s timeout: 3s retries: 30