diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8dbfae..c9de1f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -284,3 +284,71 @@ jobs: # not ship the code this workflow just tested. test "$ver" != "MISSING" -a "$ver" != "None" + publish-image: + name: push this commit's image to GHCR for the NAS + # The Synology does not build anything: it pulls. Watchtower on the NAS + # polls ghcr.io/...:latest and recreates the broker container when the + # digest moves. Same gate as the Render deploy -- the suites and the image + # smoke test above must pass first -- so the NAS never pulls a commit that + # failed CI. :sha- is pushed alongside :latest so a roll back is + # just pinning the compose file to an older tag. + needs: [sqlite, docker, version] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + run: | + # GHCR requires a lowercase repository path; the org name is not. + image="ghcr.io/${GITHUB_REPOSITORY,,}" + docker build --build-arg CASEBROKER_COMMIT="${{ github.sha }}" \ + -t "$image:latest" -t "$image:sha-${{ github.sha }}" . + docker push "$image:sha-${{ github.sha }}" + docker push "$image:latest" + + nas-deploy-check: + name: confirm the NAS picked up this commit + # Pull-based, so there is no deploy to trigger: wait for Watchtower's next + # poll and then check that /healthz reports THIS commit -- the same "is MY + # commit live" question the Render job asks, never "is something live". + # Skipped until the repo variable NAS_BROKER_URL is set, so this cannot go + # red before the NAS deployment exists. + # + # A newer push superseding this one is reported and passed rather than + # failed: the run for that newer commit is what verifies it. + needs: [publish-image] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.NAS_BROKER_URL != '' + env: + NAS_BROKER_URL: ${{ vars.NAS_BROKER_URL }} + steps: + - name: Wait for /healthz to report this commit + run: | + want="${GITHUB_SHA:0:8}" + # Watchtower polls every 10 minutes; allow two polls plus a restart. + for i in $(seq 1 50); do + # Never print the body: like the Render check, it carries + # infrastructure details that do not belong in public logs. + code=$(curl -s -o /tmp/h.json -w "%{http_code}" "$NAS_BROKER_URL/healthz" || true) + got=$(python3 -c "import json; print(json.load(open('/tmp/h.json')).get('commit'))" 2>/dev/null || echo none) + ok=$(python3 -c "import json; print(json.load(open('/tmp/h.json')).get('ok'))" 2>/dev/null || echo none) + echo "[$i] http=$code ok=$ok commit=$got (want $want)" + if [ "$code" = "200" ] && [ "$ok" = "True" ] && [ "$got" = "$want" ]; then + echo "NAS is serving $want"; exit 0 + fi + sleep 30 + done + head=$(git ls-remote "https://github.com/${GITHUB_REPOSITORY}" refs/heads/main | cut -c1-40) + if [ "$head" != "$GITHUB_SHA" ]; then + echo "::notice::main moved on to ${head:0:8} before the NAS picked up $want; that run verifies it" + exit 0 + fi + echo "::error::NAS never reported commit $want within 25 minutes"; exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3afb10e..ad45589 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,18 @@ The format follows [Keep a Changelog](https://keepachangelog.com). ## [Unreleased] +## [0.27.0] - 2026-10-04 + +### Added +- **Self-hosted deployment on the lab Synology** (`deploy/synology/`). Postgres and the broker run + as one Container Manager project; DSM's reverse proxy fronts it at ``. + The README there is the runbook, including moving the campaign off Supabase. +- **CI publishes the image to GHCR** (`publish-image`): every push to `main` that passes the suites + and the image smoke test pushes `ghcr.io/sustainableurbansystemslab/casebroker:latest` and + `:sha-`. Watchtower on the NAS pulls it; `nas-deploy-check` then waits for `/healthz` to + report that commit. It is skipped until the repo variable `NAS_BROKER_URL` is set. +- The image takes a `CASEBROKER_COMMIT` build arg, so `/healthz` names its commit off Render too. + ## [0.26.1] - 2026-10-01 ### Changed diff --git a/Dockerfile b/Dockerfile index 0326be6..ae70cd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,12 @@ RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs; INSTALL # nothing said so. Production points CASEBROKER_DB at Supabase Postgres; with # state external the service needs no disk at all. app.py warns loudly on # startup if this resolves to SQLite. +# The commit this image was built from, for /healthz. Render sets +# RENDER_GIT_COMMIT itself; an image pulled from GHCR has nothing else to say +# which commit it is, so CI passes it as a build arg. Empty (a local build) +# falls through to the other sources in app._commit(). +ARG CASEBROKER_COMMIT="" +ENV CASEBROKER_COMMIT=${CASEBROKER_COMMIT} EXPOSE 8000 HEALTHCHECK --interval=30s --timeout=5s CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/healthz').status==200 else 1)" CMD ["uvicorn", "casebroker.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/deploy/synology/.env.example b/deploy/synology/.env.example new file mode 100644 index 0000000..c916dfd --- /dev/null +++ b/deploy/synology/.env.example @@ -0,0 +1,15 @@ +# Copy to /volume1/docker/casebroker/.env on the NAS (next to compose.yaml). +# Never commit the filled-in copy. + +# The password the EXISTING data directory was initialised with (the old +# standalone `postgres` project's POSTGRES_PASSWORD). +POSTGRES_PASSWORD= +# Only if POSTGRES_PASSWORD contains @ : / ? # [ ] % -- the same password, +# percent-encoded, for the connection URL. Leave unset otherwise. +#POSTGRES_PASSWORD_URLENCODED= + +# Same value(s) as CASEBROKER_WRITE_TOKENS on the Render service, so the +# running workers keep authenticating after cut-over. +CASEBROKER_WRITE_TOKENS= +#CASEBROKER_READ_TOKENS= +#CASEBROKER_SETUP_TOKEN= diff --git a/deploy/synology/README.md b/deploy/synology/README.md new file mode 100644 index 0000000..fdeaa15 --- /dev/null +++ b/deploy/synology/README.md @@ -0,0 +1,85 @@ +# Broker on the lab Synology + +Postgres and the broker run as one Container Manager project on the NAS +(`compose.yaml` here). A push to `main` deploys itself: + +``` +push main ─► CI: tests, image smoke test ─► publish-image: ghcr.io/…/casebroker:latest + :sha- + │ +NAS: Watchtower (polls every 10 min, label-enabled) ─► pulls :latest, recreates `casebroker` + │ +CI: nas-deploy-check ─► polls $NAS_BROKER_URL/healthz until `commit` is this push +``` + +Traffic: `` ─► Cloudflare (proxied) ─► DSM reverse +proxy :443 ─► `127.0.0.1:8010` ─► broker ─► `postgres:5432`. + +## One-time setup + +1. **GHCR package public.** After the first `publish-image` run: GitHub ▸ org + ▸ Packages ▸ `casebroker` ▸ Package settings ▸ Change visibility ▸ Public. + The repo is public already; this only spares the NAS a registry login. +2. **Project folder.** `/volume1/docker/casebroker/` with `compose.yaml` and a + filled-in `.env` (from `.env.example`). +3. **Replace the standalone `postgres` project.** It uses the same data + directory and the same container name. Container Manager ▸ Project ▸ + `postgres` ▸ Stop, then Delete (the project only — the data in + `/volume1/docker/postgres/data` stays). **Never run both**: two servers on + one data directory corrupt it. +4. **Create the project.** Container Manager ▸ Project ▸ Create ▸ path + `/docker/casebroker` ▸ use the existing `compose.yaml`. Skip the web portal + step it offers. +5. **Reverse proxy.** Control Panel ▸ Login Portal ▸ Advanced ▸ Reverse Proxy ▸ + Create: source HTTPS `` :443, destination HTTP + `localhost` :8010. Custom headers: add `X-Forwarded-Proto` = `https` + (the broker sets the session cookie's Secure flag from it). +6. **DNS.** Cloudflare ▸ your zone ▸ a proxied record for the broker host, + the same as the NAS's other proxied hostnames. +7. **CI.** Repo ▸ Settings ▸ Variables ▸ `NAS_BROKER_URL` = + `https://`. Until it is set, `nas-deploy-check` is + skipped. + +## Moving the campaign off Supabase + +Workers hold leases, so do this in a quiet window. + +1. Stop the workers (or let them finish). +2. On the NAS, dump Supabase through its **session** pooler (port `5432`, not + the `6543` transaction pooler) straight into the new database: + ```bash + sudo /usr/local/bin/docker exec -i \ + -e SRC='postgresql://postgres.:@aws-0-.pooler.supabase.com:5432/postgres' \ + postgres sh -c \ + 'pg_dump --no-owner --no-acl -n public "$SRC" | psql -v ON_ERROR_STOP=1 -U admin -d casebroker' + ``` + Restore into an EMPTY `casebroker` database: if the broker has already + started once, it has created the schema, and the restore collides with it. + Stop the `casebroker` container first and `DROP DATABASE casebroker; + CREATE DATABASE casebroker;` if so. Then, still before the restore, run + `DROP SCHEMA public;` inside `casebroker`: a dump taken with `-n public` + contains its own `CREATE SCHEMA public`, which `ON_ERROR_STOP` would + otherwise abort on. +3. Start the broker; `curl https:///healthz` must say + `"ok": true`, and `casebroker doctor` should pass against it. +4. Point every worker's `CASEBROKER_URL` (machine.env, SLURM scripts) at the + new URL. Accounts and per-machine tokens moved with the data; the shared + `CASEBROKER_WRITE_TOKENS` moved via `.env`. +5. Keep Render and Supabase for a week, then retire the `deploy-smoke-test` + job and the Render service. + +## Roll back + +Pin the image in `compose.yaml` to an earlier tag +(`ghcr.io/sustainableurbansystemslab/casebroker:sha-`) and rebuild the +project. Watchtower leaves a pinned tag alone unless that tag itself moves. + +## Backups + +The data directory is not safe to copy while Postgres runs. A nightly Task +Scheduler job (root) that dumps instead: + +```bash +/usr/local/bin/docker exec postgres pg_dump -U admin -Fc casebroker \ + > /volume1/docker/casebroker/backup/casebroker_$(date +%F).dump +find /volume1/docker/casebroker/backup -name '*.dump' -mtime +14 -delete +``` diff --git a/deploy/synology/compose.yaml b/deploy/synology/compose.yaml new file mode 100644 index 0000000..b5969a9 --- /dev/null +++ b/deploy/synology/compose.yaml @@ -0,0 +1,68 @@ +# Self-hosted deployment on the lab Synology: Postgres and the broker in ONE +# Container Manager project. See README.md in this folder for the runbook. +# +# How a push to main reaches this box: CI builds the image and pushes +# ghcr.io/sustainableurbansystemslab/casebroker:latest, Watchtower (a separate +# project on the NAS, WATCHTOWER_LABEL_ENABLE=true) sees the new digest within +# its poll interval and recreates `broker` -- and only `broker`, because it is +# the only service here carrying the enable label. Postgres is upgraded by +# hand, deliberately: an unattended major-version bump cannot read the old +# data directory. +# +# TLS is not terminated here. DSM's reverse proxy (Control Panel > Login +# Portal > Advanced > Reverse Proxy) owns :443 for every hostname on +# the NAS and forwards to 127.0.0.1:8010, which is why the broker binds +# loopback only: nothing on the LAN reaches it except through the proxy. +services: + postgres: + image: postgres:17 + container_name: postgres + restart: unless-stopped + command: ["postgres", "-c", "timezone=America/New_York", "-c", "log_timezone=America/New_York"] + environment: + # Read by the image only when the data directory is EMPTY. The existing + # directory was initialised by the old standalone `postgres` project, so + # this must be that same password, not a new one -- changing it here + # changes nothing inside the database. + POSTGRES_USER: admin + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + POSTGRES_DB: app + TZ: America/New_York + volumes: + - /volume1/docker/postgres/data:/var/lib/postgresql/data + ports: + # LAN access (psql, Metabase). 5432 on the host belongs to DSM's own + # built-in Postgres. The broker does not use this; it talks to + # postgres:5432 over the project network. + - "5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U admin -d casebroker"] + interval: 10s + timeout: 5s + retries: 10 + + broker: + image: ghcr.io/sustainableurbansystemslab/casebroker:latest + container_name: casebroker + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + labels: + com.centurylinklabs.watchtower.enable: "true" + environment: + # The password goes into a URL, so any of @ : / ? # [ ] % in it must be + # percent-encoded here (python3 -c "import urllib.parse,sys; + # print(urllib.parse.quote(sys.argv[1], safe=''))" ''). + CASEBROKER_DB: postgresql://admin:${POSTGRES_PASSWORD_URLENCODED:-${POSTGRES_PASSWORD}}@postgres:5432/casebroker + # The live fleet authenticates with the shared token Render has today; + # carry the same value(s) over or every worker gets 401 at cut-over. + CASEBROKER_WRITE_TOKENS: ${CASEBROKER_WRITE_TOKENS:-} + CASEBROKER_READ_TOKENS: ${CASEBROKER_READ_TOKENS:-} + CASEBROKER_SETUP_TOKEN: ${CASEBROKER_SETUP_TOKEN:-} + # Postgres on the NAS has no free-plan ceiling; this only labels the + # dashboard's storage page. + CASEBROKER_DB_QUOTA_MB: ${CASEBROKER_DB_QUOTA_MB:-} + TZ: America/New_York + ports: + - "127.0.0.1:8010:8000" diff --git a/pyproject.toml b/pyproject.toml index ccdcb0f..5135536 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "casebroker" # back from the installed distribution metadata and everything that reports a # version (the OpenAPI document, /healthz, the dashboard badge) goes through # there. Bumping it here and tagging v is the whole release. -version = "0.26.1" +version = "0.27.0" description = "Central case broker for the v2 real-city CFD campaign: one database of cases, many workers leasing the next one to simulate." requires-python = ">=3.11" dependencies = [ diff --git a/uv.lock b/uv.lock index f0165da..73fe2e4 100644 --- a/uv.lock +++ b/uv.lock @@ -60,7 +60,7 @@ wheels = [ [[package]] name = "casebroker" -version = "0.26.1" +version = "0.27.0" source = { editable = "." } dependencies = [ { name = "duckdb" },