Add circ-celery container image to replace circ-scripts

.github/workflows/build.yml

@@ -30,6 +30,8 @@ jobs:
       scripts-meta: ${{ steps.meta-scripts.outputs.json }}
       exec-repo: ${{ steps.ghcr-repo.outputs.exec }}
       exec-meta: ${{ steps.meta-exec.outputs.json }}
+      celery-repo: ${{ steps.ghcr-repo.outputs.celery }}
+      celery-meta: ${{ steps.meta-celery.outputs.json }}
 
     # https://docs.docker.com/build/ci/github-actions/local-registry/
     services:
@@ -74,13 +76,16 @@ jobs:
           webapp="ghcr.io/$repo/circ-webapp"
           scripts="ghcr.io/$repo/circ-scripts"
           exec="ghcr.io/$repo/circ-exec"
+          celery="ghcr.io/$repo/circ-celery"
           baseimage="ghcr.io/$repo/circ-baseimage"
           echo "webapp=$webapp"
           echo "webapp=$webapp" >> "$GITHUB_OUTPUT"
           echo "scripts=$scripts"
           echo "scripts=$scripts" >> "$GITHUB_OUTPUT"
           echo "exec=$exec"
           echo "exec=$exec" >> "$GITHUB_OUTPUT"
+          echo "celery=$celery"
+          echo "celery=$celery" >> "$GITHUB_OUTPUT"
           echo "baseimage=$baseimage"
           echo "baseimage=$baseimage" >> "$GITHUB_OUTPUT"
 
@@ -222,6 +227,29 @@ jobs:
           labels: ${{ steps.meta-exec.outputs.labels }}
           outputs: type=image,"name=${{ steps.ghcr-repo.outputs.exec }}",push-by-digest=true,name-canonical=true,push=true
 
+      - name: Generate tags for circ-celery
+        id: meta-celery
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ steps.ghcr-repo.outputs.celery }}
+          tags: |
+            type=semver,pattern={{major}}.{{minor}},priority=10
+            type=semver,pattern={{version}},priority=20
+            type=ref,event=branch,priority=30
+            type=sha,priority=40
+
+      - name: Build circ-celery image
+        id: build-celery
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          target: celery
+          build-args: |
+            BASE_IMAGE=${{ steps.baseimage.outputs.tag }}
+          labels: ${{ steps.meta-celery.outputs.labels }}
+          outputs: type=image,"name=${{ steps.ghcr-repo.outputs.celery }}",push-by-digest=true,name-canonical=true,push=true
+
       - name: Export digests
         run: |
           mkdir -p ${{ runner.temp }}/digests/webapp
@@ -236,6 +264,10 @@ jobs:
           exec_digest="${{ steps.build-exec.outputs.digest }}"
           touch "${{ runner.temp }}/digests/exec/${exec_digest#sha256:}"
           echo "EXEC_DIGEST=$exec_digest"
+          mkdir -p ${{ runner.temp }}/digests/celery
+          celery_digest="${{ steps.build-celery.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/celery/${celery_digest#sha256:}"
+          echo "CELERY_DIGEST=$celery_digest"
 
       - name: Upload digests
         uses: actions/upload-artifact@v7
@@ -292,6 +324,15 @@ jobs:
           echo "$IMAGE"
           echo "WEBAPP_IMAGE=$IMAGE" >> $GITHUB_ENV
 
+      # This sets the environment variable referenced in the docker-compose file
+      # for the celery containers to the digest of the image built in the build job.
+      - name: Set celery image
+        working-directory: ${{ runner.temp }}/digests/celery
+        run: |
+          IMAGE="${{needs.build.outputs.celery-repo}}$(printf '@sha256:%s' *)"
+          echo "$IMAGE"
+          echo "CELERY_IMAGE=$IMAGE" >> $GITHUB_ENV
+
       # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
       - name: Disable network offload
         run: sudo ethtool -K eth0 tx off rx off
@@ -308,6 +349,9 @@ jobs:
       - name: Run scripts image tests
         run: ./docker/ci/test_scripts.sh scripts
 
+      - name: Run celery image tests
+        run: ./docker/ci/test_celery.sh
+
       - name: Output logs
         if: failure()
         run: docker compose logs
@@ -516,3 +560,10 @@ jobs:
           docker buildx imagetools create
           $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ needs.build.outputs.exec-meta }}')
           $(printf '${{ needs.build.outputs.exec-repo }}@sha256:%s ' *)
+
+      - name: Create manifest & push circ-celery
+        working-directory: ${{ runner.temp }}/digests/celery
+        run: >
+          docker buildx imagetools create
+          $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ needs.build.outputs.celery-meta }}')
+          $(printf '${{ needs.build.outputs.celery-repo }}@sha256:%s ' *)

README.md

@@ -33,6 +33,7 @@ Docker images created from this code are available at:
 - [circ-webapp](https://github.com/ThePalaceProject/circulation/pkgs/container/circ-webapp)
 - [circ-scripts](https://github.com/ThePalaceProject/circulation/pkgs/container/circ-scripts)
 - [circ-exec](https://github.com/ThePalaceProject/circulation/pkgs/container/circ-exec)
+- [circ-celery](https://github.com/ThePalaceProject/circulation/pkgs/container/circ-celery)
 
 Docker images are the preferred way to deploy this code in a production environment.
 
@@ -247,6 +248,28 @@ We support overriding a number of other Celery settings via environment variable
 the defaults should be sufficient. The full list of settings can be found in
 [`service/celery/configuration.py`](src/palace/manager/service/celery/configuration.py).
 
+##### `circ-celery` image
+
+The `circ-celery` image runs a single Celery process per container, so it can be deployed as the beat
+scheduler or one or more autoscaled worker pools — all from the same image. The role is chosen by the
+container command (`beat` or `worker`), and a few launch-time variables (read by the container's
+entrypoint, not by the application) configure the `worker` role:
+
+- `PALACE_CELERY_QUEUES`: Comma-separated list of queues a `worker` container should consume, e.g.
+    `high,default` (**required for the `worker` role**). Each queue's tasks are defined by
+    [`QueueNames`](src/palace/manager/service/celery/celery.py).
+- `PALACE_CELERY_CONCURRENCY`: The number of child processes for a `worker` container. This is a single
+    pool shared across all of the queues that container consumes; to give a queue its own concurrency,
+    run a separate `worker` deployment with its own `PALACE_CELERY_QUEUES`/`PALACE_CELERY_CONCURRENCY`.
+    The default is `1` (optional).
+- `PALACE_CELERY_WORKER_HOSTNAME`: The Celery `--hostname` for a `worker` container. The default is
+    `worker@%h` (optional).
+
+The `beat` role must run as a single instance (a second beat would double-fire scheduled tasks); only
+the `worker` role should be scaled. The queue-depth metrics that drive worker autoscaling
+(`QueueWaiting` / `QueueOldestAge`, in the `Celery` CloudWatch namespace) are published every minute by
+the `publish_queue_stats` task on the beat schedule, so no always-on metrics process is required.
+
 #### Redis
 
 We use Redis as the caching layer for the application. Although you can use the same Redis database for both

docker-compose.yml

@@ -1,34 +1,39 @@
 # Common CM setup
 # see: https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension
+
+# The shared environment is its own anchor so a service (e.g. the celery worker)
+# can merge it and add a few of its own variables without repeating the block.
+x-cm-environment: &cm-environment
+  SIMPLIFIED_PRODUCTION_DATABASE: "postgresql://palace:test@pg:5432/circ"
+  PALACE_SEARCH_URL: "http://os:9200"
+  PALACE_STORAGE_ACCESS_KEY: "palace"
+  PALACE_STORAGE_SECRET_KEY: "test123456789"
+  PALACE_STORAGE_ENDPOINT_URL: "http://minio:9000"
+  PALACE_STORAGE_PUBLIC_ACCESS_BUCKET: "public"
+  PALACE_STORAGE_ANALYTICS_BUCKET: "analytics"
+  PALACE_STORAGE_URL_TEMPLATE: "http://localhost:9000/{bucket}/{key}"
+  PALACE_REPORTING_NAME: "TEST CM"
+  PALACE_SECRET_KEY: "SECRET_KEY_USED_FOR_ADMIN_UI_COOKIES"
+  PALACE_PATRON_WEB_HOSTNAMES: "*"
+  PALACE_BASE_URL: "http://localhost:6500"
+  PALACE_CELERY_BROKER_URL: "redis://valkey:6379/0"
+  PALACE_CELERY_RESULT_BACKEND: "redis://valkey:6379/2"
+  PALACE_CELERY_BROKER_TRANSPORT_OPTIONS_GLOBAL_KEYPREFIX: "test"
+  PALACE_CELERY_CLOUDWATCH_STATISTICS_DRYRUN: "true"
+  PALACE_REDIS_URL: "redis://valkey:6379/1"
+  PALACE_GOOGLE_DRIVE_SERVICE_ACCOUNT_INFO_JSON: "${PALACE_GOOGLE_DRIVE_SERVICE_ACCOUNT_INFO_JSON-}"
+
+  # Set up the environment variables used for testing as well
+  PALACE_TEST_DATABASE_URL: "postgresql://palace:test@pg:5432/circ"
+  PALACE_TEST_SEARCH_URL: "http://os:9200"
+  PALACE_TEST_MINIO_URL: "http://minio:9000"
+  PALACE_TEST_MINIO_USER: "palace"
+  PALACE_TEST_MINIO_PASSWORD: "test123456789"
+  PALACE_TEST_REDIS_URL: "redis://valkey:6379/3"
+
 x-cm-variables: &cm
   platform: "${BUILD_PLATFORM-}"
-  environment:
-    SIMPLIFIED_PRODUCTION_DATABASE: "postgresql://palace:test@pg:5432/circ"
-    PALACE_SEARCH_URL: "http://os:9200"
-    PALACE_STORAGE_ACCESS_KEY: "palace"
-    PALACE_STORAGE_SECRET_KEY: "test123456789"
-    PALACE_STORAGE_ENDPOINT_URL: "http://minio:9000"
-    PALACE_STORAGE_PUBLIC_ACCESS_BUCKET: "public"
-    PALACE_STORAGE_ANALYTICS_BUCKET: "analytics"
-    PALACE_STORAGE_URL_TEMPLATE: "http://localhost:9000/{bucket}/{key}"
-    PALACE_REPORTING_NAME: "TEST CM"
-    PALACE_SECRET_KEY: "SECRET_KEY_USED_FOR_ADMIN_UI_COOKIES"
-    PALACE_PATRON_WEB_HOSTNAMES: "*"
-    PALACE_BASE_URL: "http://localhost:6500"
-    PALACE_CELERY_BROKER_URL: "redis://valkey:6379/0"
-    PALACE_CELERY_RESULT_BACKEND: "redis://valkey:6379/2"
-    PALACE_CELERY_BROKER_TRANSPORT_OPTIONS_GLOBAL_KEYPREFIX: "test"
-    PALACE_CELERY_CLOUDWATCH_STATISTICS_DRYRUN: "true"
-    PALACE_REDIS_URL: "redis://valkey:6379/1"
-    PALACE_GOOGLE_DRIVE_SERVICE_ACCOUNT_INFO_JSON: "${PALACE_GOOGLE_DRIVE_SERVICE_ACCOUNT_INFO_JSON-}"
-
-    # Set up the environment variables used for testing as well
-    PALACE_TEST_DATABASE_URL: "postgresql://palace:test@pg:5432/circ"
-    PALACE_TEST_SEARCH_URL: "http://os:9200"
-    PALACE_TEST_MINIO_URL: "http://minio:9000"
-    PALACE_TEST_MINIO_USER: "palace"
-    PALACE_TEST_MINIO_PASSWORD: "test123456789"
-    PALACE_TEST_REDIS_URL: "redis://valkey:6379/3"
+  environment: *cm-environment
 
   depends_on:
     pg:
@@ -67,6 +72,34 @@ services:
       target: scripts
     image: "${SCRIPTS_IMAGE-}"
 
+  # The celery-* services all run from the single circ-celery image; the role is
+  # the container command. In production these become the beat deployment (one
+  # replica) and the autoscaled worker deployment(s). Queue-depth metrics are
+  # published by the beat-scheduled publish_queue_stats task, so there is no
+  # separate metrics container.
+  celery-beat:
+    <<: *cm
+    build:
+      <<: *cm-build
+      target: celery
+    image: "${CELERY_IMAGE-}"
+    command: ["beat"]
+
+  celery-worker:
+    <<: *cm
+    build:
+      <<: *cm-build
+      target: celery
+    image: "${CELERY_IMAGE-}"
+    command: ["worker"]
+    # A single dev/test worker consumes every queue. In production each queue
+    # (or group) is its own deployment with its own queues/concurrency so it can
+    # be autoscaled independently.
+    environment:
+      <<: *cm-environment
+      PALACE_CELERY_QUEUES: "high,default,apply"
+      PALACE_CELERY_CONCURRENCY: "2"
+
   pg:
     image: "postgres:16"
     environment:

docker/Dockerfile

@@ -25,7 +25,7 @@
 COPY --chown=palace:palace packages/palace-opds/pyproject.toml /var/www/circulation/packages/palace-opds/
 RUN uv sync --frozen --no-dev --no-install-workspace
 
 COPY --chown=palace:palace . /var/www/circulation
 
 # Install the project itself as an editable install.
 RUN uv sync --frozen --no-dev
@@ -63,6 +63,52 @@
 
 CMD ["/sbin/my_init"]
 
+###############################################################################
+## Circ-celery Image
+##
+## A single image that runs one Celery process per container. The role is
+## selected by the container command (beat|worker), so one image backs both the
+## beat scheduler and every autoscaled worker pool. Unlike the scripts image, it
+## does NOT use runit to supervise several processes -- one process per container
+## is what lets the worker pools be scaled horizontally, and the beat scheduler
+## be pinned to a single instance.
+##
+## Like the webapp/scripts images it boots via my_init, so the shared
+## /etc/my_init.d startup scripts run first -- notably the advisory-locked
+## `initialize_instance` (alembic upgrade head) -- before the Celery role starts.
+## This keeps the migration path identical to the other images: every container
+## brings the schema to head on start, so no separate migration step is needed.
+## `--skip-runit` is used because this image has no runit services; the single
+## Celery process is my_init's main command instead.
+###############################################################################
+
+FROM common AS celery
+
+# Set the local timezone so log timestamps line up with the other images. Beat's
+# own schedule is driven by the Celery `timezone` setting, not this.
+ENV TZ=US/Eastern
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime
+
+# The Celery process runs as the `palace` user via `celery --uid`, which does not
+# reset HOME. Without this it would inherit root's HOME=/root, and libpq
+# (psycopg2) -- probing $HOME/.postgresql/postgresql.crt on connect -- would hit
+# EACCES on /root (mode 700), abort the SSL handshake, and fall back to a
+# plaintext connection that an SSL-required Postgres rejects. my_init does not
+# override an inherited HOME, so setting it here covers the --uid drop.
+ENV HOME=/home/palace
+
+COPY --chmod=755 docker/celery-entrypoint.sh /celery-entrypoint.sh
+
+VOLUME /var/log
+WORKDIR /var/www/circulation
+
+# The role is provided as the container command, e.g. `docker run circ-celery
+# worker`, with PALACE_CELERY_QUEUES / PALACE_CELERY_CONCURRENCY set for worker
+# pools. my_init runs the startup scripts (incl. the DB migration), then execs
+# the entrypoint with the role appended. See docker/celery-entrypoint.sh for the
+# supported roles and env vars.
+ENTRYPOINT ["/sbin/my_init", "--skip-runit", "--quiet", "--", "/celery-entrypoint.sh"]
+
 ###############################################################################
 ## Circ-webapp Image
 ###############################################################################

docker/README.md

@@ -77,6 +77,42 @@ $ docker run --name search_index_refresh -it \
     ghcr.io/thepalaceproject/circ-exec:main
 ```
 
+### circ-celery
+
+This image runs a single Celery process per container. Unlike `circ-scripts`, which supervises the
+beat scheduler and every worker in one container with runit, `circ-celery` runs exactly one process,
+so each role can be deployed and (for workers) autoscaled independently. The role is chosen by the
+container command: `beat` or `worker`. Queue-depth metrics that drive autoscaling are published by the
+`publish_queue_stats` task on the beat schedule, so there is no separate always-on metrics container.
+
+Like the other images, `circ-celery` boots via `my_init` and so initializes or migrates the database
+on startup (advisory-locked `alembic upgrade head`) before the Celery process starts. No separate
+migration step is required.
+
+```sh
+# The beat scheduler (run exactly one of these).
+$ docker run --name celery-beat -d \
+    -e PALACE_CELERY_BROKER_URL='redis://[host]:6379/0' \
+    -e PALACE_CELERY_RESULT_BACKEND='redis://[host]:6379/2' \
+    -e SIMPLIFIED_PRODUCTION_DATABASE='postgresql://[username]:[password]@[host]:[port]/[database_name]' \
+    ghcr.io/thepalaceproject/circ-celery:main beat
+
+# A worker pool (scale the replica count on queue depth). PALACE_CELERY_QUEUES is required.
+$ docker run --name celery-worker -d \
+    -e PALACE_CELERY_BROKER_URL='redis://[host]:6379/0' \
+    -e PALACE_CELERY_RESULT_BACKEND='redis://[host]:6379/2' \
+    -e SIMPLIFIED_PRODUCTION_DATABASE='postgresql://[username]:[password]@[host]:[port]/[database_name]' \
+    -e PALACE_CELERY_QUEUES='high,default' \
+    -e PALACE_CELERY_CONCURRENCY='8' \
+    ghcr.io/thepalaceproject/circ-celery:main worker
+```
+
+The `worker` launch variables (`PALACE_CELERY_QUEUES`, `PALACE_CELERY_CONCURRENCY`,
+`PALACE_CELERY_WORKER_HOSTNAME`) and the `cloudwatch` flush interval
+(`PALACE_CELERY_CLOUDWATCH_FLUSH_INTERVAL`) are documented in the top-level
+[`README.md`](../README.md). Logs are written to stdout/stderr rather than to files, so nothing is
+lost when an autoscaled worker is scaled away.
+
 ## Environment Variables
 
 Environment variables can be set with the `-e VARIABLE_KEY='variable_value'` option on the `docker run` command.