semaphoreui · cursor · Jul 6, 2026
diff --git a/.gitmodules b/.gitmodules
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -4,6 +4,7 @@ When creating a pull-request you should:
 
 - __Open an issue first:__ Confirm that the change or feature will be accepted
 - __Update API documentation:__ If your pull-request adding/modifying an API request, make sure you update the Swagger documentation (`api-docs.yml`)
+- __Update developer docs:__ For operational or architectural changes (runners, executors, workflows, cluster API), add or update the matching page under [`docs/developer/`](docs/developer/README.md)
 - __Run API Tests:__ If your pull request modifies the API make sure you run the integration tests using **dredd**.
 
 ## Installation in a development environment
@@ -27,12 +28,14 @@ When creating a pull-request you should:
       cd $GOPATH/src/github.com/semaphoreui
       ```
 
-2) Clone semaphore (with submodules)
+2) Clone semaphore
 
    ```
-   git clone --recursive git@github.com:semaphoreui/semaphore.git && cd semaphore
+   git clone git@github.com:semaphoreui/semaphore.git && cd semaphore
    ```
 
+   Developer notes live in [`docs/developer/`](docs/developer/README.md) (in-repo).
+
 3) Install dev dependencies
 
    ```

diff --git a/README.md b/README.md
@@ -93,6 +93,7 @@ For more installation options, visit our [Installation page](https://semaphoreui
 * [User Guide](https://docs.semaphoreui.com)
 * [API Reference](https://semaphoreui.com/api-docs)
 * [Postman Collection](https://www.postman.com/semaphoreui)
+* [Developer notes](docs/developer/README.md) — runner auth, executors, Docker images, workflows, HA cluster API, input validation
 
 ## Awesome Semaphore
 

diff --git a/api-docs.yml b/api-docs.yml
@@ -1039,6 +1039,23 @@ definitions:
           $ref: "#/definitions/TemplateVault"
       task_params:
         $ref: '#/definitions/TaskPrams'
+      jwt_params:
+        $ref: '#/definitions/TemplateJWTParams'
+
+  TemplateJWTParams:
+    type: object
+    properties:
+      enabled:
+        type: boolean
+        description: When true (and server jwt.enabled), a signed JWT is issued per task run.
+      audience:
+        type: array
+        items:
+          type: string
+        description: JWT aud claim (up to 32 entries).
+      ttl:
+        type: string
+        description: Token lifetime as a Go duration (e.g. 30m); capped by jwt.max_ttl.
 
   TemplateSurveyVar:
     type: object
@@ -1204,6 +1221,16 @@ definitions:
         format: date-time
         description: Last time the runner contacted the server.
         x-nullable: true
+      started_at:
+        type: string
+        format: date-time
+        description: Runner process start time reported on each poll (X-Runner-Started-At).
+        x-nullable: true
+      status:
+        type: string
+        enum: [online, offline]
+        description: Derived heartbeat liveness; not persisted.
+        readOnly: true
       cleaning_requested:
         type: string
         format: date-time
@@ -1246,10 +1273,7 @@ definitions:
         properties:
           token:
             type: string
-            description: Token the runner uses to authenticate.
-          private_key:
-            type: string
-            description: Generated private key, returned only when the server creates the key pair.
+            description: Token the runner uses to authenticate (X-Runner-Token).
 
   RunnerActive:
     type: object
@@ -1345,7 +1369,7 @@ definitions:
         type: integer
       kind:
         type: string
-        enum: [task, approval]
+        enum: [task, approval, note]
       convergence_mode:
         type: string
         enum: [all, any]
@@ -1404,7 +1428,7 @@ definitions:
         type: integer
       status:
         type: string
-        enum: [running, success, failed, approval]
+        enum: [running, success, failed, approval, stopped]
       start:
         type: string
         format: date-time
@@ -3214,6 +3238,22 @@ paths:
           schema:
             $ref: "#/definitions/WorkflowRunDetails"
 
+  /project/{project_id}/workflows/{workflow_id}/runs/{run_id}/stop:
+    parameters:
+      - $ref: "#/parameters/project_id"
+      - $ref: "#/parameters/workflow_id"
+      - $ref: "#/parameters/run_id"
+    post:
+      tags:
+        - workflow
+      summary: Stop workflow run
+      description: Stops every active task in the run, rejects pending approvals, and marks the run stopped.
+      responses:
+        200:
+          description: workflow run stopped
+          schema:
+            $ref: "#/definitions/WorkflowRun"
+
   /project/{project_id}/workflows/{workflow_id}/runs/{run_id}/approvals:
     parameters:
       - $ref: "#/parameters/project_id"

diff --git a/deployment/compose/README.md b/deployment/compose/README.md
@@ -65,6 +65,24 @@ A runner is considered *registered* when it has an auth token (`runner.token` in
 database). Unregistered runners appear in the UI with a filter and cannot pick up tasks
 until registration completes.
 
+### Runner liveness and status
+
+Poll-based runners send `X-Runner-Token` and `X-Runner-Started-At` on every request.
+The server updates `touched` and derives an `online` / `offline` status for the UI
+and dispatch logic. Webhook-driven runners are always treated as online.
+
+Tune server-side behaviour with the `runners` config section (env prefix
+`SEMAPHORE_RUNNERS_*`):
+
+| Setting | Default | Effect |
+| --- | --- | --- |
+| `offline_timeout_sec` | 120 | Runner marked offline; no new tasks; `starting` tasks reassigned |
+| `task_fail_timeout_sec` | 420 | `running` tasks on a silent runner are failed |
+| `reconcile_interval_sec` | 30 | How often dispatched tasks are scanned |
+
+See [docs/developer/runner-auth-and-jwt.md](../../docs/developer/runner-auth-and-jwt.md)
+for the full protocol and task-JWT details.
+
 ### Build
 
 This simply takes the currently cloned source and builds a new image including
@@ -107,15 +125,6 @@ backend.
 docker-compose <server/runner from above> -f deployment/compose/store/sqlite.yml up
 ```
 
-### SQLite
-
-This simply configures a named volume for the SQLite storage used as a database
-backend.
-
-```console
-docker-compose <server/runner from above> -f deployment/compose/store/sqlite.yml up
-```
-
 ### MariaDB
 
 This simply starts an additional container for a MariaDB instance used as a

diff --git a/deployment/docker/README.md b/deployment/docker/README.md
@@ -12,16 +12,40 @@ If you want to use [docker-compose][dockercompose] to start Semaphore you could
 also read about it on our [documentation][documentation] or take a look at our
 collection of [snippets][snippets] within this repository.
 
+## Images
+
+| Published name | Dockerfile | Role |
+| --- | --- | --- |
+| `semaphoreui/semaphore` | `deployment/docker/server/Dockerfile` | Server (API + web UI) |
+| `semaphoreui/runner` | `deployment/docker/runner/Dockerfile` | Runner process + toolchain |
+| `semaphoreui/job` | `deployment/docker/job/Dockerfile` | Task build container (Docker/K8s executors) |
+| `semaphoreui/helper` | `deployment/docker/helper/Dockerfile` | Git-clone init container (K8s executor) |
+
+The `job` image ships Ansible, Terraform, OpenTofu, Terragrunt, and Git on
+Debian slim. The `helper` image is a minimal git/SSH client image for repository
+checkout init containers. See [docs/developer/docker-images.md](../../docs/developer/docker-images.md)
+for defaults and executor configuration.
+
+`job` and `helper` are built and pushed by CI on the `develop` branch
+(`.github/workflows/dev.yml`). Server and runner images follow release tags.
+
 ## Build
 
-We have prepared multiple tasks to build an publish container images, including
+We have prepared multiple tasks to build and publish container images, including
 tasks to verify the image contains all required tools:
 
 ```console
 task docker:build
 task docker:push
 ```
 
+Build the executor images locally:
+
+```console
+docker build -f deployment/docker/job/Dockerfile -t semaphoreui/job:test .
+docker build -f deployment/docker/helper/Dockerfile -t semaphoreui/helper:test .
+```
+
 If you want to customize the image names or if you want to use [Podman][podman]
 instead of [Docker][docker] you are able to provide some set of environment
 variables to the [Task][gotask] command:

diff --git a/docs b/docs
diff --git a/docs/developer/README.md b/docs/developer/README.md
@@ -0,0 +1,16 @@
+# Developer documentation
+
+In-repo notes for contributors and operators who work directly with the Semaphore
+source tree. For end-user guides see [docs.semaphoreui.com](https://docs.semaphoreui.com).
+
+| Topic | Covers |
+| --- | --- |
+| [Runner auth and task JWT](runner-auth-and-jwt.md) | Runner bearer tokens, registration, online/offline status, fleet timeouts, per-task JWT issuance |
+| [Runner executors](runner-executors.md) | `local`, `docker`, and `k8s` executor strategies, runner config, OSS vs Pro |
+| [Docker images](docker-images.md) | `server`, `runner`, `job`, and `helper` images — purpose, contents, publishing |
+| [Workflows](workflows.md) | Pro workflow templates, runs, approvals, artifacts, API surface |
+| [HA cluster dashboard](ha-cluster-dashboard.md) | Admin cluster routes, task-state snapshot/clear, runner name in task payloads |
+| [Input validation](input-validation.md) | Playbook path checks, git URL injection prevention, branch override gate, custom roles, access key updates |
+
+When you change a public API or operational workflow, update the matching page here
+and keep `api-docs.yml` in sync (see [CONTRIBUTING.md](../../CONTRIBUTING.md)).
diff --git a/docs/developer/docker-images.md b/docs/developer/docker-images.md
@@ -0,0 +1,86 @@
+# Docker images
+
+Semaphore publishes several container images from this repository. They serve
+different roles in deployment and in the Docker/Kubernetes runner executors.
+
+## Image overview
+
+| Image | Dockerfile | Purpose |
+| --- | --- | --- |
+| `semaphoreui/semaphore` | `deployment/docker/server/Dockerfile` | Semaphore **server** (API + web UI) |
+| `semaphoreui/runner` | `deployment/docker/runner/Dockerfile` | Semaphore **runner** process + toolchain |
+| `semaphoreui/job` | `deployment/docker/job/Dockerfile` | **Task build** container for Docker/K8s executors |
+| `semaphoreui/helper` | `deployment/docker/helper/Dockerfile` | **Git-clone init** container (K8s executor) |
+
+`server` and `runner` images compile the Go binary via `task build` and bundle
+Ansible, Terraform, OpenTofu, and Terragrunt for local-style execution on the
+container host.
+
+`job` and `helper` are slim Debian-based images without the Semaphore binary.
+They are used exclusively by the Pro Docker and Kubernetes executors when a
+runner dispatches work into isolated containers/Pods.
+
+## Job image (`semaphoreui/job`)
+
+Built from `deployment/docker/job/Dockerfile` (Debian 13 slim).
+
+**Includes:** Ansible (pinned `ansible-core` + `ansible` package), OpenTofu,
+Terraform, Terragrunt, Git, OpenSSH client, Python 3, `curl`, `wget`, `unzip`.
+
+**Runs as:** `nobody` user, working directory `/workspace`.
+
+**Used for:**
+
+- Docker executor `executor.docker.image` (default `semaphoreui/job:latest`)
+- Docker executor `executor.docker.helper_image` (git clone before the build step)
+- K8s executor `executor.k8s.image` when you want the full toolchain in the Pod
+
+CI publishes this image on every `develop` build (`deploy-job` job in
+`.github/workflows/dev.yml`).
+
+## Helper image (`semaphoreui/helper`)
+
+Built from `deployment/docker/helper/Dockerfile` (Debian 13 slim).
+
+**Includes:** Git, OpenSSH client, Python 3, `curl`, `sshpass` — no Ansible or
+IaC binaries.
+
+**Runs as:** `nobody` user, working directory `/workspace`.
+
+**Used for:**
+
+- K8s executor `executor.k8s.helper_image` — init container that clones the
+  task repository before the main build container starts
+- Lighter alternative when only repository checkout is needed
+
+CI publishes this image on every `develop` build (`deploy-helper` job in
+`.github/workflows/dev.yml`).
+
+## Server and runner images
+
+See [deployment/docker/README.md](../../deployment/docker/README.md) for build
+and test tasks (`task docker:build`, `task docker:test`).
+
+Environment variables for customising image names during local builds:
+
+- `DOCKER_ORG` (default `semaphoreui`)
+- `DOCKER_SERVER` (default `semaphore`)
+- `DOCKER_RUNNER` (default `runner`)
+- `DOCKER_CMD` (default `docker`)
+
+## Choosing images for executors
+
+| Executor | Build container (`image`) | Init / clone (`helper_image`) |
+| --- | --- | --- |
+| Docker | `semaphoreui/job:latest` | `semaphoreui/job:latest` (same image) |
+| Kubernetes | `semaphoreui/job:latest` (recommended) | `semaphoreui/helper:latest` (recommended) |
+
+Override paths via `runner.executor.docker.*` or `runner.executor.k8s.*` in the
+runner config, or the corresponding `SEMAPHORE_RUNNER_DOCKER_*` /
+`SEMAPHORE_RUNNER_K8S_*` environment variables.
+
+## Related code
+
+- Runner defaults: `util/config.go` (`RunnerDockerConfig`, `RunnerK8sConfig`)
+- Executor factory: `services/runners/executor_factory.go`
+- CI publish: `.github/workflows/dev.yml` (`deploy-job`, `deploy-helper`)