diff --git a/.github/scripts/tests/test_release_workflow.py b/.github/scripts/tests/test_release_workflow.py index b608b6c0f69..c0478fea8ad 100644 --- a/.github/scripts/tests/test_release_workflow.py +++ b/.github/scripts/tests/test_release_workflow.py @@ -67,6 +67,14 @@ def test_cosign_is_verified_before_release_publication(self): ) self.assertLess(restore_position, publish_position) + def test_cosign_signing_retries_transient_service_failures(self): + self.assertEqual(self.workflow.count("for attempt in 1 2 3; do"), 3) + self.assertEqual( + self.workflow.count('echo "Cosign signing failed after $attempt attempts"'), + 3, + ) + self.assertEqual(self.workflow.count("sleep $((attempt * 15))"), 3) + def test_github_release_uses_the_git_tag_not_the_product_version(self): self.assertIn('gh release create "$GIT_TAG"', self.workflow) self.assertIn("--verify-tag", self.workflow) diff --git a/.github/workflows/build-branch.yml b/.github/workflows/build-branch.yml index 8f27d0230ce..6b386d927d0 100644 --- a/.github/workflows/build-branch.yml +++ b/.github/workflows/build-branch.yml @@ -365,7 +365,18 @@ jobs: env: DIGEST: ${{ steps.build.outputs.digest }} IMAGE: ${{ env.IMAGE_PREFIX }}${{ matrix.image }} - run: cosign sign --yes "${IMAGE}@${DIGEST}" + run: | + set -euo pipefail + for attempt in 1 2 3; do + if cosign sign --yes "${IMAGE}@${DIGEST}"; then + exit 0 + fi + if [[ "$attempt" -eq 3 ]]; then + echo "Cosign signing failed after $attempt attempts" >&2 + exit 1 + fi + sleep $((attempt * 15)) + done - name: Verify release image signature if: needs.setup.outputs.is_release == 'true' @@ -468,7 +479,18 @@ jobs: env: DIGEST: ${{ steps.build.outputs.digest }} IMAGE: ${{ env.IMAGE_PREFIX }}aio - run: cosign sign --yes "${IMAGE}@${DIGEST}" + run: | + set -euo pipefail + for attempt in 1 2 3; do + if cosign sign --yes "${IMAGE}@${DIGEST}"; then + exit 0 + fi + if [[ "$attempt" -eq 3 ]]; then + echo "Cosign signing failed after $attempt attempts" >&2 + exit 1 + fi + sleep $((attempt * 15)) + done - name: Verify release image signature if: needs.setup.outputs.is_release == 'true' @@ -614,7 +636,16 @@ jobs: fi chart_ref="ghcr.io/szymczag/charts/hangar@${chart_digest}" - cosign sign --yes "$chart_ref" + for attempt in 1 2 3; do + if cosign sign --yes "$chart_ref"; then + break + fi + if [[ "$attempt" -eq 3 ]]; then + echo "Cosign signing failed after $attempt attempts" >&2 + exit 1 + fi + sleep $((attempt * 15)) + done cosign verify \ --certificate-identity "$CERTIFICATE_IDENTITY" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ diff --git a/README.md b/README.md index 11b02e961c0..be1457456c8 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ the table marks them as available. ## Deployment - [Docker deployment](deployments/cli/community/README.md) -- [Kubernetes and Helm](docs/kubernetes/README.md) — `0.1.0-rc.28` +- [Kubernetes and Helm](docs/kubernetes/README.md) — `0.1.0-rc.29` evaluation release; production support gates remain open. - [Amazon SES, deliverability, and OpenPGP email](docs/aws-ses-email-operations.md) — secure transactional-email configuration and operations. @@ -170,7 +170,7 @@ Install Cosign, obtain the digest for the version you intend to deploy, and veri both the exact workflow identity and GitHub's OIDC issuer: ```sh -VERSION=v0.1.0-rc.28 +VERSION=v0.1.0-rc.29 GIT_TAG=hangar-$VERSION DIGEST=sha256:replace-with-the-published-digest IMAGE=ghcr.io/szymczag/hangar-api diff --git a/charts/hangar/README.md b/charts/hangar/README.md index 1e5ce28ffb2..576b4a786da 100644 --- a/charts/hangar/README.md +++ b/charts/hangar/README.md @@ -7,7 +7,7 @@ published as an OCI artifact at: oci://ghcr.io/szymczag/charts/hangar ``` -The latest published chart is `0.1.0-rc.28` (`appVersion: v0.1.0-rc.28`). Its +The latest published chart is `0.1.0-rc.29` (`appVersion: v0.1.0-rc.29`). Its evaluation profile is live-qualified on AMD64. It is a prerelease and is not yet supported for production. Release packaging stages these versions and immutable image digests from the release tag; source defaults remain fail-closed. @@ -27,17 +27,18 @@ Inspect the published chart without registry credentials: ```bash helm show chart oci://ghcr.io/szymczag/charts/hangar \ - --version 0.1.0-rc.28 + --version 0.1.0-rc.29 ``` -Do not use `0.1.0-rc.1`, `0.1.0-rc.2`, `0.1.0-rc.20`, `0.1.0-rc.24`, or -`0.1.0-rc.25`; those immutable versions were consumed by incomplete release -attempts. `0.1.0-rc.24` and `0.1.0-rc.25` each published only a subset of their -container sets and no chart. `0.1.0-rc.27` is the immediately previous complete +Do not use `0.1.0-rc.1`, `0.1.0-rc.2`, `0.1.0-rc.20`, `0.1.0-rc.24`, +`0.1.0-rc.25`, or `0.1.0-rc.28`; those immutable versions were consumed by +incomplete release attempts. `0.1.0-rc.24`, `0.1.0-rc.25`, and `0.1.0-rc.28` +each published only a subset of their container sets and no chart or GitHub +Release. `0.1.0-rc.27` is the immediately previous complete publication, but it predates the signed preview grant and transactional duplicate -admission controls in `rc.28`. +admission controls in `rc.29`. Earlier `rc.12` through `rc.17` additionally contain frontend migration failures. -Use `0.1.0-rc.28`; there is no security-equivalent +Use `0.1.0-rc.29`; there is no security-equivalent rollback target among the earlier release candidates. ## Deployment profiles @@ -114,8 +115,8 @@ digests for the five Hangar application images. The source chart intentionally contains all-zero application digests so an unstaged source checkout fails closed. -Release `0.1.0-rc.28` is available from the -[GitHub Release](https://github.com/szymczag/hangar/releases/tag/hangar-v0.1.0-rc.28) +Release `0.1.0-rc.29` is available from the +[GitHub Release](https://github.com/szymczag/hangar/releases/tag/hangar-v0.1.0-rc.29) and GHCR. Follow the [verification guide](../../docs/kubernetes/security.md) before admitting the package to a controlled environment. diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md index 7c5bf62dfbf..a9370c917cc 100644 --- a/docs/kubernetes/README.md +++ b/docs/kubernetes/README.md @@ -6,7 +6,7 @@ Hangar publishes a Helm chart for Kubernetes at: oci://ghcr.io/szymczag/charts/hangar ``` -The current release is `0.1.0-rc.28`. It is qualified for evaluation on +The current release is `0.1.0-rc.29`. It is qualified for evaluation on AMD64 Kubernetes clusters. It is not yet a supported production release. > [!IMPORTANT] @@ -44,7 +44,7 @@ only to review and help qualify the production profile. ## Compatibility -The `0.1.0-rc.28` qualification boundary is: +The `0.1.0-rc.29` qualification boundary is: | Item | Qualified boundary | | ---------------------- | ---------------------------------------------------------------- | @@ -110,23 +110,23 @@ The product, chart, and Git identifiers are deliberately different: | Identifier | Current value | | ------------------ | -------------------------------------------- | -| Product version | `v0.1.0-rc.28` | -| Helm chart version | `0.1.0-rc.28` | -| Git tag | `hangar-v0.1.0-rc.28` | -| OCI chart | `ghcr.io/szymczag/charts/hangar:0.1.0-rc.28` | - -`rc.1`, `rc.2`, `rc.20`, `rc.24`, and `rc.25` were consumed by incomplete publication -attempts. Do not use them. `rc.27` is the immediately previous complete publication, but it -predates the signed preview grant and transactional duplicate admission controls in `rc.28`. +| Product version | `v0.1.0-rc.29` | +| Helm chart version | `0.1.0-rc.29` | +| Git tag | `hangar-v0.1.0-rc.29` | +| OCI chart | `ghcr.io/szymczag/charts/hangar:0.1.0-rc.29` | + +`rc.1`, `rc.2`, `rc.20`, `rc.24`, `rc.25`, and `rc.28` were consumed by incomplete +publication attempts. Do not use them. `rc.27` is the immediately previous complete publication, but it +predates the signed preview grant and transactional duplicate admission controls in `rc.29`. Earlier `rc.12` through `rc.17` additionally contain frontend migration failures. There is no security-equivalent rollback target among the earlier release candidates. Published versions are immutable and are never repaired in place. In -particular, `rc.24` and `rc.25` each published only a subset of their container -sets and published no chart or GitHub Release. +particular, `rc.24`, `rc.25`, and `rc.28` each published only a subset of their +container sets and published no chart or GitHub Release. ## Documentation -- [Release `v0.1.0-rc.28` notes](../releases/hangar-v0.1.0-rc.28.md) — review +- [Release `v0.1.0-rc.29` notes](../releases/hangar-v0.1.0-rc.29.md) — review security changes, migrations, compatibility, limitations, and rollback. - [Install the evaluation profile](evaluation-install.md) — complete a first installation in a dedicated namespace. @@ -151,7 +151,7 @@ Pod Security, migrations, HTTPS ingress, WebSockets, positive and negative network-policy checks, dependency connectivity, object-storage persistence, an atomic upgrade, rollback-on-failure behavior, uninstall, and retained PVCs. -The public `rc.28` chart archive, OCI chart, and digest-pinned Hangar images are +The public `rc.29` chart archive, OCI chart, and digest-pinned Hangar images are anonymously downloadable. The release workflow also created provenance attestations and keyless Cosign signatures. diff --git a/docs/kubernetes/configuration.md b/docs/kubernetes/configuration.md index 257c525a80d..aa1dfbeab67 100644 --- a/docs/kubernetes/configuration.md +++ b/docs/kubernetes/configuration.md @@ -373,7 +373,7 @@ global: Use a component's `image.pullSecrets` for a narrower credential. Worker, beat, and migrator Pods use `api.image.pullSecrets` because they run the API image. Public release images currently pull anonymously, so a GHCR credential is not -required for `0.1.0-rc.28`. +required for `0.1.0-rc.29`. ## Replicas and disruption budgets @@ -400,7 +400,7 @@ spread, and a termination grace period. Component templates combine these values with their security constraints. Evaluation dependencies are fixed to `kubernetes.io/arch: amd64`. All application -images in `0.1.0-rc.28` are also AMD64-only. +images in `0.1.0-rc.29` are also AMD64-only. ## Evaluation storage diff --git a/docs/kubernetes/evaluation-install.md b/docs/kubernetes/evaluation-install.md index 309211e0286..099e10ecfaa 100644 --- a/docs/kubernetes/evaluation-install.md +++ b/docs/kubernetes/evaluation-install.md @@ -1,6 +1,6 @@ # Install Hangar for evaluation -This tutorial installs Hangar `0.1.0-rc.28` in a dedicated namespace with bundled, +This tutorial installs Hangar `0.1.0-rc.29` in a dedicated namespace with bundled, persistent PostgreSQL, Valkey, RabbitMQ, and object storage. When complete, you will have a TLS-enabled Hangar instance suitable for evaluation and compatibility testing. @@ -26,7 +26,7 @@ You need: Set the release parameters used throughout this tutorial: ```bash -export CHART_VERSION=0.1.0-rc.28 +export CHART_VERSION=0.1.0-rc.29 export RELEASE_NAME=hangar export NAMESPACE=hangar-evaluation export HANGAR_HOST=hangar-evaluation.example.com @@ -43,10 +43,10 @@ helm show chart oci://ghcr.io/szymczag/charts/hangar \ --version "$CHART_VERSION" ``` -Confirm the output reports chart version `0.1.0-rc.28`, application version -`v0.1.0-rc.28`, and the expected Kubernetes version constraint. +Confirm the output reports chart version `0.1.0-rc.29`, application version +`v0.1.0-rc.29`, and the expected Kubernetes version constraint. -For higher-assurance environments, complete [artifact verification](security.md#verify-release-010-rc28) +For higher-assurance environments, complete [artifact verification](security.md#verify-release-010-rc29) before installation. ## 2. Create a Restricted namespace @@ -82,7 +82,7 @@ Download the release-matched Secret structure into a private local file: umask 077 curl --fail --location --silent --show-error \ --output evaluation-secrets.yaml \ - https://raw.githubusercontent.com/szymczag/hangar/hangar-v0.1.0-rc.28/charts/hangar/examples/evaluation-secrets.example.yaml + https://raw.githubusercontent.com/szymczag/hangar/hangar-v0.1.0-rc.29/charts/hangar/examples/evaluation-secrets.example.yaml ``` Replace every `CHANGE_ME` value with a unique, randomly generated credential. diff --git a/docs/kubernetes/operations.md b/docs/kubernetes/operations.md index 5f38e7dc9e9..5f7345b8699 100644 --- a/docs/kubernetes/operations.md +++ b/docs/kubernetes/operations.md @@ -59,7 +59,10 @@ that one over-limit request returns `429` without creating a job or source. ## Upgrade a release -### Upgrade from `rc.27` to `rc.28` +### Upgrade from `rc.27` to `rc.29` + +`rc.28` was consumed by an incomplete publication and is not an upgrade target. +Upgrade directly from the immediately previous complete release, `rc.27`. This release closes two Todoist import admission gaps. Starting an import now requires a server-signed, 15-minute, single-use preview grant bound to the @@ -79,13 +82,13 @@ Before upgrading: 1. take a PostgreSQL backup, prove that it can be restored in isolation, and record the current Helm revision and application image digests; -2. confirm the target chart is `0.1.0-rc.28`, its application version is - `v0.1.0-rc.28`, and its signatures and digests pass the - [release verification procedure](security.md#verify-release-010-rc28); +2. confirm the target chart is `0.1.0-rc.29`, its application version is + `v0.1.0-rc.29`, and its signatures and digests pass the + [release verification procedure](security.md#verify-release-010-rc29); 3. render the existing values against the target chart and verify that only the expected release versions and immutable image digests change; and 4. deploy every application image as one coordinated Helm revision. Do not mix - `rc.27` and `rc.28` web or API images because their preview-execution request + `rc.27` and `rc.29` web or API images because their preview-execution request contract intentionally changed together. Wait for the revision-scoped migration Job to complete before admitting traffic. @@ -98,7 +101,7 @@ also fail without creating a job or retaining a source object. target; the nullable column may remain in the database. It restores the preview bypass and duplicate-confirmation race, however, so it is not a security-equivalent rollback. Prefer a forward correction and return every -application component to `rc.28` promptly. +application component to `rc.29` promptly. ### Upgrade from `rc.26` to `rc.27` diff --git a/docs/kubernetes/production-install.md b/docs/kubernetes/production-install.md index e22901e524c..9e22add6881 100644 --- a/docs/kubernetes/production-install.md +++ b/docs/kubernetes/production-install.md @@ -5,7 +5,7 @@ stateful services. It is intended for platform engineers participating in production qualification and design review. > [!CAUTION] -> Hangar `0.1.0-rc.28` is not supported for production. The production profile +> Hangar `0.1.0-rc.29` is not supported for production. The production profile > renders with secure defaults, but it has not completed the installation, > upgrade, backup/restore, failure-recovery, security-review, or compatibility > gates required for production support. Do not place critical data or users on @@ -81,7 +81,7 @@ Confirm that: ## 4. Create the namespace and TLS Secret ```bash -export CHART_VERSION=0.1.0-rc.28 +export CHART_VERSION=0.1.0-rc.29 export RELEASE_NAME=hangar export NAMESPACE=hangar export HANGAR_HOST=hangar.example.com @@ -133,7 +133,7 @@ Download the release-matched example: ```bash curl --fail --location --silent --show-error \ --output production-values.yaml \ - https://raw.githubusercontent.com/szymczag/hangar/hangar-v0.1.0-rc.28/charts/hangar/examples/production-values.yaml + https://raw.githubusercontent.com/szymczag/hangar/hangar-v0.1.0-rc.29/charts/hangar/examples/production-values.yaml ``` At minimum, set: diff --git a/docs/kubernetes/security.md b/docs/kubernetes/security.md index 6bc4c5612a3..067efb5ec7d 100644 --- a/docs/kubernetes/security.md +++ b/docs/kubernetes/security.md @@ -1,7 +1,7 @@ # Kubernetes security and artifact verification This document explains the chart's security boundaries and provides commands for -verifying the public `0.1.0-rc.28` release. +verifying the public `0.1.0-rc.29` release. ## Security model @@ -88,9 +88,9 @@ that does not enforce policies can fail open. ## Release trust chain -For `0.1.0-rc.28`, the trust chain is: +For `0.1.0-rc.29`, the trust chain is: -1. signed Git tag `hangar-v0.1.0-rc.28` identifies the source commit; +1. signed Git tag `hangar-v0.1.0-rc.29` identifies the source commit; 2. the release workflow builds AMD64 images with BuildKit SBOM and provenance; 3. GitHub creates build-provenance attestations; 4. the workflow signs image and chart digests keylessly with GitHub OIDC; @@ -101,7 +101,7 @@ For `0.1.0-rc.28`, the trust chain is: Verification must use immutable digests and the exact workflow identity. A valid signature for a different workflow, repository, tag, or issuer is not sufficient. -## Verify release `0.1.0-rc.28` +## Verify release `0.1.0-rc.29` These commands require `curl`, `sha256sum`, GitHub CLI for the GitHub attestation, and Cosign for OCI signatures. @@ -109,8 +109,8 @@ and Cosign for OCI signatures. ### 1. Download public release assets ```bash -export VERSION=0.1.0-rc.28 -export GIT_TAG=hangar-v0.1.0-rc.28 +export VERSION=0.1.0-rc.29 +export GIT_TAG=hangar-v0.1.0-rc.29 export RELEASE_URL="https://github.com/szymczag/hangar/releases/download/$GIT_TAG" mkdir "hangar-$VERSION-release" @@ -232,7 +232,7 @@ licenses, vulnerabilities, and maintenance status independently. ## Current limitations -`0.1.0-rc.28` is a prerelease. Only the evaluation profile has completed live +`0.1.0-rc.29` is a prerelease. Only the evaluation profile has completed live cluster qualification. Vulnerability and license approval, production security qualification, backup/restore, migration-failure recovery, and the complete support matrix remain open gates. diff --git a/docs/kubernetes/troubleshooting.md b/docs/kubernetes/troubleshooting.md index 57849339971..f45e5a49b62 100644 --- a/docs/kubernetes/troubleshooting.md +++ b/docs/kubernetes/troubleshooting.md @@ -60,7 +60,7 @@ Confirm the chart is public: ```bash helm show chart oci://ghcr.io/szymczag/charts/hangar \ - --version 0.1.0-rc.28 + --version 0.1.0-rc.29 ``` Confirm the node architecture: @@ -69,7 +69,7 @@ Confirm the node architecture: kubectl get nodes --label-columns kubernetes.io/arch ``` -`0.1.0-rc.28` is AMD64-only. An ARM64-only cluster cannot schedule or run the +`0.1.0-rc.29` is AMD64-only. An ARM64-only cluster cannot schedule or run the qualified images. Published charts use digest references. Inspect the failed Pod's image without diff --git a/docs/releases/hangar-v0.1.0-rc.29.md b/docs/releases/hangar-v0.1.0-rc.29.md new file mode 100644 index 00000000000..2954a77e163 --- /dev/null +++ b/docs/releases/hangar-v0.1.0-rc.29.md @@ -0,0 +1,70 @@ +## Security and privacy + +`rc.29` hardens the Todoist import review and admission boundary. The execute +endpoint now requires a short-lived, server-signed preview grant bound to the +administrator, workspace, destination project, and exact uploaded source. Each +grant carries a random nonce that can reserve at most one job, so missing, +expired, altered, cross-scope, and replayed grants fail closed before source +retention or dispatch. + +Import reservation now locks the destination project and rechecks both active +jobs and completed duplicates inside the reservation transaction. This closes a +race in which a concurrent import could complete between the earlier request +check and reservation, bypassing the explicit duplicate confirmation. + +The controls were validated with regression tests for mandatory preview, +actor/project/source binding, expiration, single use, and transactional duplicate +admission. Publication remains gated on the full API and Todoist suites, security +migrations, frontend checks, release metadata, Helm policy tests, CodeQL, +artifact attestations, and keyless signatures. + +The publication workflow now retries each keyless image and chart signing +operation up to three times with bounded backoff. This tolerates transient +Sigstore timestamp-service connection failures without skipping the mandatory +post-signature identity verification or overwriting an existing release. + +## Migrations and compatibility + +Migration `ext.0011_import_job_preview_nonce` adds a nullable unique UUID field +to import jobs. Existing jobs remain valid and require no data backfill. New jobs +created through the HTTP Todoist importer consume the nonce from their signed +preview grant; trusted internal reservation callers receive a generated nonce. + +There is no new Helm value, Secret, Kubernetes resource, public route, storage, +RBAC, or NetworkPolicy change. Existing `rc.27` values and deployment +configuration remain structurally compatible. The preview grant lifetime +defaults to 900 seconds and requires no operator action. + +Deploy all application images as one Helm revision and wait for the +revision-scoped migration Job before admitting traffic. The web and API request +contract changed together: an `rc.29` web client sends the signed grant, while +the `rc.29` API no longer accepts a client-calculated digest as proof of preview. +Do not operate mixed `rc.27` and `rc.29` web/API revisions. + +The inherited Plane baseline remains final `v1.4.0` at commit `917b23a6`. The +qualification boundary remains Kubernetes 1.30 through 1.36 (including 1.36.2), +Helm 4.2, `linux/amd64`, Restricted Pod Security Admission, TLS ingress with +WebSocket support, a `NetworkPolicy`-enforcing CNI, and persistent storage. + +The product version is `v0.1.0-rc.29`, the chart version is +`0.1.0-rc.29`, the signed Git tag is `hangar-v0.1.0-rc.29`, and the OCI +chart reference is `ghcr.io/szymczag/charts/hangar:0.1.0-rc.29`. +`rc.27` is the immediately previous complete publication. `rc.1`, `rc.2`, +`rc.20`, `rc.24`, `rc.25`, and `rc.28` were consumed by incomplete publication +attempts and are not upgrade or rollback targets. `rc.24`, `rc.25`, and `rc.28` +each published only a subset of their containers and no chart or GitHub Release. + +## Known limitations and rollback + +Hangar `rc.29` remains a prerelease qualified for evaluation rather than +production. Published images are AMD64-only. The production-profile install, +backup and restore, migration-failure recovery, vulnerability and license +approval, and full support matrix remain open qualification gates. + +`rc.27` is structurally compatible as an emergency technical rollback target; +the nullable preview-nonce column can remain in place. Rolling back restores the +superseded Todoist admission behavior, so there is no security-equivalent +rollback target. Prefer a forward correction; if availability recovery requires +rollback, move every application component together, temporarily disable Todoist +imports, and return to `rc.29` promptly. Restore a database backup only when +unrelated writes, corruption, or the incident requires point-in-time recovery.