diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 549514d4d..cd67ff98e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -66,7 +66,4 @@ jobs: - name: Apply to AKS run: | trap 'rm -f "$KUBECONFIG"' EXIT - kubectl apply -f "$RUNNER_TEMP/release-manifests/backend-deployment.yaml" -n naruon-dev - kubectl rollout status deployment/backend -n naruon-dev --timeout=120s - kubectl apply -f "$RUNNER_TEMP/release-manifests/frontend-deployment.yaml" -n naruon-dev - kubectl rollout status deployment/frontend -n naruon-dev --timeout=120s + bash scripts/deploy_runtime_manifests.sh "$RUNNER_TEMP/release-manifests" diff --git a/AGENTS.md b/AGENTS.md index cab55d306..198489546 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,6 +110,21 @@ in this repo. ## Release governance defaults +- CHANGELOG나 저장소 요약이 endpoint를 구현·배포됐다고 설명해도 현재 라우트와 + 실제 응답을 확인한다. 과거 branch의 코드를 찾으면 선행 PR과 유효 delta를 추적하고, + 문서에 적힌 `/readyz`나 root 200을 DB readiness 근거로 승계하지 않는다. +- 부분 배포 복구는 이전 image만 바꾸는 작업이 아니다. 같은 실행에서 `umask 077`로 + 이전 spec과 실제 적용 응답·readback을 확보하고 UID·spec 소유권을 확인한다. + 정방향 변경과 복구 모두 UID·resourceVersion 조건부 patch를 사용한다. 다른 writer의 + 변경, 재생성, 충돌, 응답 유실은 강제 복구나 재시도 없이 실패로 남긴다. + 서버 dry-run의 last-applied annotation을 그대로 저장하지 말고 spec만 변경한다. + 복구 후 rollout과 최종 UID·spec readback까지 확인해도 원래 배포 job은 실패다. + 두 객체의 복구가 끝난 뒤 양쪽 UID·spec을 다시 확인한다. 개별 복구 성공을 + 전체 복구 성공으로 합산하지 않으며 두 객체의 복구를 원자적이라고 표현하지 않는다. + 사전 검사는 두 대상 모두 끝내고 나서 쓴다. 두 번째 대상의 누락·tag 기반 이전 + 상태·metadata 변경을 주입해 첫 번째 대상에도 patch가 없었는지 검사한다. + 원시 snapshot·kubeconfig는 로그나 artifact로 올리지 않는다. 실제 workflow 호출의 + 부분 실패와 cleanup도 테스트하며 unit double을 실제 클러스터 검증으로 보고하지 않는다. - 배포 이미지의 tag는 게시 artifact의 동일성을 증명하지 않는다. matrix별 digest를 같은 실행·revision·attempt의 개별 artifact로 전달하고 두 runtime의 sha256을 모두 검증한 뒤 `image@sha256` manifest를 생성한다. 누락·변조·잘못된 digest는 diff --git a/backend/tests/fixtures/runtime_restore_kubectl.py b/backend/tests/fixtures/runtime_restore_kubectl.py new file mode 100644 index 000000000..3e1cf7a71 --- /dev/null +++ b/backend/tests/fixtures/runtime_restore_kubectl.py @@ -0,0 +1,83 @@ +"""Unit-only kubectl double; never opens a network connection.""" + +import copy +import json +import os +from pathlib import Path +import sys + +state_path = Path(os.environ["RESTORE_TEST_STATE"]) +test_state = json.loads(state_path.read_text()) +command_args = sys.argv[1:] +operation_name = command_args[0] +test_state["calls"].append(operation_name) +image_component = "backend" +desired_state = None +if "resources" in test_state: + if operation_name == "apply": + file_flag = "-f" if "-f" in command_args else "--filename" + desired_state = json.loads(Path(command_args[command_args.index(file_flag) + 1]).read_text()) + image_component = desired_state["metadata"]["name"] + else: + image_component = command_args[2].removeprefix("deployment/") + test_state["current"] = test_state["resources"][image_component] + test_state["operations"].append([operation_name, image_component]) +exit_code = 0 +if operation_name == "get": + if test_state["scenario"] == "missing_resource" or (test_state["scenario"] == "frontend_missing" and image_component == "frontend"): + exit_code = 1 + else: + print(json.dumps(test_state["current"])) +elif operation_name == "apply" and "--dry-run=server" in command_args: + proposed_state = copy.deepcopy(test_state["current"]) + proposed_state["metadata"].update(desired_state["metadata"]) + proposed_state["spec"] = desired_state["spec"] + proposed_state["spec"]["revisionHistoryLimit"] = 10 + proposed_state["metadata"].setdefault("annotations", {})["kubectl.kubernetes.io/last-applied-configuration"] = "unit-dry-run-annotation" + print(json.dumps(proposed_state)) +elif operation_name == "patch": + patch_path = Path(command_args[command_args.index("--patch-file") + 1]) + patch_ops = json.loads(patch_path.read_text()) + test_state["patch_ops"] = patch_ops + if test_state["scenario"] == "conflict": + test_state["current"]["metadata"]["resourceVersion"] = "external-update" + for patch_op in patch_ops: + if patch_op["op"] == "test": + current_value = test_state["current"] + for path_part in patch_op["path"].strip("/").split("/"): + current_value = current_value[path_part] + if current_value != patch_op["value"]: + exit_code = 1 + if not exit_code: + replacement = next(item["value"] for item in patch_ops if item["op"] == "replace" and item["path"] == "/spec") + test_state["current"]["spec"] = replacement + test_state["current"]["metadata"]["resourceVersion"] = "restored-version" + forward_patch = False + if "resources" in test_state: + forward_patch = image_component not in test_state["applied_components"] + state_key = "applied_components" if forward_patch else "restored_components" + test_state[state_key].append(image_component) + if test_state["scenario"] == "response_lost" or (forward_patch and image_component == "frontend" and test_state["scenario"] == "frontend_response_lost"): + exit_code = 1 + else: + print(json.dumps(test_state["current"])) +elif operation_name == "rollout": + if ( + "resources" in test_state and image_component == "frontend" + and image_component in test_state["applied_components"] + and image_component not in test_state["restored_components"] + and test_state["scenario"] in {"frontend_rollout_failed", "backend_drift", "restored_frontend_drift"} + ): + exit_code = 1 + if test_state["scenario"] == "backend_drift": + test_state["resources"]["backend"]["spec"]["replicas"] = 99 + if test_state["scenario"] == "unhealthy_restore": + exit_code = 1 + if test_state["scenario"] == "readback_drift": + test_state["current"]["spec"]["replicas"] = 99 + if test_state["scenario"] == "restored_frontend_drift" and image_component == "backend" and image_component in test_state["restored_components"]: + test_state["resources"]["frontend"]["spec"]["replicas"] = 99 +else: + exit_code = 2 +state_path.write_text(json.dumps(test_state)) +raise SystemExit(exit_code) diff --git a/backend/tests/test_release_manifest_digests.py b/backend/tests/test_release_manifest_digests.py index 020cc7e68..8195d3c73 100644 --- a/backend/tests/test_release_manifest_digests.py +++ b/backend/tests/test_release_manifest_digests.py @@ -111,8 +111,7 @@ def test_release_workflow_passes_separate_same_revision_artifacts() -> None: render = next(step for step in steps if step.get("id") == "render_manifests") assert "scripts/render_release_manifests.sh" in render["run"] apply_step = next(step for step in steps if step["name"] == "Apply to AKS") - assert '"$RUNNER_TEMP/release-manifests/backend-deployment.yaml"' in apply_step["run"] - assert '"$RUNNER_TEMP/release-manifests/frontend-deployment.yaml"' in apply_step["run"] + assert 'bash scripts/deploy_runtime_manifests.sh "$RUNNER_TEMP/release-manifests"' in apply_step["run"] @pytest.mark.parametrize("artifact_state", ["valid", "missing_backend", "missing_frontend", "invalid_backend", "invalid_frontend"]) diff --git a/backend/tests/test_runtime_deployment_restore.py b/backend/tests/test_runtime_deployment_restore.py new file mode 100644 index 000000000..2468c7d4d --- /dev/null +++ b/backend/tests/test_runtime_deployment_restore.py @@ -0,0 +1,144 @@ +"""Run conditional restoration against a network-free kubectl unit double.""" + +import copy +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +@pytest.mark.parametrize("scenario", ["restored", "uid_changed", "spec_changed", "conflict", "response_lost", "unhealthy_restore", "readback_drift", "missing_resource"]) +def test_restore_uses_owned_spec_and_atomic_resource_version( + tmp_path: Path, scenario: str +) -> None: + """Never overwrite a recreated/drifted resource or report ambiguous success.""" + before_state = { + "kind": "Deployment", + "metadata": {"name": "backend", "namespace": "naruon-dev", "uid": "unit-backend", "resourceVersion": "before-version"}, + "spec": {"replicas": 2, "template": {"spec": {"containers": [{"name": "backend", "image": "ghcr.io/unit/backend@sha256:" + "a" * 64}]}}}, + } + owned_state = copy.deepcopy(before_state) + owned_state["metadata"]["resourceVersion"] = "applied-version" + owned_state["spec"]["replicas"] = 3 + owned_state["spec"]["template"]["spec"]["containers"][0]["image"] = "ghcr.io/unit/backend@sha256:" + "b" * 64 + current_state = copy.deepcopy(owned_state) + current_state["metadata"]["resourceVersion"] = "controller-status-update" + if scenario == "uid_changed": + current_state["metadata"]["uid"] = "replacement-resource" + if scenario == "spec_changed": + current_state["spec"]["replicas"] = 7 + for file_name, payload in (("before.json", before_state), ("owned.json", owned_state)): + (tmp_path / file_name).write_text(json.dumps(payload)) + state_path = tmp_path / "unit_state.json" + state_path.write_text(json.dumps({"scenario": scenario, "current": current_state, "calls": []})) + fake_binary = tmp_path / "kubectl" + fixture_path = REPO_ROOT / "backend/tests/fixtures/runtime_restore_kubectl.py" + fake_binary.write_text(f"#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(fixture_path))} \"$@\"\n") + fake_binary.chmod(0o700) + jq_binary = shutil.which("jq") + assert jq_binary, "jq is required for the deployment contract" + result = subprocess.run( + ["bash", str(REPO_ROOT / "scripts/restore_runtime_deployment.sh"), "backend", str(tmp_path / "before.json"), str(tmp_path / "owned.json")], + env={"PATH": os.pathsep.join([str(tmp_path), str(Path(jq_binary).parent), os.defpath]), "RUNNER_TEMP": str(tmp_path), "RESTORE_TEST_STATE": str(state_path)}, + capture_output=True, text=True, check=False, + ) + final_state = json.loads(state_path.read_text()) + if scenario == "restored": + assert result.returncode == 0, result.stderr + assert final_state["current"]["spec"] == before_state["spec"] + assert final_state["calls"] == ["get", "patch", "rollout", "get"] + assert {item["path"] for item in final_state["patch_ops"] if item["op"] == "test"} >= {"/metadata/uid", "/metadata/resourceVersion"} + assert "restore_verified:backend" in result.stdout + else: + assert result.returncode != 0 + assert "restore_verified:" not in result.stdout + assert "restore_failed:" in result.stderr + if scenario in {"uid_changed", "spec_changed", "missing_resource"}: + assert "patch" not in final_state["calls"] + + +@pytest.mark.parametrize("scenario", ["deployed", "frontend_rollout_failed", "backend_drift", "frontend_response_lost", "restored_frontend_drift", "frontend_tag_baseline", "frontend_metadata_change", "frontend_missing"]) +def test_partial_deployment_restores_only_confirmed_owned_resources( + tmp_path: Path, scenario: str +) -> None: + """Run the real caller and restore helper with admission-normalized objects.""" + resource_states = {} + manifest_directory = tmp_path / "release-manifests" + manifest_directory.mkdir() + for image_component in ("backend", "frontend"): + resource_states[image_component] = { + "kind": "Deployment", + "metadata": {"name": image_component, "namespace": "naruon-dev", "uid": "unit-" + image_component, "resourceVersion": "old-version"}, + "spec": {"replicas": 2, "revisionHistoryLimit": 10, "template": {"spec": {"containers": [{"name": image_component, "image": "ghcr.io/unit/" + image_component + "@sha256:" + "a" * 64}]}}}, + } + if image_component == "frontend" and scenario == "frontend_tag_baseline": + resource_states[image_component]["spec"]["template"]["spec"]["containers"][0]["image"] = "ghcr.io/unit/frontend:old-version" + desired_state = copy.deepcopy(resource_states[image_component]) + if image_component == "frontend" and scenario == "frontend_metadata_change": + desired_state["metadata"]["labels"] = {"release_owner": "changed-owner"} + desired_state["spec"]["replicas"] = 3 + del desired_state["spec"]["revisionHistoryLimit"] + desired_state["spec"]["template"]["spec"]["containers"][0]["image"] = "ghcr.io/unit/" + image_component + "@sha256:" + "b" * 64 + (manifest_directory / f"{image_component}-deployment.yaml").write_text(json.dumps(desired_state)) + prior_states = copy.deepcopy(resource_states) + state_path = tmp_path / "unit_state.json" + state_path.write_text(json.dumps({"scenario": scenario, "resources": resource_states, "calls": [], "operations": [], "applied_components": [], "restored_components": []})) + fake_binary = tmp_path / "kubectl" + fixture_path = REPO_ROOT / "backend/tests/fixtures/runtime_restore_kubectl.py" + fake_binary.write_text(f"#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(fixture_path))} \"$@\"\n") + fake_binary.chmod(0o700) + jq_binary = shutil.which("jq") + assert jq_binary + kubeconfig_path = tmp_path / "unit_kubeconfig" + kubeconfig_path.write_text("unit-only-not-a-credential\n") + workflow = yaml.safe_load((REPO_ROOT / ".github/workflows/deploy.yml").read_text()) + apply_command = next(step["run"] for step in workflow["jobs"]["deploy"]["steps"] if step["name"] == "Apply to AKS") + result = subprocess.run( + ["bash", "-euo", "pipefail", "-c", apply_command], + cwd=REPO_ROOT, + env={"PATH": os.pathsep.join([str(tmp_path), str(Path(jq_binary).parent), os.defpath]), "RUNNER_TEMP": str(tmp_path), "RESTORE_TEST_STATE": str(state_path), "KUBECONFIG": str(kubeconfig_path)}, + capture_output=True, text=True, check=False, + ) + final_state = json.loads(state_path.read_text()) + assert not kubeconfig_path.exists() + if scenario in {"frontend_tag_baseline", "frontend_metadata_change", "frontend_missing"}: + assert result.returncode != 0 + assert final_state["applied_components"] == [] + assert final_state["restored_components"] == [] + assert "patch" not in final_state["calls"] + assert final_state["resources"] == prior_states + assert "verified" not in result.stdout + result.stderr + return + assert final_state["applied_components"] == ["backend", "frontend"] + for image_component in ("backend", "frontend"): + assert final_state["resources"][image_component]["metadata"].get("annotations", {}) == prior_states[image_component]["metadata"].get("annotations", {}) + if scenario == "deployed": + assert result.returncode == 0, result.stderr + assert final_state["restored_components"] == [] + assert "deployment_verified" in result.stdout + elif scenario == "frontend_rollout_failed": + assert result.returncode != 0 + assert final_state["restored_components"] == ["frontend", "backend"] + for image_component in ("backend", "frontend"): + assert final_state["resources"][image_component]["spec"] == prior_states[image_component]["spec"] + assert "rollback_verified" in result.stderr + elif scenario == "restored_frontend_drift": + assert result.returncode != 0 + assert final_state["restored_components"] == ["frontend", "backend"] + assert final_state["resources"]["frontend"]["spec"]["replicas"] == 99 + assert "rollback_verified" not in result.stderr + else: + assert result.returncode != 0 + assert final_state["restored_components"] == [] + assert "rollback_verified" not in result.stderr + if scenario == "backend_drift": + assert final_state["resources"]["backend"]["spec"]["replicas"] == 99 diff --git a/docs/doctoring/runtime-image-boundary-verification.md b/docs/doctoring/runtime-image-boundary-verification.md index 4a00ec68e..94a23724c 100644 --- a/docs/doctoring/runtime-image-boundary-verification.md +++ b/docs/doctoring/runtime-image-boundary-verification.md @@ -112,8 +112,106 @@ https://github.com/actions/upload-artifact/blob/043fb46d1a93c77aae656e7c1c64a875 GitHub. (n.d.-b). *Download a build artifact* [Action definition]. https://github.com/actions/download-artifact/blob/3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/action.yml +## 기존 배포 쌍의 조건부 복구 + +상태: Proposed. digest 전달 PR #1586의 +`a48aa3a3e81ba58b2fa55cd758e86b9272455b1a` 위에서 개발한다. +backend가 정상 갱신된 뒤 frontend rollout이 실패하면 두 runtime의 버전이 +갈라질 수 있다. image만 되돌리면 replica 수나 pod 설정은 새 값으로 남는다. +따라서 `scripts/deploy_runtime_manifests.sh`가 두 Deployment의 사전 정상 상태와 +전체 spec을 먼저 확인하고, `scripts/restore_runtime_deployment.sh`가 소유권이 +유지된 객체의 이전 spec을 복원하도록 실제 deploy workflow에 연결했다. + +두 객체의 server dry-run이 끝나기 전에는 쓰지 않는다. 정방향 변경도 복구도 +UID와 resourceVersion을 JSON Patch test로 검사한 뒤 spec만 교체한다. +적용 응답과 직후 readback이 일치해야 이 실행이 소유한 상태로 기록한다. +rollout 실패 시 두 객체의 소유권을 먼저 확인하고 frontend, backend 순으로 복구한다. +각 복구의 rollout과 최종 UID·spec readback이 성공해야 `restore_verified`를 출력한다. +복구 성공은 배포 성공이 아니므로 호출 workflow는 실패 상태를 유지한다. +독립 검토에서 backend 복구 중 이미 복구한 frontend가 다시 바뀌는 공백을 발견했다. +추가 회귀는 기존 코드에서 잘못된 `rollback_verified`를 재현했다 +(1 failed, 12 deselected, 31.37초). 두 복구 뒤 이전 snapshot 대비 양쪽 UID·spec을 +다시 조회하도록 고쳤다. 이 검사는 관측 시점의 확인이며 두 객체를 원자적으로 +잠그거나 마지막 조회 이후의 변경까지 막지는 않는다. +수정 후 복구 검사 전체는 13 passed, 30.10초, exit 0이었다. +후속 검사에서는 frontend의 tag 기반 이전 상태, metadata 변경, 조회 시 객체 누락을 +각각 주입했다. 세 경우 모두 backend를 포함한 양쪽 객체가 그대로이고 patch·복구· +성공 표시가 없음을 확인했다. 기존 kubectl double을 확장한 복구 검사 전체는 +16 passed, 45.72초, exit 0이었다. 이 결과는 최초 배포 지원이 아니라 사전 거부의 +검증이며 tag-to-digest 전환 절차 개발은 여전히 남아 있다. + +전체 객체 replace 대안은 기각했다. server dry-run이 추가한 last-applied annotation이 +spec 복구 뒤 남는 반례가 unit test에서 실패했다. spec-only patch로 바꾸고 명시적인 +label·annotation 변경은 사전 거부한다. 상태 갱신만으로 resourceVersion이 달라져도 +정방향 CAS는 보수적으로 실패할 수 있다. 자동 강제 덮어쓰기보다 안전한 실패를 택했다. + +제약은 명확하다. 기존의 정상·digest-pinned Deployment 두 개만 지원한다. +최초 배포, tag 기반 이전 버전, metadata migration에는 별도 승인된 절차가 필요하다. +쓰기 응답 유실은 실제 반영 여부를 확정할 수 없으므로 자동 재시도·복구하지 않는다. +다른 writer의 spec 변경이나 객체 재생성도 복구 대상이 아니다. 운영자가 현재 상태와 +승인된 release 기록을 확인해야 하며 이 도구는 모든 부분 실패의 자동 복구를 약속하지 않는다. + +snapshot은 같은 실행의 private 임시 디렉터리에서 `umask 077`로 만들고 종료 시 지운다. +원시 Kubernetes JSON, command stderr, kubeconfig를 artifact나 로그로 공개하지 않는다. +이는 지속 보관되는 사고 복구 원장이 아니다. 사고 후에도 필요한 승인된 이전 상태는 +별도의 접근 통제·보존 정책이 있는 운영 기록에서 확보해야 한다. + +검증은 실제 workflow의 Apply to AKS shell을 실행하되 kubectl을 unit double로 +대체한다. server default와 annotation 추가, backend 성공 뒤 frontend rollout 실패, +다른 writer, 응답 유실, 409에 해당하는 충돌, UID 변경, 복구 rollout 실패와 최종 +readback 불일치를 다룬다. 합성 Kubernetes 객체는 unit test에만 쓰며 클러스터 +admission·컨트롤러·네트워크 동작이 검증됐다고 주장하지 않는다. + +```sh +uv run --project backend --frozen --offline python -m pytest --noconftest \ + backend/tests/test_runtime_deployment_restore.py \ + backend/tests/test_release_manifest_digests.py \ + backend/tests/test_runtime_image_targets.py \ + backend/tests/test_release_governance.py -q -W error +shellcheck scripts/restore_runtime_deployment.sh scripts/deploy_runtime_manifests.sh +actionlint .github/workflows/deploy.yml .github/workflows/docker-publish.yml +``` + +위 구현의 로컬 검사에서 75 passed, 9.57초, exit 0을 확인했다. +Ruff·ShellCheck·actionlint도 exit 0이었다. commit 후 고정 SHA 검증은 +PR에 따로 기록한다. 실제 클러스터 쓰기는 실행하지 않았다. + +release의 동일 대상 직렬화, 오래된 release 거부, 환경 승인, DB를 포함한 readiness, +정상 인증을 거친 제품 화면의 Visual Inspection은 아직 별도로 완료해야 한다. + +## Readiness 구현과 문서의 불일치 조사 + +2026-09-07에 `82d230fa6c7f10b2f311464e59ca9b47823f40f5`를 조사했다. +DeepWiki는 CHANGELOG의 `/healthz`·`/readyz` 추가 기록으로 구현·배포를 추정했지만, +현재 `backend/main.py`에는 root 응답만 있고 언급된 `backend/core/observability.py`도 +없다. 현재 코드의 `db/session.py`에는 주 DB와 읽기 전용 engine이 따로 있으며 +request session의 종료와 application engine의 종료는 서로 다른 수명 주기다. + +실제 앱에 기존 httpx ASGITransport로 요청한 결과 `/`는 200, `/healthz`와 +`/readyz`는 404였다. `python -W error` 실행은 exit 0으로 종료됐다. +환경을 비운 뒤 unit 설정과 연결하지 않는 DB 주소를 제공했고, lifespan·worker를 +실행하거나 실제 DB·메일·provider에 접근하지 않았다. 따라서 이는 라우트 부재의 +재현이지 네트워크 서버·DB readiness 검증은 아니다. 앞선 TestClient 재현은 같은 +응답을 보였지만 StarletteDeprecationWarning이 있어 clean 근거에서 제외했다. + +로컬 Git history의 `a9214ca2`와 과거 release branch +`release/ci-cd-governance-20260509`의 tip +`3abc06f268aef6d151ddd06c54ac817277129fc4`에는 `/readyz`에서 기존 engine으로 +`SELECT 1`을 실행하는 선행 코드가 있다. 해당 commit은 관측한 보호 develop의 +ancestor가 아니다. 기존 2026-05-11 release 계획도 이 branch를 통째로 병합하지 +말고 필요한 변경을 검증해 이식하도록 기록했다. 현재 선행 PR 상태는 GitHub API +사용량 제한으로 확인하지 못했다. 선행 변경을 폐기하거나 새 owner가 없다고 단정하지 +않고 PR·승계 범위를 먼저 확인한다. 이후에는 liveness와 dependency readiness를 +분리하고, 실제 격리 PostgreSQL에서 성공·장애·연결 정리를 검증해야 한다. + ## 이미지 경계 참고 문헌 +Kubernetes Authors. (n.d.-a). *Kubernetes API concepts*. Retrieved September 7, +2026, from https://kubernetes.io/docs/reference/using-api/api-concepts/#updates-to-existing-resources + +Kubernetes Authors. (n.d.-b). *kubectl patch*. Retrieved September 7, 2026, from +https://kubernetes.io/docs/reference/kubectl/generated/kubectl_patch/ + Docker, Inc. (n.d.). *JSONArgsRecommended*. Docker Docs. https://docs.docker.com/reference/build-checks/json-args-recommended/ diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 98bc17d2a..d0e95302f 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -554,6 +554,51 @@ button, form, navigation, chart, or asynchronous data surface. | Release-train convergence | no buyer can assess a product with ~100 unconverged open PRs (102 at the 2026-08-25 snapshot) | strict gates exist but queue topology is fragmented | #1428, #1371, #1324 | all PRs classified; duplicates closed; parent-first integration; one immutable RC SHA | | Product/release truth | documentation conflicts with protected behavior/version | retry is shipped; release doc says `v0.1.0`; version is `0.14.4` | #1392, this PR | README, architecture, version, changelog, release manifest, and operator guide agree | | Independent review path | automation cannot lawfully self-approve | effective rulesets require independent post-last-push approval | #1371 | verified reviewer route and normal protected merge without bypass | +| Runtime release identity and recovery | a partially failed update can leave customer requests crossing incompatible frontend/backend versions | targeted protected observation on 2026-09-07: `develop@042b0c70531b229af3acbd0421a2f23098d848b3` still selects tags and sequentially applies two Deployments; neither manifest defines startup/readiness probes | #1365 image boundary, #1586 digest handoff, #1588 conditional spec recovery (all Proposed until protected integration) | immutable imageID evidence, first-digest migration, controlled same-target release order, actual partial-failure recovery, dependency-aware readiness and authenticated customer flow | + +#### Runtime deployment follow-up — 2026-09-07 + +This is a targeted release-safety observation, not a refresh of the historical +whole-repository inventory above. The protected SHA was fetched from GitHub's +`branches/develop` API; its `deploy.yml` and both Deployment manifests were read +at that exact revision. A Kubernetes namespace named `naruon-dev` is not evidence +of a protected GitHub deployment environment. Secrets availability alone does +not prove package eligibility, protected integration, or a deployed service. + +The owner remains Naruon's runtime deployment workflow, with organization-wide +review/release policy owned by `ContextualWisdomLab/.github`. The stack preserves +#1365 at `9b137f25f426743e18fc61125575ec7949d45db8`, #1586 at +`a48aa3a3e81ba58b2fa55cd758e86b9272455b1a`, and #1588's implementation at +`edd5e25088d9979d4d815dc0aedfebc2ceaf1057`. These open PRs are proposals, not +released contracts. Do not copy central workflows into this repository. + +The implementation evidence is deliberately narrower than the product goal: +76 local tests passed at #1588's implementation SHA; an independent execution +reproduced the final-pair drift regression's passing case. The real workflow +shell and cleanup run against a unit-only kubectl double. No actual cluster +write, admission test, release publication, or authenticated Naruon UI inspection +is established by that result. [The doctoring record](doctoring/runtime-image-boundary-verification.md#기존-배포-쌍의-조건부-복구) +explains the alternatives and failures; [the visual receipt](https://github.com/ContextualWisdomLab/naruon/pull/1588#issuecomment-5565347356) +covers only GitHub-rendered documentation. + +The next release must demonstrate each remaining condition, without relaxing +the current rollback guard to obtain a passing deployment: + +1. Preserve parent deltas and obtain exact-head required checks and independent + approval before protected integration. +2. Establish an approved initial/tag-to-digest migration with a verified prior + artifact. The proposed recovery path requires an existing healthy, + digest-pinned pair and cannot perform that migration or first installation. +3. Serialize the same deployment target, reject stale releases, and verify the + actual environment approval policy; distinct tag concurrency keys are not + a same-target lock. +4. Prove startup, database/dependency readiness, drain, and the authenticated + customer journey on the exact deployed imageIDs. Rollout success without + readiness probes is not sufficient. +5. Rehearse partial failure against an authorized real Kubernetes environment, + including concurrent writers and ambiguous responses. Two resource updates + are not atomic. Keep private recovery evidence under an approved retention + policy; ephemeral snapshots are not a durable incident recovery record. ### P0 — Connector and provider action diff --git a/scripts/deploy_runtime_manifests.sh b/scripts/deploy_runtime_manifests.sh new file mode 100644 index 000000000..8d9bea3f2 --- /dev/null +++ b/scripts/deploy_runtime_manifests.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Update an existing pair only after both server dry runs and rollback baselines. +set -euo pipefail +if [[ "$#" != 1 ]]; then + printf 'deployment_failed:invalid_arguments\n' >&2 + exit 1 +fi +manifest_directory="$1" +script_directory="$(cd "$(dirname "$0")" && pwd)" +umask 077 +snapshot_directory="$(mktemp -d "${RUNNER_TEMP:?}/naruon-deployment.XXXXXX")" +cleanup_snapshots() { + local image_component + for image_component in backend frontend; do + rm -f "$snapshot_directory/$image_component.before.json" \ + "$snapshot_directory/$image_component.proposed.json" \ + "$snapshot_directory/$image_component.forward-patch.json" \ + "$snapshot_directory/$image_component.applied.json" \ + "$snapshot_directory/$image_component.owned.json" \ + "$snapshot_directory/$image_component.current.json" + done + rm -f "$snapshot_directory/command-error" + rmdir "$snapshot_directory" +} +trap cleanup_snapshots EXIT +fail_deployment() { + printf 'deployment_failed:%s\n' "$1" >&2 + exit 1 +} +same_owned_spec() { + jq -e -s 'length == 2 and .[0].metadata.uid == .[1].metadata.uid and .[0].spec == .[1].spec' \ + "$1" "$2" > /dev/null 2> "$snapshot_directory/command-error" +} +verify_owned_pair() { + local image_component + for image_component in backend frontend; do + [[ -f "$snapshot_directory/$image_component.owned.json" ]] || continue + kubectl get deployment "$image_component" -n naruon-dev -o json \ + > "$snapshot_directory/$image_component.current.json" 2> "$snapshot_directory/command-error" || return 1 + same_owned_spec "$snapshot_directory/$image_component.owned.json" \ + "$snapshot_directory/$image_component.current.json" || return 1 + done +} +restore_owned_pair() { + local image_component + # Refuse all automatic rollback if either resource has another writer's spec. + verify_owned_pair || return 1 + for image_component in frontend backend; do + [[ -f "$snapshot_directory/$image_component.owned.json" ]] || continue + bash "$script_directory/restore_runtime_deployment.sh" "$image_component" \ + "$snapshot_directory/$image_component.before.json" \ + "$snapshot_directory/$image_component.owned.json" || return 1 + done + # Two resources are not atomic; recheck both after the last restoration. + for image_component in backend frontend; do + [[ -f "$snapshot_directory/$image_component.owned.json" ]] || continue + kubectl get deployment "$image_component" -n naruon-dev -o json \ + > "$snapshot_directory/$image_component.current.json" 2> "$snapshot_directory/command-error" || return 1 + same_owned_spec "$snapshot_directory/$image_component.before.json" \ + "$snapshot_directory/$image_component.current.json" || return 1 + done +} +for image_component in backend frontend; do + kubectl rollout status "deployment/$image_component" -n naruon-dev --timeout=120s \ + > /dev/null 2> "$snapshot_directory/command-error" || fail_deployment baseline_unhealthy + kubectl get deployment "$image_component" -n naruon-dev -o json \ + > "$snapshot_directory/$image_component.before.json" 2> "$snapshot_directory/command-error" || fail_deployment baseline_missing + kubectl apply --dry-run=server -f "$manifest_directory/$image_component-deployment.yaml" -o json \ + > "$snapshot_directory/$image_component.proposed.json" 2> "$snapshot_directory/command-error" || fail_deployment dry_run_failed + # Both objects must be existing, digest-pinned, and on the same observed + # revision. The later JSON Patch uses that resourceVersion as a server-side CAS. + jq -e -s --arg component "$image_component" ' + length == 2 and all(.[]; + .kind == "Deployment" and .metadata.name == $component and + .metadata.namespace == "naruon-dev" and + (.metadata.uid | type == "string" and length > 0) and + (.metadata.resourceVersion | type == "string" and length > 0) and + (.spec | type == "object") and + (.spec.template.spec.containers | length > 0) and + all((.spec.template.spec.containers + (.spec.template.spec.initContainers // []))[]; + .image | test("@sha256:[0-9a-f]{64}$"))) and + .[0].metadata.uid == .[1].metadata.uid and + .[0].metadata.resourceVersion == .[1].metadata.resourceVersion and + (.[0].metadata.labels // {}) == (.[1].metadata.labels // {}) and + ((.[0].metadata.annotations // {}) | del(."kubectl.kubernetes.io/last-applied-configuration")) == + ((.[1].metadata.annotations // {}) | del(."kubectl.kubernetes.io/last-applied-configuration")) + ' "$snapshot_directory/$image_component.before.json" "$snapshot_directory/$image_component.proposed.json" \ + > /dev/null 2> "$snapshot_directory/command-error" || fail_deployment baseline_or_proposal_invalid +done +for image_component in backend frontend; do + # Dry-run apply may add last-applied annotations. Only change spec; metadata + # migrations need their own contract, not an incomplete spec-only rollback. + jq -s ' + [{op:"test", path:"/metadata/uid", value:.[0].metadata.uid}, + {op:"test", path:"/metadata/resourceVersion", value:.[0].metadata.resourceVersion}, + {op:"replace", path:"/spec", value:.[1].spec}] + ' "$snapshot_directory/$image_component.before.json" "$snapshot_directory/$image_component.proposed.json" \ + > "$snapshot_directory/$image_component.forward-patch.json" 2> "$snapshot_directory/command-error" || fail_deployment patch_preparation + # A lost write response is ambiguous. Do not attempt blind rollback or retry. + kubectl patch deployment "$image_component" -n naruon-dev --type=json \ + --patch-file "$snapshot_directory/$image_component.forward-patch.json" -o json \ + > "$snapshot_directory/$image_component.applied.json" 2> "$snapshot_directory/command-error" || fail_deployment write_unconfirmed + kubectl get deployment "$image_component" -n naruon-dev -o json \ + > "$snapshot_directory/$image_component.owned.json" 2> "$snapshot_directory/command-error" || fail_deployment ownership_unconfirmed + same_owned_spec "$snapshot_directory/$image_component.applied.json" \ + "$snapshot_directory/$image_component.owned.json" || fail_deployment ownership_changed + if ! kubectl rollout status "deployment/$image_component" -n naruon-dev --timeout=120s \ + > /dev/null 2> "$snapshot_directory/command-error"; then + if restore_owned_pair; then + fail_deployment rollback_verified + fi + fail_deployment rollback_unconfirmed + fi +done +verify_owned_pair || fail_deployment final_ownership_changed +printf 'deployment_verified\n' diff --git a/scripts/restore_runtime_deployment.sh b/scripts/restore_runtime_deployment.sh new file mode 100644 index 000000000..1d13665a1 --- /dev/null +++ b/scripts/restore_runtime_deployment.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Restore one known existing deployment, never delete or force-undo a resource. +set -euo pipefail +if [[ "$#" != 3 || ( "$1" != backend && "$1" != frontend ) ]]; then + printf 'restore_failed:invalid_arguments\n' >&2 + exit 1 +fi +image_component="$1" +before_snapshot="$2" +owned_snapshot="$3" +umask 077 +restore_directory="$(mktemp -d "${RUNNER_TEMP:?}/naruon-restore.XXXXXX")" +trap 'rm -f "$restore_directory/current.json" "$restore_directory/patch.json" "$restore_directory/restored.json" "$restore_directory/command-error"; rmdir "$restore_directory"' EXIT + +fail_restore() { + printf 'restore_failed:%s:%s\n' "$image_component" "$1" >&2 + exit 1 +} + +if ! kubectl get deployment "$image_component" -n naruon-dev -o json \ + > "$restore_directory/current.json" 2> "$restore_directory/command-error"; then + fail_restore read_current +fi +# Compare server-normalized post-apply spec, not the client manifest. Status-only +# controller updates may advance resourceVersion without changing owned spec. +if ! jq -e -s --arg component "$image_component" ' + length == 3 and + all(.[]; .kind == "Deployment" and .metadata.name == $component and + .metadata.namespace == "naruon-dev" and + (.metadata.uid | type == "string" and length > 0) and + (.metadata.resourceVersion | type == "string" and length > 0) and + (.spec | type == "object")) and + .[0].metadata.uid == .[1].metadata.uid and + .[1].metadata.uid == .[2].metadata.uid and .[1].spec == .[2].spec +' "$before_snapshot" "$owned_snapshot" "$restore_directory/current.json" \ + > /dev/null 2> "$restore_directory/command-error"; then + fail_restore ownership_changed +fi +if ! jq -s ' + [{op:"test", path:"/metadata/uid", value:.[1].metadata.uid}, + {op:"test", path:"/metadata/resourceVersion", value:.[1].metadata.resourceVersion}, + {op:"replace", path:"/spec", value:.[0].spec}] +' "$before_snapshot" "$restore_directory/current.json" \ + > "$restore_directory/patch.json" 2> "$restore_directory/command-error"; then + fail_restore patch_preparation +fi +if ! kubectl patch deployment "$image_component" -n naruon-dev --type=json \ + --patch-file "$restore_directory/patch.json" -o json \ + > "$restore_directory/restored.json" 2> "$restore_directory/command-error"; then + # Includes optimistic conflict and an ambiguous response; never retry blindly. + fail_restore patch_unconfirmed +fi +if ! kubectl rollout status "deployment/$image_component" -n naruon-dev --timeout=120s \ + > /dev/null 2> "$restore_directory/command-error"; then + fail_restore rollout_unconfirmed +fi +if ! kubectl get deployment "$image_component" -n naruon-dev -o json \ + > "$restore_directory/restored.json" 2> "$restore_directory/command-error"; then + fail_restore readback_unconfirmed +fi +if ! jq -e -s ' + length == 2 and .[0].metadata.uid == .[1].metadata.uid and .[0].spec == .[1].spec +' "$before_snapshot" "$restore_directory/restored.json" \ + > /dev/null 2> "$restore_directory/command-error"; then + fail_restore readback_changed +fi +printf 'restore_verified:%s\n' "$image_component"