From a03cfcedbe9636673ba6195c3a215faf05cdd6fe Mon Sep 17 00:00:00 2001 From: Paul Razgaitis Date: Thu, 13 Aug 2026 10:45:17 -0500 Subject: [PATCH 1/4] fix(server): preserve Kubernetes sandbox security --- server/Dockerfile | 17 ++++-- .../services/k8s/egress_helper.py | 17 +++++- .../services/k8s/security_context.py | 19 ++++++- .../services/k8s/template_manager.py | 52 +++++++++++++++++++ .../tests/k8s/test_batchsandbox_provider.py | 4 ++ .../tests/k8s/test_batchsandbox_template.py | 52 +++++++++++++++++++ server/tests/k8s/test_egress_helper.py | 10 +++- 7 files changed, 160 insertions(+), 11 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index f008e5ebf..4b048f26f 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.10-slim AS builder +FROM python:3.10-alpine AS builder # Optional: inject the release version when .git is unavailable so hatch-vcs # resolves the real version instead of falling back to 0.1.0.dev0. Passed by @@ -28,9 +28,7 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ WORKDIR /app -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl ca-certificates \ - && rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache curl ca-certificates RUN curl -LsSf https://astral.sh/uv/install.sh | sh ENV PATH="/root/.local/bin:/root/.cargo/bin:${PATH}" @@ -44,7 +42,7 @@ COPY LICENSE README.md ./ # Install the project itself into the venv (deps already synced) RUN uv pip install --no-deps --editable . -FROM python:3.10-slim AS runtime +FROM python:3.10-alpine AS runtime ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PYTHONDONTWRITEBYTECODE=1 \ @@ -55,6 +53,15 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ WORKDIR /app +# The service runs entirely from the copied virtualenv. The base image's +# system setuptools is build tooling, not a runtime dependency, and its +# vendored packages needlessly expand the production vulnerability surface. +RUN rm -rf \ + /usr/local/lib/python3.10/site-packages/_distutils_hack \ + /usr/local/lib/python3.10/site-packages/pkg_resources \ + /usr/local/lib/python3.10/site-packages/setuptools \ + /usr/local/lib/python3.10/site-packages/setuptools-*.dist-info + COPY --from=builder /app/.venv /app/.venv COPY --from=builder /app/opensandbox_server /app/opensandbox_server COPY --from=builder /app/opensandbox_server/examples/example.config.k8s.toml /etc/opensandbox/config.toml diff --git a/server/opensandbox_server/services/k8s/egress_helper.py b/server/opensandbox_server/services/k8s/egress_helper.py index 6127787bb..b92de579c 100644 --- a/server/opensandbox_server/services/k8s/egress_helper.py +++ b/server/opensandbox_server/services/k8s/egress_helper.py @@ -44,11 +44,19 @@ def prep_execd_init_for_egress(exec_install_script: str) -> tuple[str, Dict[str, security context dict must be applied to the execd init container (typically via ``build_security_context_from_dict`` in ``security_context``). + A pod-level non-root UID otherwise overrides this init container and makes + the sysctl write fail with EPERM. Keep the root exception explicit and + local to the privileged init container. + Returns: - ``(prefixed_shell_script, {"privileged": True})`` + The prefixed shell script and its privileged root security context. """ script = f"set -e; echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6 && {exec_install_script}" - return script, {"privileged": True} + return script, { + "privileged": True, + "runAsNonRoot": False, + "runAsUser": 0, + } def build_security_context_for_sandbox_container( @@ -113,6 +121,11 @@ def apply_egress_to_spec( "env": env, "securityContext": { "capabilities": {"add": ["NET_ADMIN"]}, + # A pod-level non-root UID clears NET_ADMIN from the effective + # capability set. The sidecar must retain it to install the + # nftables/iptables policy, so scope the root exception here. + "runAsNonRoot": False, + "runAsUser": 0, }, "ports": [{"name": "egress-api", "containerPort": 18080}], "readinessProbe": { diff --git a/server/opensandbox_server/services/k8s/security_context.py b/server/opensandbox_server/services/k8s/security_context.py index 8f398d1ea..2d1b17100 100644 --- a/server/opensandbox_server/services/k8s/security_context.py +++ b/server/opensandbox_server/services/k8s/security_context.py @@ -41,13 +41,22 @@ def build_security_context_from_dict( ) privileged = security_context_dict.get("privileged") - - if capabilities is None and privileged is None: + run_as_non_root = security_context_dict.get("runAsNonRoot") + run_as_user = security_context_dict.get("runAsUser") + + if ( + capabilities is None + and privileged is None + and run_as_non_root is None + and run_as_user is None + ): return None return V1SecurityContext( capabilities=capabilities, privileged=privileged, + run_as_non_root=run_as_non_root, + run_as_user=run_as_user, ) @@ -72,6 +81,12 @@ def serialize_security_context_to_dict( if security_context.privileged is not None: result["privileged"] = security_context.privileged + if getattr(security_context, "run_as_non_root", None) is not None: + result["runAsNonRoot"] = security_context.run_as_non_root + + if getattr(security_context, "run_as_user", None) is not None: + result["runAsUser"] = security_context.run_as_user + if getattr(security_context, "seccomp_profile", None) is not None: sp = security_context.seccomp_profile profile_dict: Dict[str, Any] = {"type": sp.type} diff --git a/server/opensandbox_server/services/k8s/template_manager.py b/server/opensandbox_server/services/k8s/template_manager.py index 87bf03059..dd337e44b 100644 --- a/server/opensandbox_server/services/k8s/template_manager.py +++ b/server/opensandbox_server/services/k8s/template_manager.py @@ -102,7 +102,59 @@ def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any result[key] = BaseSandboxTemplateManager._deep_merge( result[key], override_value ) + elif BaseSandboxTemplateManager._are_named_object_lists( + result[key], override_value + ): + result[key] = BaseSandboxTemplateManager._merge_named_object_lists( + result[key], override_value + ) else: result[key] = BaseSandboxTemplateManager._deep_copy(override_value) return result + + @staticmethod + def _are_named_object_lists(base: Any, override: Any) -> bool: + """Return whether both values are non-empty lists keyed by unique names. + + Kubernetes uses ``name`` as the merge key for containers, init + containers, environment variables, volumes, and several related pod + fields. Restricting this behavior to unambiguously named object lists + keeps ordinary lists, such as commands and tolerations, replace-only. + """ + if not isinstance(base, list) or not isinstance(override, list): + return False + if not base or not override: + return False + + for items in (base, override): + names = [item.get("name") for item in items if isinstance(item, dict)] + if len(names) != len(items): + return False + if any(not isinstance(name, str) or not name for name in names): + return False + if len(set(names)) != len(names): + return False + + return True + + @staticmethod + def _merge_named_object_lists( + base: list[Dict[str, Any]], override: list[Dict[str, Any]] + ) -> list[Dict[str, Any]]: + """Merge runtime objects into template objects with the same name.""" + result = BaseSandboxTemplateManager._deep_copy(base) + indexes = {item["name"]: index for index, item in enumerate(result)} + + for override_item in override: + name = override_item["name"] + if name in indexes: + index = indexes[name] + result[index] = BaseSandboxTemplateManager._deep_merge( + result[index], override_item + ) + else: + indexes[name] = len(result) + result.append(BaseSandboxTemplateManager._deep_copy(override_item)) + + return result diff --git a/server/tests/k8s/test_batchsandbox_provider.py b/server/tests/k8s/test_batchsandbox_provider.py index 4d2358503..3b179198d 100644 --- a/server/tests/k8s/test_batchsandbox_provider.py +++ b/server/tests/k8s/test_batchsandbox_provider.py @@ -1787,6 +1787,8 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) caps = sidecar.get("securityContext", {}).get("capabilities", {}) assert "NET_ADMIN" in caps.get("add", []) assert sidecar.get("securityContext", {}).get("privileged") is not True + assert sidecar["securityContext"]["runAsNonRoot"] is False + assert sidecar["securityContext"]["runAsUser"] == 0 assert "command" not in sidecar assert sidecar["readinessProbe"]["httpGet"]["path"] == "/healthz" assert sidecar["readinessProbe"]["httpGet"]["port"] == 18080 @@ -1797,6 +1799,8 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) assert execd_init["name"] == "execd-installer" assert execd_init["image"] == "execd:latest" assert execd_init.get("securityContext", {}).get("privileged") is True + assert execd_init["securityContext"]["runAsNonRoot"] is False + assert execd_init["securityContext"]["runAsUser"] == 0 assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0] main = next(c for c in containers if c["name"] == "sandbox") diff --git a/server/tests/k8s/test_batchsandbox_template.py b/server/tests/k8s/test_batchsandbox_template.py index e06e8731d..4a426f1cf 100644 --- a/server/tests/k8s/test_batchsandbox_template.py +++ b/server/tests/k8s/test_batchsandbox_template.py @@ -112,6 +112,58 @@ def test_deep_merge_replaces_lists_not_merges(self): result = BatchSandboxTemplateManager._deep_merge(base, override) assert result == {"spec": {"tolerations": [{"key": "b"}]}} + + def test_deep_merge_merges_named_kubernetes_objects(self): + base = { + "spec": { + "containers": [ + { + "name": "sandbox", + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + }, + "resources": {"limits": {"memory": "6Gi"}}, + }, + {"name": "template-sidecar", "image": "template:latest"}, + ] + } + } + override = { + "spec": { + "containers": [ + { + "name": "sandbox", + "image": "runtime:latest", + "resources": {"requests": {"cpu": "100m"}}, + }, + {"name": "runtime-sidecar", "image": "runtime-sidecar:latest"}, + ] + } + } + + result = BatchSandboxTemplateManager._deep_merge(base, override) + + assert result["spec"]["containers"] == [ + { + "name": "sandbox", + "image": "runtime:latest", + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + }, + "resources": { + "limits": {"memory": "6Gi"}, + "requests": {"cpu": "100m"}, + }, + }, + {"name": "template-sidecar", "image": "template:latest"}, + {"name": "runtime-sidecar", "image": "runtime-sidecar:latest"}, + ] + + # The merge must not mutate either input template. + assert "image" not in base["spec"]["containers"][0] + assert "securityContext" not in override["spec"]["containers"][0] def test_deep_merge_none_values_do_not_override(self): base = {"spec": {"expireTime": "2024-12-31"}} diff --git a/server/tests/k8s/test_egress_helper.py b/server/tests/k8s/test_egress_helper.py index d19375fc9..68772c773 100644 --- a/server/tests/k8s/test_egress_helper.py +++ b/server/tests/k8s/test_egress_helper.py @@ -221,7 +221,7 @@ def test_handles_missing_default_action(self): assert "egress" in policy_dict def test_security_context_adds_net_admin_not_privileged(self): - """Egress sidecar uses NET_ADMIN only (IPv6 is disabled in execd init when egress is on).""" + """Egress uses the narrow NET_ADMIN + root exception, not privileged mode.""" egress_image = "opensandbox/egress:v1.1.6" network_policy = NetworkPolicy( default_action="deny", @@ -233,6 +233,8 @@ def test_security_context_adds_net_admin_not_privileged(self): security_context = container["securityContext"] assert security_context.get("privileged") is not True assert "NET_ADMIN" in security_context.get("capabilities", {}).get("add", []) + assert security_context["runAsNonRoot"] is False + assert security_context["runAsUser"] == 0 def test_no_command_uses_image_entrypoint(self): container = _egress_container( @@ -513,7 +515,11 @@ class TestPrepExecdInitForEgress: def test_returns_privileged_security_dict_and_prefixed_script(self): base = "cp ./execd /opt/opensandbox/execd" script, sc = prep_execd_init_for_egress(base) - assert sc == {"privileged": True} + assert sc == { + "privileged": True, + "runAsNonRoot": False, + "runAsUser": 0, + } assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in script assert script.endswith(base) From b3654b6168ab7f2dc27ef3a04ba92365a18353db Mon Sep 17 00:00:00 2001 From: Paul Razgaitis Date: Thu, 13 Aug 2026 10:51:47 -0500 Subject: [PATCH 2/4] fix(egress): harden runtime image --- components/egress/Dockerfile | 41 +++++++++++++----------------------- components/egress/go.mod | 12 +++++------ components/egress/go.sum | 24 ++++++++++----------- 3 files changed, 33 insertions(+), 44 deletions(-) diff --git a/components/egress/Dockerfile b/components/egress/Dockerfile index 28ec0e10e..2559ea23b 100644 --- a/components/egress/Dockerfile +++ b/components/egress/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.25.9-bookworm AS builder +FROM golang:1.26.5-alpine3.23 AS builder WORKDIR /workspace @@ -75,41 +75,30 @@ RUN cd /workspace/components/internal && \ -X 'github.com/alibaba/opensandbox/internal/version.GitCommit=${GIT_COMMIT}'" \ -o /out/opensandbox-supervisor ./cmd/supervisor -FROM debian:bookworm-slim +FROM cgr.dev/chainguard/wolfi-base@sha256:07e60ff6586b56f03c625e27b604f9f7d29498fef32f099f6560f0d207b4a056 -# iptables is needed for DNS REDIRECT; ca-certificates for TLS to upstream resolvers -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - iptables \ +# Keep only the packages required by the egress runtime. The prior diagnostic +# toolbelt materially expanded the image's vulnerability surface and is not +# used by the entrypoint or enforcement code. +RUN apk add --no-cache \ + ca-certificates \ iproute2 \ + iptables \ nftables \ - ca-certificates \ - sudo \ - curl \ - wget \ - net-tools \ - dnsutils \ - netcat-openbsd \ - iputils-ping \ - traceroute \ - telnet \ - tcpdump \ - nmap \ - htop \ procps \ - strace \ - lsof \ - python3 \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* + py3.12-pip \ + python-3.12 \ + sudo # Python mitmproxy (transparent mode): mitmdump runs as user mitmproxy; iptables skips this uid. # /var/lib/mitmproxy is mitm's home, used as the confdir (CA + config.yaml live under .mitmproxy/). -RUN useradd -r -u 10042 -d /var/lib/mitmproxy -s /usr/sbin/nologin mitmproxy \ +RUN addgroup -S -g 10042 mitmproxy \ + && adduser -S -D -H -u 10042 -G mitmproxy -h /var/lib/mitmproxy -s /sbin/nologin mitmproxy \ && mkdir -p /var/lib/mitmproxy/.mitmproxy \ && chown -R mitmproxy:mitmproxy /var/lib/mitmproxy \ - && pip3 install --no-cache-dir --break-system-packages 'mitmproxy==11.0.2' \ + && pip3.12 install --no-cache-dir 'mitmproxy==12.2.3' \ && (command -v mitmdump && mitmdump --version) \ + && rm -rf /root/.cache \ && mkdir -p /var/egress/mitmscripts # Static mitmproxy options (mode, listen_host, connection_strategy, stream_large_bodies, diff --git a/components/egress/go.mod b/components/egress/go.mod index 3799b6d44..88eea6334 100644 --- a/components/egress/go.mod +++ b/components/egress/go.mod @@ -11,7 +11,7 @@ require ( go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.uber.org/automaxprocs v1.6.0 - golang.org/x/sys v0.45.0 + golang.org/x/sys v0.46.0 k8s.io/apimachinery v0.34.2 ) @@ -35,11 +35,11 @@ require ( go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/text v0.39.0 // indirect + golang.org/x/tools v0.47.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/grpc v1.82.1 // indirect diff --git a/components/egress/go.sum b/components/egress/go.sum index 4e1cdba46..dd0369c8d 100644 --- a/components/egress/go.sum +++ b/components/egress/go.sum @@ -65,19 +65,19 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= From 7ea24061813b41863cf2ca4d9b6d84686b06c5a2 Mon Sep 17 00:00:00 2001 From: Paul Razgaitis Date: Thu, 13 Aug 2026 11:02:34 -0500 Subject: [PATCH 3/4] fix(kubernetes): preserve snapshot source content --- kubernetes/Dockerfile.image-committer | 28 ++++-- kubernetes/cmd/image-committer/main.go | 104 ++++++++++++++++---- kubernetes/cmd/image-committer/main_test.go | 74 ++++++++++++++ 3 files changed, 180 insertions(+), 26 deletions(-) diff --git a/kubernetes/Dockerfile.image-committer b/kubernetes/Dockerfile.image-committer index c4269ad64..996a1dc57 100644 --- a/kubernetes/Dockerfile.image-committer +++ b/kubernetes/Dockerfile.image-committer @@ -13,10 +13,7 @@ # limitations under the License. # Build stage -FROM golang:1.25-alpine AS builder - -# Use Aliyun mirror for faster downloads in China -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +FROM golang:1.26.5-alpine3.23@sha256:622e56dbc11a8cfe87cafa2331e9a201877271cbff918af53d3be315f3da88cc AS builder WORKDIR /workspace @@ -31,18 +28,31 @@ COPY cmd/image-committer/ cmd/image-committer/ RUN CGO_ENABLED=0 GOOS=linux go build -o /usr/local/bin/image-committer ./cmd/image-committer/ # Runtime stage -FROM alpine:3.19 +FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 -# Use Aliyun mirror for faster downloads in China -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +ARG TARGETARCH # Install nerdctl for container operations # nerdctl is used to find containers, commit rootfs, and push images. # We use nerdctl directly (not crictl or ctr) to avoid CRI API version issues. RUN apk add --no-cache \ curl \ - jq \ - nerdctl + jq + +# Install the current upstream minimal client, verify the release checksum, and +# preserve multi-platform builds. The distro package lags the nerdctl behavior +# used by the snapshot path. +RUN case "${TARGETARCH}" in \ + amd64) NERDCTL_SHA256="de3206aeb7cbd5f20f5fb1f55c1e3bf2db1be567812a8a3f5e65eba2488347ee" ;; \ + arm64) NERDCTL_SHA256="76ced9bd0d03f6140f9cf7b927958b654cb8d5ecd3c58af585d096c8bdf9d6c2" ;; \ + *) printf 'unsupported TARGETARCH: %s\n' "${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl -fsSLo /tmp/nerdctl.tgz \ + "https://github.com/containerd/nerdctl/releases/download/v2.3.5/nerdctl-2.3.5-linux-${TARGETARCH}.tar.gz" \ + && printf '%s %s\n' "${NERDCTL_SHA256}" /tmp/nerdctl.tgz | sha256sum -c - \ + && tar -xzf /tmp/nerdctl.tgz -C /usr/local/bin nerdctl \ + && rm /tmp/nerdctl.tgz \ + && nerdctl --version # Create directory for containerd socket mount RUN mkdir -p /var/run/containerd diff --git a/kubernetes/cmd/image-committer/main.go b/kubernetes/cmd/image-committer/main.go index 7717fdf16..0aaf43147 100644 --- a/kubernetes/cmd/image-committer/main.go +++ b/kubernetes/cmd/image-committer/main.go @@ -62,8 +62,9 @@ type ContainerSpec struct { } type discoveredContainer struct { - ID string - Running bool + ID string + Running bool + SourceImage string } type snapshotResult struct { @@ -163,11 +164,28 @@ func main() { fmt.Fprintf(os.Stderr, "ERROR: Failed to find container '%s': %v\n", spec.Name, err) os.Exit(1) } + sourceImage, err := getContainerImage(container.ID) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to identify the source image for container '%s': %v Ensure the sandbox container still exists, then retry the pause operation.\n", spec.Name, err) + os.Exit(1) + } + container.SourceImage = sourceImage - fmt.Printf("Container '%s' -> ID: %s (running: %t)\n", spec.Name, container.ID, container.Running) + fmt.Printf("Container '%s' -> ID: %s (running: %t, source image: %s)\n", spec.Name, container.ID, container.Running, container.SourceImage) containers[spec.Name] = container } + // Kubernetes/containerd may retain unpacked snapshots while garbage-collecting + // the compressed source layers. A committed manifest still references those + // layers, so make them locally available before attempting a registry push. + fmt.Println("\n=== Step 1b: Ensure source image content is available ===") + for _, spec := range containerSpecs { + if err := pullImageContent(containers[spec.Name].SourceImage); err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to make source image content available for container '%s': %v Ensure the snapshot registry secret can read the source image registry, then retry the pause operation.\n", spec.Name, err) + os.Exit(1) + } + } + // Step 2: Flush each running container's filesystem from inside its runtime. // This is required for VM-isolated runtimes such as Kata, where host-side // sync does not flush the guest kernel's page cache. @@ -519,6 +537,53 @@ func commitContainer(containerID, targetImage string) error { return nil } +// getContainerImage returns the source image reference recorded on a container. +func getContainerImage(containerID string) (string, error) { + const imagePrefix = "OPENSANDBOX_SOURCE_IMAGE=" + args := append(nerdctlBaseArgs(), "inspect", "--format", imagePrefix+"{{.Image}}", containerID) + output, err := commandCombinedOutput("nerdctl", args...) + if err != nil { + return "", fmt.Errorf("nerdctl inspect failed for container %s: %v, output: %s", containerID, err, strings.TrimSpace(string(output))) + } + + // nerdctl can emit harmless network-namespace warnings to stderr while + // still returning the requested image on stdout. Prefix the formatted value + // so it can be identified regardless of stdout/stderr interleaving. + for _, line := range strings.Split(string(output), "\n") { + if value, found := strings.CutPrefix(strings.TrimSpace(line), imagePrefix); found && value != "" { + return value, nil + } + } + return "", fmt.Errorf("nerdctl inspect returned an empty source image for container %s", containerID) +} + +// pullImageContent restores any compressed source layers that containerd may +// have garbage-collected after unpacking the image. nerdctl commit reuses those +// layers in the snapshot manifest, and nerdctl push requires their local blobs. +func pullImageContent(sourceImage string) error { + fmt.Printf("Ensuring source image content is available: %s...\n", sourceImage) + + imageParts := strings.Split(sourceImage, "/") + if len(imageParts) == 0 || imageParts[0] == "" { + return fmt.Errorf("invalid source image: %s", sourceImage) + } + registryHost := imageParts[0] + isInsecure := shouldUseInsecureRegistry(registryHost) + loginToRegistryIfConfigured(registryHost, isInsecure) + + pullOpts := append(nerdctlBaseArgs(), "pull", "--all-platforms") + if isInsecure { + pullOpts = append(pullOpts, "--insecure-registry") + } + pullOpts = append(pullOpts, sourceImage) + + output, err := commandCombinedOutput("nerdctl", pullOpts...) + if err != nil { + return fmt.Errorf("failed to pull source image %s: %v, output: %s", sourceImage, err, strings.TrimSpace(string(output))) + } + return nil +} + // pushImage uses nerdctl to push the image to the registry. // nerdctl push does not support --username/--password flags, so we use // nerdctl login first, then nerdctl push with --insecure-registry. @@ -534,27 +599,20 @@ func pushImage(targetImage string) error { isInsecure := shouldUseInsecureRegistry(registryHost) - // Try to login using credentials from mounted secret - credDir := "/var/run/opensandbox/registry" - configPath := filepath.Join(credDir, "config.json") - if _, err := os.Stat(configPath); err == nil { - fmt.Printf("Found registry credentials at %s\n", configPath) - if err := nerdctlLogin(configPath, registryHost, isInsecure); err != nil { - fmt.Fprintf(os.Stderr, "WARNING: nerdctl login failed: %v (will attempt push anyway)\n", err) - } - } else { - fmt.Println("No registry credentials found, assuming insecure or pre-authenticated registry") - } + loginToRegistryIfConfigured(registryHost, isInsecure) // Build push options - pushOpts := append(nerdctlBaseArgs(), "push") + // A committed sandbox image is already a complete local single-platform + // manifest. Without --all-platforms nerdctl first builds a reduced-platform + // temporary image, whose content check tries to pull this brand-new tag from + // the remote registry and fails on the expected 404. + pushOpts := append(nerdctlBaseArgs(), "push", "--all-platforms") if isInsecure { pushOpts = append(pushOpts, "--insecure-registry") } pushOpts = append(pushOpts, targetImage) - cmd := exec.Command("nerdctl", pushOpts...) - output, err := cmd.CombinedOutput() + output, err := commandCombinedOutput("nerdctl", pushOpts...) if err != nil { return fmt.Errorf("failed to push image %s: %v, output: %s", targetImage, err, string(output)) } @@ -562,6 +620,18 @@ func pushImage(targetImage string) error { return nil } +func loginToRegistryIfConfigured(registryHost string, isInsecure bool) { + configPath := filepath.Join("/var/run/opensandbox/registry", "config.json") + if _, err := os.Stat(configPath); err == nil { + fmt.Printf("Found registry credentials at %s\n", configPath) + if err := nerdctlLogin(configPath, registryHost, isInsecure); err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Registry login failed: %v Verify the snapshot registry secret grants access to %s; the operation will continue in case the registry is already authenticated.\n", err, registryHost) + } + return + } + fmt.Println("No registry credentials found, assuming insecure or pre-authenticated registry") +} + // nerdctlLogin extracts credentials from a Docker config.json and runs nerdctl login. func nerdctlLogin(configPath, registryHost string, insecure bool) error { data, err := os.ReadFile(configPath) diff --git a/kubernetes/cmd/image-committer/main_test.go b/kubernetes/cmd/image-committer/main_test.go index df9add332..db9977cf1 100644 --- a/kubernetes/cmd/image-committer/main_test.go +++ b/kubernetes/cmd/image-committer/main_test.go @@ -164,6 +164,80 @@ func TestGetContainerIDByNerdctlReturnsHelpfulErrorWhenBothLookupsAreEmpty(t *te } } +func TestGetContainerImageReturnsSourceReferenceAfterWarning(t *testing.T) { + original := commandCombinedOutput + t.Cleanup(func() { commandCombinedOutput = original }) + commandCombinedOutput = func(name string, args ...string) ([]byte, error) { + if name != "nerdctl" { + t.Fatalf("unexpected command %q", name) + } + if !contains(args, "inspect") || !contains(args, "container-1") { + t.Fatalf("unexpected inspect arguments: %v", args) + } + return []byte("time=\"2026-08-13T13:55:17Z\" level=warning msg=\"failed to inspect NetNS\"\nOPENSANDBOX_SOURCE_IMAGE=registry.example.com/quovy/sandbox:release-1\ntime=\"2026-08-13T13:55:18Z\" level=warning msg=\"cleanup warning\"\n"), nil + } + + image, err := getContainerImage("container-1") + if err != nil { + t.Fatalf("expected source image lookup to succeed, got %v", err) + } + if image != "registry.example.com/quovy/sandbox:release-1" { + t.Fatalf("unexpected source image %q", image) + } +} + +func TestPullImageContentFetchesCompressedLayers(t *testing.T) { + original := commandCombinedOutput + t.Cleanup(func() { commandCombinedOutput = original }) + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "false") + + var gotArgs []string + commandCombinedOutput = func(name string, args ...string) ([]byte, error) { + if name != "nerdctl" { + t.Fatalf("unexpected command %q", name) + } + gotArgs = append([]string(nil), args...) + return []byte("pulled"), nil + } + + image := "registry.example.com/quovy/sandbox:release-1" + if err := pullImageContent(image); err != nil { + t.Fatalf("expected source content pull to succeed, got %v", err) + } + if !contains(gotArgs, "pull") || !contains(gotArgs, image) { + t.Fatalf("unexpected pull arguments: %v", gotArgs) + } + if !contains(gotArgs, "--all-platforms") { + t.Fatalf("source pull must bypass containerd's transfer-service cache: %v", gotArgs) + } +} + +func TestPushImagePreservesCommittedManifest(t *testing.T) { + original := commandCombinedOutput + t.Cleanup(func() { commandCombinedOutput = original }) + t.Setenv("SNAPSHOT_REGISTRY_INSECURE", "false") + + var gotArgs []string + commandCombinedOutput = func(name string, args ...string) ([]byte, error) { + if name != "nerdctl" { + t.Fatalf("unexpected command %q", name) + } + gotArgs = append([]string(nil), args...) + return []byte("pushed"), nil + } + + image := "registry.example.com/quovy/sandbox:snapshot-1" + if err := pushImage(image); err != nil { + t.Fatalf("expected snapshot push to succeed, got %v", err) + } + if !contains(gotArgs, "push") || !contains(gotArgs, image) { + t.Fatalf("unexpected push arguments: %v", gotArgs) + } + if !contains(gotArgs, "--all-platforms") { + t.Fatalf("snapshot push must preserve the committed manifest: %v", gotArgs) + } +} + func contains(values []string, target string) bool { for _, value := range values { if value == target { From 51b38cf975e645e87a14c538cb5adb0add79e804 Mon Sep 17 00:00:00 2001 From: Paul Razgaitis Date: Thu, 13 Aug 2026 11:08:10 -0500 Subject: [PATCH 4/4] ci: publish hardened Quovy sandbox images --- .github/workflows/publish-quovy-images.yml | 178 +++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/publish-quovy-images.yml diff --git a/.github/workflows/publish-quovy-images.yml b/.github/workflows/publish-quovy-images.yml new file mode 100644 index 000000000..f9a9c6487 --- /dev/null +++ b/.github/workflows/publish-quovy-images.yml @@ -0,0 +1,178 @@ +name: Publish Quovy OpenSandbox images + +on: + push: + branches: [main] + paths: + - .github/workflows/publish-quovy-images.yml + - kubernetes/Dockerfile.image-committer + - kubernetes/cmd/image-committer/** + - kubernetes/go.mod + - kubernetes/go.sum + - server/Dockerfile + - server/egress/** + - server/pyproject.toml + - server/uv.lock + workflow_dispatch: + +permissions: + contents: read + id-token: write + packages: write + +concurrency: + group: publish-quovy-opensandbox-${{ github.ref }} + cancel-in-progress: false + +jobs: + publish: + name: Build, scan, and sign ${{ matrix.name }} + if: github.repository == 'Quovy/OpenSandbox' + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - name: server + context: server + dockerfile: server/Dockerfile + repository: opensandbox-server + - name: egress + context: server/egress + dockerfile: server/egress/Dockerfile + repository: opensandbox-egress + - name: image-committer + context: kubernetes + dockerfile: kubernetes/Dockerfile.image-committer + repository: opensandbox-image-committer + env: + IMAGE_REF: ghcr.io/quovy/${{ matrix.repository }} + COSIGN_CERTIFICATE_IDENTITY: https://github.com/Quovy/OpenSandbox/.github/workflows/publish-quovy-images.yml@refs/heads/main + COSIGN_VERIFICATION_OIDC_ISSUER: https://token.actions.githubusercontent.com + steps: + - name: Check out maintained source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + with: + cache-image: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 + + - name: Log in to GitHub Container Registry + env: + GHCR_TOKEN: ${{ github.token }} + run: echo "$GHCR_TOKEN" | docker login ghcr.io --username "${GITHUB_ACTOR}" --password-stdin + + - name: Build and publish immutable candidate + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.IMAGE_REF }}:sha-${{ github.sha }} + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,scope=${{ matrix.name }},mode=max + provenance: mode=max + sbom: true + + - name: Scan amd64 image and record High/Critical findings + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + env: + TRIVY_PLATFORM: linux/amd64 + with: + version: v0.66.0 + scan-type: image + image-ref: ${{ env.IMAGE_REF }}@${{ steps.build.outputs.digest }} + scanners: vuln + severity: HIGH,CRITICAL + format: json + output: trivy-${{ matrix.name }}-amd64.json + exit-code: '0' + + - name: Block Critical findings on amd64 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + env: + TRIVY_PLATFORM: linux/amd64 + with: + version: v0.66.0 + scan-type: image + image-ref: ${{ env.IMAGE_REF }}@${{ steps.build.outputs.digest }} + scanners: vuln + severity: CRITICAL + format: table + exit-code: '1' + + - name: Scan arm64 image and record High/Critical findings + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + env: + TRIVY_PLATFORM: linux/arm64 + with: + version: v0.66.0 + scan-type: image + image-ref: ${{ env.IMAGE_REF }}@${{ steps.build.outputs.digest }} + scanners: vuln + severity: HIGH,CRITICAL + format: json + output: trivy-${{ matrix.name }}-arm64.json + exit-code: '0' + + - name: Block Critical findings on arm64 + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + env: + TRIVY_PLATFORM: linux/arm64 + with: + version: v0.66.0 + scan-type: image + image-ref: ${{ env.IMAGE_REF }}@${{ steps.build.outputs.digest }} + scanners: vuln + severity: CRITICAL + format: table + exit-code: '1' + + - name: Install Cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + with: + cosign-release: v3.0.6 + + - name: Sign and verify the scanned digest + env: + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + digest_ref="${IMAGE_REF}@${IMAGE_DIGEST}" + cosign sign --yes "$digest_ref" + cosign verify \ + --certificate-identity "$COSIGN_CERTIFICATE_IDENTITY" \ + --certificate-oidc-issuer "$COSIGN_VERIFICATION_OIDC_ISSUER" \ + "$digest_ref" > "cosign-${{ matrix.name }}.json" + + - name: Upload publication evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: opensandbox-${{ matrix.name }}-${{ github.sha }} + path: | + trivy-${{ matrix.name }}-amd64.json + trivy-${{ matrix.name }}-arm64.json + cosign-${{ matrix.name }}.json + if-no-files-found: error + retention-days: 90 + + - name: Summarize immutable publication + env: + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: | + { + echo "### ${{ matrix.name }}" + echo + echo "- Source commit: \`${GITHUB_SHA}\`" + echo "- Image: \`${IMAGE_REF}@${IMAGE_DIGEST}\`" + echo "- Platforms: \`linux/amd64\`, \`linux/arm64\`" + echo "- Critical gate: passed on both platforms" + echo "- Signature: verified against the exact main-branch workflow identity" + echo "- Full High/Critical reports: attached as workflow artifacts" + } >> "$GITHUB_STEP_SUMMARY"