diff --git a/.dockerignore b/.dockerignore index cd08df2..9b900cf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ .git +.github .agents +e2e .codex runtime-cli diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a81d309 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,102 @@ +name: CI + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit: + name: Unit tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build + run: go build ./... + + - name: Test + run: go test -race -count=1 ./... + + # Several service and capabilities tests skip themselves without root: + # the direct executor integration test, the namespace probes that need + # CAP_SYS_ADMIN, and the cgroup delegation checks. The unprivileged pass + # above covers everything else, so this pass is what actually exercises + # the privileged Linux paths. + - name: Test as root + if: runner.os == 'Linux' + run: go test -exec "sudo -n" -count=1 ./... + + vet: + name: Vet and formatting + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: gofmt + run: | + unformatted="$(gofmt -l .)" + if [ -n "$unformatted" ]; then + echo "these files are not gofmt-formatted:" >&2 + echo "$unformatted" >&2 + exit 1 + fi + + - name: go vet + run: go vet ./... + + - name: go mod tidy is up to date + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum + + crossbuild: + name: Build ${{ matrix.goos }}/${{ matrix.goarch }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - {goos: linux, goarch: amd64} + - {goos: linux, goarch: arm64} + - {goos: darwin, goarch: arm64} + - {goos: windows, goarch: amd64} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # The runtime has three platform implementations behind build tags + # (linux, windows, and the unsupported fallback). Only a cross build + # keeps the two non-primary ones compiling. + - name: Build and vet + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + go build ./... + go vet ./... diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..ee3e60e --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,73 @@ +name: E2E + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + # The runtime image is identical for every case, and building it once keeps + # the matrix legs to the part that actually differs: what the host grants the + # container. + image: + name: Build runtime image + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Build + run: docker build -t mysterium-runtime:e2e . + + - name: Export + run: docker save mysterium-runtime:e2e | gzip > runtime-image.tar.gz + + - uses: actions/upload-artifact@v4 + with: + name: runtime-image + path: runtime-image.tar.gz + retention-days: 1 + + isolation: + name: ${{ matrix.case }} + needs: image + # Pinned: the selected profile depends on the runner's kernel, cgroup + # layout, and AppArmor policy, so the image is part of the expectation. + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + case: + - full + - limited-no-cgroups + - limited-no-netns + - unisolated + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: runtime-image + + - name: Load runtime image + run: docker load < runtime-image.tar.gz + + - name: Report host facts + run: | + uname -a + stat -fc %T /sys/fs/cgroup + cat /sys/fs/cgroup/cgroup.controllers + sysctl kernel.apparmor_restrict_unprivileged_userns || true + docker version --format '{{.Server.Version}}' + + - name: Run ${{ matrix.case }} + env: + E2E_REUSE_IMAGE: "1" + run: e2e/run.sh ${{ matrix.case }} diff --git a/README.md b/README.md index 0ebd21d..b05af9e 100644 --- a/README.md +++ b/README.md @@ -243,15 +243,22 @@ Build the runtime image: docker build -t mysterium-runtime:dev . ``` -Run the end-to-end demo on a native Linux Docker host. Docker Desktop is not a -supported host for this nested runtime because its VM blocks the subordinate -user namespace from mounting the workload procfs. The cgroup namespace and cgroup +Run the end-to-end demo on a Linux Docker host. The cgroup namespace and cgroup filesystem options let the demo wrapper delegate `cpu`, `memory`, and `pids` from the container cgroup to sibling workload cgroups managed by nested `runc`. The capability list provides the operational privileges used by the full profile; the outer Docker seccomp profile is disabled so `runc` can install the stricter workload seccomp profile from the generated OCI spec. +Two of the options exist only so that the subordinate user namespace can be +established, and without them the demo reaches `limited` rather than `full`: + +- `--cap-add=SETFCAP`: Linux 5.12 and later require `CAP_SETFCAP` to write a + `uid_map` that maps uid 0, which both the capability probe and `runc` do. +- `--security-opt systempaths=unconfined`: Docker's masked and read-only + `/proc` paths are locked over-mounts, and a nested user namespace may not + mount a fresh procfs while they hide part of the procfs it can already see. + ```bash docker run --rm --name mysterium-runtime-demo \ --cgroupns=host \ @@ -263,6 +270,7 @@ docker run --rm --name mysterium-runtime-demo \ --cap-add=KILL \ --cap-add=SETGID \ --cap-add=SETUID \ + --cap-add=SETFCAP \ --cap-add=NET_ADMIN \ --cap-add=SYS_CHROOT \ --cap-add=SYS_PTRACE \ @@ -270,6 +278,7 @@ docker run --rm --name mysterium-runtime-demo \ --cap-add=MKNOD \ --security-opt seccomp=unconfined \ --security-opt apparmor=unconfined \ + --security-opt systempaths=unconfined \ -e OCI_ARTIFACT="$OCI_ARTIFACT" \ -p 127.0.0.1:3000:3000 \ --entrypoint /usr/local/bin/runtime-demo \ @@ -293,6 +302,50 @@ only on loopback inside its isolated network namespace. It looks up the manifest-defined service port and bridges a local listener through `Backend.DialTCP`; it does not add a port override to the workload contract. +### Continuous integration + +[`ci.yml`](.github/workflows/ci.yml) runs the unit tests on every pull request: +`go test -race` on Linux, macOS, and Windows, a second Linux pass under `sudo` +for the tests that skip themselves without root, `gofmt`, `go vet`, `go mod +tidy`, and a cross build of every platform implementation behind a build tag. + +[`e2e.yml`](.github/workflows/e2e.yml) runs the demo workload end to end under +several isolation profiles. Each case is the same image and the same workload; +only what the host grants the runtime container changes, so the profile the +runtime selects is the thing under test: + +| Case | What the host withholds | Expected | +| --- | --- | --- | +| `full` | nothing | `full-v1` | +| `limited-no-cgroups` | the delegated cgroup v2 tree | `best-effort-v1` without cgroups | +| `limited-no-netns` | `CAP_NET_ADMIN` and `CAP_SETFCAP` | `best-effort-v1` on a shared network namespace | +| `unisolated` | `runc` | `unisolated-v1` through the direct executor | + +`limited-no-netns` withholds `CAP_SETFCAP`, and so user namespaces, along with +`CAP_NET_ADMIN`. That pairing is forced: the generated OCI spec always mounts +sysfs, and the kernel refuses a sysfs mount inside a user namespace that does +not own its network namespace, so a profile holding user namespaces without a +private network namespace cannot start a workload at all. + +Each case asserts the reported runtime level, profile name, and the feature +vector the case is defined by; that the workload answers over the proxy; and +that stopping it leaves the service passive while its desired state survives. + +The workload is served from a throwaway local registry, because the runtime +only accepts digest-pinned registry references. Docker pushes to it over +localhost, and the runtime pulls the same manifest through the docker bridge +gateway, which `go-containerregistry` treats as insecure because the address is +RFC1918. + +A single case can be run on any Linux Docker host: + +```bash +e2e/run.sh limited-no-cgroups +``` + +All four cases also pass against Docker Desktop, which needs +`E2E_ALLOW_NON_LINUX=1` to waive the platform guard. + --- ## 3. Capability Detection Strategy diff --git a/e2e/run.sh b/e2e/run.sh new file mode 100755 index 0000000..8632fb4 --- /dev/null +++ b/e2e/run.sh @@ -0,0 +1,339 @@ +#!/usr/bin/env bash +# End-to-end test: runs the demo workload inside the runtime container and +# asserts that the runtime selected the expected isolation profile, that the +# workload is reachable, and that stopping it preserves the recorded intent. +# +# e2e/run.sh +# +# Cases differ only in what the host grants the runtime container, which is the +# whole point: the runtime is supposed to pick its profile from what it finds. +# +# full every isolation mechanism available -> full-v1 +# limited-no-cgroups no delegated cgroup v2 tree -> best-effort-v1 +# limited-no-netns no CAP_NET_ADMIN, no CAP_SETFCAP -> best-effort-v1 +# unisolated no runc, direct executor only -> unisolated-v1 +# +# Needs a Linux Docker host. All four cases have also been run against Docker +# Desktop on macOS, which needs the platform guard below waived. +set -euo pipefail + +readonly case_name="${1:-}" +readonly repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +readonly runtime_image="${RUNTIME_IMAGE:-mysterium-runtime:e2e}" +readonly workload_repository="${WORKLOAD_REPOSITORY:-mysterium/nc-workload}" +readonly registry_port="${REGISTRY_PORT:-5000}" +readonly host_port="${HOST_PORT:-3000}" +readonly registry_container="mysterium-e2e-registry" +readonly runtime_container="mysterium-e2e-runtime" +readonly service_name="e2e-workload" +readonly expected_response="hello from the isolated runc workload" + +report_dir="" + +log() { printf '\n=== %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +cleanup() { + local status=$? + if [ "$status" -ne 0 ] && docker container inspect "$runtime_container" >/dev/null 2>&1; then + log "runtime container logs" + docker logs "$runtime_container" 2>&1 | tail -n 200 || true + if [ -n "$report_dir" ] && [ -f "${report_dir}/status.json" ]; then + log "reported runtime status" + cat "${report_dir}/status.json" || true + fi + # The per-probe statuses distinguish a kernel that lacks a feature from + # one that refused it, which the merged feature vector cannot. + if [ -n "$report_dir" ] && [ -f "${report_dir}/capabilities.json" ]; then + log "detected host capabilities" + jq '.detailed' <"${report_dir}/capabilities.json" || true + fi + fi + docker rm -f "$runtime_container" >/dev/null 2>&1 || true + docker rm -f "$registry_container" >/dev/null 2>&1 || true + return "$status" +} +trap cleanup EXIT + +# assert_json +assert_json() { + local file="$1" filter="$2" description="$3" + if [ ! -f "$file" ]; then + fail "${description}: ${file} was never written" + fi + if [ "$(jq -r "$filter" <"$file")" != "true" ]; then + printf 'FAIL: %s\n' "$description" >&2 + printf 'filter: %s\n' "$filter" >&2 + jq . <"$file" >&2 || cat "$file" >&2 + exit 1 + fi + printf 'ok: %s\n' "$description" +} + +wait_for_file() { + local file="$1" timeout="${2:-120}" waited=0 + while [ ! -f "$file" ]; do + if ! docker container inspect -f '{{.State.Running}}' "$runtime_container" 2>/dev/null | grep -q true; then + fail "the runtime container exited before the workload was ready" + fi + if [ "$waited" -ge "$timeout" ]; then + fail "timed out after ${timeout}s waiting for ${file}" + fi + sleep 1 + waited=$((waited + 1)) + done +} + +# Every case starts from the capability set the runtime documents as its +# operational requirement, and then takes things away. +# Not --rm: a container that dies early is the interesting case, and its logs +# have to outlive it for the cleanup trap to print them. +common_docker_args=( + --detach + --name "$runtime_container" + --cap-drop=ALL + --cap-add=CHOWN + --cap-add=DAC_OVERRIDE + --cap-add=FOWNER + --cap-add=KILL + --cap-add=SETGID + --cap-add=SETUID + --cap-add=SYS_CHROOT + # The outer seccomp profile has to be off so that runc can install the + # stricter workload profile from the generated OCI spec. + --security-opt seccomp=unconfined + --security-opt apparmor=unconfined + --publish "127.0.0.1:${host_port}:8080" +) + +isolated_docker_args=( + --cap-add=SYS_ADMIN + --cap-add=MKNOD + # runc opens the container's own mount namespace to remap its mounts, and a + # process in a different user namespace is only openable with + # CAP_SYS_PTRACE. Withholding it does not weaken isolation, it stops runc + # from starting a user-namespaced workload at all, so it stays granted even + # in the case that withholds the network namespace. + --cap-add=SYS_PTRACE + # Docker's masked and read-only /proc paths are locked over-mounts, and a + # nested user namespace may not mount a fresh procfs while they hide part + # of the procfs it can already see. + --security-opt systempaths=unconfined +) + +# Linux 5.12 and later require CAP_SETFCAP to write a uid_map that maps uid 0, +# which is exactly what the capability probe and runc's user namespace setup +# both do. Without it the runtime sees no user namespaces at all. +userns_docker_args=( + --cap-add=SETFCAP +) + +# The runtime selects a private network namespace only with both CAP_NET_ADMIN +# to configure loopback and CAP_SYS_PTRACE to open the created process's netns +# handle. Withholding CAP_NET_ADMIN alone is therefore enough to take the +# network namespace away without disturbing anything else. +netns_docker_args=( + --cap-add=NET_ADMIN +) + +cgroup_docker_args=( + --cgroupns=host + --mount type=bind,src=/sys/fs/cgroup,dst=/sys/fs/cgroup +) + +case "$case_name" in +full) + docker_args=( + "${isolated_docker_args[@]}" + "${userns_docker_args[@]}" + "${netns_docker_args[@]}" + "${cgroup_docker_args[@]}" + ) + container_env=(--env DELEGATE_CGROUPS=1 --env MINIMUM_RUNTIME_LEVEL=full) + expected_profile="full-v1" + # Full means every feature, so assert the whole vector rather than a list + # that would silently stop covering a feature added later. + status_filter=' + .level == "full" + and .profile.name == "full-v1" + and .profile.level == "full" + and ([.profile.features[]] | all) + and (.missing_for_full // [] | length) == 0 + and (.blocking_reasons // [] | length) == 0' + ;; +limited-no-cgroups) + docker_args=( + "${isolated_docker_args[@]}" + "${userns_docker_args[@]}" + "${netns_docker_args[@]}" + ) + container_env=(--env DELEGATE_CGROUPS=0 --env MINIMUM_RUNTIME_LEVEL=limited) + expected_profile="best-effort-v1" + # Without cgroups the workload process tree is controlled by the PID + # namespace alone, which is exactly the fallback the profile promises. The + # remaining mechanisms are asserted only where the limited profile + # guarantees them, so that a host with, say, no user namespaces still + # reports limited rather than failing this case. + status_filter=' + .level == "limited" + and .profile.name == "best-effort-v1" + and .profile.features.cgroups == false + and .profile.features.pid_namespaces == true + and .profile.features.mount_namespaces == true + and .profile.features.no_new_privileges == true + and (.missing_for_full | index("cgroup resource isolation")) != null + and (.blocking_reasons // [] | length) == 0' + ;; +limited-no-netns) + # Also withholds CAP_SETFCAP, which takes user namespaces away with it. + # That is not incidental: the generated OCI spec always mounts sysfs, and + # the kernel refuses a sysfs mount inside a user namespace that does not + # own its network namespace. A limited profile with user namespaces but no + # network namespace therefore cannot start a workload at all, so this case + # covers the shared-network-namespace path in the configuration a host can + # actually run. + docker_args=( + "${isolated_docker_args[@]}" + "${cgroup_docker_args[@]}" + ) + container_env=(--env DELEGATE_CGROUPS=1 --env MINIMUM_RUNTIME_LEVEL=limited) + expected_profile="best-effort-v1" + # No private network namespace: the workload gets an address from the + # managed host-loopback pool instead of 127.0.0.1, so this case is also the + # only coverage of that assignment path. + status_filter=' + .level == "limited" + and .profile.name == "best-effort-v1" + and .profile.features.network_namespaces == false + and .profile.features.user_namespaces == false + and .profile.features.cgroups == true + and .profile.features.pid_namespaces == true + and .profile.features.mount_namespaces == true + and .profile.features.no_new_privileges == true + and (.missing_for_full | index("network namespaces")) != null + and (.blocking_reasons // [] | length) == 0' + ;; +unisolated) + docker_args=() + container_env=( + --env DELEGATE_CGROUPS=0 + --env HIDE_RUNC=1 + --env MINIMUM_RUNTIME_LEVEL=unisolated + ) + expected_profile="unisolated-v1" + # The direct executor keeps only the guarantees it can apply itself. + status_filter=' + .level == "unisolated" + and .profile.name == "unisolated-v1" + and .profile.features.filesystem_jail == true + and .profile.features.non_root_user == true + and .profile.features.capabilities_dropped == true + and .profile.features.mount_namespaces == false + and .profile.features.pid_namespaces == false + and .profile.features.cgroups == false + and .profile.features.seccomp == false + and (.missing_for_limited | index("runc executable is unavailable")) != null + and (.blocking_reasons // [] | length) == 0' + ;; +*) + fail "usage: $0 " + ;; +esac + +for tool in docker jq curl; do + command -v "$tool" >/dev/null 2>&1 || fail "${tool} is required" +done +# CI runs this on a Linux runner, and a Linux host is what the results describe. +# Every case has been verified on Docker Desktop too, so the guard is a default +# rather than a hard requirement. +if [ "$(uname -s)" != "Linux" ] && [ "${E2E_ALLOW_NON_LINUX:-0}" != "1" ]; then + fail "the e2e cases expect a Linux Docker host (set E2E_ALLOW_NON_LINUX=1 to override)" +fi + +docker rm -f "$runtime_container" "$registry_container" >/dev/null 2>&1 || true + +if [ "${E2E_REUSE_IMAGE:-0}" != "1" ] || ! docker image inspect "$runtime_image" >/dev/null 2>&1; then + log "building the runtime image" + docker build -t "$runtime_image" "$repo_root" +fi + +log "publishing the workload to a local registry" +docker run --detach --name "$registry_container" \ + --publish "${registry_port}:5000" registry:2 >/dev/null +# Docker only pushes over plain HTTP to localhost, but the runtime resolves the +# same manifest through the bridge gateway, which go-containerregistry also +# treats as insecure because it is RFC1918. +local_reference="localhost:${registry_port}/${workload_repository}:e2e" +docker build -t "$local_reference" "${repo_root}/examples/nc-workload" +push_output="" +for attempt in $(seq 1 30); do + if push_output="$(docker push "$local_reference" 2>&1)"; then + break + fi + [ "$attempt" -eq 30 ] && fail "the local registry never accepted the workload image: ${push_output}" + sleep 1 +done + +# Read the digest out of this push rather than the image's RepoDigests, which +# can still carry the digest of an earlier push of the same tag. +digest="$(printf '%s\n' "$push_output" | + sed -n 's/.*digest: \(sha256:[0-9a-f]\{64\}\).*/\1/p' | tail -n 1)" +[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "the pushed workload did not resolve to a digest" + +gateway="$(docker network inspect bridge --format '{{range .IPAM.Config}}{{.Gateway}}{{end}}')" +[ -n "$gateway" ] || fail "cannot determine the docker bridge gateway address" +oci_artifact="${gateway}:${registry_port}/${workload_repository}@${digest}" + +report_dir="$(mktemp -d)" +chmod 0755 "$report_dir" + +log "running the ${case_name} case" +docker run "${common_docker_args[@]}" ${docker_args[@]+"${docker_args[@]}"} "${container_env[@]}" \ + --env "OCI_ARTIFACT=${oci_artifact}" \ + --env "REPORT_DIR=/e2e" \ + --env "SERVICE_NAME=${service_name}" \ + --mount "type=bind,src=${report_dir},dst=/e2e" \ + --mount "type=bind,src=${repo_root}/e2e/runtime-e2e,dst=/usr/local/bin/runtime-e2e,readonly" \ + --entrypoint /usr/local/bin/runtime-e2e \ + "$runtime_image" >/dev/null + +wait_for_file "${report_dir}/ready" "${READY_TIMEOUT:-180}" + +log "asserting the selected isolation profile" +assert_json "${report_dir}/status.json" "$status_filter" \ + "runtime status matches the ${case_name} expectations" +assert_json "${report_dir}/list-started.json" \ + ".[0].state == \"active\" and .[0].desired_state == \"active\" and .[0].options.isolation.name == \"${expected_profile}\"" \ + "the started workload records the ${expected_profile} profile" +assert_json "${report_dir}/list-started.json" \ + '.[0].options.service_port == 3000 and (.[0].options.process.uid == 1000) and (.[0].options.process.gid == 1000)' \ + "the workload runs as the image's non-root user on its manifest port" + +log "asserting the workload responds" +response="" +for attempt in $(seq 1 30); do + if response="$(curl --silent --fail-with-body --max-time 5 "http://127.0.0.1:${host_port}/")"; then + break + fi + [ "$attempt" -eq 30 ] && fail "the workload never answered through the proxy" + sleep 1 +done +[ "$response" = "$expected_response" ] || fail "unexpected workload response: ${response}" +printf 'ok: workload answered through the proxy\n' + +log "asserting stop keeps the recorded intent" +docker exec "$runtime_container" runtime-cli \ + -runtime-dir /var/lib/mysterium-runtime -command stop -name "$service_name" +docker exec "$runtime_container" sh -c \ + 'runtime-cli -runtime-dir /var/lib/mysterium-runtime -command list >/e2e/list-stopped.json' +assert_json "${report_dir}/list-stopped.json" \ + '.[0].state == "passive" and .[0].desired_state == "active"' \ + "a stopped workload stays passive while its desired state survives" + +if curl --silent --fail-with-body --max-time 5 "http://127.0.0.1:${host_port}/" >/dev/null 2>&1; then + fail "the workload still answered after being stopped" +fi +printf 'ok: the stopped workload is no longer reachable\n' + +log "${case_name} passed" diff --git a/e2e/runtime-e2e b/e2e/runtime-e2e new file mode 100755 index 0000000..ad3d6f0 --- /dev/null +++ b/e2e/runtime-e2e @@ -0,0 +1,89 @@ +#!/bin/sh +# Container-side end-to-end driver. It runs inside the runtime image, records +# what the runtime reported into REPORT_DIR for the host to assert on, and then +# keeps the workload reachable through the proxy until the container is stopped. +# +# The isolation profile is never requested here: it is whatever the runtime +# selects for the container the host started. The environment below only decides +# which host facilities the runtime is allowed to find. +set -eu + +: "${OCI_ARTIFACT:?set OCI_ARTIFACT to the digest-pinned workload image}" + +runtime_dir="${RUNTIME_DIR:-/var/lib/mysterium-runtime}" +service_name="${SERVICE_NAME:-e2e-workload}" +# Deliberately not the workload's own port. A workload without a private +# network namespace listens on a host-loopback address in this same namespace, +# and a wildcard bind on its port would collide with it. +listen_address="${LISTEN_ADDRESS:-0.0.0.0:8080}" +report_dir="${REPORT_DIR:-/e2e}" +minimum_level="${MINIMUM_RUNTIME_LEVEL:-limited}" +delegate_cgroups="${DELEGATE_CGROUPS:-1}" +hide_runc="${HIDE_RUNC:-0}" + +mkdir -p "$report_dir" + +# Removing runc is how the unisolated case is reached: no OCI executor, so the +# runtime falls back to the built-in direct executor. Nothing else in the +# container changes. +if [ "$hide_runc" = "1" ]; then + runc_path="$(command -v runc || true)" + if [ -n "$runc_path" ]; then + rm -f "$runc_path" + fi +fi + +# Move this process into a child cgroup first: a cgroup cannot both hold +# processes and enable controllers for its children. Once the container cgroup +# is empty, cpu, memory, and pids can be delegated to the sibling cgroups that +# nested runc creates for workloads. +delegate_cgroup_controllers() { + current_cgroup="$(awk -F: '$1 == "0" { print $3 }' /proc/self/cgroup)" + if [ -z "$current_cgroup" ] || [ "$current_cgroup" = "/" ]; then + echo "cgroup delegation needs a host cgroup namespace with a non-root container cgroup" >&2 + exit 1 + fi + + parent="/sys/fs/cgroup${current_cgroup}" + manager="${parent}/mysterium-manager" + mkdir -p "$manager" + printf '%s\n' "$$" >"${manager}/cgroup.procs" + printf '%s\n' '+cpu +memory +pids' >"${parent}/cgroup.subtree_control" + export MYSTERIUM_CGROUP_PARENT="$current_cgroup" +} + +cleanup() { + if [ -n "${proxy_pid:-}" ]; then + kill "$proxy_pid" 2>/dev/null || true + wait "$proxy_pid" 2>/dev/null || true + fi + runtime-cli -runtime-dir "$runtime_dir" -command stop -name "$service_name" || true +} +trap cleanup EXIT INT TERM + +if [ "$delegate_cgroups" = "1" ]; then + delegate_cgroup_controllers +fi + +runtime-cli -runtime-dir "$runtime_dir" -command capabilities >"${report_dir}/capabilities.json" +runtime-cli -runtime-dir "$runtime_dir" -command status >"${report_dir}/status.json" + +runtime-cli -runtime-dir "$runtime_dir" -command create \ + -name "$service_name" \ + -oci-artifact "$OCI_ARTIFACT" \ + -minimum-runtime-level "$minimum_level" +runtime-cli -runtime-dir "$runtime_dir" -command list >"${report_dir}/list-created.json" + +runtime-cli -runtime-dir "$runtime_dir" -command start -name "$service_name" +runtime-cli -runtime-dir "$runtime_dir" -command list >"${report_dir}/list-started.json" + +runtime-cli -runtime-dir "$runtime_dir" -command proxy \ + -name "$service_name" \ + -listen "$listen_address" & +proxy_pid=$! + +# The host waits for this file before asserting; everything above it must have +# succeeded for it to appear. +: >"${report_dir}/ready" + +wait "$proxy_pid"