diff --git a/.github/workflows/execd-test.yml b/.github/workflows/execd-test.yml index fb263e66f..3ff60d84b 100644 --- a/.github/workflows/execd-test.yml +++ b/.github/workflows/execd-test.yml @@ -143,6 +143,12 @@ jobs: chmod +x tests/sigterm_forward.sh ./tests/sigterm_forward.sh + - name: Lifecycle bootstrap test + if: matrix.os == 'ubuntu-latest' + timeout-minutes: 5 + working-directory: components/execd + run: bash tests/lifecycle.sh + - name: Smoke test bwrap (Docker image build + extraction) if: matrix.os == 'ubuntu-latest' shell: bash diff --git a/components/execd/bootstrap.sh b/components/execd/bootstrap.sh index b9d74dfaa..373073f84 100755 --- a/components/execd/bootstrap.sh +++ b/components/execd/bootstrap.sh @@ -16,6 +16,12 @@ set -e +EXECD_WATCHDOG_PID="" +LIFECYCLE_STATUS_DIR="" +LIFECYCLE_STATUS_FILE="" +LIFECYCLE_WATCHDOG_TIMEOUT_FILE="" +LIFECYCLE_WATCHDOG_READY_FILE="" + _forward_signal() { sig="$1" pid="$2" @@ -25,16 +31,149 @@ _forward_signal() { kill "-$sig" "$pid" 2>/dev/null || true } +_process_state() { + if [ -r "/proc/$1/stat" ]; then + sed -e 's/^.*) //' -e 's/ .*$//' "/proc/$1/stat" 2>/dev/null || true + else + ps -o stat= -p "$1" 2>/dev/null \ + | sed -n '1{s/^[[:space:]]*//; s/^\(.\).*$/\1/; p;}' \ + || true + fi +} + +_stop_execd_watchdog() { + if [ -n "${EXECD_WATCHDOG_PID:-}" ]; then + if [ -n "${LIFECYCLE_WATCHDOG_READY_FILE:-}" ] \ + && [ ! -s "$LIFECYCLE_WATCHDOG_READY_FILE" ]; then + kill -KILL "$EXECD_WATCHDOG_PID" 2>/dev/null || true + else + kill -TERM "$EXECD_WATCHDOG_PID" 2>/dev/null || true + fi + wait "$EXECD_WATCHDOG_PID" 2>/dev/null || true + EXECD_WATCHDOG_PID="" + fi +} + +_start_execd_watchdog() { + _watchdog_delay="$1" + _watchdog_message="$2" + _watchdog_mark_timeout="${3:-1}" + _watchdog_grace_delay="${4:-0}" + _watchdog_signal="${5-TERM}" + _watchdog_execd_pid="$EXECD_PID" + _watchdog_timeout_file="${LIFECYCLE_WATCHDOG_TIMEOUT_FILE:-}" + _watchdog_ready_file="${LIFECYCLE_WATCHDOG_READY_FILE:-}" + _stop_execd_watchdog + if [ "$_watchdog_mark_timeout" -eq 1 ] \ + && [ -n "$_watchdog_timeout_file" ] \ + && [ -s "$_watchdog_timeout_file" ]; then + return 1 + fi + if [ -n "$_watchdog_ready_file" ] \ + && ! ( : > "$_watchdog_ready_file" ) 2>/dev/null; then + return 1 + fi + ( + # This child must never run the parent's cleanup or shutdown traps. + trap - EXIT TERM INT + _watchdog_cancelled=0 + _watchdog_sleep_pid="" + _watchdog_spawning_sleep=0 + trap '_watchdog_cancelled=1; if [ -n "${_watchdog_sleep_pid:-}" ]; then kill -KILL "$_watchdog_sleep_pid" 2>/dev/null || true; elif [ "${_watchdog_spawning_sleep:-0}" -eq 0 ]; then exit 0; fi' TERM INT + if [ -n "$_watchdog_ready_file" ] \ + && ! printf 'ready\n' > "$_watchdog_ready_file"; then + exit 1 + fi + _watchdog_sleep() { + if [ "$_watchdog_cancelled" -ne 0 ]; then + return 1 + fi + _watchdog_spawning_sleep=1 + sleep "$1" & + _watchdog_sleep_pid=$! + _watchdog_spawning_sleep=0 + if [ "$_watchdog_cancelled" -ne 0 ]; then + kill -KILL "$_watchdog_sleep_pid" 2>/dev/null || true + fi + wait "$_watchdog_sleep_pid" || true + _watchdog_sleep_pid="" + if [ "$_watchdog_cancelled" -ne 0 ]; then + return 1 + fi + } + _watchdog_sleep "$_watchdog_delay" || exit 0 + if [ "$_watchdog_grace_delay" != "0" ]; then + _watchdog_sleep "$_watchdog_grace_delay" || exit 0 + fi + if [ "$_watchdog_cancelled" -ne 0 ]; then + exit 0 + fi + if [ "$_watchdog_mark_timeout" -eq 1 ] && [ -n "$_watchdog_timeout_file" ]; then + if ! printf 'timed-out\n' > "$_watchdog_timeout_file"; then + _forward_signal KILL "$_watchdog_execd_pid" + echo "error: failed to record lifecycle startup watchdog timeout" >&2 || true + exit 1 + fi + fi + if [ -n "$_watchdog_message" ]; then + echo "error: $_watchdog_message" >&2 || true + fi + if [ -n "$_watchdog_signal" ]; then + _forward_signal "$_watchdog_signal" "$_watchdog_execd_pid" + fi + _watchdog_sleep 10 || exit 0 + _forward_signal KILL "$_watchdog_execd_pid" + ) & + EXECD_WATCHDOG_PID=$! + if [ -n "$_watchdog_ready_file" ]; then + _watchdog_ready_attempts=0 + _watchdog_ready_limit=100 + _watchdog_ready_delay=0.1 + while [ ! -s "$_watchdog_ready_file" ] && [ "$_watchdog_ready_attempts" -lt "$_watchdog_ready_limit" ]; do + _watchdog_state="$(_process_state "$EXECD_WATCHDOG_PID")" + if [ "$_watchdog_state" = "Z" ]; then + break + fi + if ! kill -0 "$EXECD_WATCHDOG_PID" 2>/dev/null; then + break + fi + if ! sleep "$_watchdog_ready_delay" 2>/dev/null; then + # POSIX sleep only requires integer operands. Keep the same + # ten-second total bound when fractional sleep is unavailable. + _watchdog_ready_delay=1 + _watchdog_ready_limit=10 + sleep 1 + fi + _watchdog_ready_attempts=$((_watchdog_ready_attempts + 1)) + done + if [ ! -s "$_watchdog_ready_file" ]; then + kill -KILL "$EXECD_WATCHDOG_PID" 2>/dev/null || true + wait "$EXECD_WATCHDOG_PID" 2>/dev/null || true + EXECD_WATCHDOG_PID="" + return 1 + fi + fi +} + _shutdown_children() { sig="$1" + _stop_execd_watchdog _forward_signal "$sig" "${CMD_PID:-}" _forward_signal "$sig" "${EXECD_PID:-}" + if [ -n "${EXECD_PID:-}" ] && [ -n "${LIFECYCLE_STATUS_FILE:-}" ]; then + # The signal was already forwarded above; this watchdog only bounds + # graceful shutdown before escalating to KILL. + if ! _start_execd_watchdog 0 "" 0 0 ""; then + _forward_signal KILL "$EXECD_PID" + fi + fi if [ -n "${CMD_PID:-}" ]; then wait "$CMD_PID" 2>/dev/null || true fi if [ -n "${EXECD_PID:-}" ]; then wait "$EXECD_PID" 2>/dev/null || true fi + _cleanup_lifecycle_status exit 0 } @@ -49,6 +188,43 @@ is_truthy() { esac } +has_lifecycle_config() { + # Keep this in sync with pkg/lifecycle/config.go's transport env, explicit + # path env, and default persisted path. + if [ -n "$(printf '%s' "${OPEN_SANDBOX_LIFECYCLE:-}" | tr -d '[:space:]')" ]; then + return 0 + fi + if [ -n "${EXECD_LIFECYCLE_CONFIG:-}" ]; then + return 0 + fi + if [ -n "${HOME:-}" ] && [ -e "$HOME/.execd/lifecycle.toml" ]; then + return 0 + fi + return 1 +} + +_cleanup_lifecycle_status() { + _stop_execd_watchdog + if [ -n "${LIFECYCLE_STATUS_FILE:-}" ]; then + rm -f "$LIFECYCLE_STATUS_FILE" + LIFECYCLE_STATUS_FILE="" + fi + if [ -n "${LIFECYCLE_WATCHDOG_TIMEOUT_FILE:-}" ]; then + rm -f "$LIFECYCLE_WATCHDOG_TIMEOUT_FILE" + LIFECYCLE_WATCHDOG_TIMEOUT_FILE="" + fi + if [ -n "${LIFECYCLE_WATCHDOG_READY_FILE:-}" ]; then + rm -f "$LIFECYCLE_WATCHDOG_READY_FILE" + LIFECYCLE_WATCHDOG_READY_FILE="" + fi + if [ -n "${LIFECYCLE_STATUS_DIR:-}" ]; then + rmdir "$LIFECYCLE_STATUS_DIR" 2>/dev/null || true + LIFECYCLE_STATUS_DIR="" + fi +} + +trap '_cleanup_lifecycle_status' EXIT + _sudo() { if [ "$(id -u)" -eq 0 ]; then "$@" @@ -349,9 +525,147 @@ if is_truthy "${EXECD_INIT:-}"; then exec "$EXECD" --init -- "$@" fi -"$EXECD" & +if has_lifecycle_config; then + if ! LIFECYCLE_STATUS_DIR="$( + umask 077 + mktemp -d "${TMPDIR:-/tmp}/execd-lifecycle.XXXXXX" 2>/dev/null \ + || mktemp -d /tmp/execd-lifecycle.XXXXXX 2>/dev/null + )"; then + echo "error: failed to create lifecycle startup status directory" >&2 + exit 1 + fi + LIFECYCLE_STATUS_FILE="${LIFECYCLE_STATUS_DIR}/status" + LIFECYCLE_WATCHDOG_TIMEOUT_FILE="${LIFECYCLE_STATUS_DIR}/watchdog-timeout" + LIFECYCLE_WATCHDOG_READY_FILE="${LIFECYCLE_STATUS_DIR}/watchdog-ready" + if ! ( + umask 077 \ + && : > "$LIFECYCLE_STATUS_FILE" \ + && : > "$LIFECYCLE_WATCHDOG_TIMEOUT_FILE" \ + && : > "$LIFECYCLE_WATCHDOG_READY_FILE" + ); then + echo "error: failed to create lifecycle startup synchronization files" >&2 + exit 1 + fi + "$EXECD" --lifecycle-startup-status-file "$LIFECYCLE_STATUS_FILE" & +else + "$EXECD" & +fi EXECD_PID=$! +# The same long-running execd starts serving HTTP, executes preStart, then +# reports the result through this private bootstrap synchronization file. +if [ -n "$LIFECYCLE_STATUS_FILE" ]; then + if ! _start_execd_watchdog 10 "execd did not report lifecycle startup within 10 seconds"; then + echo "error: failed to arm the lifecycle startup watchdog" >&2 + _forward_signal TERM "$EXECD_PID" + _forward_signal KILL "$EXECD_PID" + wait "$EXECD_PID" 2>/dev/null || true + EXECD_PID="" + exit 1 + fi + _lifecycle_running_seen=0 + _lifecycle_done=0 + _prestart_status="" + while [ "$_lifecycle_done" -eq 0 ]; do + _lifecycle_status="" + if [ ! -r "$LIFECYCLE_STATUS_FILE" ]; then + echo "error: lifecycle startup status file is missing or unreadable" >&2 + _lifecycle_done=1 + _prestart_status=1 + elif ! { + while IFS= read -r _lifecycle_status_line; do + _lifecycle_status="$_lifecycle_status_line" + done < "$LIFECYCLE_STATUS_FILE" + } 2>/dev/null; then + echo "error: lifecycle startup status file is missing or unreadable" >&2 + _lifecycle_done=1 + _prestart_status=1 + fi + case "$_lifecycle_status" in + "running "*) + if [ "$_lifecycle_running_seen" -eq 0 ]; then + _prestart_timeout="${_lifecycle_status#running }" + # execd reports a validated positive timeout of at most ten digits. + # Treat any other value as corrupt before passing it to sleep. + case "$_prestart_timeout" in + "" | *[!0-9]* | 0* | ???????????*) _lifecycle_done=1; _prestart_status=1 ;; + *) + _lifecycle_running_seen=1 + if ! _start_execd_watchdog \ + "$_prestart_timeout" \ + "lifecycle preStart did not report completion after its timeout and 10-second grace" \ + 1 10; then + _lifecycle_done=1 + _prestart_status=1 + fi + ;; + esac + fi + ;; + "done "*) + _prestart_status="${_lifecycle_status#done }" + _lifecycle_done=1 + ;; + "") ;; + *) _lifecycle_done=1; _prestart_status=1 ;; + esac + if [ "$_lifecycle_done" -ne 0 ]; then + break + fi + _execd_state="$(_process_state "$EXECD_PID")" + if ! kill -0 "$EXECD_PID" 2>/dev/null || [ "$_execd_state" = "Z" ]; then + _stop_execd_watchdog + set +e + wait "$EXECD_PID" + _execd_status=$? + set -e + EXECD_PID="" + _cleanup_lifecycle_status + if [ "$_execd_status" -eq 0 ]; then + _execd_status=1 + fi + exit "$_execd_status" + fi + # Execd reports the effective hook timeout before running preStart. The + # external watchdog also bounds a hung daemon that never reports a result. + sleep 0.1 2>/dev/null || sleep 1 + done + _stop_execd_watchdog + if [ -n "${LIFECYCLE_WATCHDOG_TIMEOUT_FILE:-}" ] \ + && [ -s "$LIFECYCLE_WATCHDOG_TIMEOUT_FILE" ]; then + _prestart_status=1 + fi + case "${_prestart_status:-}" in + 0 | [1-9] | [1-9][0-9] | [1-9][0-9][0-9]) + if [ "$_prestart_status" -gt 255 ]; then + _prestart_status=1 + fi + ;; + *) _prestart_status=1 ;; + esac + if [ "$_prestart_status" -ne 0 ]; then + if ! _start_execd_watchdog 0 "" 0; then + echo "error: failed to start execd shutdown watchdog" >&2 + # Failing to arm the bounded escalation path must not leave execd + # running or turn this failure path into an unbounded wait. + _forward_signal TERM "$EXECD_PID" + _forward_signal KILL "$EXECD_PID" + fi + set +e + wait "$EXECD_PID" + _execd_status=$? + _stop_execd_watchdog + set -e + EXECD_PID="" + _cleanup_lifecycle_status + echo "error: lifecycle preStart failed (status $_prestart_status, execd exit $_execd_status)" >&2 + exit "$_prestart_status" + fi + _cleanup_lifecycle_status + unset _prestart_status _execd_status +fi + +unset OPEN_SANDBOX_LIFECYCLE EXECD_LIFECYCLE_CONFIG "$@" & CMD_PID=$! diff --git a/components/execd/go.mod b/components/execd/go.mod index af9301eca..bc5b7519d 100644 --- a/components/execd/go.mod +++ b/components/execd/go.mod @@ -13,6 +13,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/pelletier/go-toml/v2 v2.2.2 + github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 diff --git a/components/execd/go.sum b/components/execd/go.sum index 1ea60dc56..98558b721 100644 --- a/components/execd/go.sum +++ b/components/execd/go.sum @@ -101,6 +101,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= diff --git a/components/execd/main.go b/components/execd/main.go index a620b291b..9e8013a31 100644 --- a/components/execd/main.go +++ b/components/execd/main.go @@ -34,6 +34,7 @@ import ( "github.com/alibaba/opensandbox/execd/pkg/ebpf" "github.com/alibaba/opensandbox/execd/pkg/flag" "github.com/alibaba/opensandbox/execd/pkg/isolation" + "github.com/alibaba/opensandbox/execd/pkg/lifecycle" "github.com/alibaba/opensandbox/execd/pkg/log" "github.com/alibaba/opensandbox/execd/pkg/runtime" "github.com/alibaba/opensandbox/execd/pkg/telemetry" @@ -48,6 +49,8 @@ const ( isolatedRunnerCloseRetryInterval = 100 * time.Millisecond ) +var errStartupShutdown = errors.New("startup interrupted by shutdown") + type isolatedRunnerCloser interface { Close() error } @@ -62,6 +65,7 @@ func run() int { version.EchoVersion("OpenSandbox Execd") flag.InitFlags() + log.Init(flag.ServerLogLevel) // Load isolation config. isoCfg, err := isolation.LoadConfig(flag.IsolationConfigPath) @@ -78,6 +82,16 @@ func run() int { return 1 } + // Materialize the internal environment transport before the HTTP server + // can launch user code, then remove it from execd's process environment. + lifecycleConfig, err := lifecycle.LoadConfig() + if err != nil { + log.Error("lifecycle: config: %v", err) + return 1 + } + _ = os.Unsetenv(lifecycle.ConfigEnv) + _ = os.Unsetenv(lifecycle.ConfigPathEnv) + // Start the eBPF observation layer ([ebpf] enabled, OSEP-0018 ยง5). // The stub build reports disabled; the execd-ebpf variant attaches the // exec/connect/privilege hooks. @@ -94,12 +108,17 @@ func run() int { log.Info("isolation: available=%v isolator=%s version=%s", isolationProbe.Available, isolationProbe.Isolator, isolationProbe.Version) - log.Init(flag.ServerLogLevel) - + var startInitEntrypoint func([]string) error + var initStartupCtx context.Context + var stopInitStartupSignals context.CancelFunc if flag.InitMode { // Start after the startup probes (which run short-lived cmd.Run // children) so the reaper is the only wait4 caller from here on. - runtime.StartInitMode(flag.Args()) + initStartupCtx, stopInitStartupSignals = signal.NotifyContext( + context.Background(), os.Interrupt, syscall.SIGTERM, + ) + startInitEntrypoint = runtime.PrepareInitMode() + defer stopInitStartupSignals() } ctrl := controller.InitCodeRunner() @@ -150,18 +169,41 @@ func run() int { } engine := web.NewRouter(flag.ServerAccessToken) + if err := runHTTPServer( + engine, + startInitEntrypoint, + initStartupCtx, + stopInitStartupSignals, + lifecycleConfig, + ); err != nil { + if errors.Is(err, errStartupShutdown) { + log.Info("shutdown requested before user entrypoint started: %v", err) + return 0 + } + log.Error("execd server stopped with error: %v", err) + return 1 + } + return 0 +} + +func runHTTPServer( + engine http.Handler, + startInitEntrypoint func([]string) error, + initStartupCtx context.Context, + stopInitStartupSignals context.CancelFunc, + lifecycleConfig *lifecycle.Config, +) error { addr := fmt.Sprintf(":%d", flag.ServerPort) listener, err := net.Listen("tcp4", addr) if err != nil { - log.Error("failed to listen on %s: %v", addr, err) - return 1 + return fmt.Errorf("listen on %s: %w", addr, err) } log.Info("execd listening on %s (IPv4)", addr) // In init mode SIGTERM belongs to the init lifecycle (forward + graceful // shutdown with the entrypoint's exit status); only SIGINT cancels the // HTTP server there. ctxSignals := []os.Signal{os.Interrupt} - if !flag.InitMode { + if !flag.InitMode || len(flag.Args()) == 0 { ctxSignals = append(ctxSignals, syscall.SIGTERM) } serverCtx, stopSignals := signal.NotifyContext( @@ -169,11 +211,99 @@ func run() int { ctxSignals..., ) defer stopSignals() - if err := serveHTTPUntilShutdown(serverCtx, listener, engine); err != nil { - log.Error("execd server stopped with error: %v", err) - return 1 + var periodicManager *lifecycle.PeriodicManager + defer func() { + if periodicManager != nil { + periodicManager.Stop() + } + }() + startup := func() error { + preStartCtx := serverCtx + if flag.InitMode { + preStartCtx = initStartupCtx + if initStartupCtx.Err() != nil { + return errStartupShutdown + } + } + manager, startErr := startLifecycle( + preStartCtx, + lifecycleConfig, + flag.LifecycleStartupStatusFile, + ) + if startErr != nil { + return startErr + } + periodicManager = manager + if flag.InitMode { + stopInitStartupSignals() + if err := startInitEntrypoint(flag.Args()); err != nil { + return err + } + } + return nil } - return 0 + return serveHTTPUntilShutdown(serverCtx, listener, engine, startup) +} + +func startLifecycle( + ctx context.Context, + cfg *lifecycle.Config, + statusFile string, +) (*lifecycle.PeriodicManager, error) { + if cfg != nil && cfg.PreStart != nil { + if err := appendLifecycleStartupStatus( + statusFile, + fmt.Sprintf("running %d", int64(cfg.PreStartTimeout()/time.Second)), + ); err != nil { + return nil, err + } + if err := lifecycle.RunPreStart(ctx, cfg); err != nil { + reportErr := appendLifecycleStartupStatus(statusFile, "done 1") + if ctxErr := ctx.Err(); reportErr == nil && ctxErr != nil && errors.Is(err, ctxErr) { + return nil, errors.Join( + errStartupShutdown, + fmt.Errorf("lifecycle preStart: %w", err), + ) + } + return nil, errors.Join( + fmt.Errorf("lifecycle preStart: %w", err), + reportErr, + ) + } + } + + periodicManager, err := lifecycle.StartPeriodic(cfg) + if err != nil { + log.Error("lifecycle: periodic hooks disabled: %v", err) + periodicManager = nil + } + if err := appendLifecycleStartupStatus(statusFile, "done 0"); err != nil { + if periodicManager != nil { + periodicManager.Stop() + } + return nil, err + } + return periodicManager, nil +} + +func appendLifecycleStartupStatus(path string, status string) error { + if path == "" { + return nil + } + // Bootstrap creates and owns this private channel. Do not recreate a + // missing file: its disappearance must fail startup closed on both sides. + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return fmt.Errorf("open lifecycle startup status: %w", err) + } + if _, err := fmt.Fprintln(file, status); err != nil { + _ = file.Close() + return fmt.Errorf("write lifecycle startup status: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close lifecycle startup status: %w", err) + } + return nil } func closeIsolatedRunnerWithRetry( @@ -230,12 +360,21 @@ func serveHTTPUntilShutdown( ctx context.Context, listener net.Listener, handler http.Handler, + startup func() error, ) error { server := &http.Server{Handler: handler} serveDone := make(chan error, 1) go func() { serveDone <- server.Serve(listener) }() + if err := startup(); err != nil { + closeErr := server.Close() + serveErr := <-serveDone + if errors.Is(serveErr, http.ErrServerClosed) { + serveErr = nil + } + return errors.Join(err, closeErr, serveErr) + } select { case err := <-serveDone: diff --git a/components/execd/main_test.go b/components/execd/main_test.go index 90ab38597..4d6989f66 100644 --- a/components/execd/main_test.go +++ b/components/execd/main_test.go @@ -20,9 +20,12 @@ import ( "io" "net" "net/http" + "os" + "path/filepath" "testing" "time" + "github.com/alibaba/opensandbox/execd/pkg/lifecycle" "github.com/alibaba/opensandbox/execd/pkg/runtime" ) @@ -30,6 +33,118 @@ type fakeIsolatedRunnerCloser struct { closeFn func() error } +func TestStartLifecycleReportsStartupStatus(t *testing.T) { + tests := []struct { + name string + timeoutSeconds int + helperResult string + wantStatus string + wantError bool + }{ + {name: "default timeout", helperResult: "success", wantStatus: "running 60\ndone 0\n"}, + {name: "hook failure", timeoutSeconds: 2, helperResult: "failure", wantStatus: "running 2\ndone 1\n", wantError: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + statusFile := filepath.Join(t.TempDir(), "lifecycle-status") + if err := os.WriteFile(statusFile, nil, 0o600); err != nil { + t.Fatal(err) + } + cfg := &lifecycle.Config{PreStart: &lifecycle.Hook{ + Command: []string{ + os.Args[0], "-test.run=^TestLifecycleStartupCommandHelper$", "--", test.helperResult, + }, + TimeoutSeconds: test.timeoutSeconds, + }} + + manager, err := startLifecycle(context.Background(), cfg, statusFile) + if (err != nil) != test.wantError { + t.Fatalf("startLifecycle() error = %v, wantError %v", err, test.wantError) + } + if manager != nil { + manager.Stop() + } + raw, err := os.ReadFile(statusFile) + if err != nil { + t.Fatal(err) + } + if got := string(raw); got != test.wantStatus { + t.Fatalf("lifecycle status = %q, want %q", got, test.wantStatus) + } + }) + } +} + +func TestLifecycleStartupCommandHelper(*testing.T) { + switch os.Args[len(os.Args)-1] { + case "failure": + os.Exit(2) + case "wait": + time.Sleep(time.Hour) + } +} + +func TestStartLifecycleCancellationStatus(t *testing.T) { + for _, test := range []struct { + name string + removeStatusFile bool + }{ + {name: "reported shutdown"}, + {name: "status failure", removeStatusFile: true}, + } { + t.Run(test.name, func(t *testing.T) { + statusFile := filepath.Join(t.TempDir(), "lifecycle-status") + if err := os.WriteFile(statusFile, nil, 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := &lifecycle.Config{PreStart: &lifecycle.Hook{Command: []string{ + os.Args[0], "-test.run=^TestLifecycleStartupCommandHelper$", "--", "wait", + }}} + result := make(chan error, 1) + go func() { + _, err := startLifecycle(ctx, cfg, statusFile) + result <- err + }() + + deadline := time.After(2 * time.Second) + for { + raw, err := os.ReadFile(statusFile) + if err != nil { + t.Fatal(err) + } + if len(raw) > 0 { + break + } + select { + case <-deadline: + t.Fatal("preStart did not report running status") + case <-time.After(10 * time.Millisecond): + } + } + if test.removeStatusFile { + if err := os.Remove(statusFile); err != nil { + t.Fatal(err) + } + } + cancel() + select { + case err := <-result: + if test.removeStatusFile { + if errors.Is(err, errStartupShutdown) || !errors.Is(err, os.ErrNotExist) { + t.Fatalf("startLifecycle() error = %v, want status-file failure", err) + } + } else if !errors.Is(err, errStartupShutdown) { + t.Fatalf("startLifecycle() error = %v, want %v", err, errStartupShutdown) + } + case <-time.After(2 * time.Second): + t.Fatal("startLifecycle did not return after cancellation") + } + }) + } +} + func (f *fakeIsolatedRunnerCloser) Close() error { return f.closeFn() } @@ -165,6 +280,7 @@ func TestServeHTTPUntilShutdownReturnsAfterContextCancellation(t *testing.T) { http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }), + func() error { return nil }, ) }() @@ -186,3 +302,51 @@ func TestServeHTTPUntilShutdownReturnsAfterContextCancellation(t *testing.T) { t.Fatal("HTTP server did not stop after shutdown cancellation") } } + +func TestServeHTTPUntilShutdownServesDuringStartup(t *testing.T) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + startupStarted := make(chan struct{}) + finishStartup := make(chan struct{}) + serveDone := make(chan error, 1) + go func() { + serveDone <- serveHTTPUntilShutdown( + ctx, + listener, + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }), + func() error { + close(startupStarted) + <-finishStartup + return nil + }, + ) + }() + + <-startupStarted + response, err := http.Get("http://" + listener.Addr().String()) + if err != nil { + close(finishStartup) + cancel() + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusNoContent { + t.Fatalf("status = %d, want %d", response.StatusCode, http.StatusNoContent) + } + + close(finishStartup) + cancel() + select { + case err := <-serveDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("HTTP server did not stop after startup completed") + } +} diff --git a/components/execd/pkg/flag/flags.go b/components/execd/pkg/flag/flags.go index af66dab13..da169ad9d 100644 --- a/components/execd/pkg/flag/flags.go +++ b/components/execd/pkg/flag/flags.go @@ -48,4 +48,8 @@ var ( // subreaper) is decided by bootstrap.sh via EXECD_INIT, which passes // this flag when it execs into execd. InitMode bool + + // LifecycleStartupStatusFile is an internal bootstrap synchronization file. + // Execd writes the preStart result after its HTTP server is available. + LifecycleStartupStatusFile string ) diff --git a/components/execd/pkg/flag/parser.go b/components/execd/pkg/flag/parser.go index 685444d65..da1e45e18 100644 --- a/components/execd/pkg/flag/parser.go +++ b/components/execd/pkg/flag/parser.go @@ -43,6 +43,7 @@ func InitFlags() { JupyterIdlePollInterval = 100 * time.Millisecond IsolationConfigPath = "" InitMode = false + LifecycleStartupStatusFile = "" // First, set default values from environment variables if jupyterFromEnv := os.Getenv(jupyterHostEnv); jupyterFromEnv != "" { @@ -99,6 +100,7 @@ func InitFlags() { // Init mode must be enabled explicitly; bootstrap.sh passes it together // with EXECD_INIT so the shell's exec/background decision stays in lockstep. flag.BoolVar(&InitMode, "init", false, "Run as the sandbox init: reap children, forward signals, own the container lifecycle") + flag.StringVar(&LifecycleStartupStatusFile, "lifecycle-startup-status-file", "", "Write the internal lifecycle startup result to this file") // Parse flags - these will override environment variables if provided flag.Parse() diff --git a/components/execd/pkg/isolation/config.go b/components/execd/pkg/isolation/config.go index e9f0ac7b0..57536686f 100644 --- a/components/execd/pkg/isolation/config.go +++ b/components/execd/pkg/isolation/config.go @@ -62,6 +62,8 @@ var execdConfigEnvBlacklist = []string{ "JUPYTER_TOKEN", "EXECD_ISOLATION_CONFIG", "EXECD_ENVS", + "OPEN_SANDBOX_LIFECYCLE", + "EXECD_LIFECYCLE_CONFIG", } // ExecdConfigEnvBlacklist returns a copy of the execd config env names. diff --git a/components/execd/pkg/lifecycle/config.go b/components/execd/pkg/lifecycle/config.go new file mode 100644 index 000000000..887288b58 --- /dev/null +++ b/components/execd/pkg/lifecycle/config.go @@ -0,0 +1,256 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + toml "github.com/pelletier/go-toml/v2" + "github.com/robfig/cron/v3" +) + +const ( + ConfigEnv = "OPEN_SANDBOX_LIFECYCLE" + ConfigPathEnv = "EXECD_LIFECYCLE_CONFIG" + + defaultTimeout = 60 * time.Second + configVersion = 1 +) + +// Config is the creation-time sandbox lifecycle configuration consumed by +// bootstrap.sh and execd. JSON is used for environment transport while the +// persisted in-sandbox representation is TOML. +type Config struct { + Version int `json:"version" toml:"version"` + PreStart *Hook `json:"preStart,omitempty" toml:"preStart,omitempty"` + Periodic []PeriodicHook `json:"periodic,omitempty" toml:"periodic,omitempty"` +} + +type Hook struct { + Command []string `json:"command" toml:"command"` + TimeoutSeconds int `json:"timeoutSeconds,omitempty" toml:"timeout_seconds,omitempty"` +} + +type PeriodicHook struct { + Name string `json:"name" toml:"name"` + Schedule string `json:"schedule" toml:"schedule"` + Command []string `json:"command" toml:"command"` + TimeoutSeconds int `json:"timeoutSeconds,omitempty" toml:"timeout_seconds,omitempty"` +} + +func (h Hook) timeout() time.Duration { + if h.TimeoutSeconds == 0 { + return defaultTimeout + } + return time.Duration(h.TimeoutSeconds) * time.Second +} + +// PreStartTimeout returns the effective timeout for the configured preStart +// hook, or zero when no preStart hook is configured. +func (c *Config) PreStartTimeout() time.Duration { + if c == nil || c.PreStart == nil { + return 0 + } + return c.PreStart.timeout() +} + +func (h PeriodicHook) hook() Hook { + return Hook{Command: h.Command, TimeoutSeconds: h.TimeoutSeconds} +} + +// LoadConfig prefers and atomically persists the injected environment config. +// When the transport is absent, it reads the persisted config instead. +func LoadConfig() (*Config, error) { + raw := strings.TrimSpace(os.Getenv(ConfigEnv)) + if raw != "" { + cfg, err := decodeConfig([]byte(raw)) + if err != nil { + return nil, fmt.Errorf("decode %s: %w", ConfigEnv, err) + } + path, err := resolveConfigPath() + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, fmt.Errorf("create lifecycle config directory: %w", err) + } + if err := persistConfig(path, cfg); err != nil { + return nil, err + } + return cfg, nil + } + + path, err := resolveConfigPath() + if err != nil { + return nil, nil //nolint:nilerr,nilnil // no transport and no home means hooks are optional + } + if raw, err := os.ReadFile(path); err == nil { + cfg, decodeErr := decodeConfig(raw) + if decodeErr != nil { + return nil, fmt.Errorf("invalid persisted lifecycle config %s: %w", path, decodeErr) + } + return cfg, nil + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("read lifecycle config %s: %w", path, err) + } + + return nil, nil //nolint:nilnil // lifecycle hooks are optional +} + +func decodeConfig(raw []byte) (*Config, error) { + var cfg Config + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 { + return nil, errors.New("empty lifecycle config") + } + var err error + if trimmed[0] == '{' { + err = json.Unmarshal(trimmed, &cfg) + } else { + err = toml.Unmarshal(trimmed, &cfg) + } + if err != nil { + return nil, err + } + if cfg.Version == 0 { + cfg.Version = configVersion + } + if err := cfg.validate(); err != nil { + return nil, err + } + return &cfg, nil +} + +func (c *Config) validate() error { + if c.Version != configVersion { + return fmt.Errorf("unsupported lifecycle config version %d", c.Version) + } + if c.PreStart != nil { + if err := validateHook("preStart", *c.PreStart); err != nil { + return err + } + } + seen := make(map[string]struct{}, len(c.Periodic)) + for index := range c.Periodic { + periodic := &c.Periodic[index] + periodic.Name = strings.TrimSpace(periodic.Name) + periodic.Schedule = strings.TrimSpace(periodic.Schedule) + if periodic.Name == "" { + return errors.New("periodic hook name must not be blank") + } + if _, ok := seen[periodic.Name]; ok { + return fmt.Errorf("duplicate periodic hook name %q", periodic.Name) + } + seen[periodic.Name] = struct{}{} + if periodic.Schedule == "" { + return fmt.Errorf("periodic hook %q schedule must not be blank", periodic.Name) + } + schedule := periodic.Schedule + descriptor := schedule + if strings.HasPrefix(descriptor, "TZ=") || strings.HasPrefix(descriptor, "CRON_TZ=") { + space := strings.IndexByte(descriptor, ' ') + if space < 0 { + return fmt.Errorf("periodic hook %q has invalid schedule", periodic.Name) + } + descriptor = strings.TrimSpace(descriptor[space+1:]) + } + if _, err := cron.ParseStandard(schedule); err != nil { + return fmt.Errorf("periodic hook %q has invalid schedule: %w", periodic.Name, err) + } + if strings.HasPrefix(descriptor, "@every ") { + interval, err := time.ParseDuration(strings.TrimSpace(strings.TrimPrefix(descriptor, "@every "))) + if err != nil || interval < time.Second || interval%time.Second != 0 { + return fmt.Errorf("periodic hook %q @every interval must be a whole number of seconds", periodic.Name) + } + } + if err := validateHook("periodic "+periodic.Name, periodic.hook()); err != nil { + return err + } + } + return nil +} + +func validateHook(name string, hook Hook) error { + if len(hook.Command) == 0 || strings.TrimSpace(hook.Command[0]) == "" { + return fmt.Errorf("%s command must not be empty", name) + } + if hook.TimeoutSeconds < 0 { + return fmt.Errorf("%s timeoutSeconds must not be negative", name) + } + const maxTimeoutSeconds = int64((time.Duration(1<<63 - 1)) / time.Second) + if int64(hook.TimeoutSeconds) > maxTimeoutSeconds { + return fmt.Errorf("%s timeoutSeconds is too large", name) + } + return nil +} + +func resolveConfigPath() (string, error) { + if configuredPath := os.Getenv(ConfigPathEnv); configuredPath != "" { + return configuredPath, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolve lifecycle config home directory: %w", err) + } + return filepath.Join(home, ".execd", "lifecycle.toml"), nil +} + +func persistConfig(path string, cfg *Config) error { + raw, err := toml.Marshal(cfg) + if err != nil { + return fmt.Errorf("encode lifecycle TOML: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".lifecycle-*.tmp") + if err != nil { + return fmt.Errorf("create lifecycle config temp file: %w", err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return fmt.Errorf("chmod lifecycle config temp file: %w", err) + } + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return fmt.Errorf("write lifecycle config temp file: %w", err) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("sync lifecycle config temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close lifecycle config temp file: %w", err) + } + if err := os.Rename(tmpName, path); err != nil { + return fmt.Errorf("replace lifecycle config: %w", err) + } + if runtime.GOOS != "windows" { + // The rename already made the fsynced contents live. Directory sync is + // best-effort because some container filesystems reject it with EINVAL. + if dir, err := os.Open(filepath.Dir(path)); err == nil { + _ = dir.Sync() + _ = dir.Close() + } + } + return nil +} diff --git a/components/execd/pkg/lifecycle/config_test.go b/components/execd/pkg/lifecycle/config_test.go new file mode 100644 index 000000000..ea9139414 --- /dev/null +++ b/components/execd/pkg/lifecycle/config_test.go @@ -0,0 +1,210 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfigMaterializesEnvironmentConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "state", "lifecycle.toml") + t.Setenv(ConfigPathEnv, path) + t.Setenv(ConfigEnv, `{ + "version": 1, + "preStart": {"command": ["sh", "-c", "echo ready"], "timeoutSeconds": 5}, + "periodic": [{"name": "sync", "schedule": "@every 5m", "command": ["sync"]}] +}`) + + cfg, err := LoadConfig() + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Equal(t, []string{"sh", "-c", "echo ready"}, cfg.PreStart.Command) + require.FileExists(t, path) + + t.Setenv(ConfigEnv, "") + reloaded, err := LoadConfig() + require.NoError(t, err) + require.NotNil(t, reloaded) + assert.Equal(t, "sync", reloaded.Periodic[0].Name) +} + +func TestLoadConfigMaterializesEnvironmentConfigUnderHome(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv(ConfigPathEnv, "") + t.Setenv(ConfigEnv, `{"preStart":{"command":["true"]}}`) + + cfg, err := LoadConfig() + + require.NoError(t, err) + require.NotNil(t, cfg) + require.FileExists(t, filepath.Join(home, ".execd", "lifecycle.toml")) +} + +func TestLoadConfigRejectsUnwritableDefaultPath(t *testing.T) { + home := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(home, ".execd"), []byte("not a directory"), 0o600)) + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv(ConfigPathEnv, "") + t.Setenv(ConfigEnv, `{"preStart":{"command":["true"]}}`) + + cfg, err := LoadConfig() + + assert.Nil(t, cfg) + require.ErrorContains(t, err, "create lifecycle config directory") +} + +func TestLoadConfigRejectsUnwritableExplicitPath(t *testing.T) { + parent := filepath.Join(t.TempDir(), "not-a-directory") + require.NoError(t, os.WriteFile(parent, []byte("file"), 0o600)) + t.Setenv(ConfigPathEnv, filepath.Join(parent, "lifecycle.toml")) + t.Setenv(ConfigEnv, `{"preStart":{"command":["true"]}}`) + + cfg, err := LoadConfig() + + assert.Nil(t, cfg) + require.ErrorContains(t, err, "create lifecycle config directory") +} + +func TestLoadConfigEnvironmentOverridesPersistedConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "lifecycle.toml") + require.NoError(t, os.WriteFile(path, []byte(`version = 1 +[preStart] +command = ["old"] +`), 0o600)) + t.Setenv(ConfigPathEnv, path) + t.Setenv(ConfigEnv, `{"preStart":{"command":["new"]}}`) + + cfg, err := LoadConfig() + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Equal(t, []string{"new"}, cfg.PreStart.Command) + + t.Setenv(ConfigEnv, "") + reloaded, err := LoadConfig() + require.NoError(t, err) + require.NotNil(t, reloaded) + assert.Equal(t, []string{"new"}, reloaded.PreStart.Command) +} + +func TestDecodeConfigRejectsDuplicatePeriodicNames(t *testing.T) { + _, err := decodeConfig([]byte(`{ + "periodic": [ + {"name": "sync", "schedule": "@hourly", "command": ["true"]}, + {"name": "sync", "schedule": "@daily", "command": ["true"]} + ] +}`)) + + require.ErrorContains(t, err, `duplicate periodic hook name "sync"`) +} + +func TestDecodeConfigRejectsInvalidPeriodicSchedule(t *testing.T) { + _, err := decodeConfig([]byte(`{ + "periodic": [{"name": "sync", "schedule": "61 * * * *", "command": ["true"]}] +}`)) + + require.ErrorContains(t, err, `periodic hook "sync" has invalid schedule`) +} + +func TestDecodeConfigNormalizesPeriodicIdentityAndSchedule(t *testing.T) { + cfg, err := decodeConfig([]byte(`{ + "periodic": [{"name": " sync ", "schedule": " @every 1m ", "command": ["true"]}] +}`)) + + require.NoError(t, err) + assert.Equal(t, "sync", cfg.Periodic[0].Name) + assert.Equal(t, "@every 1m", cfg.Periodic[0].Schedule) +} + +func TestDecodeConfigRejectsNonWholeSecondEveryInterval(t *testing.T) { + for _, schedule := range []string{ + "@every 500ms", + "TZ=UTC @every 500ms", + "CRON_TZ=UTC @every 1500ms", + } { + t.Run(schedule, func(t *testing.T) { + _, err := decodeConfig([]byte(`{ + "periodic": [{"name": "sync", "schedule": "` + schedule + `", "command": ["true"]}] +}`)) + + require.ErrorContains(t, err, `periodic hook "sync" @every interval must be a whole number of seconds`) + }) + } +} + +func TestDecodeConfigRejectsTimezoneWithoutSchedule(t *testing.T) { + _, err := decodeConfig([]byte(`{ + "periodic": [{"name": "sync", "schedule": "TZ=UTC", "command": ["true"]}] +}`)) + + require.ErrorContains(t, err, `periodic hook "sync" has invalid schedule`) +} + +func TestDecodeConfigRejectsOverflowingTimeout(t *testing.T) { + _, err := decodeConfig([]byte(`{ + "preStart": {"command": ["true"], "timeoutSeconds": 9223372037} +}`)) + + require.ErrorContains(t, err, "timeoutSeconds is too large") +} + +func TestLoadConfigRejectsInvalidPersistedConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "lifecycle.toml") + require.NoError(t, os.WriteFile(path, []byte("not valid TOML ="), 0o600)) + t.Setenv(ConfigPathEnv, path) + t.Setenv(ConfigEnv, "") + + cfg, err := LoadConfig() + require.ErrorContains(t, err, "invalid persisted lifecycle config") + assert.Nil(t, cfg) +} + +func TestLoadConfigReturnsNilWhenNotConfigured(t *testing.T) { + t.Setenv(ConfigPathEnv, filepath.Join(t.TempDir(), "lifecycle.toml")) + t.Setenv(ConfigEnv, "") + + cfg, err := LoadConfig() + require.NoError(t, err) + assert.Nil(t, cfg) + _, statErr := os.Stat(os.Getenv(ConfigPathEnv)) + assert.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestLoadConfigWithoutHome(t *testing.T) { + t.Setenv(ConfigPathEnv, "") + t.Setenv(ConfigEnv, "") + t.Setenv("HOME", "") + t.Setenv("USERPROFILE", "") + t.Setenv("HOMEDRIVE", "") + t.Setenv("HOMEPATH", "") + + cfg, err := LoadConfig() + + require.NoError(t, err) + assert.Nil(t, cfg) + + t.Setenv(ConfigEnv, `{"preStart":{"command":["true"]}}`) + cfg, err = LoadConfig() + + assert.Nil(t, cfg) + require.ErrorContains(t, err, "resolve lifecycle config home directory") +} diff --git a/components/execd/pkg/lifecycle/periodic.go b/components/execd/pkg/lifecycle/periodic.go new file mode 100644 index 000000000..7a2cccf2d --- /dev/null +++ b/components/execd/pkg/lifecycle/periodic.go @@ -0,0 +1,106 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/robfig/cron/v3" + + "github.com/alibaba/opensandbox/execd/pkg/log" +) + +type PeriodicManager struct { + cron *cron.Cron + ctx context.Context + cancel context.CancelFunc + running map[string]*atomic.Bool + entryIDs map[string]cron.EntryID +} + +func StartPeriodic(cfg *Config) (*PeriodicManager, error) { + if cfg == nil || len(cfg.Periodic) == 0 { + return nil, nil //nolint:nilnil // no manager is needed when periodic hooks are not configured + } + + ctx, cancel := context.WithCancel(context.Background()) + manager := &PeriodicManager{ + cron: cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger))), + ctx: ctx, + cancel: cancel, + running: make(map[string]*atomic.Bool, len(cfg.Periodic)), + entryIDs: make(map[string]cron.EntryID, len(cfg.Periodic)), + } + for _, configured := range cfg.Periodic { + hook := configured + manager.running[hook.Name] = &atomic.Bool{} + entryID, err := manager.cron.AddFunc(hook.Schedule, func() { manager.run(hook) }) + if err != nil { + cancel() + return nil, fmt.Errorf("parse periodic hook %q schedule: %w", hook.Name, err) + } + manager.entryIDs[hook.Name] = entryID + } + manager.cron.Start() + return manager, nil +} + +func (m *PeriodicManager) run(periodic PeriodicHook) { + running := m.running[periodic.Name] + if !running.CompareAndSwap(false, true) { + log.Warn("lifecycle: periodic hook %q skipped because its previous run is still active", periodic.Name) + return + } + releaseRunning := true + defer func() { + if releaseRunning { + running.Store(false) + } + }() + + log.Info("lifecycle: periodic hook %q started", periodic.Name) + hook := periodic.hook() + result := RunHook(m.ctx, hook) + if result.Incomplete { + releaseRunning = false + m.cron.Remove(m.entryIDs[periodic.Name]) + log.Error("lifecycle: periodic hook %q did not exit after cancellation; future runs disabled", periodic.Name) + return + } + if result.TimedOut { + log.Warn("lifecycle: periodic hook %q timed out after %s", periodic.Name, hook.timeout()) + return + } + if result.Err != nil && m.ctx.Err() != nil { + log.Info("lifecycle: periodic hook %q canceled during shutdown", periodic.Name) + return + } + if result.Err != nil { + log.Warn("lifecycle: periodic hook %q failed exit_code=%d duration=%s: %v", periodic.Name, result.ExitCode, result.Duration, result.Err) + return + } + log.Info("lifecycle: periodic hook %q completed exit_code=0 duration=%s", periodic.Name, result.Duration) +} + +func (m *PeriodicManager) Stop() { + if m == nil { + return + } + m.cancel() + ctx := m.cron.Stop() + <-ctx.Done() +} diff --git a/components/execd/pkg/lifecycle/process_unix.go b/components/execd/pkg/lifecycle/process_unix.go new file mode 100644 index 000000000..1c8c68208 --- /dev/null +++ b/components/execd/pkg/lifecycle/process_unix.go @@ -0,0 +1,32 @@ +//go:build !windows + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "os/exec" + "syscall" +) + +func prepareCommand(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func terminateCommand(cmd *exec.Cmd) { + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } +} diff --git a/components/execd/pkg/lifecycle/process_windows.go b/components/execd/pkg/lifecycle/process_windows.go new file mode 100644 index 000000000..b7eb13bdd --- /dev/null +++ b/components/execd/pkg/lifecycle/process_windows.go @@ -0,0 +1,54 @@ +//go:build windows + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "context" + "os/exec" + "path/filepath" + "strconv" + "time" + + "github.com/alibaba/opensandbox/execd/pkg/log" + "golang.org/x/sys/windows" +) + +const taskkillTimeout = 2 * time.Second + +func prepareCommand(_ *exec.Cmd) {} + +func terminateCommand(cmd *exec.Cmd) { + if cmd.Process != nil { + killCtx, killCancel := context.WithTimeout(context.Background(), taskkillTimeout) + defer killCancel() + if systemDirectory, err := windows.GetSystemDirectory(); err == nil { + if err := exec.CommandContext( + killCtx, + filepath.Join(systemDirectory, "taskkill.exe"), + "/T", + "/F", + "/PID", + strconv.Itoa(cmd.Process.Pid), + ).Run(); err != nil { + log.Warn("lifecycle: terminate Windows process tree for pid %d: %v", cmd.Process.Pid, err) + } + } else { + log.Warn("lifecycle: resolve Windows system directory for taskkill: %v", err) + } + _ = cmd.Process.Kill() + } +} diff --git a/components/execd/pkg/lifecycle/runner.go b/components/execd/pkg/lifecycle/runner.go new file mode 100644 index 000000000..beaa60b39 --- /dev/null +++ b/components/execd/pkg/lifecycle/runner.go @@ -0,0 +1,98 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" + "github.com/alibaba/opensandbox/execd/pkg/runtime" +) + +type Result struct { + ExitCode int + Duration time.Duration + TimedOut bool + Incomplete bool + Err error +} + +func RunHook(parent context.Context, hook Hook) Result { + if len(hook.Command) == 0 { + return Result{ExitCode: -1, Err: errors.New("hook command must not be empty")} + } + ctx, cancel := context.WithTimeout(parent, hook.timeout()) + defer cancel() + + cmd := exec.Command(hook.Command[0], hook.Command[1:]...) + cmd.Env = sanitizedHookEnvironment() + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + prepareCommand(cmd) + + started := time.Now() + exitCode, err := runtime.RunManagedCommand(ctx, cmd, func() { terminateCommand(cmd) }) + + result := Result{ExitCode: exitCode, Duration: time.Since(started), Err: err} + if err != nil && errors.Is(err, context.DeadlineExceeded) { + result.TimedOut = true + } + result.Incomplete = errors.Is(err, runtime.ErrManagedCommandCancelTimeout) + return result +} + +func sanitizedHookEnvironment() []string { + blocked := make(map[string]struct{}) + for _, name := range isolation.ExecdConfigEnvBlacklist() { + blocked[strings.ToUpper(name)] = struct{}{} + } + + env := os.Environ() + filtered := make([]string, 0, len(env)) + for _, entry := range env { + name, _, ok := strings.Cut(entry, "=") + if !ok { + continue + } + if _, found := blocked[strings.ToUpper(name)]; found { + continue + } + filtered = append(filtered, entry) + } + return filtered +} + +func RunPreStart(ctx context.Context, cfg *Config) error { + if cfg == nil || cfg.PreStart == nil { + return nil + } + result := RunHook(ctx, *cfg.PreStart) + if result.TimedOut { + return fmt.Errorf("command %v timed out after %s: %w", cfg.PreStart.Command, cfg.PreStart.timeout(), result.Err) + } + if result.Err != nil { + if result.ExitCode >= 0 { + return fmt.Errorf("command %v failed with exit code %d: %w", cfg.PreStart.Command, result.ExitCode, result.Err) + } + return fmt.Errorf("command %v failed: %w", cfg.PreStart.Command, result.Err) + } + return nil +} diff --git a/components/execd/pkg/lifecycle/runner_test.go b/components/execd/pkg/lifecycle/runner_test.go new file mode 100644 index 000000000..222dc35c4 --- /dev/null +++ b/components/execd/pkg/lifecycle/runner_test.go @@ -0,0 +1,150 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/alibaba/opensandbox/execd/pkg/isolation" + "github.com/alibaba/opensandbox/execd/pkg/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + log.Init(6) + os.Exit(m.Run()) +} + +func TestRunPreStartExecutesConfiguredCommand(t *testing.T) { + marker := filepath.Join(t.TempDir(), "pre-started") + t.Setenv(ConfigPathEnv, filepath.Join(t.TempDir(), "lifecycle.toml")) + t.Setenv(ConfigEnv, `{"preStart":{"command":["sh","-c","touch `+marker+`"]}}`) + + cfg, err := LoadConfig() + require.NoError(t, err) + require.NoError(t, RunPreStart(context.Background(), cfg)) + require.FileExists(t, marker) +} + +func TestRunHookReturnsExitCode(t *testing.T) { + result := RunHook(context.Background(), Hook{Command: []string{"sh", "-c", "exit 17"}}) + + require.Error(t, result.Err) + assert.Equal(t, 17, result.ExitCode) + assert.False(t, result.TimedOut) +} + +func TestRunHookStripsExecdConfigurationEnvironment(t *testing.T) { + marker := filepath.Join(t.TempDir(), "hook-env") + t.Setenv("LIFECYCLE_HOOK_ENV_HELPER", "1") + t.Setenv("LIFECYCLE_HOOK_ENV_MARKER", marker) + for _, name := range isolation.ExecdConfigEnvBlacklist() { + t.Setenv(name, "secret") + } + + result := RunHook(context.Background(), Hook{ + Command: []string{os.Args[0], "-test.run=TestLifecycleHookEnvironmentHelper"}, + }) + + require.NoError(t, result.Err) + raw, err := os.ReadFile(marker) + require.NoError(t, err) + assert.Empty(t, string(raw)) +} + +func TestLifecycleHookEnvironmentHelper(t *testing.T) { + if os.Getenv("LIFECYCLE_HOOK_ENV_HELPER") != "1" { + return + } + marker := os.Getenv("LIFECYCLE_HOOK_ENV_MARKER") + require.NotEmpty(t, marker) + + leaked := make([]string, 0) + for _, name := range isolation.ExecdConfigEnvBlacklist() { + if _, ok := os.LookupEnv(name); ok { + leaked = append(leaked, name) + } + } + require.NoError(t, os.WriteFile(marker, []byte(strings.Join(leaked, "\n")), 0o600)) +} + +func TestRunHookKillsTimedOutProcess(t *testing.T) { + marker := filepath.Join(t.TempDir(), "descendant-survived") + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + result := RunHook(ctx, Hook{Command: []string{"sh", "-c", `(sleep 0.2; touch "$1") & wait`, "_", marker}}) + + require.Error(t, result.Err) + assert.True(t, result.TimedOut) + assert.Less(t, result.Duration, time.Second) + time.Sleep(300 * time.Millisecond) + _, err := os.Stat(marker) + assert.ErrorIs(t, err, os.ErrNotExist) +} + +func TestPeriodicManagerSkipsOverlappingRun(t *testing.T) { + marker := filepath.Join(t.TempDir(), "runs") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + manager := &PeriodicManager{ + ctx: ctx, + cancel: cancel, + running: map[string]*atomic.Bool{"sync": {}}, + } + hook := PeriodicHook{ + Name: "sync", + Schedule: "@every 1m", + Command: []string{"sh", "-c", "echo start >> " + marker + "; sleep 0.2; echo end >> " + marker}, + } + + done := make(chan struct{}) + go func() { + manager.run(hook) + close(done) + }() + require.Eventually(t, func() bool { return manager.running["sync"].Load() }, time.Second, 10*time.Millisecond) + manager.run(hook) + <-done + + raw, err := os.ReadFile(marker) + require.NoError(t, err) + assert.Equal(t, "start\nend\n", string(raw)) +} + +func TestStartPeriodicRunsConfiguredSchedule(t *testing.T) { + marker := filepath.Join(t.TempDir(), "scheduled") + t.Setenv(ConfigPathEnv, filepath.Join(t.TempDir(), "lifecycle.toml")) + t.Setenv(ConfigEnv, `{"periodic":[{"name":"sync","schedule":"@every 1s","command":["sh","-c","touch `+marker+`"]}]}`) + + cfg, err := LoadConfig() + require.NoError(t, err) + manager, err := StartPeriodic(cfg) + require.NoError(t, err) + require.NotNil(t, manager) + defer manager.Stop() + + require.Eventually(t, func() bool { + _, err := os.Stat(marker) + return err == nil + }, 2500*time.Millisecond, 50*time.Millisecond) +} diff --git a/components/execd/pkg/runtime/initmode_linux.go b/components/execd/pkg/runtime/initmode_linux.go index 666c41bc5..96520a415 100644 --- a/components/execd/pkg/runtime/initmode_linux.go +++ b/components/execd/pkg/runtime/initmode_linux.go @@ -233,6 +233,8 @@ func (r *reaper) drain() { // share one launch path regardless of mode. type managedProcess struct { cmd *exec.Cmd + stateMu sync.Mutex + exited bool preReap func() noHardening bool stripEnv []string // nil = default blacklist; explicit list overrides @@ -260,16 +262,44 @@ func (mp *managedProcess) deliver(ws syscall.WaitStatus) { func (mp *managedProcess) Wait() error { if initReaper == nil { - return mp.cmd.Wait() + return waitCommandWithExitBarrier(mp.cmd, func(_ error) { + // Success marks exit before reap. A failed barrier cannot prove + // ownership, so also disable signaling rather than risk PID reuse. + mp.stateMu.Lock() + mp.exited = true + mp.stateMu.Unlock() + }) } <-mp.done return mp.exitErr } +func (mp *managedProcess) Cancel(cancel func()) { + if initReaper == nil { + mp.stateMu.Lock() + defer mp.stateMu.Unlock() + if !mp.exited { + cancel() + } + return + } + + // Keep the reaper lock across signal delivery so the PID/PGID cannot be + // recycled. cancel must only deliver a signal and must not block. + initReaper.mu.Lock() + defer initReaper.mu.Unlock() + if initReaper.owned[mp.pid()] == mp { + cancel() + } +} + // ExitCode returns the process exit code, or -1 if it has not exited (or was // killed by a signal), matching os.ProcessState.ExitCode semantics. func (mp *managedProcess) ExitCode() int { if initReaper == nil { + if mp.cmd.ProcessState == nil { + return -1 + } return mp.cmd.ProcessState.ExitCode() } select { @@ -303,11 +333,15 @@ func withoutHardening() launchOption { // bootstrapEnv overrides the env strip for the user entrypoint: its scripts // may need JUPYTER_TOKEN/EXECD_ENVS to configure themselves (e.g. the -// code-interpreter entrypoint), but EXECD_ACCESS_TOKEN must never reach the -// long-lived entrypoint (its Jupyter kernels are user code). +// code-interpreter entrypoint), but credentials and lifecycle transport must +// never reach the long-lived entrypoint (its Jupyter kernels are user code). func bootstrapEnv() launchOption { return func(mp *managedProcess) { - mp.stripEnv = []string{"EXECD_ACCESS_TOKEN"} + mp.stripEnv = []string{ + "EXECD_ACCESS_TOKEN", + "OPEN_SANDBOX_LIFECYCLE", + "EXECD_LIFECYCLE_CONFIG", + } } } @@ -383,12 +417,10 @@ func exitStatusError(ws syscall.WaitStatus) error { return &processExitError{code: -1, msg: fmt.Sprintf("signal: %v", ws.Signal())} } -// StartInitMode activates init duties: non-dumpable self, subreaper fallback -// when not PID 1, the reaper, the user entrypoint, signal forwarding, and the -// container lifecycle owner. It returns once the entrypoint is launched; the -// process is torn down via os.Exit when the entrypoint exits or SIGTERM -// arrives. -func StartInitMode(entryArgs []string) { +// PrepareInitMode activates the init/reaper duties and registers signal +// handling before any managed child starts. The returned function launches +// the user entrypoint after execd has started serving and preStart succeeds. +func PrepareInitMode() func([]string) error { if err := unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil { log.Warn("init: PR_SET_DUMPABLE(0) failed: %v", err) } @@ -410,21 +442,53 @@ func StartInitMode(entryArgs []string) { // hitting the runtime default handler. sigCh := make(chan os.Signal, 8) signal.Notify(sigCh, initForwardedSignals...) + entryCh := make(chan *managedProcess, 1) + safego.Go(func() { forwardInitSignalsWhenReady(entryCh, sigCh) }) - entry := launchEntrypoint(entryArgs) - if entry == nil { - signal.Stop(sigCh) - return + return func(entryArgs []string) error { + if len(entryArgs) == 0 { + log.Warn("init: --init set but no user command provided; no entrypoint to supervise") + entryCh <- nil + return nil + } + entry, err := launchEntrypoint(entryArgs) + if err != nil { + entryCh <- nil + return err + } + entryCh <- entry + safego.Go(func() { waitEntrypointExit(entry) }) + return nil } - safego.Go(func() { forwardInitSignals(entry, sigCh) }) - safego.Go(func() { waitEntrypointExit(entry) }) } -func launchEntrypoint(args []string) *managedProcess { - if len(args) == 0 { - log.Warn("init: --init set but no user command provided; no entrypoint to supervise") - return nil +func forwardInitSignalsWhenReady( + entryCh <-chan *managedProcess, + sigCh chan os.Signal, +) { + termPending := false + for { + select { + case entry := <-entryCh: + if entry == nil { + signal.Stop(sigCh) + return + } + if termPending { + terminateInit(entry) + return + } + forwardInitSignals(entry, sigCh) + return + case sig := <-sigCh: + if sig == syscall.SIGTERM { + termPending = true + } + } } +} + +func launchEntrypoint(args []string) (*managedProcess, error) { cmd := exec.Command(args[0], args[1:]...) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Stdin = os.Stdin @@ -432,11 +496,10 @@ func launchEntrypoint(args []string) *managedProcess { cmd.Stderr = os.Stderr mp, err := launchManaged(cmd, bootstrapEnv()) if err != nil { - log.Error("init: failed to start user entrypoint %q: %v", args[0], err) - os.Exit(1) + return nil, fmt.Errorf("start user entrypoint %q: %w", args[0], err) } log.Info("init: user entrypoint started pid=%d argv=%v", mp.pid(), args) - return mp + return mp, nil } // waitEntrypointExit owns the container lifecycle: when the entrypoint exits, diff --git a/components/execd/pkg/runtime/initmode_linux_test.go b/components/execd/pkg/runtime/initmode_linux_test.go index 0946e0161..08f60fa98 100644 --- a/components/execd/pkg/runtime/initmode_linux_test.go +++ b/components/execd/pkg/runtime/initmode_linux_test.go @@ -17,6 +17,7 @@ package runtime import ( + "context" "errors" "os" "os/exec" @@ -363,3 +364,19 @@ func TestManagedProcessWithoutReaperUsesCmdWait(t *testing.T) { t.Fatalf("ExitCode = %d, want 4", mp.ExitCode()) } } + +func TestRunManagedCommandWithReaper(t *testing.T) { + startReaperForTest(t) + cmd := exec.Command("sh", "-c", "exit 17") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + exitCode, err := RunManagedCommand(context.Background(), cmd, func() { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + }) + if err == nil { + t.Fatal("RunManagedCommand error = nil, want exit status 17") + } + if exitCode != 17 { + t.Fatalf("RunManagedCommand exit code = %d, want 17", exitCode) + } +} diff --git a/components/execd/pkg/runtime/initmode_other.go b/components/execd/pkg/runtime/initmode_other.go index 16a40daa5..e740c0df7 100644 --- a/components/execd/pkg/runtime/initmode_other.go +++ b/components/execd/pkg/runtime/initmode_other.go @@ -22,9 +22,10 @@ package runtime import "github.com/alibaba/opensandbox/execd/pkg/log" -// StartInitMode is unsupported off Linux; execd keeps today's behavior. -func StartInitMode(entryArgs []string) { +// PrepareInitMode is unsupported off Linux; execd keeps today's behavior. +func PrepareInitMode() func([]string) error { log.Warn("init mode is unsupported on this platform; continuing without init duties") + return func([]string) error { return nil } } // InitModeReport reports the init mode actually in effect for the diff --git a/components/execd/pkg/runtime/managed_command.go b/components/execd/pkg/runtime/managed_command.go new file mode 100644 index 000000000..0e8a63881 --- /dev/null +++ b/components/execd/pkg/runtime/managed_command.go @@ -0,0 +1,67 @@ +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "context" + "errors" + "os/exec" + "time" +) + +const managedCommandCancelWait = 5 * time.Second + +var ErrManagedCommandCancelTimeout = errors.New("timed out waiting for canceled command to exit") + +type managedCommand interface { + Wait() error + ExitCode() int + Cancel(func()) +} + +// RunManagedCommand runs cmd through execd's init-mode-aware child tracker. +// cancel must promptly terminate cmd and any descendants when ctx is canceled; +// it may be called while the init reaper lock is held. +func RunManagedCommand(ctx context.Context, cmd *exec.Cmd, cancel func()) (int, error) { + if err := ctx.Err(); err != nil { + return -1, err + } + process, err := startManagedCommand(cmd) + if err != nil { + return -1, err + } + + done := make(chan error, 1) + go func() { done <- process.Wait() }() + + select { + case err := <-done: + return process.ExitCode(), err + case <-ctx.Done(): + // Prefer a result that completed concurrently with cancellation. + select { + case err := <-done: + return process.ExitCode(), err + default: + } + process.Cancel(cancel) + select { + case err := <-done: + return process.ExitCode(), errors.Join(ctx.Err(), err) + case <-time.After(managedCommandCancelWait): + return -1, errors.Join(ctx.Err(), ErrManagedCommandCancelTimeout) + } + } +} diff --git a/components/execd/pkg/runtime/managed_command_linux.go b/components/execd/pkg/runtime/managed_command_linux.go new file mode 100644 index 000000000..0356c0637 --- /dev/null +++ b/components/execd/pkg/runtime/managed_command_linux.go @@ -0,0 +1,23 @@ +//go:build linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import "os/exec" + +func startManagedCommand(cmd *exec.Cmd) (managedCommand, error) { + return launchManaged(cmd) +} diff --git a/components/execd/pkg/runtime/managed_command_other.go b/components/execd/pkg/runtime/managed_command_other.go new file mode 100644 index 000000000..8be702ab5 --- /dev/null +++ b/components/execd/pkg/runtime/managed_command_other.go @@ -0,0 +1,53 @@ +//go:build !linux + +// Copyright 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runtime + +import ( + "os/exec" + "sync/atomic" +) + +type directManagedCommand struct { + cmd *exec.Cmd + exited atomic.Bool +} + +func (c *directManagedCommand) Wait() error { + err := c.cmd.Wait() + c.exited.Store(true) + return err +} + +func (c *directManagedCommand) ExitCode() int { + if c.cmd.ProcessState == nil { + return -1 + } + return c.cmd.ProcessState.ExitCode() +} + +func (c *directManagedCommand) Cancel(cancel func()) { + if !c.exited.Load() { + cancel() + } +} + +func startManagedCommand(cmd *exec.Cmd) (managedCommand, error) { + if err := cmd.Start(); err != nil { + return nil, err + } + return &directManagedCommand{cmd: cmd}, nil +} diff --git a/components/execd/tests/lifecycle.sh b/components/execd/tests/lifecycle.sh new file mode 100755 index 000000000..ec0c0af20 --- /dev/null +++ b/components/execd/tests/lifecycle.sh @@ -0,0 +1,606 @@ +#!/bin/bash +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +BOOTSTRAP="$ROOT_DIR/bootstrap.sh" +TESTDIR="$(mktemp -d)" +export HOME="$TESTDIR/home" +mkdir -p "$HOME" +BOOTSTRAP_PID="" +cleanup() { + if [ -n "$BOOTSTRAP_PID" ]; then + kill -TERM "$BOOTSTRAP_PID" 2>/dev/null || true + wait "$BOOTSTRAP_PID" 2>/dev/null || true + fi + rm -rf "$TESTDIR" +} +trap cleanup EXIT + +assert_status_dir_empty() { + leaked="$(ls -A "$STATUS_DIR")" + if [ -n "$leaked" ]; then + echo "FAIL: leaked lifecycle temp entries: $leaked" >&2 + exit 1 + fi +} + +EXECD_STUB="$TESTDIR/execd" +cat > "$EXECD_STUB" <<'STUB' +#!/bin/sh +stub_fail() { + printf 'execd test stub: %s\n' "$1" >&2 + exit 90 +} + +status_file="" +while [ "$#" -gt 0 ]; do + case "$1" in + --lifecycle-startup-status-file) + if [ -z "${2:-}" ]; then + stub_fail "missing value for --lifecycle-startup-status-file" + fi + status_file="$2" + shift 2 + ;; + --lifecycle-startup-status-file=*) + status_file="${1#*=}" + if [ -z "$status_file" ]; then + stub_fail "missing value for --lifecycle-startup-status-file" + fi + shift + ;; + *) + stub_fail "unexpected argument: $1" + ;; + esac +done +[ -n "${EXECD_MARKER:-}" ] || stub_fail "stub requires EXECD_MARKER" +[ -n "${EXECD_READY_MARKER:-}" ] || stub_fail "stub requires EXECD_READY_MARKER" +[ -n "${SEQUENCE_FILE:-}" ] || stub_fail "stub requires SEQUENCE_FILE" +touch "$EXECD_MARKER" +printf 'execd-started\n' >> "$SEQUENCE_FILE" +if [ -n "$status_file" ]; then + sleep 0.05 + touch "$EXECD_READY_MARKER" + printf 'execd-ready\n' >> "$SEQUENCE_FILE" + if [ "${EXECD_DIE_BEFORE_STATUS:-}" = "1" ]; then + exit 17 + fi + if [ "${EXECD_HANG_BEFORE_STATUS:-}" = "1" ]; then + [ -n "${EXECD_HANG_TERMINATED_MARKER:-}" ] \ + || stub_fail "hang case requires EXECD_HANG_TERMINATED_MARKER" + trap 'touch "$EXECD_HANG_TERMINATED_MARKER"; exit 0' TERM INT + if [ "${EXECD_REMOVE_STATUS_FILE:-}" = "1" ]; then + rm -f "$status_file" + fi + while true; do sleep 1; done + fi + lifecycle_config="" + lifecycle_transport="$(printf '%s' "${OPEN_SANDBOX_LIFECYCLE:-}" | tr -d '[:space:]')" + if [ -z "$lifecycle_transport" ]; then + lifecycle_config="${EXECD_LIFECYCLE_CONFIG:-}" + if [ -z "$lifecycle_config" ]; then + if [ -z "${HOME:-}" ]; then + stub_fail "stub requires HOME to resolve the default lifecycle config" + fi + lifecycle_config="$HOME/.execd/lifecycle.toml" + fi + fi + lifecycle_config_missing=0 + if [ -n "${lifecycle_config:-}" ] && [ ! -e "$lifecycle_config" ]; then + lifecycle_config_missing=1 + elif [ -n "${lifecycle_config:-}" ] && [ ! -f "$lifecycle_config" ]; then + exit 1 + fi + prestart_status=0 + if [ "$lifecycle_config_missing" != "1" ] && [ "${EXECD_NO_PRESTART:-}" != "1" ]; then + test -f "$status_file" || stub_fail "stub requires lifecycle status file" + if [ "${EXECD_HANG_AFTER_RUNNING:-}" = "1" ]; then + [ -n "${EXECD_HANG_TERMINATED_MARKER:-}" ] \ + || stub_fail "hang case requires EXECD_HANG_TERMINATED_MARKER" + if [ "${EXECD_IGNORE_TERM:-}" = "1" ]; then + trap 'touch "$EXECD_HANG_TERMINATED_MARKER"' TERM INT + elif [ "${EXECD_REPORT_SUCCESS_ON_TERM:-}" = "1" ]; then + trap 'trap - TERM INT; if [ -f "$status_file" ]; then printf "done 0\n" >> "$status_file"; fi; touch "$EXECD_HANG_TERMINATED_MARKER"; sleep 2; exit 0' TERM INT + else + trap 'touch "$EXECD_HANG_TERMINATED_MARKER"; exit 0' TERM INT + fi + fi + printf 'running %s\n' "${PRESTART_TIMEOUT_SECONDS:-60}" >> "$status_file" + if [ -n "${EXECD_RUNNING_MARKER:-}" ]; then + touch "$EXECD_RUNNING_MARKER" + fi + if [ "${EXECD_HANG_AFTER_RUNNING:-}" = "1" ]; then + while true; do sleep 1; done + fi + if [ -n "${PRESTART_DELAY_SECONDS:-}" ]; then + sleep "$PRESTART_DELAY_SECONDS" + fi + if [ "${PRESTART_BLOCK:-}" = "1" ]; then + trap 'touch "$PRESTART_TERMINATED_MARKER"; exit 0' TERM INT + touch "$PRESTART_MARKER" + printf 'preStart\n' >> "$SEQUENCE_FILE" + while true; do sleep 1; done + fi + touch "$PRESTART_MARKER" + printf 'preStart\n' >> "$SEQUENCE_FILE" + prestart_status="${PRESTART_EXIT_CODE:-0}" + fi + test -f "$status_file" || stub_fail "stub requires lifecycle status file" + if [ -n "${PRESTART_STATUS_RAW+x}" ]; then + printf 'done %s\n' "$PRESTART_STATUS_RAW" >> "$status_file" + else + printf 'done %s\n' "$prestart_status" >> "$status_file" + fi + if [ "${EXECD_STATUS_STAY_ALIVE:-}" = "1" ]; then + trap 'touch "$STATUS_TERMINATED_MARKER"; exit 0' TERM INT + while true; do sleep 1; done + fi + if [ "$prestart_status" -ne 0 ]; then + exit "$prestart_status" + fi +fi +trap 'exit 0' TERM INT +while true; do sleep 1; done +STUB +chmod +x "$EXECD_STUB" + +USER_SCRIPT="$TESTDIR/user.sh" +cat > "$USER_SCRIPT" <<'USER' +#!/bin/sh +set -e +if [ "${EXPECT_PRESTART_MARKER:-1}" = "1" ]; then + test -f "$PRESTART_MARKER" +fi +test -f "$EXECD_READY_MARKER" +test -z "${OPEN_SANDBOX_LIFECYCLE:-}" +test -z "${EXECD_LIFECYCLE_CONFIG:-}" +test -f "$EXECD_MARKER" +touch "$USER_MARKER" +printf 'user\n' >> "$SEQUENCE_FILE" +USER +chmod +x "$USER_SCRIPT" + +PRESTART_MARKER="$TESTDIR/prestart" +EXECD_MARKER="$TESTDIR/execd-started" +EXECD_READY_MARKER="$TESTDIR/execd-ready" +USER_MARKER="$TESTDIR/user-started" +SEQUENCE_FILE="$TESTDIR/sequence" +STATUS_DIR="$TESTDIR/status" +mkdir "$STATUS_DIR" +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" + +test -f "$PRESTART_MARKER" +test -f "$EXECD_MARKER" +test -f "$EXECD_READY_MARKER" +test -f "$USER_MARKER" +test "$(cat "$SEQUENCE_FILE")" = "$(printf 'execd-started\nexecd-ready\npreStart\nuser')" +assert_status_dir_empty +echo "PASS: preStart completed before the user entrypoint" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +OPEN_SANDBOX_LIFECYCLE='{"periodic":[{"name":"sync","schedule":"@hourly","command":["true"]}]}' \ +EXECD="$EXECD_STUB" \ +EXECD_NO_PRESTART=1 \ +EXPECT_PRESTART_MARKER=0 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" + +test ! -f "$PRESTART_MARKER" +test -f "$USER_MARKER" +test "$(cat "$SEQUENCE_FILE")" = "$(printf 'execd-started\nexecd-ready\nuser')" +assert_status_dir_empty +echo "PASS: periodic-only lifecycle starts without a preStart running status" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +# Keep this delay above bootstrap's 10-second initial startup watchdog. +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +PRESTART_TIMEOUT_SECONDS=30 \ +PRESTART_DELAY_SECONDS=11 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" + +test -f "$PRESTART_MARKER" +test -f "$EXECD_MARKER" +test -f "$EXECD_READY_MARKER" +test -f "$USER_MARKER" +test "$(cat "$SEQUENCE_FILE")" = "$(printf 'execd-started\nexecd-ready\npreStart\nuser')" +assert_status_dir_empty +echo "PASS: preStart completion may exceed the initial startup watchdog" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +set +e +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +PRESTART_EXIT_CODE=42 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 42 +test -f "$EXECD_MARKER" +test -f "$EXECD_READY_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: preStart failure stops execd and prevents the user entrypoint from starting" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +PRESTART_TERMINATED_MARKER="$TESTDIR/prestart-terminated" +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +PRESTART_BLOCK=1 \ +PRESTART_TIMEOUT_SECONDS=300 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +PRESTART_TERMINATED_MARKER="$PRESTART_TERMINATED_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" & +BOOTSTRAP_PID=$! + +i=0 +while [ ! -f "$PRESTART_MARKER" ] && [ "$i" -lt 50 ]; do + sleep 0.1 + i=$((i + 1)) +done +test -f "$PRESTART_MARKER" +kill -TERM "$BOOTSTRAP_PID" +wait "$BOOTSTRAP_PID" || true +BOOTSTRAP_PID="" +test -f "$PRESTART_TERMINATED_MARKER" +test -f "$EXECD_MARKER" +test -f "$EXECD_READY_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: termination during preStart is forwarded to the hook" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +EXECD_RUNNING_MARKER="$TESTDIR/execd-running" +EXECD_IGNORED_TERM_MARKER="$TESTDIR/execd-ignored-term" +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +EXECD_HANG_AFTER_RUNNING=1 \ +EXECD_IGNORE_TERM=1 \ +EXECD_HANG_TERMINATED_MARKER="$EXECD_IGNORED_TERM_MARKER" \ +EXECD_RUNNING_MARKER="$EXECD_RUNNING_MARKER" \ +PRESTART_TIMEOUT_SECONDS=300 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" & +BOOTSTRAP_PID=$! + +i=0 +while [ ! -f "$EXECD_RUNNING_MARKER" ] && [ "$i" -lt 50 ]; do + sleep 0.1 + i=$((i + 1)) +done +test -f "$EXECD_RUNNING_MARKER" +kill -TERM "$BOOTSTRAP_PID" +i=0 +# bootstrap gives execd 10 seconds before KILL; allow ample CI scheduling slack. +while kill -0 "$BOOTSTRAP_PID" 2>/dev/null && [ "$i" -lt 300 ]; do + sleep 0.1 + i=$((i + 1)) +done +if kill -0 "$BOOTSTRAP_PID" 2>/dev/null; then + kill -KILL "$BOOTSTRAP_PID" 2>/dev/null || true + wait "$BOOTSTRAP_PID" 2>/dev/null || true + BOOTSTRAP_PID="" + echo "FAIL: bootstrap did not bound execd shutdown after TERM" >&2 + exit 1 +fi +wait "$BOOTSTRAP_PID" || true +BOOTSTRAP_PID="" +test -f "$EXECD_IGNORED_TERM_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: termination during preStart kills an unresponsive execd" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +set +e +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +EXECD_DIE_BEFORE_STATUS=1 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 17 +test -f "$EXECD_MARKER" +test -f "$EXECD_READY_MARKER" +test ! -f "$PRESTART_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: execd exit before lifecycle status fails startup without leaking the status file" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +EXECD_HANG_TERMINATED_MARKER="$TESTDIR/execd-hang-terminated" +set +e +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +EXECD_HANG_BEFORE_STATUS=1 \ +EXECD_HANG_TERMINATED_MARKER="$EXECD_HANG_TERMINATED_MARKER" \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 1 +test -f "$EXECD_HANG_TERMINATED_MARKER" +test ! -f "$PRESTART_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: lifecycle startup watchdog terminates a hung execd" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +EXECD_RUNNING_HANG_TERMINATED_MARKER="$TESTDIR/execd-running-hang-terminated" +set +e +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +EXECD_HANG_AFTER_RUNNING=1 \ +EXECD_HANG_TERMINATED_MARKER="$EXECD_RUNNING_HANG_TERMINATED_MARKER" \ +EXECD_REPORT_SUCCESS_ON_TERM=1 \ +PRESTART_TIMEOUT_SECONDS=1 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 1 +test -f "$EXECD_RUNNING_HANG_TERMINATED_MARKER" +test ! -f "$PRESTART_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: lifecycle hook watchdog timeout cannot be overwritten by a late success" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +EXECD_INVALID_RUNNING_TERMINATED_MARKER="$TESTDIR/execd-invalid-running-terminated" +set +e +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +EXECD_HANG_AFTER_RUNNING=1 \ +EXECD_HANG_TERMINATED_MARKER="$EXECD_INVALID_RUNNING_TERMINATED_MARKER" \ +PRESTART_TIMEOUT_SECONDS=10000000000 \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 1 +test -f "$EXECD_INVALID_RUNNING_TERMINATED_MARKER" +test ! -f "$PRESTART_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: malformed lifecycle running status fails closed" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +EXECD_MISSING_STATUS_TERMINATED_MARKER="$TESTDIR/execd-missing-status-terminated" +set +e +OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ +EXECD="$EXECD_STUB" \ +EXECD_HANG_BEFORE_STATUS=1 \ +EXECD_REMOVE_STATUS_FILE=1 \ +EXECD_HANG_TERMINATED_MARKER="$EXECD_MISSING_STATUS_TERMINATED_MARKER" \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 1 +test -f "$EXECD_MISSING_STATUS_TERMINATED_MARKER" +test ! -f "$PRESTART_MARKER" +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: missing lifecycle status file fails closed and terminates execd" + +STATUS_TERMINATED_MARKER="$TESTDIR/status-terminated" +for invalid_status in garbled 999 999999999999999999999999; do + rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" "$STATUS_TERMINATED_MARKER" + set +e + OPEN_SANDBOX_LIFECYCLE='{"preStart":{"command":["true"]}}' \ + EXECD="$EXECD_STUB" \ + PRESTART_STATUS_RAW="$invalid_status" \ + EXECD_STATUS_STAY_ALIVE=1 \ + STATUS_TERMINATED_MARKER="$STATUS_TERMINATED_MARKER" \ + PRESTART_MARKER="$PRESTART_MARKER" \ + EXECD_MARKER="$EXECD_MARKER" \ + EXECD_READY_MARKER="$EXECD_READY_MARKER" \ + USER_MARKER="$USER_MARKER" \ + SEQUENCE_FILE="$SEQUENCE_FILE" \ + TMPDIR="$STATUS_DIR" \ + BOOTSTRAP_CMD="$USER_SCRIPT" \ + "$BOOTSTRAP" + status=$? + set -e + + test "$status" -eq 1 + test -f "$STATUS_TERMINATED_MARKER" + test ! -f "$USER_MARKER" + assert_status_dir_empty +done +echo "PASS: malformed lifecycle status fails closed and terminates a still-running execd" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +PERSISTED_CONFIG="$HOME/.execd/lifecycle.toml" +mkdir -p "$(dirname "$PERSISTED_CONFIG")" +printf 'version = 1\n[preStart]\ncommand = ["true"]\n' > "$PERSISTED_CONFIG" +EXECD_LIFECYCLE_CONFIG='' \ +EXECD="$EXECD_STUB" \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$USER_SCRIPT" \ +"$BOOTSTRAP" + +test -f "$PRESTART_MARKER" +test -f "$EXECD_MARKER" +test -f "$EXECD_READY_MARKER" +test -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: persisted lifecycle config triggers preStart" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +SANITIZE_USER_SCRIPT="$TESTDIR/sanitize-user.sh" +cat > "$SANITIZE_USER_SCRIPT" <<'USER' +#!/bin/sh +set -e +test -z "${OPEN_SANDBOX_LIFECYCLE:-}" +test -z "${EXECD_LIFECYCLE_CONFIG:-}" +i=0 +while [ ! -f "$EXECD_MARKER" ] && [ "$i" -lt 50 ]; do + sleep 0.1 2>/dev/null || sleep 1 + i=$((i + 1)) +done +test -f "$EXECD_MARKER" +touch "$USER_MARKER" +USER +chmod +x "$SANITIZE_USER_SCRIPT" +test -f "$PERSISTED_CONFIG" +OPEN_SANDBOX_LIFECYCLE='' \ +EXECD_LIFECYCLE_CONFIG="$TESTDIR/missing-lifecycle.toml" \ +EXECD="$EXECD_STUB" \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$SANITIZE_USER_SCRIPT" \ +"$BOOTSTRAP" + +test -f "$USER_MARKER" +test ! -f "$PRESTART_MARKER" +assert_status_dir_empty +echo "PASS: explicit missing lifecycle config does not fall back and internal environment is stripped" + +rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" +CONFIG_DIR_PATH="$TESTDIR/config-as-dir" +mkdir -p "$CONFIG_DIR_PATH" +set +e +OPEN_SANDBOX_LIFECYCLE='' \ +EXECD_LIFECYCLE_CONFIG="$CONFIG_DIR_PATH" \ +EXECD="$EXECD_STUB" \ +PRESTART_MARKER="$PRESTART_MARKER" \ +EXECD_MARKER="$EXECD_MARKER" \ +EXECD_READY_MARKER="$EXECD_READY_MARKER" \ +USER_MARKER="$USER_MARKER" \ +SEQUENCE_FILE="$SEQUENCE_FILE" \ +TMPDIR="$STATUS_DIR" \ +BOOTSTRAP_CMD="$SANITIZE_USER_SCRIPT" \ +"$BOOTSTRAP" +status=$? +set -e + +test "$status" -eq 1 +test ! -f "$USER_MARKER" +assert_status_dir_empty +echo "PASS: invalid lifecycle config path fails before the user entrypoint starts" + +rm -f "$PERSISTED_CONFIG" +for lifecycle_home in "$HOME" ''; do + lifecycle_transport='' + [ -z "$lifecycle_home" ] || lifecycle_transport=' ' + rm -f "$PRESTART_MARKER" "$EXECD_MARKER" "$EXECD_READY_MARKER" "$USER_MARKER" "$SEQUENCE_FILE" + HOME="$lifecycle_home" \ + OPEN_SANDBOX_LIFECYCLE="$lifecycle_transport" \ + EXECD_LIFECYCLE_CONFIG='' \ + EXECD="$EXECD_STUB" \ + PRESTART_MARKER="$PRESTART_MARKER" \ + EXECD_MARKER="$EXECD_MARKER" \ + EXECD_READY_MARKER="$EXECD_READY_MARKER" \ + USER_MARKER="$USER_MARKER" \ + SEQUENCE_FILE="$SEQUENCE_FILE" \ + TMPDIR="$STATUS_DIR" \ + BOOTSTRAP_CMD="$SANITIZE_USER_SCRIPT" \ + "$BOOTSTRAP" + + test -f "$USER_MARKER" + test ! -f "$PRESTART_MARKER" + assert_status_dir_empty +done +echo "PASS: missing default config does not affect sandboxes without lifecycle hooks" diff --git a/docs/components/execd.md b/docs/components/execd.md index 45202bcc7..23089a44c 100644 --- a/docs/components/execd.md +++ b/docs/components/execd.md @@ -216,6 +216,18 @@ override it. | `OPENSANDBOX_ID` | Authoritative sandbox id stamped into eBPF audit records (`sandbox_id`) and metrics; the server injects it on Docker/Kubernetes task-template paths. Kubernetes pool allocations that skip the task template (default entrypoint, no env, no init mode) cannot inject it, and the eBPF layer reports `unsupported` attribution on that path. | | `OPENSANDBOX_EXECD_METRICS_EXTRA_ATTRS` | Optional extra metric attrs (`k=v,k2=v2`). | +### Lifecycle hook trust boundary + +`execd` runs configured `preStart` and `periodic` commands directly as its own +OS user in the sandbox's existing container namespaces. Lifecycle hooks do not +use isolated-session confinement. + +Lifecycle hooks are trusted setup and maintenance code, not a security or +policy boundary. The default persisted config at +`$HOME/.execd/lifecycle.toml` is writable by execd's user, and a root sandbox +workload can modify or remove it. Do not use hooks for tamper-resistant +auditing or mandatory controls against sandbox workloads. + ### Isolation Config File Isolated sessions read an optional TOML file given by `--isolation-config`