Skip to content

fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly #2370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
politerealism wants to merge 4 commits into
base: main
Choose a base branch
from
+463 −15
Open

fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly #2370

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8d0099f
fix(proxy): retry with backoff on transient accept errors instead of …
politerealism Jul 20, 2026
26a00e8
fix(sandbox): terminate sandbox when proxy accept loop exits unexpect…
politerealism Jul 20, 2026
4e915ed
test(proxy): add unit tests for exit detection drop-guard and take_ex…
politerealism Jul 20, 2026
6a1fe36
fix(proxy): classify terminal vs transient accept errors and exit on …
politerealism Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 98 additions & 6 deletions crates/openshell-sandbox/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ mod sidecar_control;

use miette::{IntoDiagnostic, Result, WrapErr};
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::AtomicU32;
use std::time::Duration;
Expand Down Expand Up @@ -361,7 +362,7 @@ pub async fn run_sandbox(
#[cfg(not(target_os = "linux"))]
drop(bypass_activity_tx);

let networking = if network_enabled {
let mut networking = if network_enabled {
#[cfg(target_os = "linux")]
let proxy_bind_ip = netns
.as_ref()
Expand Down Expand Up @@ -628,6 +629,18 @@ pub async fn run_sandbox(
.zip(bootstrap.proxy_ca_bundle_path.clone())
});

let proxy_exited: Pin<Box<dyn Future<Output = ()> + Send>> =
if let Some(rx) = networking
.as_mut()
.and_then(|n| n.proxy.as_mut())
.map(|p| p.take_exit_receiver())
{
Box::pin(async { let _ = rx.await; })
} else {
Box::pin(std::future::pending())
};
tokio::pin!(proxy_exited);

let exit_code = if process_enabled {
let ca_file_paths = networking
.as_ref()
Expand Down Expand Up @@ -707,9 +720,41 @@ pub async fn run_sandbox(
"authoritative network-sidecar control channel closed"
));
}
_ = &mut proxy_exited => {
ocsf_emit!(
AppLifecycleBuilder::new(ocsf_ctx())
.activity(ActivityId::Fail)
.severity(SeverityId::High)
.status(StatusId::Failure)
.message(
"Proxy accept loop exited unexpectedly; terminating sandbox"
)
.build()
);
return Err(miette::miette!(
"proxy accept loop exited unexpectedly"
));
}
}
} else {
process.await?
tokio::select! {
result = process => result?,
_ = &mut proxy_exited => {
ocsf_emit!(
AppLifecycleBuilder::new(ocsf_ctx())
.activity(ActivityId::Fail)
.severity(SeverityId::High)
.status(StatusId::Failure)
.message(
"Proxy accept loop exited unexpectedly; terminating sandbox"
)
.build()
);
return Err(miette::miette!(
"proxy accept loop exited unexpectedly"
));
}
}
}
} else {
// Network-only sidecar mode: keep the proxy and its background
Expand All @@ -725,15 +770,62 @@ pub async fn run_sandbox(
warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar");
1
}
_ = &mut proxy_exited => {
ocsf_emit!(
AppLifecycleBuilder::new(ocsf_ctx())
.activity(ActivityId::Fail)
.severity(SeverityId::High)
.status(StatusId::Failure)
.message(
"Proxy accept loop exited unexpectedly; terminating sandbox"
)
.build()
);
return Err(miette::miette!(
"proxy accept loop exited unexpectedly"
));
}
}
} else {
wait_for_shutdown_signal().await;
0
tokio::select! {
() = wait_for_shutdown_signal() => 0,
_ = &mut proxy_exited => {
ocsf_emit!(
AppLifecycleBuilder::new(ocsf_ctx())
.activity(ActivityId::Fail)
.severity(SeverityId::High)
.status(StatusId::Failure)
.message(
"Proxy accept loop exited unexpectedly; terminating sandbox"
)
.build()
);
return Err(miette::miette!(
"proxy accept loop exited unexpectedly"
));
}
}
}
#[cfg(not(target_os = "linux"))]
{
wait_for_shutdown_signal().await;
0
tokio::select! {
() = wait_for_shutdown_signal() => 0,
_ = &mut proxy_exited => {
ocsf_emit!(
AppLifecycleBuilder::new(ocsf_ctx())
.activity(ActivityId::Fail)
.severity(SeverityId::High)
.status(StatusId::Failure)
.message(
"Proxy accept loop exited unexpectedly; terminating sandbox"
)
.build()
);
return Err(miette::miette!(
"proxy accept loop exited unexpectedly"
));
}
}
}
};

Expand Down
102 changes: 98 additions & 4 deletions crates/openshell-supervisor-network/src/proxy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,11 +174,11 @@ impl InferenceContext {
}
}

#[derive(Debug)]
pub struct ProxyHandle {
#[allow(dead_code)]
http_addr: Option<SocketAddr>,
join: JoinHandle<()>,
exited_rx: tokio::sync::oneshot::Receiver<()>,
}

impl ProxyHandle {
Expand Down Expand Up @@ -240,7 +240,13 @@ impl ProxyHandle {
);
}

let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>();
let join = tokio::spawn(async move {
// Hold the sender for the lifetime of this task — when the task
// exits (panic, abort, or loop break), the sender drops and the
// receiver fires, notifying the sandbox that the proxy is gone.
let _proxy_exit_guard = exited_tx;

// Wait for the OPA engine's symlink resolution reload to complete
// before accepting connections. This prevents requests from
// observing a generation transition mid-flight, which would cause
Expand All @@ -265,9 +271,11 @@ impl ProxyHandle {
}
}

let mut consecutive_fd_errors: u32 = 0;
loop {
match listener.accept().await {
Ok((stream, _addr)) => {
consecutive_fd_errors = 0;
let opa = opa_engine.clone();
let cache = identity_cache.clone();
let spid = entrypoint_pid.clone();
Expand Down Expand Up @@ -314,14 +322,34 @@ impl ProxyHandle {
});
}
Err(err) => {
// EMFILE (24) / ENFILE (23) indicate FD exhaustion —
// back off exponentially to let connections drain.
let is_fd_exhaustion = matches!(
err.raw_os_error(),
Some(24) | Some(23)

@johntmyers johntmyers Jul 21, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gator-agent

Warning: Replace numeric errno values with libc::EMFILE and libc::ENFILE through a platform-aware helper. raw_os_error() values are platform-specific, so non-Linux targets may miss FD exhaustion and use the wrong backoff and severity. Rebasing onto the final #2369 helper should address this.

);
let (severity, backoff) = if is_fd_exhaustion {
consecutive_fd_errors = consecutive_fd_errors.saturating_add(1);
let backoff_ms = 100u64
.saturating_mul(
1u64 << consecutive_fd_errors.min(6).saturating_sub(1),

@johntmyers johntmyers Jul 21, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gator-agent

Warning: consecutive_fd_errors.min(6) limits the exponent to 5, so this tops out at 3,200 ms and the 5,000 ms cap is unreachable. Allow the next exponent before applying the cap and add a unit test covering the complete backoff sequence.

)
.min(5_000);
(SeverityId::Medium, std::time::Duration::from_millis(backoff_ms))
} else {
(SeverityId::Low, std::time::Duration::from_millis(100))
};
let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx())
.activity(ActivityId::Fail)
.severity(SeverityId::Low)
.severity(severity)
.status(StatusId::Failure)
.message(format!("Proxy accept error: {err}"))
.message(format!(
"Proxy accept error (retrying in {}ms): {err}",
backoff.as_millis(),
))
.build();
ocsf_emit!(event);
break;
tokio::time::sleep(backoff).await;
}
}
}
Expand All @@ -330,13 +358,21 @@ impl ProxyHandle {
Ok(Self {
http_addr: Some(local_addr),
join,
exited_rx,
})
}

#[allow(dead_code)]
pub const fn http_addr(&self) -> Option<SocketAddr> {
self.http_addr
}

/// Take the exit notification receiver for health monitoring.
/// Resolves when the proxy accept loop task exits for any reason.
pub fn take_exit_receiver(&mut self) -> tokio::sync::oneshot::Receiver<()> {
let (_tx, dummy_rx) = tokio::sync::oneshot::channel();

@johntmyers johntmyers Jul 21, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gator-agent

Warning: Store exited_rx as an Option and make take_exit_receiver() return Option or Result. The dropped dummy sender makes a second call indistinguishable from a real proxy exit, which can falsely terminate a healthy sandbox; the added test currently codifies that footgun.

std::mem::replace(&mut self.exited_rx, dummy_rx)
}
}

impl Drop for ProxyHandle {
Expand Down Expand Up @@ -9827,4 +9863,62 @@ network_policies:
}
}
}

#[tokio::test]
async fn test_exit_receiver_fires_when_task_exits() {
let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>();

@johntmyers johntmyers Jul 21, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gator-agent

Warning: These tests recreate standalone Tokio channels, so they still pass if the production exit guard or run_sandbox selection is removed. Exercise an actual ProxyHandle task exit and add a focused sandbox-level test proving proxy notification returns the expected error and cancels the workload.

let handle = tokio::spawn(async move {
let _guard = exited_tx;
});
handle.await.unwrap();
// The sender was dropped when the task completed, so the receiver
// should resolve immediately with an Err (sender dropped).
assert!(exited_rx.await.is_err());
}

#[tokio::test]
async fn test_exit_receiver_fires_when_task_is_aborted() {
let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>();
let handle = tokio::spawn(async move {
let _guard = exited_tx;
std::future::pending::<()>().await;
});
handle.abort();
// Abort drops the task's locals, including the sender guard.
assert!(exited_rx.await.is_err());
}

#[tokio::test]
async fn test_take_exit_receiver_returns_real_receiver() {
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
let join = tokio::spawn(std::future::pending::<()>());
let mut handle = ProxyHandle {
http_addr: None,
join,
exited_rx: rx,
};
let mut taken = handle.take_exit_receiver();
// Sender still alive — receiver should not be ready yet.
assert!(taken.try_recv().is_err());
// Drop the original sender — the taken receiver should resolve.
drop(tx);
assert!(taken.await.is_err());
}

#[tokio::test]
async fn test_take_exit_receiver_second_call_returns_instantly() {
let (_tx, rx) = tokio::sync::oneshot::channel::<()>();
let join = tokio::spawn(std::future::pending::<()>());
let mut handle = ProxyHandle {
http_addr: None,
join,
exited_rx: rx,
};
let _first = handle.take_exit_receiver();
let second = handle.take_exit_receiver();
// The dummy's sender was dropped inside take_exit_receiver, so
// the second receiver resolves immediately — this demonstrates
// the double-call hazard.
assert!(second.await.is_err());
}
}
Loading