Fix: enforce single root Remote L3 startup deadline; split attach vs runtime timeout

docs/remote-l3-worker-design.md

@@ -346,8 +346,10 @@ from simpler.worker import RemoteCallable, RemoteWorkerSpec, Worker
 
 w4 = Worker(level=4)
 
+# endpoint host must be a numeric IP (or "localhost"); hostnames are rejected
+# because getaddrinfo resolution is unbounded and could pin startup.
 l3 = RemoteWorkerSpec(
-    endpoint="node17:19073",
+    endpoint="10.0.0.17:19073",
     platform="a2a3",
     runtime="tensormap_and_ringbuffer",
     device_ids=list(range(16)),

python/bindings/worker_bind.h

@@ -402,15 +402,16 @@ inline void bind_worker(nb::module_ &m) {
             "add_remote_l3_socket",
             [](Worker &self, int32_t worker_id, uint64_t session_id, const std::string &transport_name,
                const std::string &host, uint16_t port, const std::string &health_host, uint16_t health_port,
-               double timeout_s) {
+               double attach_timeout_s, double runtime_timeout_s) {
                 nb::gil_scoped_release release;
                 self.add_remote_l3_socket(
-                    worker_id, session_id, transport_name, host, port, health_host, health_port, timeout_s
+                    worker_id, session_id, transport_name, host, port, health_host, health_port, attach_timeout_s,
+                    runtime_timeout_s
                 );
             },
             nb::arg("worker_id"), nb::arg("session_id"), nb::arg("transport_name"), nb::arg("host"), nb::arg("port"),
-            nb::arg("health_host"), nb::arg("health_port"), nb::arg("timeout_s") = 30.0,
-            "Register a REMOTE_L3 endpoint after the session reports HELLO READY."
+            nb::arg("health_host"), nb::arg("health_port"), nb::arg("attach_timeout_s") = 30.0,
+            nb::arg("runtime_timeout_s") = 30.0, "Register a REMOTE_L3 endpoint after the session reports HELLO READY."
         )
 
         // Release the GIL while starting the Scheduler thread so another Python

python/simpler/remote_l3_session.py

@@ -182,6 +182,19 @@ def _startup_remaining_s(manifest: dict[str, Any]) -> float:
     return remaining_s
 
 
+def _startup_deadline(manifest: dict[str, Any]) -> float:
+    # The daemon and this runner share CLOCK_MONOTONIC (same host), so when the
+    # daemon supplies an absolute deadline the runner uses it directly — its
+    # remaining is measured post-spawn, and its deadline cannot exceed the
+    # daemon's. Only a pre-P0.3 daemon omits it; fall back to the duration then.
+    if "startup_deadline_monotonic" in manifest:
+        deadline = float(manifest["startup_deadline_monotonic"])
+        if not math.isfinite(deadline):
+            raise ValueError("manifest startup_deadline_monotonic must be a finite monotonic timestamp")
+        return deadline
+    return time.monotonic() + _startup_remaining_s(manifest)
+
+
 def _health_loop(sock: socket.socket, stop: threading.Event, session_id: int, worker_id: int) -> None:
     conn: socket.socket | None = None
     sock.settimeout(0.2)
@@ -924,23 +937,28 @@ def run_session(manifest: dict[str, Any], ready_fd: int) -> int:
     stop_health = threading.Event()
     health_thread: threading.Thread | None = None
     try:
-        session_timeout_s = _session_timeout_s(manifest)
-        startup_remaining_s = _startup_remaining_s(manifest)
+        # Validate the runtime command timeout wire value up front (rejects a
+        # malformed session_timeout_s); the command lane itself idle-waits blocking.
+        _session_timeout_s(manifest)
+        # Establish this runner's single absolute deadline before any startup work
+        # (registry install, inner init) so every stage draws from it. It is the
+        # daemon's shared-clock deadline when supplied (so spawn/import time is
+        # already charged), else derived from the remaining duration.
+        startup_deadline = _startup_deadline(manifest)
         manifest_dispatch_registry = _install_manifest_dispatcher_registry(manifest)
         # Register the inner L3 callables before init() so they are frozen into
         # the eager startup snapshot and uploaded to the chip children before
         # those children publish INIT_READY. init() is the single, eager startup
         # point: it forks and readies the whole inner L3->L2 tree, so the runner
         # reports ready (below) only once that subtree is up.
         manifest_inner_handles = _install_manifest_inner_registry(manifest, inner_worker)
-        # Bound the inner startup by the parent's remaining startup budget
-        # (rebuilt against this host's own monotonic clock — the parent's
-        # absolute deadline is not comparable across machines), not a fresh full
+        # Bound the inner startup by the runner's single deadline established
+        # above (which already charges registry-install time), not a fresh full
         # session_timeout_s. Mark it non-root so its chip/sub children inherit
         # this runner's process group (see start_new_session in the daemon)
         # rather than splitting into their own — the daemon reaps the whole
         # L3->L2 subtree with one killpg on the runner.
-        inner_worker.init(_startup_deadline=time.monotonic() + startup_remaining_s)
+        inner_worker.init(_startup_deadline=startup_deadline)
 
         listen_host = str(manifest.get("listen_host", "127.0.0.1"))
         command_sock = _bind_listener(listen_host)
@@ -965,8 +983,22 @@ def run_session(manifest: dict[str, Any], ready_fd: int) -> int:
             },
         )
 
-        command_sock.settimeout(session_timeout_s)
+        # Waiting for the parent to attach is still the attach phase, so bound it
+        # by the remaining startup budget — not session_timeout_s. Otherwise a
+        # tiny runtime timeout could drop the parent before it connects, and a
+        # large one could keep the runner alive well past the root deadline if the
+        # parent is lost.
+        attach_remaining = startup_deadline - time.monotonic()
+        if attach_remaining <= 0:
+            raise TimeoutError("remote L3 runner: startup deadline exceeded before parent attach")
+        command_sock.settimeout(attach_remaining)
         conn, _addr = command_sock.accept()
+        # Force the command connection blocking, explicitly: accept() inherits
+        # socket.getdefaulttimeout(), which a user module imported during registry
+        # install could have set. A finite timeout would tear down a healthy but
+        # idle session; the loop instead idle-waits blocking and ends when the
+        # parent closes the command socket (read_frame sees EOF).
+        conn.settimeout(None)
         with conn:
             _run_command_loop(conn, manifest, inner_worker, manifest_inner_handles, manifest_dispatch_registry)
         return 0

python/simpler/remote_l3_worker.py

@@ -87,9 +87,8 @@ def _startup_remaining_s(manifest: dict[str, Any]) -> float:
     return remaining_s
 
 
-def _read_runner_ready(fd: int, timeout_s: float) -> dict[str, Any]:
+def _read_runner_ready(fd: int, deadline: float) -> dict[str, Any]:
     chunks = bytearray()
-    deadline = time.monotonic() + timeout_s
     while True:
         remaining = deadline - time.monotonic()
         if remaining <= 0:
@@ -154,18 +153,30 @@ def _start_session(manifest: dict[str, Any]) -> tuple[dict[str, Any], subprocess
     # manifest tempfile, runner Popen) exists: the runner is never launched only
     # to die on an invalid session_timeout_s or startup_remaining_s.
     _session_timeout_s(manifest)
-    # The runner must publish ready within the parent's remaining startup budget,
-    # not a fresh full command timeout.
-    timeout_s = _startup_remaining_s(manifest)
+    # One local absolute deadline for this session's startup: the runner-ready
+    # wait below and the budget handed to the runner both derive from it, so the
+    # daemon's own setup and the runner's init cannot each restart the full slice.
+    deadline = time.monotonic() + _startup_remaining_s(manifest)
     ready_r, ready_w = os.pipe()
     manifest_path = ""
     proc: subprocess.Popen[Any] | None = None
     try:
+        runner_remaining = deadline - time.monotonic()
+        if runner_remaining <= 0:
+            raise TimeoutError("remote L3 session: startup deadline exceeded before runner launch")
+        runner_manifest = dict(manifest)
+        # Hand the runner this daemon's ABSOLUTE monotonic deadline. Daemon and
+        # runner share CLOCK_MONOTONIC (same host, Popen-spawned), so the runner
+        # derives its remaining post-spawn and its deadline equals this one — the
+        # spawn/import/registry time is charged, not restarted from a frozen
+        # duration. startup_remaining_s stays for a pre-P0.3 runner's fallback.
+        runner_manifest["startup_deadline_monotonic"] = deadline
+        runner_manifest["startup_remaining_s"] = runner_remaining
         with tempfile.NamedTemporaryFile(
             "w", encoding="utf-8", prefix="simpler-remote-l3-", suffix=".json", delete=False
         ) as f:
             manifest_path = f.name
-            json.dump(manifest, f, sort_keys=True)
+            json.dump(runner_manifest, f, sort_keys=True)
         proc = subprocess.Popen(
             [
                 sys.executable,
@@ -185,7 +196,7 @@ def _start_session(manifest: dict[str, Any]) -> tuple[dict[str, Any], subprocess
         os.close(ready_w)
         ready_w = -1
         try:
-            ready = _read_runner_ready(ready_r, timeout_s)
+            ready = _read_runner_ready(ready_r, deadline)
         except BaseException:
             _wait_or_kill_runner(proc)
             raise