diff --git a/docs/v6/guides/running-an-eval.mdx b/docs/v6/guides/running-an-eval.mdx
index 15c309f3c..c080693a7 100644
--- a/docs/v6/guides/running-an-eval.mdx
+++ b/docs/v6/guides/running-an-eval.mdx
@@ -130,7 +130,7 @@ change to `env.py` or the tasks:
| Runtime | Where the env runs |
| --- | --- |
-| `LocalRuntime("env.py")` | A child process on your machine |
+| `LocalRuntime("env.py")` | In this process, on your machine |
| `DockerRuntime("my-env")` | A fresh local container per rollout |
| `ModalRuntime("my-env")` | A fresh [Modal](https://modal.com) sandbox per rollout |
| `DaytonaRuntime("my-env")` | A fresh [Daytona](https://daytona.io) sandbox per rollout |
diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx
index 6a5f51bbd..ca853a8a7 100644
--- a/docs/v6/reference/cli.mdx
+++ b/docs/v6/reference/cli.mdx
@@ -105,7 +105,7 @@ For a platform taskset, pass its name or id directly: `hud eval "My Tasks" claud
| `--config`, `-c` | Agent config `key=value` (repeatable). |
| `--verbose`, `-v` | Show agent logs (step progress, tool calls) for batch runs too. |
| `--very-verbose`, `-vv` | Debug-level logs. |
-| `--runtime` | Placement: `local`, `hud` (HUD runtime tunnel), or `tcp://host:port`. Defaults to `local` for a tasks file; platform tasksets default to remote hosted execution. |
+| `--runtime` | Placement: `local` (a child process per rollout, serving the env source — `SubprocessRuntime`), `hud` (HUD runtime tunnel), or `tcp://host:port`. Defaults to `local` for a tasks file; platform tasksets default to remote hosted execution. |
| `--remote` | Run the whole rollout remotely on the HUD platform. |
| `--yes`, `-y` | Skip confirmation prompt. |
diff --git a/docs/v6/reference/environment.mdx b/docs/v6/reference/environment.mdx
index 6a38f8c93..ad1a60418 100644
--- a/docs/v6/reference/environment.mdx
+++ b/docs/v6/reference/environment.mdx
@@ -59,7 +59,7 @@ and [Tasks & Tasksets](/v6/reference/tasks).
You rarely serve by hand - `hud eval`, [`task.run()`](/v6/reference/tasks), and `Taskset.run()` bring
the environment up for you, and the [runtime](/v6/reference/runtime) you pass decides where. Serving
itself belongs to `hud.environment.server`, the entry point every substrate runs: a
-[`LocalRuntime`](/v6/reference/runtime#localruntime) child process, a container CMD, or `hud serve`.
+[`SubprocessRuntime`](/v6/reference/runtime#subprocessruntime) child process, a container CMD, or `hud serve`.
| Function | Description |
|----------|-------------|
diff --git a/docs/v6/reference/runtime.mdx b/docs/v6/reference/runtime.mdx
index 1db44baa1..102760d83 100644
--- a/docs/v6/reference/runtime.mdx
+++ b/docs/v6/reference/runtime.mdx
@@ -18,7 +18,8 @@ await taskset.run(agent, runtime=LocalRuntime("env.py")) # serve env.py locall
| Runtime | Where the env runs | When to reach for it |
|---------|--------------------|----------------------|
-| `LocalRuntime("env.py")` | A child process from your source | Fastest iteration; local development |
+| `LocalRuntime("env.py")` | This process, loaded fresh from your source per rollout | Fastest iteration; local development |
+| `SubprocessRuntime("env.py")` | A child process from your source | Local runs isolated from your process |
| `DockerRuntime("my-env")` | A fresh local container per rollout | Reproducibility and parity with production |
| `ModalRuntime("my-env")` | A fresh [Modal](https://modal.com/) sandbox per rollout | Cloud scale, no infra to manage |
| `DaytonaRuntime("my-env")` | A fresh [Daytona](https://www.daytona.io/) sandbox per rollout | Cloud scale on Daytona |
@@ -30,10 +31,17 @@ Most runtimes are on the top-level package (`from hud import LocalRuntime, Docke
HostedRuntime, Runtime`); `ModalRuntime` and `DaytonaRuntime` import from `hud.eval`.
-**Omit `runtime=` and it's inferred** from each task's `_source`, the file its template was defined
-in. When every task shares one `_source`, that source is served locally as `LocalRuntime(source)`;
-otherwise (mixed sources, or rows loaded from a file or the platform with no source) it falls back to
-`HUDRuntime()`. Pass a runtime explicitly the moment you want something else.
+**You can usually omit `runtime=`.** A run without one uses what HUD already knows:
+
+- a taskset loaded from Python source (`Taskset.from_file` / `from_module`) runs against that source
+- a platform taskset (`Taskset.from_api`) runs on the platform
+- otherwise, if the envs your tasks name are defined in files you've imported, each rollout gets a
+ fresh env served from its file — so `my_task().run(agent)` just works in the project that defines
+ the env
+
+When none of these apply, `run` raises and lists the runtimes you can pass — it never silently picks
+one. If two imported files define an env with the same name, that's also an error; disambiguate by
+passing the instance you mean: `runtime=LocalRuntime(env)`.
To deploy an environment to the platform and run against it, see
@@ -74,7 +82,30 @@ The constructor for each built-in runtime:
### `LocalRuntime`
```python
-LocalRuntime(path, *, env=None, ready_timeout=120.0)
+LocalRuntime(source, *, env=None, ready_timeout=120.0)
+```
+
+Serves a fresh env per rollout, in this process, over the same control channel as every placement.
+`source` is any pointer to the env:
+
+- **a `.py` file or directory** that declares it, imported fresh per rollout (sibling imports
+ resolve). **`env`** pins one name when the source declares several; it defaults to the placed
+ task's env.
+- **a live `Environment`** declared at module level - its declaring module's file is the recipe;
+ the instance itself is never served, so every rollout is still fresh.
+- **a `(task) -> Environment` constructor** for envs built in code (integrations, parameterized
+ envs), called fresh per rollout with the placed row.
+
+`ready_timeout` bounds `@env.initialize` startup. The freshness boundary is the env's own source:
+it is re-imported per rollout, while modules it imports follow normal Python import caching and are
+shared process-wide - state kept in helper modules persists across rollouts. Env hooks run in this
+process and share its event loop - keep envs async, or use `SubprocessRuntime` / `DockerRuntime`
+when rollouts need whole-process isolation.
+
+### `SubprocessRuntime`
+
+```python
+SubprocessRuntime(path, *, env=None, ready_timeout=120.0)
```
- **`path`** - `.py` file (or directory) that declares the env. The child's working directory is the source's directory, so sibling imports and relative data paths resolve.
diff --git a/docs/v6/start/overview.mdx b/docs/v6/start/overview.mdx
index 35ef95bbd..3b5818278 100644
--- a/docs/v6/start/overview.mdx
+++ b/docs/v6/start/overview.mdx
@@ -157,7 +157,7 @@ hud eval env.py claude --runtime hud # same env, executed on HUD's hosted inf
```python
from hud.eval import LocalRuntime, DockerRuntime, ModalRuntime, HUDRuntime
-LocalRuntime("env.py") # local child process - fastest iteration
+LocalRuntime("env.py") # in this process - fastest iteration
DockerRuntime("my-env") # a fresh container per rollout
ModalRuntime("my-env") # a Modal cloud sandbox per rollout
HUDRuntime() # HUD's hosted infra (after `hud deploy`)
diff --git a/hud/__init__.py b/hud/__init__.py
index 268724cae..76f4b5e54 100644
--- a/hud/__init__.py
+++ b/hud/__init__.py
@@ -24,6 +24,7 @@
RuntimeGPU,
RuntimeLimits,
RuntimeResources,
+ SubprocessRuntime,
SyncPlan,
Task,
Taskset,
@@ -49,6 +50,7 @@
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
+ "SubprocessRuntime",
"SyncPlan",
"Task",
"Taskset",
diff --git a/hud/cli/eval.py b/hud/cli/eval.py
index 39afd6edf..0a071b53b 100644
--- a/hud/cli/eval.py
+++ b/hud/cli/eval.py
@@ -708,7 +708,7 @@ def _python_defines_environment(path: Path) -> bool:
def _spawn_target(source: Path) -> Path:
- """The path the ``LocalRuntime`` provider serves.
+ """The path the ``SubprocessRuntime`` provider serves.
Directories and env-defining ``.py`` files are served as-is. Task-only
sources (``tasks.py`` importing from ``env.py``) resolve to a sibling
@@ -736,7 +736,7 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
``--remote`` submits every rollout for platform-hosted execution; a
``tcp://`` url attaches to an env served elsewhere.
"""
- from hud.eval import HostedRuntime, HUDRuntime, LocalRuntime, Runtime
+ from hud.eval import HostedRuntime, HUDRuntime, Runtime, SubprocessRuntime
if cfg.remote:
require_api_key("run remote hosted evals")
@@ -744,7 +744,7 @@ def _resolve_placement(cfg: EvalConfig, source_path: Path | None) -> Any:
if cfg.runtime == "local":
if source_path is None:
raise ValueError("local placement requires a local source path")
- return LocalRuntime(_spawn_target(source_path))
+ return SubprocessRuntime(_spawn_target(source_path))
if cfg.runtime == "hud":
require_api_key("run HUD runtime tunnel evals")
return HUDRuntime()
diff --git a/hud/cli/task.py b/hud/cli/task.py
index 59a0637e4..3e15470b2 100644
--- a/hud/cli/task.py
+++ b/hud/cli/task.py
@@ -94,7 +94,7 @@ def _resolve(
"""
from contextlib import nullcontext
- from hud.eval.runtime import LocalRuntime, Runtime
+ from hud.eval.runtime import Runtime, SubprocessRuntime
attach = url
if attach is None and source is None:
@@ -118,7 +118,7 @@ def _resolve(
hud_console.error(f"No task matching {task!r} (available: {available})")
raise typer.Exit(1)
selected = matches[0]
- placement = LocalRuntime(_spawn_target(source or "."))(selected)
+ placement = SubprocessRuntime(_spawn_target(source or "."))(selected)
return selected.id, args or selected.args, placement
diff --git a/hud/environment/env.py b/hud/environment/env.py
index 07e1afd80..b3072e78c 100644
--- a/hud/environment/env.py
+++ b/hud/environment/env.py
@@ -120,13 +120,7 @@ def __call__(self, *args: P.args, **kwargs: P.kwargs) -> EvalTask:
from hud.eval.task import Task
bound = self.sig.bind(*args, **kwargs)
- task = Task(env=self.env.name, id=self.id, args=dict(bound.arguments))
- # Record where this template was defined so ``task.run()`` can default to
- # serving that source locally (in-process only; never crosses the wire).
- source = inspect.getsourcefile(self.func)
- if source is not None:
- task._source = source
- return task
+ return Task(env=self.env.name, id=self.id, args=dict(bound.arguments))
class Environment(LegacyEnvMixin):
diff --git a/hud/environment/server.py b/hud/environment/server.py
index e18f64a43..f4e264c22 100644
--- a/hud/environment/server.py
+++ b/hud/environment/server.py
@@ -6,7 +6,7 @@
(:func:`bind`), and the full serving lifecycle (:func:`serve`) — backing
daemons up, control channel bound (announcing the port on stdout as
``HUD_SERVE_PORT=``), daemons down. Every substrate shape runs it: the
-:class:`~hud.eval.runtime.LocalRuntime` child process, a container CMD, and
+:class:`~hud.eval.runtime.SubprocessRuntime` child process, a container CMD, and
``hud serve``.
"""
diff --git a/hud/eval/__init__.py b/hud/eval/__init__.py
index 83f7970ab..2824645c7 100644
--- a/hud/eval/__init__.py
+++ b/hud/eval/__init__.py
@@ -14,9 +14,9 @@
row.)
Placement is passed at execution time (see :mod:`.runtime`): ``LocalRuntime`` a
-local source, ``DockerRuntime`` an image, ``Runtime(url)`` an env served
-elsewhere, ``HUDRuntime`` a HUD runtime tunnel, or ``HostedRuntime`` to run the
-whole rollout remotely on the platform::
+local source served in-process, ``DockerRuntime`` an image,
+``Runtime(url)`` an env served elsewhere, ``HUDRuntime`` a HUD runtime tunnel,
+or ``HostedRuntime`` to run the whole rollout remotely on the platform::
from hud.eval import LocalRuntime, Taskset
@@ -46,6 +46,7 @@
RuntimeGPU,
RuntimeLimits,
RuntimeResources,
+ SubprocessRuntime,
)
from .sync import SyncPlan
from .task import Task
@@ -68,6 +69,7 @@
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
+ "SubprocessRuntime",
"SyncPlan",
"Task",
"Taskset",
diff --git a/hud/eval/chat.py b/hud/eval/chat.py
index 836f41713..17fee982d 100644
--- a/hud/eval/chat.py
+++ b/hud/eval/chat.py
@@ -132,8 +132,9 @@ async def send(self, message: MessageContent) -> Trace:
)
if self._runtime is None:
raise RuntimeError(
- "Chat needs a runtime to converse against — pass an env placement, "
- 'e.g. runtime=Runtime("tcp://...") or runtime=LocalRuntime("env.py").'
+ "Chat needs a runtime to converse against — pass an env placement: "
+ 'LocalRuntime("env.py") (a source file), LocalRuntime(env) (a live env), '
+ "Runtime(url) (a served substrate), or HUDRuntime() (your deployed env)."
)
if self.job is None: # one job spans the whole conversation
self.job = await Job.start(self._task.id)
diff --git a/hud/eval/runtime.py b/hud/eval/runtime.py
index eea8bba34..a05e16ab1 100644
--- a/hud/eval/runtime.py
+++ b/hud/eval/runtime.py
@@ -6,8 +6,12 @@
transparent, so "co-located" (loopback) and "split" (agent here, env
elsewhere) are the same code, differing only in the url.
-- :class:`LocalRuntime` — runs a subprocess serving the row's env from a ``.py``
- source (the path is always given, never recovered from a live object).
+- :class:`LocalRuntime` — serve a fresh env per rollout, in this process,
+ from any pointer to it: a ``.py`` source path, a live module-level
+ :class:`Environment` (its declaring file is the recipe), or a
+ ``(task) -> Environment`` constructor.
+- :class:`SubprocessRuntime` — serve the row's env from a ``.py`` source in a
+ child process, when the env should not share the orchestrator's fate.
- :class:`DockerRuntime` — ``docker run``s an image whose CMD serves the channel.
- ``Runtime(url)`` — the ``nullcontext`` of providers: yields itself, a
*borrowed, shared* substrate provisioned elsewhere (env served anywhere —
@@ -49,7 +53,7 @@
from .run import Grade, Run, rollout
if TYPE_CHECKING:
- from collections.abc import AsyncIterator, Mapping, Sequence
+ from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence
from hud.agents.base import Agent
from hud.environment.env import Environment
@@ -154,7 +158,204 @@ def _modal_image_from_uri(modal: Any, image_uri: str) -> Any:
class LocalRuntime:
- """The local provider: serve the placed row's env from *path* in a child process.
+ """The local provider: serve a fresh env per rollout, in this process.
+
+ *source* points at the env in whatever form you have:
+
+ - a ``.py`` file or directory — imported fresh per acquisition (sibling
+ imports resolve); *env* pins one name when several are declared,
+ defaulting to the placed task's env
+ - a live :class:`~hud.environment.Environment` — shorthand for its
+ declaring file; the instance itself is never served
+ - a ``(task) -> Environment`` callable — called per acquisition with the
+ placed row
+
+ ::
+
+ runtime = LocalRuntime("env.py")
+ runtime = LocalRuntime(env)
+ runtime = LocalRuntime(lambda task: build_env(task.env))
+
+ ``ready_timeout`` bounds ``@env.initialize`` startup. Freshness covers
+ the env's own source; modules it imports are cached as usual and shared
+ across rollouts. Hooks share this process's event loop, so blocking env
+ code stalls concurrent rollouts — use :class:`SubprocessRuntime` or
+ :class:`DockerRuntime` for process isolation, and ``Runtime(url)`` to
+ attach to a substrate served elsewhere.
+ """
+
+ def __init__(
+ self,
+ source: str | Path | Environment | Callable[[Task], Environment],
+ *,
+ env: str | None = None,
+ ready_timeout: float = 120.0,
+ ) -> None:
+ from hud.environment.env import Environment as _Environment
+
+ self.ready_timeout = ready_timeout
+ # A live instance may have been mutated since its module was imported;
+ # verify the fresh copy still declares its templates, so drift fails
+ # at acquisition with the cause named instead of "unknown task" later.
+ expected_templates: frozenset[str] = frozenset()
+ if isinstance(source, _Environment):
+ file = _declaring_file(source, env or source.name)
+ if file is None:
+ raise TypeError(
+ f"LocalRuntime: env {source.name!r} is not rebuilt by importing "
+ "any file this process has loaded (constructed in a function or "
+ "notebook cell, or declared inside a package using relative "
+ "imports); pass its constructor instead: "
+ "LocalRuntime(lambda task: )"
+ )
+ expected_templates = frozenset(source.tasks)
+ source, env = file, env or source.name
+ self._source_dir: Path | None = None
+ if isinstance(source, (str, Path)):
+ path, pinned = Path(source).resolve(), env
+ self._source_dir = path if path.is_dir() else path.parent
+ from hud.environment import load_environment
+
+ def _load(task: Task) -> _Environment:
+ loaded = load_environment(path, name=pinned or task.env)
+ missing = expected_templates - loaded.tasks.keys()
+ if missing:
+ raise ValueError(
+ f"env {loaded.name!r} loaded from {path} lacks template(s) "
+ f"{sorted(missing)} present on the live instance — it was "
+ "modified after import; pass a constructor instead: "
+ "LocalRuntime(lambda task: )"
+ )
+ return loaded
+
+ self._build: Callable[[Task], _Environment] = _load
+ elif callable(source):
+ if env is not None:
+ raise TypeError("LocalRuntime: env= applies only to source paths")
+ self._build = source
+ else:
+ raise TypeError(
+ f"LocalRuntime: expected a source path, a live Environment, or a "
+ f"(task) -> Environment constructor; got {source!r}"
+ )
+
+ @asynccontextmanager
+ async def __call__(self, task: Task) -> AsyncIterator[Runtime]:
+ from hud.environment.env import Environment as _Environment
+
+ if task.runtime_config is not None:
+ raise ValueError("LocalRuntime does not support task runtime_config")
+ # The source dir stays importable for the whole acquisition, not just
+ # the initial import, so a template can lazily import a sibling
+ # module at run time (as it could under the child-process runtime).
+ # Always insert-and-remove one entry: balanced under concurrency.
+ if self._source_dir is not None:
+ sys.path.insert(0, str(self._source_dir))
+ try:
+ try:
+ env = self._build(task)
+ except RuntimeError as e:
+ # The source ran an event loop at import — usually an unguarded
+ # top-level run call; name the actual mistake.
+ if "running event loop" not in str(e):
+ raise
+ raise RuntimeError(
+ "the env source ran async code while being imported to place a "
+ 'rollout — guard top-level run calls with `if __name__ == "__main__":`'
+ ) from e
+ if not isinstance(env, _Environment):
+ raise TypeError(f"LocalRuntime: constructor returned {env!r}, not an Environment")
+ async with _local(env, ready_timeout=self.ready_timeout) as runtime:
+ yield runtime
+ finally:
+ if self._source_dir is not None:
+ with contextlib.suppress(ValueError):
+ sys.path.remove(str(self._source_dir))
+
+
+def _live_envs() -> Iterator[tuple[Environment, str]]:
+ """Envs declared in loaded, file-backed modules' globals, with their files.
+
+ The in-memory counterpart of scanning ``.py`` sources on disk
+ (:func:`~hud.environment.load_environment`): an env found here can be
+ served fresh by re-importing its file. Envs in modules without a file
+ (a notebook ``__main__``) are not yielded — re-import could not
+ reconstruct them.
+ """
+ from hud.environment.env import Environment as _Environment
+
+ for module in list(sys.modules.values()):
+ module_file = getattr(module, "__file__", None)
+ module_vars = getattr(module, "__dict__", None)
+ if not module_file or not isinstance(module_vars, dict):
+ continue
+ for value in list(module_vars.values()):
+ if isinstance(value, _Environment):
+ yield value, module_file
+
+
+def _declaring_file(env: Environment, name: str) -> Path | None:
+ """A file whose fresh import re-declares *env*, else None.
+
+ Candidate files hold the instance in their module globals, but a holder
+ may be a re-exporter (``from .env import env`` in a package
+ ``__init__``, a tasks file re-exporting its env): validate each by
+ loading it fresh — a declarer yields a *new* instance under *name*, a
+ re-exporter yields the same live one (or fails to import standalone).
+ ``__init__.py`` holders are tried last.
+ """
+ from hud.environment import load_environment
+
+ candidates = dict.fromkeys(Path(file) for live, file in _live_envs() if live is env)
+ for file in sorted(candidates, key=lambda f: f.name == "__init__.py"):
+ try:
+ probe = load_environment(file, name=name)
+ except Exception as e:
+ logger.debug("candidate %s does not rebuild env %r: %s", file, name, e)
+ continue
+ if probe is not env:
+ return file
+ return None
+
+
+def _declared_env(name: str) -> Environment | None:
+ """The one live env named *name*, else None; two distinct ones raise.
+
+ The same instance re-exported across modules is one match; distinct envs
+ claiming one name are ambiguous.
+ """
+ matches = {id(env): env for env, _ in _live_envs() if env.name == name}
+ if len(matches) > 1:
+ files = sorted({file for env, file in _live_envs() if env.name == name})
+ raise ValueError(
+ f"env name {name!r} is declared by multiple live environments "
+ f"({', '.join(files)}); pass runtime= explicitly — the exact "
+ "instance disambiguates: runtime=LocalRuntime(env)"
+ )
+ return next(iter(matches.values()), None)
+
+
+def _declared_names(source: Path) -> set[str]:
+ """Env names a ``.py`` source (file or directory) itself declares.
+
+ A fresh execution of the source yields *new* instances for envs it
+ declares; an env it merely imports is the already-live one and does not
+ count — importing the source again could not rebuild it.
+ """
+ from hud.environment.env import Environment as _Environment
+ from hud.utils.modules import iter_modules
+
+ live = {id(env) for env, _ in _live_envs()}
+ return {
+ value.name
+ for module in iter_modules(source)
+ for value in vars(module).values()
+ if isinstance(value, _Environment) and id(value) not in live
+ }
+
+
+class SubprocessRuntime:
+ """The child-process provider: serve the placed row's env from *path*.
Each acquisition runs ``python -m hud.environment.server --env
name`` — the same serving entry point a container CMD runs — on an
@@ -167,8 +368,8 @@ class LocalRuntime:
The child's working directory is the source's directory, so sibling
imports and relative data paths resolve; ``@env.initialize`` daemons start
in the child and die with it. Because the source is re-imported in the
- child, a script spawning itself (``LocalRuntime(__file__)``) must keep top-level
- run calls under ``if __name__ == "__main__":``.
+ child, a script spawning itself (``SubprocessRuntime(__file__)``) must keep
+ top-level run calls under ``if __name__ == "__main__":``.
"""
def __init__(
@@ -185,9 +386,9 @@ def __init__(
@asynccontextmanager
async def __call__(self, task: Task) -> AsyncIterator[Runtime]:
if task.runtime_config is not None:
- raise ValueError("LocalRuntime does not support task runtime_config")
+ raise ValueError("SubprocessRuntime does not support task runtime_config")
if not self.source.exists():
- raise FileNotFoundError(f"LocalRuntime: source not found: {self.source}")
+ raise FileNotFoundError(f"SubprocessRuntime: source not found: {self.source}")
cmd = [sys.executable, "-m", "hud.environment.server", str(self.source)]
cmd += ["--env", self.env or task.env]
proc = await create_process_group_exec(
@@ -603,30 +804,34 @@ async def _docker(*args: str, check: bool = True) -> tuple[str, str]:
@asynccontextmanager
-async def _local(env: Environment) -> AsyncIterator[Runtime]:
+async def _local(env: Environment, *, ready_timeout: float | None = None) -> AsyncIterator[Runtime]:
"""Substrate-side serving: a live env owned by *this* process, as a runtime.
- Not a placement the engine offers (the orchestrator never serves an env
- in-process), so deliberately not a ``Provider`` — it serves a live object,
- not a placed row. Code already running *inside* a placed substrate adapts
- it (``AgentTool`` sub-rollouts: ``runtime=lambda _: _local(env)``); test
- harnesses enter it directly.
+ One env lifecycle (start → serve → stop) around one bound control
+ channel; ``ready_timeout`` bounds ``env.start()`` (initialize
+ hooks/daemons). ``LocalRuntime`` enters this per acquisition with the
+ fresh env it built; test harnesses enter it directly with a live one.
"""
from hud.environment.server import bind
- await env.start()
- server = await bind(env, "127.0.0.1", 0)
- host, port = server.sockets[0].getsockname()[:2]
- serve_task = asyncio.create_task(server.serve_forever())
+ # start() inside the try: a failed or timed-out initialize hook still gets
+ # its already-started daemons torn down by stop() (best-effort per hook).
try:
- yield Runtime(f"tcp://{host}:{port}")
+ started = env.start()
+ await (asyncio.wait_for(started, ready_timeout) if ready_timeout is not None else started)
+ server = await bind(env, "127.0.0.1", 0)
+ host, port = server.sockets[0].getsockname()[:2]
+ serve_task = asyncio.create_task(server.serve_forever())
+ try:
+ yield Runtime(f"tcp://{host}:{port}")
+ finally:
+ serve_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await serve_task
+ server.close()
+ with contextlib.suppress(Exception):
+ await server.wait_closed()
finally:
- serve_task.cancel()
- with contextlib.suppress(asyncio.CancelledError):
- await serve_task
- server.close()
- with contextlib.suppress(Exception):
- await server.wait_closed()
await env.stop()
@@ -1011,4 +1216,5 @@ async def ws_to_tcp() -> None:
"RuntimeGPU",
"RuntimeLimits",
"RuntimeResources",
+ "SubprocessRuntime",
]
diff --git a/hud/eval/task.py b/hud/eval/task.py
index ab3363ae2..2f6eb2814 100644
--- a/hud/eval/task.py
+++ b/hud/eval/task.py
@@ -24,7 +24,7 @@
import json
from typing import TYPE_CHECKING, Any
-from pydantic import BaseModel, Field, PrivateAttr
+from pydantic import BaseModel, Field
from .runtime import RuntimeConfig
@@ -40,8 +40,8 @@ class Task(BaseModel):
Pure data — holds no execution state, so one ``Task`` can drive many
concurrent rollouts. ``run`` it for a graded :class:`~hud.eval.job.Job`;
- placement comes from ``runtime=`` (a provider), else the source the task was
- minted from (local), else the HUD runtime tunnel by ``env`` name.
+ placement comes from ``runtime=`` (a provider), else the HUD runtime
+ tunnel by ``env`` name.
"""
env: str = Field(min_length=1)
@@ -57,12 +57,6 @@ class Task(BaseModel):
#: supported subset into their native launch shape or reject it.
runtime_config: RuntimeConfig | None = None
- #: In-process only: the source file the template was defined in, captured
- #: when a template factory mints the task. Lets ``run`` default to serving
- #: that source locally. Excluded from the wire (a row loaded from JSON has
- #: none, and falls back to HUD runtime tunnel placement).
- _source: str | None = PrivateAttr(default=None)
-
def default_slug(self) -> str:
"""A stable slug from the task id, disambiguated by an args hash when present."""
if not self.args:
@@ -90,8 +84,8 @@ async def run(
open ``job`` from :meth:`Job.start` to accumulate into), ``group``
repeats sharing a group_id, ``max_concurrent`` capping parallelism —
over a taskset of one. ``runtime`` is the placement; left unset it
- serves the task's source locally when minted in-process, else falls
- back to the HUD runtime tunnel by ``env`` name.
+ falls back to the HUD runtime tunnel by ``env`` name. For a local
+ run, pass one explicitly (``runtime=LocalRuntime("env.py")``).
"""
from .taskset import Taskset # circular: taskset -> sync -> task
diff --git a/hud/eval/taskset.py b/hud/eval/taskset.py
index a7f7e9cea..2ef0d223a 100644
--- a/hud/eval/taskset.py
+++ b/hud/eval/taskset.py
@@ -23,7 +23,7 @@
from .job import Job, job_enter
from .run import rollout
-from .runtime import HostedRuntime, HUDRuntime, LocalRuntime
+from .runtime import HostedRuntime, HUDRuntime, LocalRuntime, _declared_env, _declared_names
from .sync import fetch_taskset_tasks, resolve_taskset_id
if TYPE_CHECKING:
@@ -208,6 +208,37 @@ def environment_names(self) -> set[str]:
"""Return env names referenced by tasks in this taskset."""
return {task.env for task in self}
+ def _resolve_placement(self) -> Provider | HUDRuntime:
+ if self.origin and self.origin.startswith("module:"):
+ # The origin claims the rows only if it actually declares their
+ # envs (a tasks-only module importing its envs from elsewhere
+ # does not) — and it serves as the exact path, so a same-named
+ # variant in a sibling file is never dragged in.
+ source = Path(self.origin[len("module:") :])
+ if self.environment_names() <= _declared_names(source):
+ return LocalRuntime(source)
+ if self.origin and self.origin.startswith("api:"):
+ return HUDRuntime()
+ declared = {name: _declared_env(name) for name in self.environment_names()}
+ if declared and all(declared.values()):
+ providers = {
+ name: LocalRuntime(env) for name, env in declared.items() if env is not None
+ }
+ logger.info(
+ "no runtime given: serving %s fresh from their declaring modules",
+ ", ".join(sorted(providers)),
+ )
+ return lambda task: providers[task.env](task)
+ missing = sorted(name for name, env in declared.items() if env is None)
+ raise ValueError(
+ f"no placement for env(s) {', '.join(missing) or ''}: pass runtime= — "
+ 'LocalRuntime("env.py") (a source file), LocalRuntime(env) (a live env), '
+ "LocalRuntime(build) (a (task) -> Environment constructor), Runtime(url) "
+ "(a served substrate), or HUDRuntime() (your deployed env). A row taken "
+ "from a loaded taskset keeps its placement when run through it: "
+ 'taskset.filter(["slug"]).run(...)'
+ )
+
async def run(
self,
agent: Agent,
@@ -224,7 +255,12 @@ async def run(
placement: a :class:`~hud.eval.runtime.Provider` (the env served
somewhere, the agent loop driven here by :func:`~hud.eval.run.rollout`),
or :class:`~hud.eval.runtime.HostedRuntime` to run each rollout remotely
- on the platform (left unset: HUD tunnel by env name). One provider serves a mixed-env
+ on the platform. Left unset, what is already known decides: a
+ taskset loaded from local ``.py`` source serves that source's
+ directory; a platform taskset runs on the platform; rows naming envs
+ declared in imported modules serve each fresh from its file; anything
+ else raises, naming the forms to pass. One provider serves a
+ mixed-env
taskset and can size each substrate per row. Registers one HUD job as
the platform receipt and reports each run's trace under it — or, given
an open ``job`` (:meth:`Job.start`), accumulates this batch into it
@@ -264,17 +300,17 @@ async def run(
# Placement is chosen once for the batch: HostedRuntime delegates the
# whole rollout to the platform, anything else is a Provider driven
- # locally by rollout().
- # No runtime: serve the tasks' shared source locally if they were minted
- # in-process from one file (the common authoring case); otherwise (mixed
- # or wire-loaded rows with no source) default to the HUD runtime tunnel.
- if runtime is None:
- sources = {t._source for t in task_list if t._source is not None}
- runtime = LocalRuntime(next(iter(sources))) if len(sources) == 1 else None
- placement = runtime if runtime is not None else HUDRuntime()
+ # locally by rollout(). No runtime: what the taskset or this process
+ # already knows decides (rows never carry placement) — a loaded
+ # taskset runs where it came from; rows naming envs declared in
+ # imported modules serve each fresh from its file; anything else is
+ # an error naming the forms to pass.
+ # An empty taskset schedules nothing, so it needs no placement.
+ placement = runtime if runtime is not None or not task_list else self._resolve_placement()
sem = asyncio.Semaphore(max_concurrent) if max_concurrent else None
async def _run(task: Task, group_id: str) -> Run:
+ assert placement is not None # only reached when tasks were expanded
if isinstance(placement, HostedRuntime):
return await placement.run(task, agent, job_id=job_id, group_id=group_id)
return await rollout(
diff --git a/hud/eval/tests/test_chat.py b/hud/eval/tests/test_chat.py
index 68b6bdaf0..47220ccfb 100644
--- a/hud/eval/tests/test_chat.py
+++ b/hud/eval/tests/test_chat.py
@@ -1,6 +1,6 @@
"""``Chat`` — multi-turn conversation runner over a task.
-Turn tests place each turn's rollout with ``runtime=LocalRuntime(env_file)`` — a pure-data
+Turn tests place each turn's rollout with ``runtime=SubprocessRuntime(env_file)`` — a pure-data
``Task`` row against a chat-style env served from a child process.
"""
@@ -13,7 +13,7 @@
from mcp.types import TextContent
from hud.agents.base import Agent
-from hud.eval import LocalRuntime, Task
+from hud.eval import SubprocessRuntime, Task
from hud.eval.chat import Chat, _content_to_blocks
if TYPE_CHECKING:
@@ -89,7 +89,7 @@ class TestSend:
async def test_send_runs_a_turn_and_stores_prompt_message_format(
self, chat_env_file: Path
) -> None:
- chat = Chat(_chat_task(), _EchoAgent(), runtime=LocalRuntime(chat_env_file))
+ chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file))
trace = await chat.send("hello")
@@ -107,7 +107,7 @@ async def test_send_runs_a_turn_and_stores_prompt_message_format(
assert assistant_msg["content"]["text"] == "echo:hello"
async def test_one_job_spans_the_conversation(self, chat_env_file: Path) -> None:
- chat = Chat(_chat_task(), _EchoAgent(), runtime=LocalRuntime(chat_env_file))
+ chat = Chat(_chat_task(), _EchoAgent(), runtime=SubprocessRuntime(chat_env_file))
await chat.send("hello")
await chat.send("again")
@@ -125,7 +125,7 @@ class _Boom(Agent):
async def __call__(self, run: Any) -> None:
raise RuntimeError("agent exploded")
- chat = Chat(_chat_task(), _Boom(), runtime=LocalRuntime(chat_env_file))
+ chat = Chat(_chat_task(), _Boom(), runtime=SubprocessRuntime(chat_env_file))
with pytest.raises(RuntimeError, match="agent exploded"):
await chat.send("hello")
diff --git a/hud/eval/tests/test_local_runtime.py b/hud/eval/tests/test_local_runtime.py
new file mode 100644
index 000000000..fd8ec4240
--- /dev/null
+++ b/hud/eval/tests/test_local_runtime.py
@@ -0,0 +1,398 @@
+"""Local placement: LocalRuntime and the no-runtime resolution ladder.
+
+LocalRuntime serves a fresh env per rollout from any pointer to it — a source
+path (throwaway import), a live module-level env (its declaring file is the
+recipe), or a ``(task) -> Environment`` constructor. With no runtime, a run
+uses what is already known: taskset origin, then envs declared in imported
+modules, else a loud error. Everything crosses the real control channel —
+these tests drive the rollout engine end to end.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from collections.abc import AsyncGenerator # noqa: TC003 - env.template resolves at runtime
+from typing import Any, cast
+
+import pytest
+
+from hud.agents.base import Agent
+from hud.environment import Environment
+from hud.eval import LocalRuntime, Task, Taskset
+from hud.eval.run import rollout
+
+_SUMS_ENV = """\
+from hud import Environment
+
+env = Environment("{name}")
+
+
+@env.template(id="add")
+async def add(a: int, b: int):
+ answer = yield f"add:{{a}}:{{b}}"
+ yield 1.0 if answer == str(a + b) else 0.0
+"""
+
+
+@pytest.fixture
+def imported_env(tmp_path, request):
+ """Write an env module, import it for real, and clean it up after.
+
+ The module stays in ``sys.modules`` for the test's duration — the state a
+ user's ``from env import add`` leaves behind.
+ """
+ module_name = f"_sums_mod_{request.node.name}"
+ file = tmp_path / f"{module_name}.py"
+ file.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
+ spec = importlib.util.spec_from_file_location(module_name, file)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = module
+ spec.loader.exec_module(module)
+ yield module
+ del sys.modules[module_name]
+
+
+def _sums_env(name: str = "sums") -> Environment:
+ env = Environment(name)
+
+ @env.template(id="add")
+ async def add(a: int, b: int) -> AsyncGenerator[Any, Any]:
+ answer = yield f"add:{a}:{b}"
+ yield 1.0 if answer == str(a + b) else 0.0
+
+ return env
+
+
+class _FnAgent(Agent):
+ """Stateless agent: answers each run by applying ``fn`` to ``run.prompt``."""
+
+ def __init__(self, fn: Any) -> None:
+ self._fn = fn
+
+ async def __call__(self, run: Any) -> None:
+ run.trace.content = self._fn(run.prompt)
+
+
+def _solve_add(prompt: str) -> str:
+ _, a, b = prompt.split(":")
+ return str(int(a) + int(b))
+
+
+# ─── LocalRuntime: the three pointer forms ─────────────────────────────
+
+
+async def test_source_path_serves_a_fresh_env_per_rollout(tmp_path) -> None:
+ # LOADS is module state: a fresh throwaway import per acquisition means
+ # every rollout sees exactly one load.
+ (tmp_path / "env.py").write_text(
+ "from hud import Environment\n\n"
+ "LOADS = []\n"
+ 'env = Environment("sums")\n\n\n'
+ '@env.template(id="add")\nasync def add(a: int, b: int):\n'
+ " LOADS.append(1)\n"
+ ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n'
+ " yield 1.0 if answer == str(a + b) else 0.0\n",
+ encoding="utf-8",
+ )
+
+ def _solve(prompt: str) -> str:
+ _, a, b, loads = prompt.split(":")
+ assert loads == "1"
+ return str(int(a) + int(b))
+
+ job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run(
+ _FnAgent(_solve),
+ runtime=LocalRuntime(tmp_path / "env.py"),
+ group=2,
+ )
+
+ assert [run.reward for run in job.runs] == [1.0, 1.0]
+
+
+async def test_live_env_pointer_resolves_to_its_declaring_file(imported_env) -> None:
+ run = await rollout(
+ Task(env="sums", id="add", args={"a": 2, "b": 3}),
+ _FnAgent(_solve_add),
+ runtime=LocalRuntime(imported_env.env),
+ )
+
+ assert run.reward == 1.0
+
+
+def test_live_env_without_a_declaring_file_is_rejected() -> None:
+ with pytest.raises(TypeError, match="constructor instead"):
+ LocalRuntime(_sums_env())
+
+
+async def test_constructor_builds_fresh_per_rollout_from_the_row() -> None:
+ built: list[str] = []
+
+ def env_for(task: Task) -> Environment:
+ built.append(task.env)
+ return _sums_env(task.env)
+
+ job = await Task(env="sums", id="add", args={"a": 1, "b": 2}).run(
+ _FnAgent(_solve_add),
+ runtime=LocalRuntime(env_for),
+ group=3,
+ max_concurrent=3,
+ )
+
+ assert all(run.reward == 1.0 for run in job.runs)
+ assert built == ["sums", "sums", "sums"]
+
+
+async def test_source_missing_env_name_fails_loudly(tmp_path) -> None:
+ (tmp_path / "env.py").write_text(
+ 'from hud import Environment\n\nenv = Environment("sums")\n',
+ encoding="utf-8",
+ )
+ provider = LocalRuntime(tmp_path / "env.py")
+
+ with pytest.raises(ValueError, match="no Environment named 'other'"):
+ async with provider(Task(env="other", id="add")):
+ pass
+
+
+# ─── the no-runtime resolution ladder ──────────────────────────────────
+
+
+async def test_module_loaded_taskset_serves_its_source_by_default(tmp_path, request) -> None:
+ (tmp_path / "env.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
+ (tmp_path / "tasks.py").write_text(
+ "from env import add\n\ntasks = [add(a=2, b=3), add(a=4, b=5)]\n",
+ encoding="utf-8",
+ )
+ # tasks.py's `from env import add` imports env.py normally, so it outlives
+ # the throwaway tasks module.
+ request.addfinalizer(lambda: sys.modules.pop("env", None))
+
+ taskset = Taskset.from_module(tmp_path / "tasks.py")
+ job = await taskset.run(_FnAgent(_solve_add))
+
+ assert len(job.runs) == 2
+ assert all(run.reward == 1.0 for run in job.runs)
+
+
+async def test_minted_tasks_resolve_a_declared_env_by_name(imported_env) -> None:
+ job = await Taskset("sums", [imported_env.add(a=2, b=3), imported_env.add(a=4, b=5)]).run(
+ _FnAgent(_solve_add)
+ )
+
+ assert len(job.runs) == 2
+ assert all(run.reward == 1.0 for run in job.runs)
+
+
+async def test_no_placement_fails_with_the_forms_to_pass() -> None:
+ with pytest.raises(ValueError, match="no placement for env"):
+ await Task(env="ghost", id="add").run(_FnAgent(_solve_add))
+
+
+async def test_ambiguous_env_name_fails_loudly(imported_env, tmp_path, request) -> None:
+ module_name = f"_sums_rival_{request.node.name}"
+ file = tmp_path / f"{module_name}.py"
+ file.write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
+ spec = importlib.util.spec_from_file_location(module_name, file)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = module
+ try:
+ spec.loader.exec_module(module)
+ with pytest.raises(ValueError, match="LocalRuntime\\(env\\)"):
+ await Task(env="sums", id="add").run(_FnAgent(_solve_add))
+ finally:
+ del sys.modules[module_name]
+
+
+def test_rejects_a_non_pointer_argument() -> None:
+ with pytest.raises(TypeError, match="expected a source path"):
+ LocalRuntime(cast("Any", 42))
+
+
+async def test_failed_startup_still_runs_shutdown_hooks() -> None:
+ from hud.eval.runtime import _local
+
+ env = _sums_env()
+ lifecycle: list[str] = []
+
+ @env.initialize
+ async def _up() -> None:
+ lifecycle.append("up")
+
+ @env.shutdown
+ async def _down() -> None:
+ lifecycle.append("down")
+
+ @env.initialize
+ async def _boom() -> None:
+ raise RuntimeError("daemon failed to start")
+
+ with pytest.raises(RuntimeError, match="daemon failed to start"):
+ async with _local(env):
+ pass
+
+ assert lifecycle == ["up", "down"]
+
+
+async def test_tasks_only_module_resolves_envs_imported_from_elsewhere(
+ tmp_path, monkeypatch, request
+) -> None:
+ # The env lives in a separate importable package, not next to tasks.py:
+ # the origin declares no envs, so resolution falls through to the live
+ # env the tasks module imported.
+ env_dir = tmp_path / "pkg"
+ env_dir.mkdir()
+ (env_dir / "sums_envmod.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
+ tasks_dir = tmp_path / "tasks"
+ tasks_dir.mkdir()
+ (tasks_dir / "tasks.py").write_text(
+ "from sums_envmod import add\n\ntasks = [add(a=2, b=3)]\n",
+ encoding="utf-8",
+ )
+ monkeypatch.syspath_prepend(str(env_dir))
+ request.addfinalizer(lambda: sys.modules.pop("sums_envmod", None))
+
+ taskset = Taskset.from_module(tasks_dir / "tasks.py")
+ job = await taskset.run(_FnAgent(_solve_add))
+
+ assert [run.reward for run in job.runs] == [1.0]
+
+
+async def test_single_file_taskset_never_drags_in_a_same_named_sibling(tmp_path) -> None:
+ (tmp_path / "env_a.py").write_text(
+ _SUMS_ENV.format(name="sums") + "\ntasks = [add(a=2, b=3)]\n", encoding="utf-8"
+ )
+ (tmp_path / "env_b.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
+
+ taskset = Taskset.from_module(tmp_path / "env_a.py")
+ job = await taskset.run(_FnAgent(_solve_add))
+
+ assert [run.reward for run in job.runs] == [1.0]
+
+
+async def test_empty_taskset_runs_without_a_placement() -> None:
+ job = await Taskset("empty", []).run(_FnAgent(_solve_add))
+
+ assert job.runs == []
+
+
+async def test_reexporting_tasks_module_does_not_claim_the_env(
+ tmp_path, monkeypatch, request
+) -> None:
+ # tasks.py re-exports the env object alongside the factory: the origin
+ # must not claim it (re-import of tasks.py would reuse the cached env
+ # module) — each rollout rebuilds the env from its real file instead.
+ env_dir = tmp_path / "pkg"
+ env_dir.mkdir()
+ (env_dir / "sums_reexp_envmod.py").write_text(
+ "from hud import Environment\n\n"
+ "LOADS = []\n"
+ 'env = Environment("sums")\n\n\n'
+ '@env.template(id="add")\nasync def add(a: int, b: int):\n'
+ " LOADS.append(1)\n"
+ ' answer = yield f"add:{a}:{b}:{len(LOADS)}"\n'
+ " yield 1.0 if answer == str(a + b) else 0.0\n",
+ encoding="utf-8",
+ )
+ tasks_dir = tmp_path / "tasks"
+ tasks_dir.mkdir()
+ (tasks_dir / "tasks.py").write_text(
+ "from sums_reexp_envmod import add, env\n\ntasks = [add(a=2, b=3)]\n",
+ encoding="utf-8",
+ )
+ monkeypatch.syspath_prepend(str(env_dir))
+ request.addfinalizer(lambda: sys.modules.pop("sums_reexp_envmod", None))
+
+ def _solve(prompt: str) -> str:
+ _, a, b, loads = prompt.split(":")
+ assert loads == "1" # a fresh env module per rollout, not the cached one
+ return str(int(a) + int(b))
+
+ taskset = Taskset.from_module(tasks_dir / "tasks.py")
+ job = await taskset.run(_FnAgent(_solve), group=2)
+
+ assert [run.reward for run in job.runs] == [1.0, 1.0]
+
+
+async def test_package_reexported_env_pointer_uses_the_declaring_submodule(
+ tmp_path, monkeypatch, request
+) -> None:
+ pkg = tmp_path / "sums_pkg"
+ pkg.mkdir()
+ (pkg / "__init__.py").write_text("from .env_mod import env\n", encoding="utf-8")
+ (pkg / "env_mod.py").write_text(_SUMS_ENV.format(name="sums"), encoding="utf-8")
+ monkeypatch.syspath_prepend(str(tmp_path))
+ request.addfinalizer(
+ lambda: [sys.modules.pop(m, None) for m in ("sums_pkg", "sums_pkg.env_mod")]
+ )
+ import importlib
+
+ sums_pkg = importlib.import_module("sums_pkg")
+
+ run = await rollout(
+ Task(env="sums", id="add", args={"a": 2, "b": 3}),
+ _FnAgent(_solve_add),
+ runtime=LocalRuntime(sums_pkg.env),
+ )
+
+ assert run.reward == 1.0
+
+
+async def test_template_can_lazily_import_a_sibling_module(tmp_path) -> None:
+ (tmp_path / "lazy_helper.py").write_text("ANSWER_SUFFIX = ':ok'\n", encoding="utf-8")
+ (tmp_path / "env.py").write_text(
+ "from hud import Environment\n\n"
+ 'env = Environment("sums")\n\n\n'
+ '@env.template(id="add")\nasync def add(a: int, b: int):\n'
+ " import lazy_helper\n"
+ ' answer = yield f"add:{a}:{b}{lazy_helper.ANSWER_SUFFIX}"\n'
+ " yield 1.0 if answer == str(a + b) else 0.0\n",
+ encoding="utf-8",
+ )
+
+ def _solve(prompt: str) -> str:
+ _, a, b, ok = prompt.split(":")
+ assert ok == "ok"
+ return str(int(a) + int(b))
+
+ job = await Task(env="sums", id="add", args={"a": 2, "b": 3}).run(
+ _FnAgent(_solve),
+ runtime=LocalRuntime(tmp_path / "env.py"),
+ group=2,
+ max_concurrent=2,
+ )
+
+ assert [run.reward for run in job.runs] == [1.0, 1.0]
+
+
+# ─── seam defenses ─────────────────────────────────────────────────────
+
+
+@pytest.mark.filterwarnings("ignore::RuntimeWarning") # the broken source leaks its coroutine
+async def test_unguarded_run_call_in_source_names_the_mistake(tmp_path) -> None:
+ (tmp_path / "env.py").write_text(
+ "import asyncio\n\nfrom hud import Environment\n\n"
+ 'env = Environment("sums")\n\n'
+ "asyncio.run(asyncio.sleep(0))\n",
+ encoding="utf-8",
+ )
+ provider = LocalRuntime(tmp_path / "env.py")
+
+ with pytest.raises(RuntimeError, match='if __name__ == "__main__"'):
+ async with provider(Task(env="sums", id="add")):
+ pass
+
+
+async def test_live_env_mutated_after_import_fails_with_the_cause(imported_env) -> None:
+ @imported_env.env.template(id="patched")
+ async def patched() -> Any:
+ answer = yield "noop"
+ yield 1.0 if answer else 0.0
+
+ provider = LocalRuntime(imported_env.env)
+
+ with pytest.raises(ValueError, match="modified after import"):
+ async with provider(Task(env="sums", id="patched")):
+ pass
diff --git a/hud/eval/tests/test_rollout.py b/hud/eval/tests/test_rollout.py
index 0d8bb5166..3473d5a1c 100644
--- a/hud/eval/tests/test_rollout.py
+++ b/hud/eval/tests/test_rollout.py
@@ -1,7 +1,7 @@
"""The rollout engine: ``rollout(task, agent)`` and its schedulers.
These drive the engine end-to-end through the real placement path: a pure-data
-``Task`` row plus ``runtime=LocalRuntime(env_file)`` — a child process serves the env, the
+``Task`` row plus ``runtime=SubprocessRuntime(env_file)`` — a child process serves the env, the
engine connects over the wire, the agent answers, grading comes back. The
engine contract is a graded :class:`Run` with a trace id (always under a job —
there are no standalone traces), and failure isolation that never raises: a
@@ -31,7 +31,7 @@
from hud.agents.openai_compatible import OpenAIChatAgent
from hud.agents.types import OpenAIChatConfig
from hud.environment import Environment
-from hud.eval import Job, LocalRuntime, Task, Taskset
+from hud.eval import Job, SubprocessRuntime, Task, Taskset
from hud.eval.run import Run, rollout
from hud.eval.runtime import _local
@@ -155,7 +155,7 @@ async def _wait_for_pid_inactive(pid: int, max_wait: float = 2.0) -> bool:
async def test_rollout_returns_graded_run_with_trace_id(env_file: Path) -> None:
- run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=LocalRuntime(env_file))
+ run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file))
assert run.reward == 1.0
assert run.trace.content == "5"
@@ -249,7 +249,7 @@ async def start_child():
try:
with pytest.raises(RuntimeError, match="startup boom"):
- async with LocalRuntime(env_file, ready_timeout=2.0)(Task(env="leaky", id="noop")):
+ async with SubprocessRuntime(env_file, ready_timeout=2.0)(Task(env="leaky", id="noop")):
pass
pid = int(pid_file.read_text())
assert await _wait_for_pid_inactive(pid)
@@ -263,7 +263,7 @@ async def test_mid_run_failure_keeps_the_real_run_and_its_evidence(env_file: Pat
def boom(prompt: str) -> str:
raise RuntimeError("agent exploded")
- run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=LocalRuntime(env_file))
+ run = await rollout(_add_task(2, 3), _FnAgent(boom), runtime=SubprocessRuntime(env_file))
assert run.trace.is_error
assert "agent exploded" in (run.trace.error or "")
@@ -291,7 +291,7 @@ async def test_mid_run_failure_still_grades_best_effort(env_file: Path) -> None:
# The agent answers correctly, then fails. The env is still alive, so the
# run is graded best-effort: the reward is captured even though it errored.
run = await rollout(
- _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=LocalRuntime(env_file)
+ _add_task(2, 3), _AnswerThenBoomAgent(_solve_add), runtime=SubprocessRuntime(env_file)
)
assert run.trace.is_error
@@ -358,7 +358,7 @@ def placer(task: TaskRow) -> Any:
# The scheduler half of placement: the row is the request, so a
# provider can size/route each substrate per task.
placed.append(f"{task.env}/{task.id}:{task.args['a']}")
- return LocalRuntime(env_file)(task)
+ return SubprocessRuntime(env_file)(task)
run = await rollout(_add_task(2, 3), _FnAgent(_solve_add), runtime=placer)
@@ -367,7 +367,7 @@ def placer(task: TaskRow) -> Any:
async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None:
- job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=LocalRuntime(env_file))
+ job = await _add_task(2, 3).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(env_file))
(run,) = job.runs
assert job.reward == 1.0
@@ -377,7 +377,7 @@ async def test_task_run_schedules_a_single_task_job(env_file: Path) -> None:
async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None:
job = await _add_task(1, 2).run(
- _FnAgent(_solve_add), runtime=LocalRuntime(env_file), group=2, max_concurrent=1
+ _FnAgent(_solve_add), runtime=SubprocessRuntime(env_file), group=2, max_concurrent=1
)
assert job.group == 2
@@ -388,7 +388,7 @@ async def test_task_run_has_taskset_scheduling_semantics(env_file: Path) -> None
async def test_open_job_spans_multiple_scheduler_calls(env_file: Path) -> None:
session = await Job.start("session", group=2)
- provider = LocalRuntime(env_file)
+ provider = SubprocessRuntime(env_file)
job1 = await _add_task(1, 1).run(_FnAgent(_solve_add), runtime=provider, job=session)
job2 = await _add_task(2, 2).run(_FnAgent(_solve_add), runtime=provider, job=session)
@@ -434,7 +434,7 @@ async def test_one_spawn_serves_each_rows_env_in_a_mixed_taskset(
# One provider, two envs: each acquisition serves the row it was called
# with (the task ids only exist on their own env, so a misplacement
# would fail the rollout).
- job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=LocalRuntime(path))
+ job = await Taskset("zoo", rows).run(_FnAgent(_solve_add), runtime=SubprocessRuntime(path))
assert [run.reward for run in job.runs] == [1.0, 1.0]
assert [run.prompt for run in job.runs] == ["alpha:1:2", "beta:3:4"]
@@ -444,7 +444,7 @@ async def test_rollout_threads_job_and_group_ids(env_file: Path) -> None:
run = await rollout(
_add_task(1, 1),
_FnAgent(_solve_add),
- runtime=LocalRuntime(env_file),
+ runtime=SubprocessRuntime(env_file),
job_id="j1",
group_id="g1",
)
diff --git a/hud/eval/tests/test_task.py b/hud/eval/tests/test_task.py
index cf258a0d7..5d0de0586 100644
--- a/hud/eval/tests/test_task.py
+++ b/hud/eval/tests/test_task.py
@@ -137,7 +137,7 @@ def test_row_validation_rejects_malformed_entries() -> None:
# ─── placement ─────────────────────────────────────────────────────────
-async def test_no_placement_defaults_to_hud_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_platform_taskset_defaults_to_hud_runtime(monkeypatch: pytest.MonkeyPatch) -> None:
import hud.eval.taskset as taskset_mod
seen: dict[str, object] = {}
@@ -150,8 +150,9 @@ async def fake_rollout(task: Task, agent: Agent, **kwargs: object) -> Run:
monkeypatch.setattr(taskset_mod, "rollout", fake_rollout)
- v = Task(env="hosted-env", id="solve", args={"n": 1})
- job = await v.run(cast("Agent", object()))
+ task = Task(env="hosted-env", id="solve", args={"n": 1})
+ taskset = taskset_mod.Taskset("hosted", [task], origin="api:ts_123")
+ job = await taskset.run(cast("Agent", object()))
(run,) = job.runs
assert run.trace.status == "completed"
diff --git a/hud/tests/test_init.py b/hud/tests/test_init.py
index b11dc88d2..144bed28f 100644
--- a/hud/tests/test_init.py
+++ b/hud/tests/test_init.py
@@ -55,6 +55,7 @@ def test_all_exports_available(self):
"RuntimeLimits",
"RuntimeResources",
"LocalRuntime",
+ "SubprocessRuntime",
"SyncPlan",
"Task",
"Taskset",
diff --git a/hud/tests/test_init_module.py b/hud/tests/test_init_module.py
index 45b458642..79a470974 100644
--- a/hud/tests/test_init_module.py
+++ b/hud/tests/test_init_module.py
@@ -35,6 +35,7 @@ def test_all_exports(self):
"RuntimeLimits",
"RuntimeResources",
"LocalRuntime",
+ "SubprocessRuntime",
"SyncPlan",
"Task",
"Taskset",