From a7a19ac3ae3fa902218feae4d15d475187361d0d Mon Sep 17 00:00:00 2001 From: Trevor Manz Date: Thu, 27 Aug 2026 13:38:02 -0400 Subject: [PATCH 1/9] Provision sandbox environments with uv sync Sandboxed notebooks get their environments from a two-stage resolution: `uv export --script` flattens the metadata to requirements lines, and a second command re-resolves them with the `[tool.uv]` tables reduced to command-line flags. The flags cannot express index names, default ordering, explicit indexes, or source pins, so scripts that resolve correctly under `uv run` fail to open in the sandbox (#10547). Export failures are silently discarded on top. These changes add `marimo._environments.environment` to provision an environment in one step. `sync()` hands the script to `uv sync --script`, so uv resolves the metadata with its full semantics, and returns a frozen handle read from uv's JSON report. ```py env = environment.sync(script, cwd=notebook_dir) subprocess.Popen([env.python, ...], env=env.process_env()) if env.requires_restart(previous): ... # relaunch the kernel ``` The environment is uv's own script environment, shared with `uv run notebook.py`. Synchronizing is idempotent and cheap when nothing changed, so callers synchronize before every launch instead of tracking staleness. A uv older than 0.7.21 (the JSON report) is rejected with an upgrade hint. Callers migrate in the following changes. --- marimo/_environments/environment.py | 243 +++++++++++++++++++++ tests/_environments/test_environment.py | 269 ++++++++++++++++++++++++ 2 files changed, 512 insertions(+) create mode 100644 marimo/_environments/environment.py create mode 100644 tests/_environments/test_environment.py diff --git a/marimo/_environments/environment.py b/marimo/_environments/environment.py new file mode 100644 index 00000000000..57cd31dcdd3 --- /dev/null +++ b/marimo/_environments/environment.py @@ -0,0 +1,243 @@ +# Copyright 2026 Marimo. All rights reserved. +"""Provision script environments with uv. + +`sync()` makes a script's environment match its PEP 723 metadata and +returns a frozen `Environment`: the interpreter to launch, the environment +root, and the action uv took. Synchronizing is idempotent and cheap when +nothing changed, so callers synchronize before every launch instead of +tracking staleness themselves. + +uv owns resolution, so index configuration, sources, and credentials in +the metadata apply exactly as they do for `uv run script.py`. The +environment is uv's own script environment, shared with `uv run`. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +import msgspec + +from marimo import _loggers +from marimo._environments.uv import UvError, uv + +if TYPE_CHECKING: + from collections.abc import Mapping + +LOGGER = _loggers.marimo_logger() + +Action = Literal["created", "updated", "replaced", "unchanged"] + +# `uv sync --script --output-format json` reports the environment path and +# interpreter on stdout. The flag landed in uv 0.7.21. +MINIMUM_UV_VERSION = (0, 7, 21) + + +class UvUnsupportedVersionError(UvError): + """The installed uv predates the script-environment interface.""" + + def __init__(self, found: str) -> None: + minimum = ".".join(str(part) for part in MINIMUM_UV_VERSION) + super().__init__( + f"uv {minimum} or newer is required to manage sandbox " + f"environments; found uv {found}. " + "Upgrade with `uv self update`." + ) + + +class UvSyncReportError(UvError): + """uv synchronized the script but its report was unreadable.""" + + +@dataclass(frozen=True) +class Environment: + """A synchronized script environment. + + A process launched from a previous handle must be relaunched when + `requires_restart` says so; an `updated` environment keeps its + interpreter, and newly installed packages are importable without a + relaunch. + """ + + python: str + root: str + action: Action + + def requires_restart(self, previous: Environment | None) -> bool: + """Whether a process launched from `previous` must be relaunched.""" + if previous is None: + return False + return self.action == "replaced" or self.python != previous.python + + def process_env( + self, base: Mapping[str, str] | None = None + ) -> dict[str, str]: + """Environment variables for a process running in this environment. + + Package operations inside the process must target this environment, + not an enclosing project or virtualenv, and subprocesses the + process spawns by name must resolve this environment's tools + first, as they would under `uv run`. + """ + env = dict(os.environ if base is None else base) + env["VIRTUAL_ENV"] = self.root + env.pop("UV_PROJECT_ENVIRONMENT", None) + bin_dir = _venv_bin_dir(self.root) + path = env.get("PATH") + env["PATH"] = bin_dir if not path else bin_dir + os.pathsep + path + return env + + +def sync( + script: str, + *, + cwd: str | None = None, + python_override: str | None = None, +) -> Environment: + """Make the script's environment match its metadata. + + `cwd` is the directory uv runs from, normally the notebook's own + directory so directory-scoped uv configuration applies. The + `python_override` wins over the script's `requires-python`; html-wasm + export needs the environment's interpreter to match Pyodide even when + the script declares something else. Raises `UvCommandError` subclasses + on resolution or synchronization failure and never mutates `script`. + """ + ensure_supported_uv() + args = [ + "sync", + "--script", + script, + "--compile-bytecode", + "--output-format", + "json", + ] + if python_override is not None: + args.extend(["--python", python_override]) + completed = uv(args, env=_sync_env(), cwd=cwd) + return _parse_report(completed.stdout) + + +def ensure_supported_uv() -> None: + """Raise `UvUnsupportedVersionError` for a uv below the minimum.""" + version = _uv_version() + parsed = _parse_version(version) + if parsed is None: + # An unparsable version is likely newer than anything we know; + # let the actual command fail if the interface is missing. + LOGGER.debug("Could not parse uv version: %s", version) + return + if parsed < MINIMUM_UV_VERSION: + raise UvUnsupportedVersionError(version) + + +def _uv_version() -> str: + """The version string of the invoked uv. + + Probed per call: the probe is cheap next to any synchronization, and + caching would pin a stale answer across `uv self update` or a changed + `UV` environment variable. + """ + # "uv 0.7.21 (a1b2c3d4 2025-01-01)" + stdout = uv(["--version"]).stdout.strip() + return stdout.removeprefix("uv ").split(" ")[0] + + +def _parse_version(version: str) -> tuple[int, ...] | None: + match = re.match(r"(\d+)\.(\d+)\.(\d+)", version) + if match is None: + return None + return tuple(int(part) for part in match.groups()) + + +# The report schema is labeled preview upstream; only the fields consumed +# here are declared, and drift raises `UvSyncReportError` rather than +# guessing. +_ACTIONS: dict[str, Action] = { + "create": "created", + "update": "updated", + "replace": "replaced", + "check": "unchanged", +} + + +class _ReportPython(msgspec.Struct): + path: str + + +class _ReportEnvironment(msgspec.Struct): + path: str + python: _ReportPython + + +class _ReportSync(msgspec.Struct): + environment: _ReportEnvironment + action: str + + +class _Report(msgspec.Struct): + sync: _ReportSync + + +def _parse_report(stdout: str) -> Environment: + try: + report = msgspec.json.decode(stdout, type=_Report) + except msgspec.DecodeError as error: + raise UvSyncReportError( + "uv synchronized the script but did not report its environment" + ) from error + reported_python = report.sync.environment.python.path + root = report.sync.environment.path + raw_action = report.sync.action + action = _ACTIONS.get(raw_action) + if action is None: + # An unknown action still names a usable environment; interpreter + # identity drives restarts, so treat it as an in-place update. + LOGGER.debug("Unknown uv sync action: %s", raw_action) + action = "updated" + # The report spells the interpreter differently across actions (e.g. + # bin/python on create, bin/python3 on check). Pick a stable name from + # the root so interpreter identity is comparable across syncs. + python = _venv_python(root) or reported_python + return Environment(python=python, root=root, action=action) + + +def _venv_python(root: str) -> str | None: + """The environment's interpreter, preferring the unversioned name. + + Symlinks are deliberately not resolved: bin/python commonly links to + the base interpreter, and resolving it would launch outside the + environment. + """ + bin_dir = _venv_bin_dir(root) + candidates: tuple[str, ...] + if os.name == "nt": + candidates = (os.path.join(bin_dir, "python.exe"),) + else: + candidates = ( + os.path.join(bin_dir, "python"), + os.path.join(bin_dir, "python3"), + ) + for candidate in candidates: + if os.path.exists(candidate): + return candidate + return None + + +def _venv_bin_dir(root: str) -> str: + return os.path.join(root, "Scripts" if os.name == "nt" else "bin") + + +def _sync_env() -> dict[str, str]: + """Environment for `uv sync --script` invocations. + + Script environments are selected by the script and its metadata; an + enclosing project or virtualenv must not redirect synchronization. + """ + env = dict(os.environ) + env.pop("VIRTUAL_ENV", None) + env.pop("UV_PROJECT_ENVIRONMENT", None) + return env diff --git a/tests/_environments/test_environment.py b/tests/_environments/test_environment.py new file mode 100644 index 00000000000..d21b2625baf --- /dev/null +++ b/tests/_environments/test_environment.py @@ -0,0 +1,269 @@ +# Copyright 2026 Marimo. All rights reserved. +from __future__ import annotations + +import json +import os +import stat +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest + +from marimo._environments import environment +from marimo._environments.environment import ( + Environment, + UvUnsupportedVersionError, + ensure_supported_uv, + sync, +) +from marimo._environments.uv import ( + UvError, + UvMissingScriptMetadataError, + UvResolutionError, + is_uv_available, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def _supports_sync() -> bool: + if not is_uv_available(): + return False + try: + ensure_supported_uv() + except UvError: + return False + return True + + +SUPPORTS_SYNC = _supports_sync() + +REQUIRES_PYTHON = f">={sys.version_info[0]}.{sys.version_info[1]}" + +EMPTY_SCRIPT = f"""\ +# /// script +# requires-python = "{REQUIRES_PYTHON}" +# dependencies = [] +# /// +""" + + +def test_requires_restart() -> None: + first = Environment( + python="/env/bin/python", root="/env", action="created" + ) + unchanged = Environment( + python="/env/bin/python", root="/env", action="unchanged" + ) + updated = Environment( + python="/env/bin/python", root="/env", action="updated" + ) + replaced = Environment( + python="/env/bin/python", root="/env", action="replaced" + ) + moved = Environment( + python="/other/bin/python", root="/other", action="unchanged" + ) + + assert not first.requires_restart(None) + assert not unchanged.requires_restart(first) + assert not updated.requires_restart(first) + assert replaced.requires_restart(first) + assert moved.requires_restart(first) + + +def test_process_env_targets_the_environment() -> None: + env = Environment(python="/env/bin/python", root="/env", action="created") + base = {"UV_PROJECT_ENVIRONMENT": "/project", "PATH": "/usr/bin"} + + child = env.process_env(base) + + assert child["VIRTUAL_ENV"] == "/env" + assert "UV_PROJECT_ENVIRONMENT" not in child + # The environment's tools resolve first, as under `uv run`. + bin_dir = os.path.join("/env", "Scripts" if os.name == "nt" else "bin") + assert child["PATH"] == f"{bin_dir}{os.pathsep}/usr/bin" + # The base mapping is not mutated. + assert base["UV_PROJECT_ENVIRONMENT"] == "/project" + + +@pytest.mark.skipif( + sys.platform == "win32", reason="shell stub is not executable on Windows" +) +def test_old_uv_is_rejected( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + stub = tmp_path / "uv" + stub.write_text('#!/bin/sh\necho "uv 0.5.0 (stub)"\n') + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("UV", str(stub)) + + with pytest.raises(UvUnsupportedVersionError, match="0.5.0"): + sync(str(tmp_path / "nb.py")) + + +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +def test_sync_creates_and_reuses_the_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # An isolated cache keeps test environments out of the user's real + # uv cache; tmp_path is unique per run, so they would accumulate. + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + script = tmp_path / "nb.py" + script.write_text(EMPTY_SCRIPT, encoding="utf-8") + + first = sync(str(script), cwd=str(tmp_path)) + assert first.action == "created" + assert os.path.exists(first.python) + assert first.python.startswith(first.root) + + again = sync(str(script), cwd=str(tmp_path)) + assert again.action == "unchanged" + assert again.python == first.python + assert not again.requires_restart(first) + + +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +def test_sync_requires_script_metadata(tmp_path: Path) -> None: + script = tmp_path / "plain.py" + script.write_text("print('hi')\n", encoding="utf-8") + with pytest.raises(UvMissingScriptMetadataError): + sync(str(script)) + + +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +def test_resolution_failure_carries_the_solver_message( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("UV_OFFLINE", "1") + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + script = tmp_path / "nb.py" + script.write_text( + "# /// script\n" + f'# requires-python = "{REQUIRES_PYTHON}"\n' + '# dependencies = ["definitely-not-a-real-pkg-xyz==99.99"]\n' + "# ///\n", + encoding="utf-8", + ) + with pytest.raises(UvResolutionError) as excinfo: + sync(str(script), cwd=str(tmp_path)) + assert "definitely-not-a-real-pkg-xyz" in excinfo.value.stderr + + +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +@pytest.mark.slow +@pytest.mark.network +def test_sync_honors_index_semantics(tmp_path: Path) -> None: + """Repro from marimo-team/marimo#10547 (requires network access). + + pip-install-test==0.0.3 exists on test.pypi.org but not on pypi.org. + The metadata pins it to a named explicit index; delegating to + `uv sync --script` honors the pin, where flattening the indexes into + command-line flags could not. + """ + script = tmp_path / "nb.py" + script.write_text( + "# /// script\n" + f'# requires-python = ">={sys.version_info[0]}.{sys.version_info[1]}"\n' + "# dependencies = [\n" + '# "pip-install-test==0.0.3",\n' + "# ]\n" + "#\n" + "# [tool.uv.sources]\n" + '# pip-install-test = { index = "testpypi" }\n' + "#\n" + "# [[tool.uv.index]]\n" + '# url = "https://pypi.org/simple/"\n' + "# default = true\n" + "#\n" + "# [[tool.uv.index]]\n" + '# name = "testpypi"\n' + '# url = "https://test.pypi.org/simple/"\n' + "# explicit = true\n" + "# ///\n", + encoding="utf-8", + ) + + result = sync(str(script), cwd=str(tmp_path)) + + site_packages = _site_packages(result.root) + assert any( + entry.startswith("pip_install_test") for entry in site_packages + ), site_packages + + +def _site_packages(root: str) -> list[str]: + for dirpath, dirnames, _ in os.walk(root): + if os.path.basename(dirpath) == "site-packages": + del dirnames[:] + return os.listdir(dirpath) + return [] + + +def test_unreadable_report_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(environment, "ensure_supported_uv", lambda: None) + + class Completed: + stdout = "not json" + + monkeypatch.setattr( + environment, "uv", lambda *_args, **_kwargs: Completed() + ) + with pytest.raises(environment.UvSyncReportError): + sync("nb.py") + + +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +def test_sync_accepts_a_python_override( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + script = tmp_path / "nb.py" + script.write_text(EMPTY_SCRIPT, encoding="utf-8") + override = f"{sys.version_info[0]}.{sys.version_info[1]}" + + env = sync(str(script), cwd=str(tmp_path), python_override=override) + + reported = subprocess.run( + [env.python, "-c", "import sys; print(sys.version_info[:2])"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert reported == str(tuple(sys.version_info[:2])) + + +@pytest.mark.parametrize( + ("raw_action", "action"), + [ + ("create", "created"), + ("update", "updated"), + ("replace", "replaced"), + ("check", "unchanged"), + ("something-new", "updated"), + ], +) +def test_report_actions_map_to_the_handle( + monkeypatch: pytest.MonkeyPatch, raw_action: str, action: str +) -> None: + report = { + "sync": { + "environment": { + "path": "/env", + "python": {"path": "/env/bin/python"}, + }, + "action": raw_action, + } + } + + class Completed: + stdout = json.dumps(report) + + monkeypatch.setattr(environment, "ensure_supported_uv", lambda: None) + monkeypatch.setattr( + environment, "uv", lambda *_args, **_kwargs: Completed() + ) + + assert sync("nb.py").action == action From 6e9b9029fbe350a5a3a061698f5b60463eed51ae Mon Sep 17 00:00:00 2001 From: Trevor Manz Date: Thu, 27 Aug 2026 14:49:25 -0400 Subject: [PATCH 2/9] Launch processes from script environments A synchronized script environment is only useful if marimo can run the server and kernels inside it. The previous change created environments; nothing yet plans a process in one, and markdown notebooks have no script for uv to synchronize at all. These changes add `launch()`, which plans `python ` inside an environment, and `materialized_for_environment()`, which gives every notebook a script uv can synchronize. ```py with script_metadata.materialized_for_environment(path) as target: env = environment.sync(target.path, cwd=target.directory) plan = environment.launch(env, ["-m", "marimo", *args], overlay=extras) subprocess.Popen(plan.argv, env=plan.env) ``` Without an overlay the plan invokes the environment's interpreter directly. Overlay requirements ride `uv run --with`, resolved into a cached side environment for that process only, so the script environment stays exactly what the manifest declares and `uv run notebook.py` sees the same environment marimo used. A markdown or Quarto notebook materializes its header verbatim to the stable carrier `.marimo-v1-.py` next to the notebook, deleted on exit. uv keys a script environment on the script's absolute path, so the stable name maps one notebook to one environment across sessions. Callers migrate in the following changes. --- marimo/_environments/environment.py | 110 ++++++++++++++------ marimo/_environments/script_metadata.py | 98 +++++++++++++++-- tests/_environments/test_environment.py | 52 +++++++++ tests/_environments/test_script_metadata.py | 107 ++++++++++++++++++- 4 files changed, 319 insertions(+), 48 deletions(-) diff --git a/marimo/_environments/environment.py b/marimo/_environments/environment.py index 57cd31dcdd3..f7e32afd896 100644 --- a/marimo/_environments/environment.py +++ b/marimo/_environments/environment.py @@ -2,14 +2,11 @@ """Provision script environments with uv. `sync()` makes a script's environment match its PEP 723 metadata and -returns a frozen `Environment`: the interpreter to launch, the environment -root, and the action uv took. Synchronizing is idempotent and cheap when -nothing changed, so callers synchronize before every launch instead of -tracking staleness themselves. - -uv owns resolution, so index configuration, sources, and credentials in -the metadata apply exactly as they do for `uv run script.py`. The -environment is uv's own script environment, shared with `uv run`. +returns a frozen `Environment`. Synchronizing is idempotent and cheap +when nothing changed; callers synchronize before every launch. uv owns +resolution, so the metadata's indexes, sources, and credentials apply +exactly as they do for `uv run script.py`, and the environment is the +same one `uv run` uses. """ from __future__ import annotations @@ -22,10 +19,10 @@ import msgspec from marimo import _loggers -from marimo._environments.uv import UvError, uv +from marimo._environments.uv import UvError, require_uv_bin, uv if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Mapping, Sequence LOGGER = _loggers.marimo_logger() @@ -56,10 +53,8 @@ class UvSyncReportError(UvError): class Environment: """A synchronized script environment. - A process launched from a previous handle must be relaunched when - `requires_restart` says so; an `updated` environment keeps its - interpreter, and newly installed packages are importable without a - relaunch. + An `updated` environment keeps its interpreter; newly installed + packages are importable without a relaunch. """ python: str @@ -75,12 +70,10 @@ def requires_restart(self, previous: Environment | None) -> bool: def process_env( self, base: Mapping[str, str] | None = None ) -> dict[str, str]: - """Environment variables for a process running in this environment. + """Environment variables for a process in this environment. - Package operations inside the process must target this environment, - not an enclosing project or virtualenv, and subprocesses the - process spawns by name must resolve this environment's tools - first, as they would under `uv run`. + Sets `VIRTUAL_ENV`, drops `UV_PROJECT_ENVIRONMENT`, and puts the + environment's bin directory first on `PATH`, as `uv run` would. """ env = dict(os.environ if base is None else base) env["VIRTUAL_ENV"] = self.root @@ -91,20 +84,71 @@ def process_env( return env +@dataclass(frozen=True) +class ProcessPlan: + """A command ready to run inside a script environment.""" + + argv: tuple[str, ...] + env: dict[str, str] + + +def launch( + environment: Environment, + args: Sequence[str], + *, + overlay: Sequence[str] = (), + base_env: Mapping[str, str] | None = None, +) -> ProcessPlan: + """Plans running `python ` inside the environment. + + With no overlay, the plan invokes the environment's interpreter + directly. Overlay requirements (PEP 508, or `-e ` for an + editable install) are layered via `uv run --with` into a cached side + environment for this process only; the script environment and the + manifest are never modified. + """ + env = environment.process_env(base_env) + if not overlay: + return ProcessPlan(argv=(environment.python, *args), env=env) + + with_args: list[str] = [] + for requirement in overlay: + if requirement.startswith("-e "): + with_args.extend(["--with-editable", requirement[3:].strip()]) + else: + with_args.extend(["--with", requirement]) + return ProcessPlan( + argv=( + require_uv_bin(), + "run", + # The script environment is VIRTUAL_ENV in `env`; --active + # makes uv layer on top of it instead of ignoring it. + "--active", + "--no-project", + "--python", + environment.python, + *with_args, + "--", + "python", + *args, + ), + env=env, + ) + + def sync( script: str, *, cwd: str | None = None, python_override: str | None = None, ) -> Environment: - """Make the script's environment match its metadata. - - `cwd` is the directory uv runs from, normally the notebook's own - directory so directory-scoped uv configuration applies. The - `python_override` wins over the script's `requires-python`; html-wasm - export needs the environment's interpreter to match Pyodide even when - the script declares something else. Raises `UvCommandError` subclasses - on resolution or synchronization failure and never mutates `script`. + """Makes the script's environment match its metadata. + + Runs uv from `cwd`, normally the notebook's directory, so + directory-scoped uv configuration applies. `python_override` wins + over the script's `requires-python` (html-wasm export pins the + Pyodide interpreter). Raises `UvCommandError` on failure and never + mutates `script`. """ ensure_supported_uv() args = [ @@ -135,11 +179,10 @@ def ensure_supported_uv() -> None: def _uv_version() -> str: - """The version string of the invoked uv. + """The invoked uv's version string. - Probed per call: the probe is cheap next to any synchronization, and - caching would pin a stale answer across `uv self update` or a changed - `UV` environment variable. + Probed per call; caching would pin a stale answer across + `uv self update` or a changed `UV` environment variable. """ # "uv 0.7.21 (a1b2c3d4 2025-01-01)" stdout = uv(["--version"]).stdout.strip() @@ -208,9 +251,8 @@ def _parse_report(stdout: str) -> Environment: def _venv_python(root: str) -> str | None: """The environment's interpreter, preferring the unversioned name. - Symlinks are deliberately not resolved: bin/python commonly links to - the base interpreter, and resolving it would launch outside the - environment. + Symlinks are not resolved: bin/python commonly links to the base + interpreter, and resolving it would launch outside the environment. """ bin_dir = _venv_bin_dir(root) candidates: tuple[str, ...] diff --git a/marimo/_environments/script_metadata.py b/marimo/_environments/script_metadata.py index fe5356c9f06..a855fe150ac 100644 --- a/marimo/_environments/script_metadata.py +++ b/marimo/_environments/script_metadata.py @@ -277,23 +277,52 @@ def _sweep_carriers(directory: str, path: str) -> None: @contextlib.contextmanager -def _carrier(notebook: str, content: str) -> Iterator[str]: +def _carrier( + notebook: str, content: str, *, stable: bool = False +) -> Iterator[str]: """A sidecar script uv can operate on, next to the notebook. uv anchors relative paths in script metadata to the script's own - directory, so a manifest that lives in frontmatter must be - materialized beside its notebook before uv can act on it. The - versioned, deterministic name (`.marimo-v-..py`) - marks the file as marimo's: safe to overwrite, sweep, and ignore. - Entry sweeps carriers stranded by killed processes; exit always - removes the carrier, best effort. + directory and keys a script environment on the script's absolute + path, so a manifest that lives in frontmatter must be materialized + beside its notebook before uv can act on it. The versioned, + deterministic name (`.marimo-v-[.].py`) marks the + file as marimo's: safe to overwrite, sweep, and ignore. A stable + carrier maps one notebook to one environment across sessions; a + unique one keeps concurrent edits apart. Entry sweeps carriers + stranded by killed processes; exit always removes the carrier, best + effort. """ absolute = os.path.abspath(notebook) directory = os.path.dirname(absolute) _sweep_carriers(directory, absolute) - descriptor, target = tempfile.mkstemp( - dir=directory, prefix=f"{_carrier_prefix(absolute)}.", suffix=".py" - ) + try: + if stable: + target = os.path.join(directory, f"{_carrier_prefix(absolute)}.py") + descriptor = os.open( + target, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 + ) + else: + descriptor, target = tempfile.mkstemp( + dir=directory, + prefix=f"{_carrier_prefix(absolute)}.", + suffix=".py", + ) + except OSError: + # A read-only notebook directory cannot hold the carrier. Fall + # back to a deterministic path in the temp directory; the + # environment stays stable, but relative paths in the metadata + # and directory-scoped uv configuration no longer resolve + # against the notebook's directory. + target = _fallback_carrier_path(absolute) + LOGGER.warning( + "Notebook directory is not writable; materializing the " + "manifest at %s", + target, + ) + descriptor = os.open( + target, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600 + ) try: with os.fdopen(descriptor, "w", encoding="utf-8") as f: f.write(content) @@ -305,6 +334,21 @@ def _carrier(notebook: str, content: str) -> Iterator[str]: LOGGER.debug("Could not remove carrier %s", target) +def _fallback_carrier_path(absolute: str) -> str: + """A deterministic carrier path outside the notebook's directory. + + uv keys a script environment on the script's absolute path, so the + fallback must be a pure function of the notebook's path. + """ + import hashlib + + digest = hashlib.sha256(absolute.encode("utf-8")).hexdigest()[:16] + stem = re.sub(r"[^A-Za-z0-9._-]+", "-", Path(absolute).stem)[:64] + return os.path.join( + tempfile.gettempdir(), f".marimo-v1-{stem}-{digest}.py" + ) + + def _edit_frontmatter(path: str, edit: Callable[[str, str], None]) -> None: """Edits metadata carried in markdown frontmatter. @@ -338,3 +382,37 @@ def _edit_frontmatter(path: str, edit: Callable[[str, str], None]) -> None: document = ["---", header.strip(), "---", front.body] with open(path, "w", encoding="utf-8") as f: f.write("\n".join(document)) + + +@dataclass(frozen=True) +class MaterializedScript: + """A script uv can operate on and the directory to run uv from.""" + + path: str + directory: str + + +@contextlib.contextmanager +def materialized_for_environment(path: str) -> Iterator[MaterializedScript]: + """Materializes a notebook's manifest for environment operations. + + A Python notebook is its own script, so its environment is the one + `uv run notebook.py` uses and nothing is created. A markdown or + Quarto notebook writes its header verbatim to the stable carrier + `.marimo-v-.py` next to the notebook, deleted on exit. uv + keys a script environment on the script's absolute path, so the + stable name maps one notebook to one environment, reconciled in + place across sessions. Concurrent writers produce identical content. + """ + absolute = os.path.abspath(path) + directory = os.path.dirname(absolute) + if not path.endswith((".md", ".qmd")): + yield MaterializedScript(path=absolute, directory=directory) + return + + content = ( + "# Generated by marimo; safe to delete.\n" + + _read_frontmatter(absolute).header + ) + with _carrier(absolute, content, stable=True) as target: + yield MaterializedScript(path=target, directory=directory) diff --git a/tests/_environments/test_environment.py b/tests/_environments/test_environment.py index d21b2625baf..ce59f2fe30b 100644 --- a/tests/_environments/test_environment.py +++ b/tests/_environments/test_environment.py @@ -15,6 +15,7 @@ Environment, UvUnsupportedVersionError, ensure_supported_uv, + launch, sync, ) from marimo._environments.uv import ( @@ -267,3 +268,54 @@ class Completed: ) assert sync("nb.py").action == action + + +def test_launch_without_overlay_is_direct() -> None: + env = Environment(python="/env/bin/python", root="/env", action="created") + + plan = launch(env, ["-m", "marimo"], base_env={"PATH": "/usr/bin"}) + + assert plan.argv == ("/env/bin/python", "-m", "marimo") + assert plan.env["VIRTUAL_ENV"] == "/env" + + +@pytest.mark.network +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +def test_launch_overlay_chains_without_mutating( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Overlay packages import alongside manifest packages, and neither + the script environment nor the manifest records them.""" + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + pkg = tmp_path / "localpkg" + (pkg / "localpkg").mkdir(parents=True) + (pkg / "localpkg" / "__init__.py").write_text("value = 1\n") + (pkg / "pyproject.toml").write_text( + '[project]\nname = "localpkg"\nversion = "0.1.0"\n' + ) + script = tmp_path / "nb.py" + script.write_text( + "# /// script\n" + f'# requires-python = "{REQUIRES_PYTHON}"\n' + '# dependencies = ["six"]\n' + "# ///\n", + encoding="utf-8", + ) + + env = sync(str(script), cwd=str(tmp_path)) + plan = launch( + env, + ["-c", "import six, idna, localpkg; print('chained')"], + overlay=["idna", f"-e {pkg}"], + ) + + result = subprocess.run( + list(plan.argv), env=plan.env, capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr + assert "chained" in result.stdout + # The script environment itself has only the manifest's packages. + entries = _site_packages(env.root) + assert any(entry.startswith("six") for entry in entries) + assert not any(entry.startswith("idna") for entry in entries) + assert "idna" not in script.read_text() diff --git a/tests/_environments/test_script_metadata.py b/tests/_environments/test_script_metadata.py index 2d2ad37c666..b4350d6ac3c 100644 --- a/tests/_environments/test_script_metadata.py +++ b/tests/_environments/test_script_metadata.py @@ -3,16 +3,14 @@ import platform import subprocess -from typing import TYPE_CHECKING +import sys +from pathlib import Path import pytest from marimo._environments import script_metadata from marimo._environments.uv import UvNotFoundError, is_uv_available -if TYPE_CHECKING: - from pathlib import Path - HAS_UV = is_uv_available() BLOCK_WITH_TOOL_TABLES = """\ @@ -290,3 +288,104 @@ def test_edits_accept_relative_notebook_paths( project = script_metadata.loads((notebooks / "nb.py").read_text()) assert project is not None assert project["dependencies"] == [] + + +def test_materialize_python_notebook_is_itself(tmp_path: Path) -> None: + script = tmp_path / "nb.py" + script.write_text("# /// script\n# dependencies = []\n# ///\n") + + with script_metadata.materialized_for_environment( + str(script) + ) as materialized: + assert materialized.path == str(script) + assert materialized.directory == str(tmp_path) + assert script.exists() + + +def test_materialize_markdown_carrier_is_adjacent_and_stable( + tmp_path: Path, +) -> None: + """The carrier sits next to the notebook under a versioned, + deterministic name, carries the header verbatim, and is deleted on + exit.""" + notebook = tmp_path / "notebook.md" + notebook.write_text( + """--- +pyproject: | + dependencies = [] + + [tool.uv.sources] + mylib = { path = "./lib" } +--- + +# Hello +""" + ) + + with script_metadata.materialized_for_environment(str(notebook)) as first: + assert first.directory == str(tmp_path) + carrier = Path(first.path) + assert carrier.parent == tmp_path + assert carrier.name == ".marimo-v1-notebook.md.py" + # The header is verbatim: relative paths are uv's to anchor. + assert 'path = "./lib"' in carrier.read_text() + assert not carrier.exists() + + with script_metadata.materialized_for_environment(str(notebook)) as second: + assert second.path == first.path + + +def test_stranded_carriers_are_swept(tmp_path: Path) -> None: + """A stray from a killed process is removed on the next operation; + a fresh carrier (a concurrent process's) is spared.""" + import os as _os + import time as _time + + notebook = tmp_path / "notebook.md" + notebook.write_text( + "---\npyproject: |\n dependencies = []\n---\n\n# Hi\n" + ) + stale = tmp_path / ".marimo-v1-notebook.md.stranded.py" + stale.write_text("# stray\n") + old = _time.time() - 3600 + _os.utime(stale, (old, old)) + fresh = tmp_path / ".marimo-v1-notebook.md.inflight.py" + fresh.write_text("# in flight\n") + + with script_metadata.materialized_for_environment(str(notebook)): + pass + + assert not stale.exists() + assert fresh.exists() + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows ignores POSIX directory permissions", +) +def test_materialize_falls_back_when_directory_is_read_only( + tmp_path: Path, +) -> None: + """A read-only notebook directory materializes the carrier at a + deterministic temp path instead of failing.""" + import os as _os + import stat as _stat + + notebook = tmp_path / "notebook.md" + notebook.write_text( + "---\npyproject: |\n dependencies = []\n---\n\n# Hi\n" + ) + _os.chmod(tmp_path, _stat.S_IRUSR | _stat.S_IXUSR) + try: + with script_metadata.materialized_for_environment( + str(notebook) + ) as first: + assert Path(first.path).parent != tmp_path + assert Path(first.path).exists() + fallback = first.path + with script_metadata.materialized_for_environment( + str(notebook) + ) as second: + assert second.path == fallback + finally: + _os.chmod(tmp_path, 0o700) From 2620649d7efa91ce5ef98154c1739a9816a70c1e Mon Sep 17 00:00:00 2001 From: Trevor Manz Date: Thu, 27 Aug 2026 16:18:43 -0400 Subject: [PATCH 3/9] Launch sandboxed marimo from the script environment Single-file sandbox mode wraps the whole process in a `uv run` command built from flattened requirements: `uv export --script` output, re-resolved through bare index flags. The flags cannot express index names, default ordering, explicit indexes, or source pins, so scripts that resolve under `uv run` fail to open in the sandbox. Every entry point also discards the inner process's exit code. These changes launch the sandbox from the notebook's script environment instead. The manifest is synchronized once, with uv's full semantics, and marimo rides the runtime overlay pinned to the running version. ```py with script_metadata.materialized_for_environment(name) as target: handle = environment.sync(target.path, cwd=target.directory) plan = environment.launch(handle, cmd, overlay=overlay, base_env=env) ``` A target without a metadata block, such as a new notebook, runs in an ephemeral environment (`uv run --isolated`), so packages installed during the session die with it. Exit codes propagate at every entry point. `construct_uv_command` and `construct_uv_flags` remain as deprecated shims for the quarto plugin. Multi-file sandbox mode keeps its existing path and moves in the next change. Closes #10547 --- marimo/_cli/cli.py | 17 ++-- marimo/_cli/export/commands.py | 34 ++++--- marimo/_cli/export/session.py | 3 +- marimo/_cli/export/thumbnail.py | 18 ++-- marimo/_cli/sandbox.py | 135 ++++++++++++++++++++++------ marimo/_environments/environment.py | 54 +++++++++-- tests/_cli/test_cli.py | 2 + tests/_cli/test_cli_export.py | 4 +- tests/_cli/test_sandbox.py | 115 ++++++++++++++++++++++++ 9 files changed, 314 insertions(+), 68 deletions(-) diff --git a/marimo/_cli/cli.py b/marimo/_cli/cli.py index 0b7fff5b339..e5d3e74512a 100644 --- a/marimo/_cli/cli.py +++ b/marimo/_cli/cli.py @@ -540,8 +540,11 @@ def edit( if sandbox_mode is SandboxMode.SINGLE: from marimo._cli.sandbox import run_in_sandbox - run_in_sandbox(sys.argv[1:], name=name, additional_features=["lsp"]) - return + sys.exit( + run_in_sandbox( + sys.argv[1:], name=name, additional_features=["lsp"] + ) + ) # Multi-file sandbox: use IPC kernels with per-notebook sandboxed venvs if sandbox_mode is SandboxMode.MULTI: @@ -747,8 +750,11 @@ def new( from marimo._cli.sandbox import run_in_sandbox # TODO: consider adding recommended as well - run_in_sandbox(sys.argv[1:], name=None, additional_features=["lsp"]) - return + sys.exit( + run_in_sandbox( + sys.argv[1:], name=None, additional_features=["lsp"] + ) + ) workspace: NotebookWorkspace | None = None @@ -1234,8 +1240,7 @@ def run( sandbox=sandbox, name=validated_paths[0] ) if sandbox_mode is SandboxMode.SINGLE: - run_in_sandbox(sys.argv[1:], name=validated_paths[0]) - return + sys.exit(run_in_sandbox(sys.argv[1:], name=validated_paths[0])) # Multi-file sandbox: use IPC kernels with per-notebook sandboxed venvs if sandbox_mode is SandboxMode.MULTI: diff --git a/marimo/_cli/export/commands.py b/marimo/_cli/export/commands.py index d5cfb4914b5..d3f31762853 100644 --- a/marimo/_cli/export/commands.py +++ b/marimo/_cli/export/commands.py @@ -295,8 +295,7 @@ def html( sandbox = maybe_prompt_run_in_sandbox(name) if sandbox: - run_in_sandbox(sys.argv[1:], name=name) - return + sys.exit(run_in_sandbox(sys.argv[1:], name=name)) cli_args = parse_args(args) @@ -379,8 +378,7 @@ def script( Export a marimo notebook as a flat script, in topological order. """ if sandbox: - run_in_sandbox(sys.argv[1:], name=name) - return + sys.exit(run_in_sandbox(sys.argv[1:], name=name)) def export_callback(file_path: MarimoPath) -> ExportResult: try: @@ -462,8 +460,7 @@ def md( Export a marimo notebook as a code fenced markdown document. """ if sandbox: - run_in_sandbox(sys.argv[1:], name=name) - return + sys.exit(run_in_sandbox(sys.argv[1:], name=name)) filename = str(output) if output is not None else None @@ -570,12 +567,13 @@ def ipynb( sandbox = maybe_prompt_run_in_sandbox(name) if sandbox: - run_in_sandbox( - sys.argv[1:], - name=name, - additional_deps=["nbformat"], + sys.exit( + run_in_sandbox( + sys.argv[1:], + name=name, + additional_deps=["nbformat"], + ) ) - return try: DependencyManager.nbformat.require( @@ -760,12 +758,13 @@ def pdf( export_deps = ["nbformat"] # Adding webpdf extras to sandbox even if `webpdf` is False, since standard PDF export may fall back to it. export_deps.append("nbconvert[webpdf]") - run_in_sandbox( - sys.argv[1:], - name=name, - additional_deps=export_deps, + sys.exit( + run_in_sandbox( + sys.argv[1:], + name=name, + additional_deps=export_deps, + ) ) - return try: DependencyManager.require_many( @@ -1026,8 +1025,7 @@ def html_wasm( sandbox = maybe_prompt_run_in_sandbox(name) if sandbox: - run_in_sandbox(sys.argv[1:], name=name) - return + sys.exit(run_in_sandbox(sys.argv[1:], name=name)) out_dir = output filename = "index.html" diff --git a/marimo/_cli/export/session.py b/marimo/_cli/export/session.py index f0080209d4b..0a09e1dadf7 100644 --- a/marimo/_cli/export/session.py +++ b/marimo/_cli/export/session.py @@ -340,8 +340,7 @@ def session( notebook, force_overwrite=force_overwrite ): return - run_in_sandbox(sys.argv[1:], name=str(name)) - return + sys.exit(run_in_sandbox(sys.argv[1:], name=str(name))) asyncio_run( _export_sessions( diff --git a/marimo/_cli/export/thumbnail.py b/marimo/_cli/export/thumbnail.py index c076554fc68..4f3333a0ed8 100644 --- a/marimo/_cli/export/thumbnail.py +++ b/marimo/_cli/export/thumbnail.py @@ -104,14 +104,16 @@ def _bootstrap_thumbnail_sandbox( ) -> None: from marimo._cli.sandbox import run_in_sandbox - run_in_sandbox( - args, - name=name, - additional_deps=_thumbnail_sandbox_deps, - extra_env={ - _sandbox_bootstrapped_env: "1", - _sandbox_mode_env: sandbox_mode.value, - }, + sys.exit( + run_in_sandbox( + args, + name=name, + additional_deps=_thumbnail_sandbox_deps, + extra_env={ + _sandbox_bootstrapped_env: "1", + _sandbox_mode_env: sandbox_mode.value, + }, + ) ) diff --git a/marimo/_cli/sandbox.py b/marimo/_cli/sandbox.py index 9081c3927dc..f530de64c94 100644 --- a/marimo/_cli/sandbox.py +++ b/marimo/_cli/sandbox.py @@ -19,9 +19,10 @@ from marimo._cli.errors import MarimoCLIMissingDependencyError from marimo._cli.print import bold, echo, green, muted from marimo._config.settings import GLOBAL_SETTINGS -from marimo._environments import script_metadata +from marimo._environments import environment, script_metadata from marimo._environments.uv import ( UvCommandError, + UvError, UvMissingScriptMetadataError, UvNotFoundError, find_uv_bin, @@ -40,7 +41,8 @@ class SandboxMode(Enum): """Sandbox mode for marimo notebooks. - - SINGLE: Single-file sandbox (wraps entire process with uv run) + - SINGLE: Single-file sandbox (the server runs from the notebook's + script environment) - MULTI: Multi-file sandbox (IPC kernels with per-notebook venvs) """ @@ -107,7 +109,8 @@ def resolve_sandbox_mode( Returns: - None: No sandboxing - - SandboxMode.SINGLE: Single-file sandbox (wrap with uv run) + - SandboxMode.SINGLE: Single-file sandbox (server in the script + environment) - SandboxMode.MULTI: Multi-file sandbox (IPC kernels with per-notebook venvs) When sandbox is None, prompts the user if the notebook has sandbox metadata @@ -131,7 +134,7 @@ def resolve_sandbox_mode( # Sandbox enabled - determine mode based on target type # Directory or home page -> multi-file sandbox (IPC kernels) - # Single file -> single-file sandbox (uv run wrapper) + # Single file -> single-file sandbox (server in the script environment) return SandboxMode.MULTI if is_directory else SandboxMode.SINGLE @@ -258,7 +261,9 @@ def construct_uv_flags( additional_deps: list[str], python_version_override: str | None = None, ) -> list[str]: - # NB. Used in quarto plugin + # Deprecated: retained for the quarto plugin. marimo launches + # sandboxes from the script environment instead; the flags built here + # flatten `[[tool.uv.index]]` semantics (#10547). # If name if a filepath, parse the dependencies from the file dependencies = _resolve_requirements_txt_lines(pyproject) @@ -332,6 +337,7 @@ def construct_uv_command( additional_deps: list[str], python_version_override: str | None = None, ) -> list[str]: + """Deprecated: retained for the quarto plugin.""" cmd = ["marimo"] + args if "--sandbox" in cmd: cmd.remove("--sandbox") @@ -364,6 +370,28 @@ def construct_uv_command( return uv_cmd + cmd +def _runtime_overlay( + additional_features: list[DepFeatures], + additional_deps: list[str], +) -> list[str]: + """The requirements marimo layers into a sandbox launch. + + marimo itself rides the overlay, pinned to the running version or as + an editable install from a development checkout, so the inner server + matches the CLI regardless of what the manifest's `marimo` resolves + to. Overlay entries never enter the manifest. + """ + if is_editable("marimo"): + LOGGER.info("Using editable of marimo for sandbox") + marimo_dep = f"-e {get_marimo_dir()}" + elif additional_features: + features = ",".join(additional_features) + marimo_dep = f"marimo[{features}]=={__version__}" + else: + marimo_dep = f"marimo=={__version__}" + return [marimo_dep, *additional_deps] + + def run_in_sandbox( args: list[str], *, @@ -374,14 +402,17 @@ def run_in_sandbox( python_version_override: str | None = None, pyodide_constraints: bool = False, ) -> int: - """Run marimo in a sandboxed uv environment. + """Runs marimo inside the notebook's script environment. - This wraps the marimo command with `uv run` to create an isolated - virtual environment with the notebook's dependencies. + Synchronizes the environment from the notebook's metadata, then + launches `python -m marimo ` from it with marimo layered on + top. uv resolves the metadata with its full semantics, so indexes, + sources, and credentials behave as they do for `uv run notebook.py`, + and the environment is shared with it. A target without a metadata + block runs in an ephemeral environment instead. Used for "single" sandbox mode (marimo edit --sandbox notebook.py). - For "multi" sandbox mode (directory), see IPCKernelManagerImpl which - creates per-notebook sandboxed kernels. + For "multi" sandbox mode (directory), see IPCKernelManagerImpl. """ try: require_uv_bin() @@ -394,7 +425,7 @@ def run_in_sandbox( # Ensure marimo and the python version are in the script metadata before # running. Adding marimo is best-effort: the sandbox still runs without - # it, since the requirements normalization injects marimo. + # it, since the runtime overlay carries marimo. if name is not None and name.endswith(".py"): try: script_metadata.ensure_marimo(name) @@ -404,15 +435,9 @@ def run_in_sandbox( LOGGER.warning(f"Failed to add marimo to script metadata: {e}") script_metadata.ensure_requires_python(name) - uv_cmd = construct_uv_command( - args, - name, - additional_features or [], - additional_deps or [], - python_version_override=python_version_override, - ) - - echo(f"Running in a sandbox: {muted(' '.join(uv_cmd))}", err=True) + cmd = ["-m", "marimo", *args] + if "--sandbox" in cmd: + cmd.remove("--sandbox") env = os.environ.copy() env["MARIMO_MANAGE_SCRIPT_METADATA"] = "true" @@ -436,6 +461,7 @@ def run_in_sandbox( constraint_tmp.close() constraint_path = constraint_tmp.name if write_constraint_file(constraint_path): + # Resolution happens in the child uv process; see below. env["UV_CONSTRAINT"] = constraint_path def cleanup_constraint_file() -> None: @@ -446,20 +472,79 @@ def cleanup_constraint_file() -> None: atexit.register(cleanup_constraint_file) + overlay = _runtime_overlay( + additional_features or [], additional_deps or [] + ) + + # Explicit override > metadata requires-python (uv reads it) > host. + pyproject = ( + PyProjectReader.from_filename(name) + if name is not None + else PyProjectReader({}, config_path=None) + ) + python_request = python_version_override or ( + None if pyproject.python_version else platform.python_version() + ) + + # An interpreter override (html-wasm pins the Pyodide version) must + # not replace the notebook's shared script environment; it resolves + # ephemerally with the notebook's dependencies layered instead. + overridden = python_version_override is not None or pyodide_constraints + + plan: environment.ProcessPlan | None = None + if name is not None and not overridden and os.path.isfile(name): + try: + with script_metadata.materialized_for_environment(name) as target: + handle = environment.sync( + target.path, + cwd=target.directory, + python_override=python_request, + ) + echo( + f"Using script environment: {muted(handle.root)}", + err=True, + ) + plan = environment.launch( + handle, cmd, overlay=overlay, base_env=env + ) + except UvMissingScriptMetadataError: + # No metadata block yet; run ephemerally below. + pass + except UvError as e: + echo(str(e), err=True) + return getattr(e, "returncode", None) or 1 + + if plan is None: + if overridden and name is not None and os.path.isfile(name): + overlay = [ + line + for line in _resolve_requirements_txt_lines(pyproject) + if line.strip() and not is_marimo_dependency(line) + ] + overlay + plan = environment.launch_isolated( + cmd, + overlay=overlay, + python=python_request or platform.python_version(), + base_env=env, + ) + + echo(f"Running in a sandbox: {muted(' '.join(plan.argv))}", err=True) + if sys.platform == "win32": # The console already delivers Ctrl-C to uv and the inner server; # forwarding CTRL_C_EVENT would rebroadcast to the whole console, # including ourselves (#4842). Let the inner server drive shutdown. signal.signal(signal.SIGINT, signal.SIG_IGN) - process = subprocess.Popen(uv_cmd, env=env) + process = subprocess.Popen(plan.argv, env=plan.env) else: - # On Unix, run `uv` in its own session so that (a) the tty no + # On Unix, run the child in its own session so that (a) the tty no # longer delivers SIGINT/SIGTERM to it directly and (b) we can # signal the whole subtree with a single killpg. The signal # handlers below are then the sole path for forwarding signals - # from the CLI down to uv, the inner marimo server, and the - # kernel. - process = subprocess.Popen(uv_cmd, env=env, start_new_session=True) + # from the CLI down to the inner marimo server and the kernel. + process = subprocess.Popen( + plan.argv, env=plan.env, start_new_session=True + ) def handler(sig: int, frame: object) -> None: del frame diff --git a/marimo/_environments/environment.py b/marimo/_environments/environment.py index f7e32afd896..bff7915342d 100644 --- a/marimo/_environments/environment.py +++ b/marimo/_environments/environment.py @@ -111,12 +111,6 @@ def launch( if not overlay: return ProcessPlan(argv=(environment.python, *args), env=env) - with_args: list[str] = [] - for requirement in overlay: - if requirement.startswith("-e "): - with_args.extend(["--with-editable", requirement[3:].strip()]) - else: - with_args.extend(["--with", requirement]) return ProcessPlan( argv=( require_uv_bin(), @@ -127,7 +121,42 @@ def launch( "--no-project", "--python", environment.python, - *with_args, + *_with_args(overlay), + "--", + "python", + *args, + ), + env=env, + ) + + +def launch_isolated( + args: Sequence[str], + *, + overlay: Sequence[str], + python: str, + base_env: Mapping[str, str] | None = None, +) -> ProcessPlan: + """Plans `python ` in an ephemeral environment. + + For targets without a manifest, such as a new notebook. uv resolves + the overlay into a cached environment and runs the process in an + ephemeral copy, so packages installed during the session die with + it and nothing persists per invocation. + """ + env = dict(os.environ if base_env is None else base_env) + env.pop("VIRTUAL_ENV", None) + env.pop("UV_PROJECT_ENVIRONMENT", None) + return ProcessPlan( + argv=( + require_uv_bin(), + "run", + "--isolated", + "--no-project", + "--compile-bytecode", + "--python", + python, + *_with_args(overlay), "--", "python", *args, @@ -136,6 +165,17 @@ def launch( ) +def _with_args(overlay: Sequence[str]) -> list[str]: + """`uv run` flags for overlay requirements; `-e ` is editable.""" + with_args: list[str] = [] + for requirement in overlay: + if requirement.startswith("-e "): + with_args.extend(["--with-editable", requirement[3:].strip()]) + else: + with_args.extend(["--with", requirement]) + return with_args + + def sync( script: str, *, diff --git a/tests/_cli/test_cli.py b/tests/_cli/test_cli.py index 58166b3ea65..99c4a287d0d 100644 --- a/tests/_cli/test_cli.py +++ b/tests/_cli/test_cli.py @@ -1388,6 +1388,7 @@ def test_cli_sandbox_edit_new_file() -> None: with patch( "marimo._cli.sandbox.run_in_sandbox" ) as mock_run_in_sandbox: + mock_run_in_sandbox.return_value = 0 result = runner.invoke( cli_main, ["edit", path, "--headless", "--no-token", "--sandbox"], @@ -1921,6 +1922,7 @@ def assert_custom_config(contents: bytes | None) -> None: with patch( "marimo._cli.sandbox.run_in_sandbox" ) as mock_run_in_sandbox: + mock_run_in_sandbox.return_value = 0 result = runner.invoke( cli_main, ["new", "--sandbox", "--headless", "--no-token"], diff --git a/tests/_cli/test_cli_export.py b/tests/_cli/test_cli_export.py index 356e67aace6..9c6d547bb6a 100644 --- a/tests/_cli/test_cli_export.py +++ b/tests/_cli/test_cli_export.py @@ -662,7 +662,7 @@ def test_cli_export_html_sandbox(temp_marimo_file: str) -> None: output = p.stderr.decode() # Check for sandbox message assert "Running in a sandbox" in output - assert "run --isolated" in output + assert "Using script environment" in output html = normalize_index_html(output) html = _normalize_html_path(html, temp_marimo_file) assert '' not in html @@ -1295,7 +1295,7 @@ def test_cli_export_ipynb_sandbox(temp_marimo_file: str) -> None: output = p.stderr.decode() # Check for sandbox message assert "Running in a sandbox" in output - assert "run --isolated" in output + assert "Using script environment" in output @staticmethod @pytest.mark.skipif( diff --git a/tests/_cli/test_sandbox.py b/tests/_cli/test_sandbox.py index 0e35880dfc7..a8b201a4cda 100644 --- a/tests/_cli/test_sandbox.py +++ b/tests/_cli/test_sandbox.py @@ -727,3 +727,118 @@ def test_python_version_override_without_metadata(tmp_path: Path) -> None: ) python_idx = uv_cmd.index("--python") assert uv_cmd[python_idx + 1] == "3.12" + + +def _supports_sync() -> bool: + from marimo._environments.environment import ensure_supported_uv + from marimo._environments.uv import UvError, is_uv_available + + if not is_uv_available(): + return False + try: + ensure_supported_uv() + except UvError: + return False + return True + + +SUPPORTS_SYNC = _supports_sync() + + +@pytest.fixture +def _restore_signal_handlers(): + """run_in_sandbox installs forwarding handlers; undo them.""" + import signal + + saved = { + sig: signal.getsignal(sig) + for name in ("SIGINT", "SIGTERM", "SIGHUP") + if (sig := getattr(signal, name, None)) is not None + } + yield + for sig, handler in saved.items(): + signal.signal(sig, handler) + + +@pytest.mark.network +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +@pytest.mark.skipif( + os.name == "nt", reason="signal forwarding differs on Windows" +) +@pytest.mark.usefixtures("_restore_signal_handlers") +def test_run_in_sandbox_from_script_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The provisioned path: a markdown notebook's manifest is + synchronized and marimo launches from the script environment.""" + from marimo._cli.sandbox import run_in_sandbox + + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path.parent / "uv-cache")) + + notebook = tmp_path / "notebook.md" + notebook.write_text( + """--- +pyproject: | + dependencies = [] +--- + +# Hello +""", + encoding="utf-8", + ) + + code = run_in_sandbox(["--version"], name=str(notebook)) + + assert code == 0 + # The carrier is deleted after synchronization. + assert sorted(p.name for p in tmp_path.iterdir()) == ["notebook.md"] + + +@pytest.mark.network +@pytest.mark.skipif(not SUPPORTS_SYNC, reason="uv >= 0.7.21 required") +@pytest.mark.skipif( + os.name == "nt", reason="signal forwarding differs on Windows" +) +@pytest.mark.usefixtures("_restore_signal_handlers") +def test_run_in_sandbox_without_a_manifest() -> None: + """No target means no manifest: marimo runs ephemerally.""" + from marimo._cli.sandbox import run_in_sandbox + + code = run_in_sandbox(["--version"], name=None) + + assert code == 0 + + +def test_sandbox_exit_codes_propagate(tmp_path: Path) -> None: + """Every sandbox entry point exits with the inner process's code.""" + from unittest.mock import patch as mock_patch + + from click.testing import CliRunner + + from marimo._cli.cli import main as cli_main + + notebook = tmp_path / "nb.py" + notebook.write_text( + '# /// script\n# dependencies = ["numpy"]\n# ///\n', encoding="utf-8" + ) + runner = CliRunner() + + for command, target in ( + ( + ["edit", str(notebook), "--sandbox", "--headless", "--no-token"], + "marimo._cli.sandbox.run_in_sandbox", + ), + ( + ["export", "html", str(notebook), "--sandbox"], + "marimo._cli.export.commands.run_in_sandbox", + ), + ): + with ( + mock_patch(target, return_value=3), + mock_patch( + "marimo._cli.sandbox.maybe_prompt_run_in_sandbox", + return_value=True, + ), + ): + result = runner.invoke(cli_main, command) + assert result.exit_code == 3, (command, result.output) From 3cfc5eca06faab24bfcfd9bf7e75e25cc36e5fef Mon Sep 17 00:00:00 2001 From: Trevor Manz Date: Thu, 27 Aug 2026 16:41:12 -0400 Subject: [PATCH 4/9] Launch IPC kernels from script environments Multi-file sandbox mode builds a fresh venv per notebook per session with `uv venv --seed` and `uv pip install -r`, from the same flattened requirements as single-file mode but with no index configuration at all. The venv is rebuilt on every session, editable-install failures reduce to warnings, and pinned kernel dependencies can conflict with the notebook's own pins in one requirements file. These changes provision IPC kernels, run-mode app hosts, and export runners from the notebook's script environment. The environment is synchronized once with uv's full semantics, reused across sessions, and the kernel launches from a plan that layers marimo and its IPC dependencies without recording them in the manifest. ```py handle = sync_notebook(filename) plan = launch(handle, args, overlay=runtime_overlay( additional_deps=get_ipc_kernel_deps() )) ``` The overlay policy moves to `marimo._environments.overlay` so session code does not reach into the CLI. A notebook without a metadata block runs in an ephemeral environment, and configured `[tool.marimo.venv]` environments keep their existing path. `build_sandbox_venv`, `get_sandbox_requirements`, and `cleanup_sandbox_dir` are gone; uv owns environment lifecycle and caching. --- marimo/_cli/export/_common.py | 68 ++++++--- marimo/_cli/export/session.py | 17 ++- marimo/_cli/export/thumbnail.py | 17 ++- marimo/_cli/sandbox.py | 180 +--------------------- marimo/_environments/environment.py | 21 +++ marimo/_environments/overlay.py | 42 ++++++ marimo/_ipc/launch_kernel.py | 6 + marimo/_runtime/runtime.py | 4 +- marimo/_session/app_host/host.py | 33 ++-- marimo/_session/app_host/main.py | 4 +- marimo/_session/app_host/pool.py | 40 +++-- marimo/_session/managers/ipc.py | 184 +++++++++++++++++------ marimo/_utils/subprocess.py | 16 +- tests/_cli/test_cli_export_session.py | 8 +- tests/_cli/test_sandbox.py | 124 --------------- tests/_session/app_host/test_app_host.py | 158 +++++++++---------- tests/_session/managers/test_ipc.py | 52 +++++++ 17 files changed, 470 insertions(+), 504 deletions(-) create mode 100644 marimo/_environments/overlay.py diff --git a/marimo/_cli/export/_common.py b/marimo/_cli/export/_common.py index 30c578ab271..eead718ffa1 100644 --- a/marimo/_cli/export/_common.py +++ b/marimo/_cli/export/_common.py @@ -3,6 +3,7 @@ import json import subprocess +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -16,6 +17,8 @@ if TYPE_CHECKING: from collections.abc import Iterable + from marimo._environments.environment import Environment + def is_multi_target(paths: list[Path]) -> bool: return len(paths) > 1 or any(path.is_dir() for path in paths) @@ -45,42 +48,67 @@ def collect_notebooks(paths: Iterable[Path]) -> list[MarimoPath]: return [notebooks[k] for k in sorted(notebooks)] +@dataclass(frozen=True) +class SandboxTarget: + """Where a sandboxed export runs. + + `environment` is the notebook's script environment, or None for a + notebook without a metadata block, which runs ephemerally. + """ + + environment: Environment | None + + class SandboxVenvPool: + """Caches synchronized script environments by notebook path.""" + def __init__(self) -> None: - self._envs: dict[tuple[str, ...], tuple[str, str]] = {} + self._targets: dict[str, SandboxTarget] = {} - def get_python(self, notebook_path: str) -> str: - from marimo._cli.sandbox import ( - build_sandbox_venv, - get_sandbox_requirements, - ) + def get_target(self, notebook_path: str) -> SandboxTarget: + from marimo._environments.environment import sync_notebook + from marimo._environments.uv import UvMissingScriptMetadataError - requirements = tuple(get_sandbox_requirements(notebook_path)) - existing = self._envs.get(requirements) + key = str(Path(notebook_path).resolve()) + existing = self._targets.get(key) if existing is not None: - return existing[1] + return existing - sandbox_dir, venv_python = build_sandbox_venv(notebook_path) - self._envs[requirements] = (sandbox_dir, venv_python) - return venv_python + try: + target = SandboxTarget(environment=sync_notebook(key)) + except UvMissingScriptMetadataError: + target = SandboxTarget(environment=None) + self._targets[key] = target + return target def close(self) -> None: - from marimo._cli.sandbox import cleanup_sandbox_dir - - for sandbox_dir, _ in self._envs.values(): - cleanup_sandbox_dir(sandbox_dir) - self._envs.clear() + # uv owns the environments; there is nothing to remove. + self._targets.clear() def run_python_subprocess( *, - venv_python: str, + sandbox: SandboxTarget, script: str, payload: dict[str, Any], action: str, ) -> str: + import platform + + from marimo._environments.environment import launch, launch_isolated + from marimo._environments.overlay import runtime_overlay + + args = ["-c", script, json.dumps(payload)] + overlay = runtime_overlay() + if sandbox.environment is not None: + plan = launch(sandbox.environment, args, overlay=overlay) + else: + plan = launch_isolated( + args, overlay=overlay, python=platform.python_version() + ) result = subprocess.run( - [venv_python, "-c", script, json.dumps(payload)], + list(plan.argv), + env=plan.env, check=False, capture_output=True, text=True, @@ -89,7 +117,7 @@ def run_python_subprocess( stderr = result.stderr.strip() raise click.ClickException( f"Failed to {action} in sandbox.\n\n" - f"Command:\n\n {venv_python} -c