From e1076f2e7521cb47cd46a15c84475b15683e4296 Mon Sep 17 00:00:00 2001 From: Hemingway_G <1145205269@qq.com> Date: Sun, 5 Jul 2026 17:34:23 +0800 Subject: [PATCH] Add opt-in Windows SWE-bench harness compatibility --- miniswerouter/cli.py | 6 + miniswerouter/harness/run_eval.py | 2 + miniswerouter/harness/run_instance.py | 6 +- swerouter/cli.py | 6 + swerouter/harness/container_runner.py | 151 +++++++++++++-- swerouter/harness/run_eval.py | 2 + swerouter/harness/run_instance.py | 6 +- tests/test_windows_compat_harness.py | 252 ++++++++++++++++++++++++++ 8 files changed, 416 insertions(+), 15 deletions(-) create mode 100644 tests/test_windows_compat_harness.py diff --git a/miniswerouter/cli.py b/miniswerouter/cli.py index 04d18d1..91c7b5b 100644 --- a/miniswerouter/cli.py +++ b/miniswerouter/cli.py @@ -231,6 +231,7 @@ def _cmd_run(args: argparse.Namespace) -> int: run_id=args.run_id, force_rerun=args.force_rerun, rm_image=args.rm_image, + windows_compat=args.windows_compat, pool_path=Path(args.pool) if args.pool else DEFAULT_POOL, pricing_path=Path(args.pricing) if args.pricing else DEFAULT_PRICING, ttl_path=Path(args.ttl) if args.ttl else DEFAULT_TTL, @@ -459,6 +460,11 @@ def _build_parser() -> argparse.ArgumentParser: run.add_argument("--run-id", default="miniswerouter_default") run.add_argument("--force-rerun", action="store_true") run.add_argument("--rm-image", action="store_true") + run.add_argument( + "--windows-compat", + action="store_true", + help="Opt in to Windows-local SWE-bench compatibility shims. Linux/default behavior is unchanged when unset.", + ) run.add_argument( "--pool", default=None, diff --git a/miniswerouter/harness/run_eval.py b/miniswerouter/harness/run_eval.py index a3cee8e..a323055 100644 --- a/miniswerouter/harness/run_eval.py +++ b/miniswerouter/harness/run_eval.py @@ -69,6 +69,7 @@ class EvalRequest: force_rerun: bool = False rm_image: bool = False image_namespace: str | None = "swebench" + windows_compat: bool = False @dataclass @@ -232,6 +233,7 @@ def _execute_one(instance_id: str) -> InstanceResult: run_id=request.run_id, rm_image=request.rm_image, image_namespace=request.image_namespace, + windows_compat=request.windows_compat, ) return run_instance(req) diff --git a/miniswerouter/harness/run_instance.py b/miniswerouter/harness/run_instance.py index 64d2734..239037d 100644 --- a/miniswerouter/harness/run_instance.py +++ b/miniswerouter/harness/run_instance.py @@ -263,6 +263,7 @@ class RunInstanceRequest: force_rebuild: bool = False rm_image: bool = False image_namespace: str | None = DEFAULT_IMAGE_NAMESPACE + windows_compat: bool = False # Extra litellm model_kwargs forwarded to every pool member's LitellmModel. # Callers typically leave this empty; mini's canonical defaults # (``drop_params=True, temperature=0.0, parallel_tool_calls=True``) are @@ -354,7 +355,9 @@ def run_instance(request: RunInstanceRequest) -> InstanceResult: dataset_split=request.dataset_split, ) test_spec = make_test_spec_for_instance( - instance, image_namespace=request.image_namespace + instance, + image_namespace=request.image_namespace, + windows_compat=request.windows_compat, ) handle = SwebenchContainerHandle( @@ -428,6 +431,7 @@ def run_instance(request: RunInstanceRequest) -> InstanceResult: timeout_sec=request.eval_timeout_sec, rm_image=request.rm_image, model_name=MINI_EVAL_MODEL_NAME, + windows_compat=request.windows_compat, ) if agent_error is not None: diff --git a/swerouter/cli.py b/swerouter/cli.py index 3ab2dff..fb12783 100644 --- a/swerouter/cli.py +++ b/swerouter/cli.py @@ -216,6 +216,7 @@ def _cmd_run(args: argparse.Namespace) -> int: run_id=args.run_id, force_rerun=args.force_rerun, rm_image=args.rm_image, + windows_compat=args.windows_compat, ) summary = run_eval(req, router_label=args.router_label) print( @@ -423,6 +424,11 @@ def _build_parser() -> argparse.ArgumentParser: run.add_argument("--run-id", default="swerouter_default") run.add_argument("--force-rerun", action="store_true") run.add_argument("--rm-image", action="store_true") + run.add_argument( + "--windows-compat", + action="store_true", + help="Opt in to Windows-local SWE-bench compatibility shims. Linux/default behavior is unchanged when unset.", + ) run.set_defaults(func=_cmd_run) score = sub.add_parser("score", help="Score an existing run directory.") diff --git a/swerouter/harness/container_runner.py b/swerouter/harness/container_runner.py index 5d5afc2..d95428d 100644 --- a/swerouter/harness/container_runner.py +++ b/swerouter/harness/container_runner.py @@ -69,14 +69,33 @@ def make_test_spec_for_instance( instance: dict[str, Any], *, image_namespace: str | None = DEFAULT_IMAGE_NAMESPACE, + windows_compat: bool = False, ) -> Any: """Build the upstream ``TestSpec`` used by the container builder and the official evaluator. ``image_namespace`` pulls pre-built images from Docker Hub when set (saves the multi-hour local rebuild path).""" - from swebench.harness.test_spec.test_spec import make_test_spec + if not windows_compat: + from swebench.harness.test_spec.test_spec import make_test_spec - return make_test_spec(instance, namespace=image_namespace) + return make_test_spec(instance, namespace=image_namespace) + + import inspect + + try: + from swebench.harness.test_spec.test_spec import make_test_spec + except ModuleNotFoundError as exc: + if exc.name != "swebench.harness.test_spec.test_spec": + raise + from swebench.harness.test_spec import make_test_spec + + make_test_spec_params = inspect.signature(make_test_spec).parameters + if ( + "namespace" in make_test_spec_params + or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in make_test_spec_params.values()) + ): + return make_test_spec(instance, namespace=image_namespace) + return make_test_spec(instance) # --------------------------------------------------------------------------- @@ -307,6 +326,53 @@ def _ingest_report_json(report_path: Path, instance_id: str) -> EvalReport: ) +def _copy_to_container_posix(container: Any, src: Path, dst: Any) -> None: + """Copy ``src`` to a POSIX path inside ``container``. + + SWE-bench may pass ``Path("/eval.sh")`` to its copy helper. On Windows + that can become ``WindowsPath("\\eval.sh")`` before reaching the Linux + container. This shim is used only when Windows compatibility mode is + explicitly enabled. + """ + + import io + import posixpath + import shlex + import tarfile + + src_path = Path(src) + dst_path = str(dst).replace("\\", "/") + parent = posixpath.dirname(dst_path) + basename = posixpath.basename(dst_path) + if not dst_path.startswith("/") or not parent or not basename: + raise ValueError(f"destination path must be an absolute file path: {dst!r}") + + if parent != "/": + mkdir_cmd = f"mkdir -p {shlex.quote(parent)}" + res = container.exec_run(["/bin/sh", "-lc", mkdir_cmd]) + if int(getattr(res, "exit_code", 1) or 0) != 0: + out = getattr(res, "output", b"") + if isinstance(out, bytes): + out = out.decode("utf-8", errors="replace") + raise RuntimeError(f"failed to create container directory {parent}: {out}") + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + if basename.endswith(".sh"): + data = src_path.read_bytes().replace(b"\r\n", b"\n").replace(b"\r", b"\n") + info = tarfile.TarInfo(name=basename) + stat_result = src_path.stat() + info.size = len(data) + info.mode = stat_result.st_mode + info.mtime = stat_result.st_mtime + tar.addfile(info, io.BytesIO(data)) + else: + tar.add(src_path, arcname=basename) + buf.seek(0) + if not container.put_archive(parent, buf.read()): + raise RuntimeError(f"failed to copy {src_path} to container path {dst_path}") + + def run_upstream_eval( *, test_spec: Any, @@ -317,6 +383,7 @@ def run_upstream_eval( timeout_sec: int = 1800, rm_image: bool = False, model_name: str = DEFAULT_EVAL_MODEL_NAME, + windows_compat: bool = False, ) -> EvalReport: """Grade ``patch_text`` against ``instance_id`` using upstream's pipeline. @@ -330,7 +397,6 @@ def run_upstream_eval( """ from swebench.harness.constants import LOG_REPORT, RUN_EVALUATION_LOG_DIR - from swebench.harness.run_evaluation import run_instance as upstream_run_instance if not patch_text.strip(): return EvalReport( @@ -353,16 +419,30 @@ def run_upstream_eval( Path(RUN_EVALUATION_LOG_DIR) / run_id / model_name / instance_id / LOG_REPORT ) try: - upstream_run_instance( - test_spec=test_spec, - pred=pred, - rm_image=rm_image, - force_rebuild=False, - client=client, - run_id=run_id, - timeout=timeout_sec, - rewrite_reports=False, - ) + if windows_compat: + _run_upstream_eval_windows_compat( + test_spec=test_spec, + pred=pred, + rm_image=rm_image, + client=client, + run_id=run_id, + timeout_sec=timeout_sec, + ) + else: + from swebench.harness.run_evaluation import ( + run_instance as upstream_run_instance, + ) + + upstream_run_instance( + test_spec=test_spec, + pred=pred, + rm_image=rm_image, + force_rebuild=False, + client=client, + run_id=run_id, + timeout=timeout_sec, + rewrite_reports=False, + ) except Exception as ex: return EvalReport( resolved=False, @@ -373,6 +453,51 @@ def run_upstream_eval( return _ingest_report_json(report_path, instance_id) +def _run_upstream_eval_windows_compat( + *, + test_spec: Any, + pred: dict[str, str], + rm_image: bool, + client: Any, + run_id: str, + timeout_sec: int, +) -> None: + """Run upstream eval with Windows-only compatibility shims enabled.""" + + import inspect + import swebench.harness.run_evaluation as run_evaluation_module + + upstream_run_instance = run_evaluation_module.run_instance + kwargs: dict[str, Any] = { + "test_spec": test_spec, + "pred": pred, + "rm_image": rm_image, + "force_rebuild": False, + "client": client, + "run_id": run_id, + "timeout": timeout_sec, + } + upstream_params = inspect.signature(upstream_run_instance).parameters + if ( + "rewrite_reports" in upstream_params + or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in upstream_params.values()) + ): + kwargs["rewrite_reports"] = False + + sentinel = object() + original_copy_to_container = getattr( + run_evaluation_module, "copy_to_container", sentinel + ) + run_evaluation_module.copy_to_container = _copy_to_container_posix + try: + upstream_run_instance(**kwargs) + finally: + if original_copy_to_container is sentinel: + delattr(run_evaluation_module, "copy_to_container") + else: + run_evaluation_module.copy_to_container = original_copy_to_container + + __all__ = [ "DEFAULT_DATASET_NAME", "DEFAULT_DATASET_SPLIT", diff --git a/swerouter/harness/run_eval.py b/swerouter/harness/run_eval.py index 95855f9..589da02 100644 --- a/swerouter/harness/run_eval.py +++ b/swerouter/harness/run_eval.py @@ -60,6 +60,7 @@ class EvalRequest: force_rerun: bool = False rm_image: bool = False image_namespace: str | None = "swebench" + windows_compat: bool = False @dataclass @@ -217,6 +218,7 @@ def _execute_one(instance_id: str) -> InstanceResult: run_id=request.run_id, rm_image=request.rm_image, image_namespace=request.image_namespace, + windows_compat=request.windows_compat, ) return run_instance(req) diff --git a/swerouter/harness/run_instance.py b/swerouter/harness/run_instance.py index 0f53d1b..1fd194c 100644 --- a/swerouter/harness/run_instance.py +++ b/swerouter/harness/run_instance.py @@ -109,6 +109,7 @@ class RunInstanceRequest: # pre-built instance images from Docker Hub under that namespace (the # official registry is ``swebench``). Avoids multi-hour local rebuilds. image_namespace: str | None = DEFAULT_IMAGE_NAMESPACE + windows_compat: bool = False @dataclass @@ -310,7 +311,9 @@ def run_instance(request: RunInstanceRequest) -> InstanceResult: dataset_split=request.dataset_split, ) test_spec = make_test_spec_for_instance( - instance, image_namespace=request.image_namespace + instance, + image_namespace=request.image_namespace, + windows_compat=request.windows_compat, ) llm = LLMClient( @@ -371,6 +374,7 @@ def run_instance(request: RunInstanceRequest) -> InstanceResult: timeout_sec=request.eval_timeout_sec, rm_image=request.rm_image, model_name=DEFAULT_EVAL_MODEL_NAME, + windows_compat=request.windows_compat, ) if agent_summary is not None: diff --git a/tests/test_windows_compat_harness.py b/tests/test_windows_compat_harness.py new file mode 100644 index 0000000..271dd52 --- /dev/null +++ b/tests/test_windows_compat_harness.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import io +import sys +import tarfile +import types +from typing import Any, Callable + +from swerouter.harness import container_runner + + +def _install_eval_modules( + monkeypatch: Any, + tmp_path: Any, + run_instance: Callable[..., None], + *, + copy_to_container: object | None, +) -> types.ModuleType: + swebench = types.ModuleType("swebench") + harness = types.ModuleType("swebench.harness") + constants = types.ModuleType("swebench.harness.constants") + constants.LOG_REPORT = "report.json" + constants.RUN_EVALUATION_LOG_DIR = str(tmp_path) + run_evaluation = types.ModuleType("swebench.harness.run_evaluation") + run_evaluation.run_instance = run_instance + if copy_to_container is not None: + run_evaluation.copy_to_container = copy_to_container + + swebench.harness = harness + harness.constants = constants + harness.run_evaluation = run_evaluation + + monkeypatch.setitem(sys.modules, "swebench", swebench) + monkeypatch.setitem(sys.modules, "swebench.harness", harness) + monkeypatch.setitem(sys.modules, "swebench.harness.constants", constants) + monkeypatch.setitem( + sys.modules, "swebench.harness.run_evaluation", run_evaluation + ) + return run_evaluation + + +def _install_test_spec_modules( + monkeypatch: Any, + make_test_spec: Callable[..., object], + *, + nested_module: bool, +) -> None: + swebench = types.ModuleType("swebench") + harness = types.ModuleType("swebench.harness") + test_spec_pkg = types.ModuleType("swebench.harness.test_spec") + test_spec_pkg.__path__ = [] + + swebench.harness = harness + harness.test_spec = test_spec_pkg + monkeypatch.setitem(sys.modules, "swebench", swebench) + monkeypatch.setitem(sys.modules, "swebench.harness", harness) + monkeypatch.setitem(sys.modules, "swebench.harness.test_spec", test_spec_pkg) + + if nested_module: + test_spec_mod = types.ModuleType("swebench.harness.test_spec.test_spec") + test_spec_mod.make_test_spec = make_test_spec + test_spec_pkg.test_spec = test_spec_mod + monkeypatch.setitem( + sys.modules, "swebench.harness.test_spec.test_spec", test_spec_mod + ) + else: + test_spec_pkg.make_test_spec = make_test_spec + monkeypatch.delitem( + sys.modules, "swebench.harness.test_spec.test_spec", raising=False + ) + + +def test_default_eval_path_does_not_patch_copy_to_container(monkeypatch: Any, tmp_path: Any) -> None: + calls: list[dict[str, Any]] = [] + original_copy = object() + module_box: dict[str, types.ModuleType] = {} + + def run_instance(**kwargs: Any) -> None: + assert module_box["module"].copy_to_container is original_copy + calls.append(kwargs) + + module_box["module"] = _install_eval_modules( + monkeypatch, + tmp_path, + run_instance, + copy_to_container=original_copy, + ) + + report = container_runner.run_upstream_eval( + test_spec=object(), + instance_id="example__repo-1", + patch_text="diff --git a/file.py b/file.py\n", + run_id="default_eval", + client=object(), + ) + + assert calls + assert calls[0]["rewrite_reports"] is False + assert module_box["module"].copy_to_container is original_copy + assert report.error and "eval report not produced" in report.error + + +def test_windows_eval_path_patches_and_restores_copy_to_container(monkeypatch: Any, tmp_path: Any) -> None: + calls: list[dict[str, Any]] = [] + original_copy = object() + module_box: dict[str, types.ModuleType] = {} + + def run_instance(**kwargs: Any) -> None: + assert ( + module_box["module"].copy_to_container + is container_runner._copy_to_container_posix + ) + calls.append(kwargs) + + module_box["module"] = _install_eval_modules( + monkeypatch, + tmp_path, + run_instance, + copy_to_container=original_copy, + ) + + report = container_runner.run_upstream_eval( + test_spec=object(), + instance_id="example__repo-1", + patch_text="diff --git a/file.py b/file.py\n", + run_id="windows_eval", + client=object(), + windows_compat=True, + ) + + assert calls + assert calls[0]["rewrite_reports"] is False + assert module_box["module"].copy_to_container is original_copy + assert report.error and "eval report not produced" in report.error + + +def test_windows_copy_shim_uses_posix_container_paths(tmp_path: Any) -> None: + class ExecResult: + exit_code = 0 + output = b"" + + class FakeContainer: + def __init__(self) -> None: + self.exec_calls: list[list[str]] = [] + self.archives: list[tuple[str, bytes]] = [] + + def exec_run(self, cmd: list[str]) -> ExecResult: + self.exec_calls.append(cmd) + return ExecResult() + + def put_archive(self, path: str, data: bytes) -> bool: + self.archives.append((path, data)) + return True + + src = tmp_path / "eval.sh" + src.write_bytes(b"#!/bin/sh\r\necho ok\r\n") + container = FakeContainer() + + container_runner._copy_to_container_posix(container, src, "\\tmp\\eval.sh") + + assert container.exec_calls == [["/bin/sh", "-lc", "mkdir -p /tmp"]] + assert len(container.archives) == 1 + parent, data = container.archives[0] + assert parent == "/tmp" + with tarfile.open(fileobj=io.BytesIO(data), mode="r") as tar: + member = tar.getmember("eval.sh") + assert tar.extractfile(member).read() == b"#!/bin/sh\necho ok\n" + + +def test_make_test_spec_default_uses_nested_module(monkeypatch: Any) -> None: + calls: list[tuple[dict[str, str], str | None]] = [] + + def make_test_spec(instance: dict[str, str], namespace: str | None = None) -> str: + calls.append((instance, namespace)) + return "spec" + + _install_test_spec_modules(monkeypatch, make_test_spec, nested_module=True) + + spec = container_runner.make_test_spec_for_instance( + {"instance_id": "i"}, image_namespace="swebench" + ) + + assert spec == "spec" + assert calls == [({"instance_id": "i"}, "swebench")] + + +def test_make_test_spec_windows_compat_supports_old_import_and_signature( + monkeypatch: Any, +) -> None: + calls: list[dict[str, str]] = [] + + def make_test_spec(instance: dict[str, str]) -> str: + calls.append(instance) + return "old-spec" + + _install_test_spec_modules(monkeypatch, make_test_spec, nested_module=False) + + spec = container_runner.make_test_spec_for_instance( + {"instance_id": "i"}, + image_namespace="swebench", + windows_compat=True, + ) + + assert spec == "old-spec" + assert calls == [{"instance_id": "i"}] + + +def test_make_test_spec_windows_compat_keeps_namespace_when_supported( + monkeypatch: Any, +) -> None: + calls: list[tuple[dict[str, str], str | None]] = [] + + def make_test_spec(instance: dict[str, str], namespace: str | None = None) -> str: + calls.append((instance, namespace)) + return "new-spec" + + _install_test_spec_modules(monkeypatch, make_test_spec, nested_module=True) + + spec = container_runner.make_test_spec_for_instance( + {"instance_id": "i"}, + image_namespace="swebench", + windows_compat=True, + ) + + assert spec == "new-spec" + assert calls == [({"instance_id": "i"}, "swebench")] + + +def test_run_cli_windows_compat_flag_defaults_false_and_can_be_enabled() -> None: + from miniswerouter.cli import _build_parser as build_mini_parser + from swerouter.cli import _build_parser as build_swe_parser + + required_args = [ + "run", + "--router-import", + "pkg.mod:Router.from_cli_args", + "--router-label", + "example", + "--output-dir", + "runs/example", + ] + + assert build_mini_parser().parse_args(required_args).windows_compat is False + assert build_swe_parser().parse_args(required_args).windows_compat is False + assert ( + build_mini_parser().parse_args(required_args + ["--windows-compat"]).windows_compat + is True + ) + assert ( + build_swe_parser().parse_args(required_args + ["--windows-compat"]).windows_compat + is True + )