From 3b49597b6c7f7aa4b6b6b2043e04a81e76b3db04 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 6 May 2026 01:13:41 +0200 Subject: [PATCH 01/10] git remotes --- conan/api/conan_api.py | 12 +- conan/api/subapi/graph.py | 3 +- conan/internal/api/profile/profile_loader.py | 7 +- conan/internal/graph/git_remotes_resolver.py | 51 +++ conan/internal/graph/proxy.py | 34 +- conan/internal/graph/range_resolver.py | 23 +- conan/internal/model/git_remotes.py | 88 ++++++ conan/internal/model/profile.py | 10 + test/functional/graph/test_git_remotes.py | 308 +++++++++++++++++++ 9 files changed, 529 insertions(+), 7 deletions(-) create mode 100644 conan/internal/graph/git_remotes_resolver.py create mode 100644 conan/internal/model/git_remotes.py create mode 100644 test/functional/graph/test_git_remotes.py diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index adf9ce68a3c..17900725935 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -205,17 +205,20 @@ def _flags_plugin(self): return mod.flags_map return None - def get_loader(self): + def get_loader(self, git_remotes=None): ws_editables = self._conan_api.workspace.packages() editable_packages = self._editable_packages.update_copy(ws_editables) legacy_update = self.global_conf.get("core:update_policy", choices=["legacy"]) # This proxy is caching information proxy = ConanProxy(self.cache, self.remote_manager, editable_packages, - legacy_update=legacy_update) + legacy_update=legacy_update, + git_remotes=git_remotes, + hook_manager=self.hook_manager, + global_conf=self.global_conf) # This is caching too range_resolver = RangeResolver(self.cache, self.remote_manager, self.global_conf, - editable_packages) + editable_packages, git_remotes=git_remotes) cmd_wrap = CmdWrapper(HomePaths(self._conan_api.home_folder).wrapper_path) conanfile_helpers = ConanFileHelpers(self._requester, cmd_wrap, self.global_conf, @@ -224,4 +227,7 @@ def get_loader(self): pyreq_loader = PyRequireLoader(proxy, range_resolver, self.global_conf) # This is caching too! loader = ConanFileLoader(pyreq_loader, conanfile_helpers) + # loader is created after proxy due to PyRequireLoader depending on proxy; + # set it here inside the factory before returning + proxy.loader = loader return proxy, range_resolver, loader, None diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index f44161afe2e..e581adfc0f7 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -183,7 +183,8 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo assert profile_host is not None assert profile_build is not None - proxy, range_resolver, loader, _ = self._helpers.get_loader() + proxy, range_resolver, loader, _ = self._helpers.get_loader( + git_remotes=profile_host.git_remotes) remotes = remotes or [] cache = self._helpers.cache diff --git a/conan/internal/api/profile/profile_loader.py b/conan/internal/api/profile/profile_loader.py index 1428a079b51..e193b37eeda 100644 --- a/conan/internal/api/profile/profile_loader.py +++ b/conan/internal/api/profile/profile_loader.py @@ -233,7 +233,7 @@ def get_profile(profile_text, base_profile=None): "platform_tool_requires", "settings", "options", "conf", "buildenv", "runenv", "replace_requires", "replace_tool_requires", - "runner"]) + "runner", "git_remotes"]) # Parse doc sections into Conan model, Settings, Options, etc settings, package_settings = _ProfileValueParser._parse_settings(doc) options = Options.loads(doc.options) if doc.options else None @@ -315,6 +315,11 @@ def load_replace(doc_replace_requires): runner = _ProfileValueParser._parse_key_value(doc.runner) if doc.runner else {} base_profile.runner.update(runner) + + if doc.git_remotes: + from conan.internal.model.git_remotes import GitRemotes + base_profile.git_remotes.update(GitRemotes.loads(doc.git_remotes)) + return base_profile @staticmethod diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py new file mode 100644 index 00000000000..5816bf4dd43 --- /dev/null +++ b/conan/internal/graph/git_remotes_resolver.py @@ -0,0 +1,51 @@ +import hashlib +import os + +from conan.api.output import ConanOutput +from conan.errors import ConanException +from conan.internal.api.export import cmd_export +from conan.internal.util.files import rmdir +from conan.internal.util.runners import detect_runner + + +class GitRemotesResolver: + + def __init__(self, cache, global_conf): + self._cache = cache + self._global_conf = global_conf + self._clones_base = os.path.join(cache.store, "git_clones") + + def clone_and_export(self, ref, git_spec, loader, hook_manager, force_clone=False): + clone_folder = self._clone_folder(git_spec) + if force_clone and os.path.exists(clone_folder): + rmdir(clone_folder) + if not os.path.exists(clone_folder): + self._do_clone(git_spec, clone_folder) + conanfile_path = os.path.join(clone_folder, "conanfile.py") + if not os.path.exists(conanfile_path): + raise ConanException( + f"conanfile.py not found at root of git repo '{git_spec.url}'") + return cmd_export(loader, self._cache, hook_manager, self._global_conf, + conanfile_path, ref.name, str(ref.version), + ref.user, ref.channel, graph_lock=None, remotes=None) + + def _clone_folder(self, git_spec): + key = f"{git_spec.url}:{git_spec.ref or ''}" + h = hashlib.md5(key.encode()).hexdigest()[:12] + return os.path.join(self._clones_base, h) + + def _do_clone(self, git_spec, clone_folder): + output = ConanOutput() + os.makedirs(clone_folder, exist_ok=True) + output.info(f"Cloning git repository '{git_spec.url}'...") + ret, out = detect_runner(f'git clone "{git_spec.url}" "{clone_folder}"') + if ret != 0: + rmdir(clone_folder) + raise ConanException(f"git clone failed for '{git_spec.url}':\n{out}") + if git_spec.ref: + output.info(f"Checking out git ref '{git_spec.ref}'...") + ret, out = detect_runner( + f'git -C "{clone_folder}" checkout {git_spec.ref}') + if ret != 0: + raise ConanException( + f"git checkout '{git_spec.ref}' failed for '{git_spec.url}':\n{out}") diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index 7b5a05f6358..021209e7026 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -10,13 +10,18 @@ class ConanProxy: - def __init__(self, cache, remote_manager, editable_packages, legacy_update=None): + def __init__(self, cache, remote_manager, editable_packages, legacy_update=None, + git_remotes=None, loader=None, hook_manager=None, global_conf=None): # collaborators self._editable_packages = editable_packages self._cache = cache self._remote_manager = remote_manager self._resolved = {} # Cache of the requested recipes to optimize calls self._legacy_update = legacy_update + self._git_remotes = git_remotes + self.loader = loader # set by get_loader() after loader is created (ordering constraint) + self._hook_manager = hook_manager + self._global_conf = global_conf def get_recipe(self, ref, remotes, update, check_update): """ @@ -49,6 +54,13 @@ def _get_recipe(self, reference, remotes, update, check_update): recipe_layout = self._cache.recipe_layout(reference) ref = recipe_layout.reference # latest revision if it was not defined except ConanException: + # NOT in local cache → try git_remotes before real remotes + if self._git_remotes: + git_spec = self._git_remotes.get(reference) + if git_spec is not None: + output.info(f"Not found in local cache, resolving from git remote " + f"'{git_spec.url}'") + return self._clone_export_recipe(reference, git_spec, force=False, output=output) # NOT in disk, must be retrieved from remotes # we will only check all servers for latest revision if we did a --update layout, remote = self._download_recipe(reference, remotes, output, update, check_update) @@ -58,9 +70,19 @@ def _get_recipe(self, reference, remotes, update, check_update): # TODO: cache2.0: check with new --update flows # TODO: If the revision is given, then we don't need to check for updates? if not (check_update or should_update_reference(reference, update)): + if self._git_remotes and self._git_remotes.get(reference) is not None: + output.info(f"Found in cache (configured via git remote " + f"'{self._git_remotes.get(reference).url}')") status = RECIPE_INCACHE return recipe_layout, status, None + # Update needed → git_remotes takes precedence over real remotes + if self._git_remotes and should_update_reference(reference, update): + git_spec = self._git_remotes.get(reference) + if git_spec is not None: + output.info(f"Updating from git remote '{git_spec.url}'...") + return self._clone_export_recipe(reference, git_spec, force=True, output=output) + # Need to check updates remote, remote_ref = self._find_newest_recipe_in_remotes(reference, remotes, update, check_update) @@ -174,6 +196,16 @@ def _download(self, ref, remote): output.info("Downloaded recipe revision %s" % ref.revision) return recipe_layout + def _clone_export_recipe(self, reference, git_spec, force, output): + from conan.internal.graph.git_remotes_resolver import GitRemotesResolver + resolver = GitRemotesResolver(self._cache, self._global_conf) + if git_spec.ref: + output.info(f" git ref: {git_spec.ref}") + new_ref, _ = resolver.clone_and_export( + reference, git_spec, self.loader, self._hook_manager, force_clone=force) + recipe_layout = self._cache.recipe_layout(new_ref) + return recipe_layout, RECIPE_UPDATED if force else RECIPE_DOWNLOADED, None + def should_update_reference(reference, update): if update is None: diff --git a/conan/internal/graph/range_resolver.py b/conan/internal/graph/range_resolver.py index f5938d489f5..f2ad5764eb6 100644 --- a/conan/internal/graph/range_resolver.py +++ b/conan/internal/graph/range_resolver.py @@ -7,7 +7,7 @@ class RangeResolver: - def __init__(self, cache, remote_manager, global_conf, editable_packages): + def __init__(self, cache, remote_manager, global_conf, editable_packages, git_remotes=None): self._cache = cache self._editable_packages = editable_packages self._remote_manager = remote_manager @@ -15,6 +15,7 @@ def __init__(self, cache, remote_manager, global_conf, editable_packages): self._cached_remote_found = {} # dict {ref (pkg/*): {remote_name: results (pkg/1, pkg/2)}} self.resolved_ranges = {} self._resolve_prereleases = global_conf.get('core.version_ranges:resolve_prereleases') + self._git_remotes = git_remotes def resolve(self, require, base_conanref, remotes, update): try: @@ -44,6 +45,12 @@ def resolve(self, require, base_conanref, remotes, update): search_ref = RecipeReference(ref.name, "*", ref.user, ref.channel) resolved_ref = self._resolve_local(search_ref, version_range) + + git_resolved_ref = self._resolve_git_remotes(search_ref, version_range) + if git_resolved_ref is not None: + if resolved_ref is None or git_resolved_ref.version > resolved_ref.version: + resolved_ref = git_resolved_ref + if resolved_ref is None or should_update_reference(search_ref, update): remote_resolved_ref = self._resolve_remote(search_ref, version_range, remotes, update) if resolved_ref is None or (remote_resolved_ref is not None and @@ -106,6 +113,20 @@ def _resolve_remote(self, search_ref, version_range, remotes, update): self._resolve_prereleases) return resolved_version + def _resolve_git_remotes(self, search_ref, version_range): + if not self._git_remotes: + return None + candidates = [] + for key_str in self._git_remotes.entries: + candidate_ref = RecipeReference.loads(key_str) + if (candidate_ref.name == search_ref.name and + candidate_ref.user == search_ref.user and + candidate_ref.channel == search_ref.channel): + candidates.append(candidate_ref) + if candidates: + return self._resolve_version(version_range, candidates, self._resolve_prereleases) + return None + @staticmethod def _resolve_version(version_range, refs_found, resolve_prereleases): for ref in reversed(sorted(refs_found)): diff --git a/conan/internal/model/git_remotes.py b/conan/internal/model/git_remotes.py new file mode 100644 index 00000000000..6b65b431e50 --- /dev/null +++ b/conan/internal/model/git_remotes.py @@ -0,0 +1,88 @@ +from conan.api.model import RecipeReference +from conan.errors import ConanException + + +class GitRemoteSpec: + def __init__(self, url, ref=None): + self.url = url + self.ref = ref # branch, tag, or commit (optional) + + @staticmethod + def loads(value): + value = value.strip() + if "@" in value: + # Split on the LAST @, treating it as a ref separator when the part + # before it contains a "/" (i.e. it looks like a URL or file path). + # This handles: https://..., file:///..., C:/..., /home/..., + # and even SSH URLs like git@github.com:user/repo.git@branch. + # A bare "git@github.com:user/repo.git" (no extra @) is left intact + # because rfind finds the only @, and "git" before it has no "/". + idx = value.rfind("@") + before = value[:idx] + after = value[idx + 1:] + if "/" in before and after: + return GitRemoteSpec(before, after) + return GitRemoteSpec(value) + + def dumps(self): + if self.ref: + return f"{self.url}@{self.ref}" + return self.url + + def __repr__(self): + return self.dumps() + + +class GitRemotes: + def __init__(self): + self._entries = {} # "name/version" → GitRemoteSpec + + def update(self, other): + self._entries.update(other._entries) + + def get(self, ref): + return self._entries.get(f"{ref.name}/{ref.version}") + + @property + def entries(self): + return self._entries + + @staticmethod + def loads(text): + result = GitRemotes() + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if ":" not in line: + raise ConanException(f"[git_remotes] invalid entry '{line}': " + f"expected 'name/version: url'") + # Split on first colon that is followed by space (to avoid splitting on http://) + # Use ": " as separator, fall back to ":" if needed + if ": " in line: + key, value = line.split(": ", 1) + else: + key, value = line.split(":", 1) + value = value.strip() + key = key.strip() + # Validate key looks like name/version + try: + ref = RecipeReference.loads(key) + if ref.name is None or ref.version is None: + raise ConanException(f"[git_remotes] key '{key}' must be 'name/version'") + except Exception as e: + raise ConanException(f"[git_remotes] invalid key '{key}': {e}") + result._entries[key] = GitRemoteSpec.loads(value) + return result + + def dumps(self): + lines = [] + for key, spec in self._entries.items(): + lines.append(f"{key}: {spec.dumps()}") + return "\n".join(lines) + + def serialize(self): + return {k: v.dumps() for k, v in self._entries.items()} + + def __bool__(self): + return bool(self._entries) diff --git a/conan/internal/model/profile.py b/conan/internal/model/profile.py index e452536eb9f..ef7e3a63aba 100644 --- a/conan/internal/model/profile.py +++ b/conan/internal/model/profile.py @@ -4,6 +4,7 @@ from conan.errors import ConanException from conan.tools.env.environment import ProfileEnvironment from conan.internal.model.conf import ConfDefinition +from conan.internal.model.git_remotes import GitRemotes from conan.internal.model.options import Options from conan.api.model import RecipeReference @@ -26,6 +27,7 @@ def __init__(self): self.buildenv = ProfileEnvironment() self.runenv = ProfileEnvironment() self.runner = {} + self.git_remotes = GitRemotes() # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -60,6 +62,9 @@ def _serialize_tool_requires(): if self.platform_requires: result["platform_requires"] = [str(t) for t in self.platform_requires] + if self.git_remotes: + result["git_remotes"] = self.git_remotes.serialize() + return result @property @@ -123,6 +128,10 @@ def dumps(self): result.append("[runenv]") result.append(self.runenv.dumps()) + if self.git_remotes: + result.append("[git_remotes]") + result.append(self.git_remotes.dumps()) + if result and result[-1] != "": result.append("") @@ -150,6 +159,7 @@ def compose_profile(self, other): self.replace_requires.update(other.replace_requires) self.replace_tool_requires.update(other.replace_tool_requires) + self.git_remotes.update(other.git_remotes) runner_type = self.runner.get("type") other_runner_type = other.runner.get("type") diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py new file mode 100644 index 00000000000..7959c2ce74f --- /dev/null +++ b/test/functional/graph/test_git_remotes.py @@ -0,0 +1,308 @@ +import textwrap + +import pytest + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.scm import create_local_git_repo +from conan.test.utils.tools import TestClient + + +def _header_lib(name, version): + """Generate a header-only conanfile (no binary required)""" + return str(GenConanfile(name, version).with_package_type("header-library")) + + +@pytest.mark.tool("git") +class TestGitRemotesBasic: + + def test_basic_resolution_from_git(self): + """Package not in cache: profile git_remote entry clones and exports it""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("zlib", "1.2.11")} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.2.11: {repo_url} + """) + c.save({ + "profile": profile, + "conanfile.py": str(GenConanfile().with_requires("zlib/1.2.11")), + }) + c.run("install . -pr=profile --build=missing") + assert "resolving from git remote" in c.out + assert "zlib/1.2.11" in c.out + + def test_cache_first_no_reclone_on_second_run(self): + """Second install reuses cache — no re-clone""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("zlib", "1.2.11")} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.2.11: {repo_url} + """) + c.save({ + "profile": profile, + "conanfile.py": str(GenConanfile().with_requires("zlib/1.2.11")), + }) + c.run("install . -pr=profile --build=missing") + assert "resolving from git remote" in c.out + + c.run("install . -pr=profile --build=missing") + assert "Found in cache (configured via git remote" in c.out + assert "Cloning" not in c.out + + def test_update_flag_forces_reclone(self): + """--update forces a re-clone from git""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("zlib", "1.2.11")} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.2.11: {repo_url} + """) + c.save({ + "profile": profile, + "conanfile.py": str(GenConanfile().with_requires("zlib/1.2.11")), + }) + c.run("install . -pr=profile --build=missing") + c.run("install . -pr=profile --build=missing --update") + assert "Updating from git remote" in c.out + assert "Cloning" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesRef: + + def test_branch_ref(self): + """Profile entry with @branch clones and checks out that branch""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("mypkg", "1.0")}, + branch="dev", + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + mypkg/1.0: {repo_url}@dev + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("mypkg/1.0"), + }) + c.run("install . -pr=profile --build=missing") + assert "mypkg/1.0" in c.out + assert "git ref: dev" in c.out + + def test_tag_ref(self): + """Profile entry with @tag clones and checks out that tag""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("mypkg", "2.0")}, + tags=["v2.0"], + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + mypkg/2.0: {repo_url}@v2.0 + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("mypkg/2.0"), + }) + c.run("install . -pr=profile --build=missing") + assert "mypkg/2.0" in c.out + assert "git ref: v2.0" in c.out + + def test_commit_ref(self): + """Profile entry with @ checks out that exact commit""" + repo_url, commit = create_local_git_repo( + files={"conanfile.py": _header_lib("mypkg", "3.0")}, + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + mypkg/3.0: {repo_url}@{commit} + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("mypkg/3.0"), + }) + c.run("install . -pr=profile --build=missing") + assert "mypkg/3.0" in c.out + assert f"git ref: {commit}" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesVersionRange: + + def test_version_range_resolved_via_git_remotes(self): + """Version range resolves to a version defined in [git_remotes]""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("zlib", "1.3.0")} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.3.0: {repo_url} + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("zlib/[>=1.0 <2.0]"), + }) + c.run("install . -pr=profile --build=missing") + assert "zlib/1.3.0" in c.out + assert "resolving from git remote" in c.out + + def test_no_match_falls_through(self): + """Non-matching package is not handled by git_remotes""" + repo_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("pkga", "1.0")} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + pkga/1.0: {repo_url} + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("pkgb/1.0"), + }) + c.run("install . -pr=profile --build=missing", assert_error=True) + assert "pkgb/1.0" in c.out + # pkga git_remote must not have been invoked + assert "Resolving from git remote" not in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesProfileComposition: + + def test_profile_composition_last_wins(self): + """When two profiles define the same key, the last profile's URL wins""" + repo_url1, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("zlib", "1.2.11")} + ) + repo_url2, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("zlib", "1.2.11")} + ) + c = TestClient(light=True) + profile1 = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.2.11: {repo_url1} + """) + profile2 = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.2.11: {repo_url2} + """) + c.save({ + "profile1": profile1, + "profile2": profile2, + "conanfile.py": GenConanfile().with_requires("zlib/1.2.11"), + }) + c.run("install . -pr=profile1 -pr=profile2 --build=missing") + assert "zlib/1.2.11" in c.out + assert repo_url2 in c.out + + def test_profile_composition_additive(self): + """Two profiles with different keys: both entries are available""" + repo_url_a, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("pkga", "1.0")} + ) + repo_url_b, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("pkgb", "2.0")} + ) + c = TestClient(light=True) + profile1 = textwrap.dedent(f"""\ + [git_remotes] + pkga/1.0: {repo_url_a} + """) + profile2 = textwrap.dedent(f"""\ + [git_remotes] + pkgb/2.0: {repo_url_b} + """) + conanfile = textwrap.dedent("""\ + from conan import ConanFile + class Consumer(ConanFile): + requires = "pkga/1.0", "pkgb/2.0" + """) + c.save({ + "profile1": profile1, + "profile2": profile2, + "conanfile.py": conanfile, + }) + c.run("install . -pr=profile1 -pr=profile2 --build=missing") + assert "pkga/1.0" in c.out + assert "pkgb/2.0" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesErrors: + + def test_missing_conanfile_in_repo(self): + """Repo without conanfile.py gives a clear error message""" + repo_url, _ = create_local_git_repo( + files={"README.md": "# hello"} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + zlib/1.2.11: {repo_url} + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("zlib/1.2.11"), + }) + c.run("install . -pr=profile --build=missing", assert_error=True) + assert "conanfile.py not found" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesProfileShow: + + def test_profile_show_displays_git_remotes_section(self): + """conan profile show includes the [git_remotes] section""" + c = TestClient(light=True) + profile = textwrap.dedent("""\ + [git_remotes] + zlib/1.2.11: https://github.com/example/zlib.git@main + """) + c.save({"myprofile": profile}) + c.run("profile show -pr=myprofile") + assert "[git_remotes]" in c.out + assert "zlib/1.2.11: https://github.com/example/zlib.git@main" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesTransitive: + + def test_transitive_deps_both_from_git_remotes(self): + """Pkg A from git_remotes requires pkg B, which also has a git_remotes entry""" + repo_b_url, _ = create_local_git_repo( + files={"conanfile.py": _header_lib("pkgb", "1.0")} + ) + conanfile_a = textwrap.dedent("""\ + from conan import ConanFile + class PkgA(ConanFile): + name = "pkga" + version = "1.0" + package_type = "header-library" + requires = "pkgb/1.0" + """) + repo_a_url, _ = create_local_git_repo( + files={"conanfile.py": conanfile_a} + ) + c = TestClient(light=True) + profile = textwrap.dedent(f"""\ + [git_remotes] + pkga/1.0: {repo_a_url} + pkgb/1.0: {repo_b_url} + """) + c.save({ + "profile": profile, + "conanfile.py": GenConanfile().with_requires("pkga/1.0"), + }) + c.run("install . -pr=profile --build=missing") + assert "pkga/1.0" in c.out + assert "pkgb/1.0" in c.out + assert c.out.count("resolving from git remote") == 2 From d4507c180454a17c99a697f2dfd945294c8f9dae Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 6 May 2026 13:36:38 +0200 Subject: [PATCH 02/10] wip --- conan/internal/graph/git_remotes_resolver.py | 3 ++- conan/internal/graph/proxy.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py index 5816bf4dd43..d9099839b17 100644 --- a/conan/internal/graph/git_remotes_resolver.py +++ b/conan/internal/graph/git_remotes_resolver.py @@ -34,7 +34,8 @@ def _clone_folder(self, git_spec): h = hashlib.md5(key.encode()).hexdigest()[:12] return os.path.join(self._clones_base, h) - def _do_clone(self, git_spec, clone_folder): + @staticmethod + def _do_clone(git_spec, clone_folder): output = ConanOutput() os.makedirs(clone_folder, exist_ok=True) output.info(f"Cloning git repository '{git_spec.url}'...") diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index 021209e7026..480c8b65116 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -70,9 +70,6 @@ def _get_recipe(self, reference, remotes, update, check_update): # TODO: cache2.0: check with new --update flows # TODO: If the revision is given, then we don't need to check for updates? if not (check_update or should_update_reference(reference, update)): - if self._git_remotes and self._git_remotes.get(reference) is not None: - output.info(f"Found in cache (configured via git remote " - f"'{self._git_remotes.get(reference).url}')") status = RECIPE_INCACHE return recipe_layout, status, None From d6beb4b2b816a529e6b13fe1881f2a977b8e1954 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 6 May 2026 14:34:46 +0200 Subject: [PATCH 03/10] wip --- conan/api/conan_api.py | 12 ++--- conan/api/subapi/graph.py | 6 +-- conan/internal/graph/git_remotes_resolver.py | 14 +++-- conan/internal/graph/graph_builder.py | 57 +++++++++++++++++++- conan/internal/graph/proxy.py | 31 +---------- conan/internal/graph/range_resolver.py | 23 +------- 6 files changed, 74 insertions(+), 69 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 17900725935..adf9ce68a3c 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -205,20 +205,17 @@ def _flags_plugin(self): return mod.flags_map return None - def get_loader(self, git_remotes=None): + def get_loader(self): ws_editables = self._conan_api.workspace.packages() editable_packages = self._editable_packages.update_copy(ws_editables) legacy_update = self.global_conf.get("core:update_policy", choices=["legacy"]) # This proxy is caching information proxy = ConanProxy(self.cache, self.remote_manager, editable_packages, - legacy_update=legacy_update, - git_remotes=git_remotes, - hook_manager=self.hook_manager, - global_conf=self.global_conf) + legacy_update=legacy_update) # This is caching too range_resolver = RangeResolver(self.cache, self.remote_manager, self.global_conf, - editable_packages, git_remotes=git_remotes) + editable_packages) cmd_wrap = CmdWrapper(HomePaths(self._conan_api.home_folder).wrapper_path) conanfile_helpers = ConanFileHelpers(self._requester, cmd_wrap, self.global_conf, @@ -227,7 +224,4 @@ def get_loader(self, git_remotes=None): pyreq_loader = PyRequireLoader(proxy, range_resolver, self.global_conf) # This is caching too! loader = ConanFileLoader(pyreq_loader, conanfile_helpers) - # loader is created after proxy due to PyRequireLoader depending on proxy; - # set it here inside the factory before returning - proxy.loader = loader return proxy, range_resolver, loader, None diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index e581adfc0f7..75e9b630bbb 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -183,13 +183,13 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo assert profile_host is not None assert profile_build is not None - proxy, range_resolver, loader, _ = self._helpers.get_loader( - git_remotes=profile_host.git_remotes) + proxy, range_resolver, loader, _ = self._helpers.get_loader() remotes = remotes or [] cache = self._helpers.cache builder = DepsGraphBuilder(proxy, loader, range_resolver, cache, remotes, - update, check_update, self._helpers.global_conf) + update, check_update, self._helpers.global_conf, + git_remotes=profile_host.git_remotes) deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) return deps_graph diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py index d9099839b17..6561a9471fe 100644 --- a/conan/internal/graph/git_remotes_resolver.py +++ b/conan/internal/graph/git_remotes_resolver.py @@ -4,18 +4,18 @@ from conan.api.output import ConanOutput from conan.errors import ConanException from conan.internal.api.export import cmd_export +from conan.internal.model.conf import ConfDefinition from conan.internal.util.files import rmdir from conan.internal.util.runners import detect_runner class GitRemotesResolver: - def __init__(self, cache, global_conf): + def __init__(self, cache): self._cache = cache - self._global_conf = global_conf self._clones_base = os.path.join(cache.store, "git_clones") - def clone_and_export(self, ref, git_spec, loader, hook_manager, force_clone=False): + def clone_and_export(self, ref, git_spec, loader, force_clone=False): clone_folder = self._clone_folder(git_spec) if force_clone and os.path.exists(clone_folder): rmdir(clone_folder) @@ -25,7 +25,13 @@ def clone_and_export(self, ref, git_spec, loader, hook_manager, force_clone=Fals if not os.path.exists(conanfile_path): raise ConanException( f"conanfile.py not found at root of git repo '{git_spec.url}'") - return cmd_export(loader, self._cache, hook_manager, self._global_conf, + + class MyHookManager: + def execute(self, method_name, conanfile): + pass + hook_manager = MyHookManager() + global_conf = ConfDefinition() + return cmd_export(loader, self._cache, hook_manager, global_conf, conanfile_path, ref.name, str(ref.version), ref.user, ref.channel, graph_lock=None, remotes=None) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 67e7f9263bb..bfacff9cb19 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -23,7 +23,8 @@ class DepsGraphBuilder: ALLOW_ALIAS = False - def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update, global_conf): + def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update, global_conf, + git_remotes=None): self._proxy = proxy self._loader = loader self._resolver = resolver @@ -32,6 +33,7 @@ def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update self._update = update self._check_update = check_update self._resolve_prereleases = global_conf.get('core.version_ranges:resolve_prereleases') + self._git_remotes = git_remotes def load_graph(self, root_node, profile_host, profile_build, graph_lock=None): assert profile_host is not None @@ -241,6 +243,7 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h if not resolved: self._resolve_alias(node, require, alias, graph) self._resolve_replace_requires(node, require, profile_build, profile_host, graph) + self._prefetch_git_remote(require) if graph_lock: graph_lock.resolve_overrides(require, node.context) node.transitive_deps[require] = TransitiveRequirement(require, node=None) @@ -294,6 +297,58 @@ def _resolve_recipe(self, ref, graph_lock): check_update=self._check_update) return layout, dep_conanfile, recipe_status, remote + def _prefetch_git_remote(self, require): + """Ensure git_remotes entries matching this require are in the local cache before + range resolution and proxy lookup run. Mirrors the _resolve_replace_requires pattern: + acting early in _initialize_requires so the rest of graph resolution is transparent.""" + if not self._git_remotes: + return + version_range = require.version_range + if version_range is not None: + # For version ranges, export every git_remotes candidate matching the package name + # so _resolve_local() can pick the best version from cache afterward. + for key_str, git_spec in self._git_remotes.entries.items(): + from conan.api.model import RecipeReference + candidate_ref = RecipeReference.loads(key_str) + if (candidate_ref.name == require.ref.name and + candidate_ref.user == require.ref.user and + candidate_ref.channel == require.ref.channel): + self._export_from_git_remote(candidate_ref, git_spec) + else: + git_spec = self._git_remotes.get(require.ref) + if git_spec is not None: + self._export_from_git_remote(require.ref, git_spec) + + def _export_from_git_remote(self, ref, git_spec): + """Clone the git repo (if needed) and export its conanfile.py into the local cache.""" + from conan.api.output import ConanOutput + from conan.internal.graph.git_remotes_resolver import GitRemotesResolver + from conan.internal.graph.proxy import should_update_reference + + output = ConanOutput(scope=str(ref)) + force_clone = bool(should_update_reference(ref, self._update)) + + if not force_clone: + try: + if ref.revision: + self._cache.recipe_layout(ref) + else: + self._cache.recipe_layout_latest(ref) + output.info(f"Found in cache (configured via git remote '{git_spec.url}')") + return # Already in cache, nothing to do + except Exception: + pass # Not in cache — proceed with clone+export + + if force_clone: + output.info(f"Updating from git remote '{git_spec.url}'...") + else: + output.info(f"Not found in local cache, resolving from git remote '{git_spec.url}'") + if git_spec.ref: + output.info(f" git ref: {git_spec.ref}") + + resolver = GitRemotesResolver(self._cache) + resolver.clone_and_export(ref, git_spec, self._loader, force_clone=force_clone) + @staticmethod def _resolved_system(node, require, profile_build, profile_host, resolve_prereleases): profile = profile_build if node.context == CONTEXT_BUILD else profile_host diff --git a/conan/internal/graph/proxy.py b/conan/internal/graph/proxy.py index 480c8b65116..7b5a05f6358 100644 --- a/conan/internal/graph/proxy.py +++ b/conan/internal/graph/proxy.py @@ -10,18 +10,13 @@ class ConanProxy: - def __init__(self, cache, remote_manager, editable_packages, legacy_update=None, - git_remotes=None, loader=None, hook_manager=None, global_conf=None): + def __init__(self, cache, remote_manager, editable_packages, legacy_update=None): # collaborators self._editable_packages = editable_packages self._cache = cache self._remote_manager = remote_manager self._resolved = {} # Cache of the requested recipes to optimize calls self._legacy_update = legacy_update - self._git_remotes = git_remotes - self.loader = loader # set by get_loader() after loader is created (ordering constraint) - self._hook_manager = hook_manager - self._global_conf = global_conf def get_recipe(self, ref, remotes, update, check_update): """ @@ -54,13 +49,6 @@ def _get_recipe(self, reference, remotes, update, check_update): recipe_layout = self._cache.recipe_layout(reference) ref = recipe_layout.reference # latest revision if it was not defined except ConanException: - # NOT in local cache → try git_remotes before real remotes - if self._git_remotes: - git_spec = self._git_remotes.get(reference) - if git_spec is not None: - output.info(f"Not found in local cache, resolving from git remote " - f"'{git_spec.url}'") - return self._clone_export_recipe(reference, git_spec, force=False, output=output) # NOT in disk, must be retrieved from remotes # we will only check all servers for latest revision if we did a --update layout, remote = self._download_recipe(reference, remotes, output, update, check_update) @@ -73,13 +61,6 @@ def _get_recipe(self, reference, remotes, update, check_update): status = RECIPE_INCACHE return recipe_layout, status, None - # Update needed → git_remotes takes precedence over real remotes - if self._git_remotes and should_update_reference(reference, update): - git_spec = self._git_remotes.get(reference) - if git_spec is not None: - output.info(f"Updating from git remote '{git_spec.url}'...") - return self._clone_export_recipe(reference, git_spec, force=True, output=output) - # Need to check updates remote, remote_ref = self._find_newest_recipe_in_remotes(reference, remotes, update, check_update) @@ -193,16 +174,6 @@ def _download(self, ref, remote): output.info("Downloaded recipe revision %s" % ref.revision) return recipe_layout - def _clone_export_recipe(self, reference, git_spec, force, output): - from conan.internal.graph.git_remotes_resolver import GitRemotesResolver - resolver = GitRemotesResolver(self._cache, self._global_conf) - if git_spec.ref: - output.info(f" git ref: {git_spec.ref}") - new_ref, _ = resolver.clone_and_export( - reference, git_spec, self.loader, self._hook_manager, force_clone=force) - recipe_layout = self._cache.recipe_layout(new_ref) - return recipe_layout, RECIPE_UPDATED if force else RECIPE_DOWNLOADED, None - def should_update_reference(reference, update): if update is None: diff --git a/conan/internal/graph/range_resolver.py b/conan/internal/graph/range_resolver.py index f2ad5764eb6..f5938d489f5 100644 --- a/conan/internal/graph/range_resolver.py +++ b/conan/internal/graph/range_resolver.py @@ -7,7 +7,7 @@ class RangeResolver: - def __init__(self, cache, remote_manager, global_conf, editable_packages, git_remotes=None): + def __init__(self, cache, remote_manager, global_conf, editable_packages): self._cache = cache self._editable_packages = editable_packages self._remote_manager = remote_manager @@ -15,7 +15,6 @@ def __init__(self, cache, remote_manager, global_conf, editable_packages, git_re self._cached_remote_found = {} # dict {ref (pkg/*): {remote_name: results (pkg/1, pkg/2)}} self.resolved_ranges = {} self._resolve_prereleases = global_conf.get('core.version_ranges:resolve_prereleases') - self._git_remotes = git_remotes def resolve(self, require, base_conanref, remotes, update): try: @@ -45,12 +44,6 @@ def resolve(self, require, base_conanref, remotes, update): search_ref = RecipeReference(ref.name, "*", ref.user, ref.channel) resolved_ref = self._resolve_local(search_ref, version_range) - - git_resolved_ref = self._resolve_git_remotes(search_ref, version_range) - if git_resolved_ref is not None: - if resolved_ref is None or git_resolved_ref.version > resolved_ref.version: - resolved_ref = git_resolved_ref - if resolved_ref is None or should_update_reference(search_ref, update): remote_resolved_ref = self._resolve_remote(search_ref, version_range, remotes, update) if resolved_ref is None or (remote_resolved_ref is not None and @@ -113,20 +106,6 @@ def _resolve_remote(self, search_ref, version_range, remotes, update): self._resolve_prereleases) return resolved_version - def _resolve_git_remotes(self, search_ref, version_range): - if not self._git_remotes: - return None - candidates = [] - for key_str in self._git_remotes.entries: - candidate_ref = RecipeReference.loads(key_str) - if (candidate_ref.name == search_ref.name and - candidate_ref.user == search_ref.user and - candidate_ref.channel == search_ref.channel): - candidates.append(candidate_ref) - if candidates: - return self._resolve_version(version_range, candidates, self._resolve_prereleases) - return None - @staticmethod def _resolve_version(version_range, refs_found, resolve_prereleases): for ref in reversed(sorted(refs_found)): From 83195d8dcefbdfff4d68c84ea721fe97d8444b59 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 8 May 2026 12:50:57 +0200 Subject: [PATCH 04/10] wip --- conan/api/subapi/graph.py | 3 +- conan/internal/api/profile/profile_loader.py | 7 +- conan/internal/graph/git_remotes_resolver.py | 43 ++- conan/internal/graph/graph_builder.py | 60 ++-- conan/internal/model/git_remotes.py | 88 ----- conan/internal/model/profile.py | 10 - conan/internal/model/requires.py | 21 +- conan/test/assets/genconanfile.py | 3 +- conan/test/utils/scm.py | 2 +- test/functional/graph/test_git_remotes.py | 317 +++++-------------- 10 files changed, 142 insertions(+), 412 deletions(-) delete mode 100644 conan/internal/model/git_remotes.py diff --git a/conan/api/subapi/graph.py b/conan/api/subapi/graph.py index 75e9b630bbb..f44161afe2e 100644 --- a/conan/api/subapi/graph.py +++ b/conan/api/subapi/graph.py @@ -188,8 +188,7 @@ def load_graph(self, root_node, profile_host, profile_build, lockfile=None, remo remotes = remotes or [] cache = self._helpers.cache builder = DepsGraphBuilder(proxy, loader, range_resolver, cache, remotes, - update, check_update, self._helpers.global_conf, - git_remotes=profile_host.git_remotes) + update, check_update, self._helpers.global_conf) deps_graph = builder.load_graph(root_node, profile_host, profile_build, lockfile) return deps_graph diff --git a/conan/internal/api/profile/profile_loader.py b/conan/internal/api/profile/profile_loader.py index e193b37eeda..1428a079b51 100644 --- a/conan/internal/api/profile/profile_loader.py +++ b/conan/internal/api/profile/profile_loader.py @@ -233,7 +233,7 @@ def get_profile(profile_text, base_profile=None): "platform_tool_requires", "settings", "options", "conf", "buildenv", "runenv", "replace_requires", "replace_tool_requires", - "runner", "git_remotes"]) + "runner"]) # Parse doc sections into Conan model, Settings, Options, etc settings, package_settings = _ProfileValueParser._parse_settings(doc) options = Options.loads(doc.options) if doc.options else None @@ -315,11 +315,6 @@ def load_replace(doc_replace_requires): runner = _ProfileValueParser._parse_key_value(doc.runner) if doc.runner else {} base_profile.runner.update(runner) - - if doc.git_remotes: - from conan.internal.model.git_remotes import GitRemotes - base_profile.git_remotes.update(GitRemotes.loads(doc.git_remotes)) - return base_profile @staticmethod diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py index 6561a9471fe..fc5b752acff 100644 --- a/conan/internal/graph/git_remotes_resolver.py +++ b/conan/internal/graph/git_remotes_resolver.py @@ -4,7 +4,6 @@ from conan.api.output import ConanOutput from conan.errors import ConanException from conan.internal.api.export import cmd_export -from conan.internal.model.conf import ConfDefinition from conan.internal.util.files import rmdir from conan.internal.util.runners import detect_runner @@ -15,44 +14,42 @@ def __init__(self, cache): self._cache = cache self._clones_base = os.path.join(cache.store, "git_clones") - def clone_and_export(self, ref, git_spec, loader, force_clone=False): - clone_folder = self._clone_folder(git_spec) + def clone_and_export(self, ref, url, git_ref, loader, force_clone=False): + clone_folder = self._clone_folder(url, git_ref) if force_clone and os.path.exists(clone_folder): rmdir(clone_folder) if not os.path.exists(clone_folder): - self._do_clone(git_spec, clone_folder) + self._do_clone(url, git_ref, clone_folder) conanfile_path = os.path.join(clone_folder, "conanfile.py") if not os.path.exists(conanfile_path): raise ConanException( - f"conanfile.py not found at root of git repo '{git_spec.url}'") - - class MyHookManager: - def execute(self, method_name, conanfile): - pass - hook_manager = MyHookManager() - global_conf = ConfDefinition() - return cmd_export(loader, self._cache, hook_manager, global_conf, + f"conanfile.py not found at root of git repo '{url}'") + + class _NoopHooks: + def execute(self, *a, **kw): pass + + from conan.internal.model.conf import ConfDefinition + return cmd_export(loader, self._cache, _NoopHooks(), ConfDefinition(), conanfile_path, ref.name, str(ref.version), ref.user, ref.channel, graph_lock=None, remotes=None) - def _clone_folder(self, git_spec): - key = f"{git_spec.url}:{git_spec.ref or ''}" + def _clone_folder(self, url, git_ref): + key = f"{url}:{git_ref or ''}" h = hashlib.md5(key.encode()).hexdigest()[:12] return os.path.join(self._clones_base, h) @staticmethod - def _do_clone(git_spec, clone_folder): + def _do_clone(url, git_ref, clone_folder): output = ConanOutput() os.makedirs(clone_folder, exist_ok=True) - output.info(f"Cloning git repository '{git_spec.url}'...") - ret, out = detect_runner(f'git clone "{git_spec.url}" "{clone_folder}"') + output.info(f"Cloning git repository '{url}'...") + ret, out = detect_runner(f'git clone "{url}" "{clone_folder}"') if ret != 0: rmdir(clone_folder) - raise ConanException(f"git clone failed for '{git_spec.url}':\n{out}") - if git_spec.ref: - output.info(f"Checking out git ref '{git_spec.ref}'...") - ret, out = detect_runner( - f'git -C "{clone_folder}" checkout {git_spec.ref}') + raise ConanException(f"git clone failed for '{url}':\n{out}") + if git_ref: + output.info(f"Checking out git ref '{git_ref}'...") + ret, out = detect_runner(f'git -C "{clone_folder}" checkout {git_ref}') if ret != 0: raise ConanException( - f"git checkout '{git_spec.ref}' failed for '{git_spec.url}':\n{out}") + f"git checkout '{git_ref}' failed for '{url}':\n{out}") diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index bfacff9cb19..6084c4536ad 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -23,8 +23,7 @@ class DepsGraphBuilder: ALLOW_ALIAS = False - def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update, global_conf, - git_remotes=None): + def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update, global_conf): self._proxy = proxy self._loader = loader self._resolver = resolver @@ -33,7 +32,6 @@ def __init__(self, proxy, loader, resolver, cache, remotes, update, check_update self._update = update self._check_update = check_update self._resolve_prereleases = global_conf.get('core.version_ranges:resolve_prereleases') - self._git_remotes = git_remotes def load_graph(self, root_node, profile_host, profile_build, graph_lock=None): assert profile_host is not None @@ -298,56 +296,46 @@ def _resolve_recipe(self, ref, graph_lock): return layout, dep_conanfile, recipe_status, remote def _prefetch_git_remote(self, require): - """Ensure git_remotes entries matching this require are in the local cache before - range resolution and proxy lookup run. Mirrors the _resolve_replace_requires pattern: - acting early in _initialize_requires so the rest of graph resolution is transparent.""" - if not self._git_remotes: + """If the requirement declares a git= source, clone and export it into the local cache + before range resolution and proxy lookup run. This mirrors the _resolve_replace_requires + pattern — acting early in _initialize_requires so the rest of graph resolution is + transparent (proxy just finds the recipe in cache as usual).""" + if require.git is None: return - version_range = require.version_range - if version_range is not None: - # For version ranges, export every git_remotes candidate matching the package name - # so _resolve_local() can pick the best version from cache afterward. - for key_str, git_spec in self._git_remotes.entries.items(): - from conan.api.model import RecipeReference - candidate_ref = RecipeReference.loads(key_str) - if (candidate_ref.name == require.ref.name and - candidate_ref.user == require.ref.user and - candidate_ref.channel == require.ref.channel): - self._export_from_git_remote(candidate_ref, git_spec) - else: - git_spec = self._git_remotes.get(require.ref) - if git_spec is not None: - self._export_from_git_remote(require.ref, git_spec) - def _export_from_git_remote(self, ref, git_spec): - """Clone the git repo (if needed) and export its conanfile.py into the local cache.""" from conan.api.output import ConanOutput from conan.internal.graph.git_remotes_resolver import GitRemotesResolver from conan.internal.graph.proxy import should_update_reference + ref = require.ref + if ref.revision: + raise ConanException(f"Ref {ref.revision} cannot be specified with git") + git = require.git # raw string: "url" or "url@ref" + idx = git.rsplit("@", 1) + url, git_ref = idx if len(idx) == 2 else (idx[0], None) + output = ConanOutput(scope=str(ref)) - force_clone = bool(should_update_reference(ref, self._update)) + force_clone = should_update_reference(ref, self._update) - if not force_clone: + if force_clone: + output.info(f"Updating from git remote '{url}'...") + else: try: if ref.revision: self._cache.recipe_layout(ref) else: self._cache.recipe_layout_latest(ref) - output.info(f"Found in cache (configured via git remote '{git_spec.url}')") + output.info(f"Found in cache (configured via git remote '{url}')") return # Already in cache, nothing to do - except Exception: + except ConanException: pass # Not in cache — proceed with clone+export + output.info(f"Not found in local cache, resolving from git remote '{url}'") - if force_clone: - output.info(f"Updating from git remote '{git_spec.url}'...") - else: - output.info(f"Not found in local cache, resolving from git remote '{git_spec.url}'") - if git_spec.ref: - output.info(f" git ref: {git_spec.ref}") + if git_ref: + output.info(f" git ref: {git_ref}") - resolver = GitRemotesResolver(self._cache) - resolver.clone_and_export(ref, git_spec, self._loader, force_clone=force_clone) + GitRemotesResolver(self._cache).clone_and_export(ref, url, git_ref, self._loader, + force_clone=force_clone) @staticmethod def _resolved_system(node, require, profile_build, profile_host, resolve_prereleases): diff --git a/conan/internal/model/git_remotes.py b/conan/internal/model/git_remotes.py deleted file mode 100644 index 6b65b431e50..00000000000 --- a/conan/internal/model/git_remotes.py +++ /dev/null @@ -1,88 +0,0 @@ -from conan.api.model import RecipeReference -from conan.errors import ConanException - - -class GitRemoteSpec: - def __init__(self, url, ref=None): - self.url = url - self.ref = ref # branch, tag, or commit (optional) - - @staticmethod - def loads(value): - value = value.strip() - if "@" in value: - # Split on the LAST @, treating it as a ref separator when the part - # before it contains a "/" (i.e. it looks like a URL or file path). - # This handles: https://..., file:///..., C:/..., /home/..., - # and even SSH URLs like git@github.com:user/repo.git@branch. - # A bare "git@github.com:user/repo.git" (no extra @) is left intact - # because rfind finds the only @, and "git" before it has no "/". - idx = value.rfind("@") - before = value[:idx] - after = value[idx + 1:] - if "/" in before and after: - return GitRemoteSpec(before, after) - return GitRemoteSpec(value) - - def dumps(self): - if self.ref: - return f"{self.url}@{self.ref}" - return self.url - - def __repr__(self): - return self.dumps() - - -class GitRemotes: - def __init__(self): - self._entries = {} # "name/version" → GitRemoteSpec - - def update(self, other): - self._entries.update(other._entries) - - def get(self, ref): - return self._entries.get(f"{ref.name}/{ref.version}") - - @property - def entries(self): - return self._entries - - @staticmethod - def loads(text): - result = GitRemotes() - for line in text.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - if ":" not in line: - raise ConanException(f"[git_remotes] invalid entry '{line}': " - f"expected 'name/version: url'") - # Split on first colon that is followed by space (to avoid splitting on http://) - # Use ": " as separator, fall back to ":" if needed - if ": " in line: - key, value = line.split(": ", 1) - else: - key, value = line.split(":", 1) - value = value.strip() - key = key.strip() - # Validate key looks like name/version - try: - ref = RecipeReference.loads(key) - if ref.name is None or ref.version is None: - raise ConanException(f"[git_remotes] key '{key}' must be 'name/version'") - except Exception as e: - raise ConanException(f"[git_remotes] invalid key '{key}': {e}") - result._entries[key] = GitRemoteSpec.loads(value) - return result - - def dumps(self): - lines = [] - for key, spec in self._entries.items(): - lines.append(f"{key}: {spec.dumps()}") - return "\n".join(lines) - - def serialize(self): - return {k: v.dumps() for k, v in self._entries.items()} - - def __bool__(self): - return bool(self._entries) diff --git a/conan/internal/model/profile.py b/conan/internal/model/profile.py index ef7e3a63aba..e452536eb9f 100644 --- a/conan/internal/model/profile.py +++ b/conan/internal/model/profile.py @@ -4,7 +4,6 @@ from conan.errors import ConanException from conan.tools.env.environment import ProfileEnvironment from conan.internal.model.conf import ConfDefinition -from conan.internal.model.git_remotes import GitRemotes from conan.internal.model.options import Options from conan.api.model import RecipeReference @@ -27,7 +26,6 @@ def __init__(self): self.buildenv = ProfileEnvironment() self.runenv = ProfileEnvironment() self.runner = {} - self.git_remotes = GitRemotes() # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -62,9 +60,6 @@ def _serialize_tool_requires(): if self.platform_requires: result["platform_requires"] = [str(t) for t in self.platform_requires] - if self.git_remotes: - result["git_remotes"] = self.git_remotes.serialize() - return result @property @@ -128,10 +123,6 @@ def dumps(self): result.append("[runenv]") result.append(self.runenv.dumps()) - if self.git_remotes: - result.append("[git_remotes]") - result.append(self.git_remotes.dumps()) - if result and result[-1] != "": result.append("") @@ -159,7 +150,6 @@ def compose_profile(self, other): self.replace_requires.update(other.replace_requires) self.replace_tool_requires.update(other.replace_tool_requires) - self.git_remotes.update(other.git_remotes) runner_type = self.runner.get("type") other_runner_type = other.runner.get("type") diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index 611384e1813..2860c63b3ec 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -10,7 +10,7 @@ class Requirement: def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visible=None, transitive_headers=None, transitive_libs=None, test=None, package_id_mode=None, force=None, override=None, direct=None, options=None, no_skip=False, - consistent=None): + consistent=None, git=None): # * prevents the usage of more positional parameters, always ref + **kwargs # By default this is a generic library requirement self.ref = ref @@ -39,6 +39,8 @@ def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visib self.skip = False self.required_nodes = set() # store which intermediate nodes are required, to compute "Skip" self.no_skip = no_skip + # git source: raw string "url" or "url@ref"; parsed at use time in DepsGraphBuilder + self.git = git # computed ones, not default ones self.consistent_policy_new = False if self.visible and not self.consistent: @@ -462,10 +464,10 @@ def __init__(self, requires): self._requires = requires def __call__(self, ref, package_id_mode=None, visible=False, run=True, options=None, - override=None): + override=None, git=None): # TODO: Check which arguments could be user-defined self._requires.tool_require(ref, package_id_mode=package_id_mode, visible=visible, run=run, - options=options, override=override) + options=options, override=override, git=git) class TestRequirements: @@ -473,8 +475,8 @@ class TestRequirements: def __init__(self, requires): self._requires = requires - def __call__(self, ref, run=None, options=None, force=None): - self._requires.test_require(ref, run=run, options=options, force=force) + def __call__(self, ref, run=None, options=None, force=None, git=None): + self._requires.test_require(ref, run=run, options=options, force=force, git=git) class Requirements: @@ -576,7 +578,7 @@ def build_require(self, ref, raise_if_duplicated=True, package_id_mode=None, vis raise ConanException("Duplicated requirement: {}".format(ref)) self._requires[req] = req - def test_require(self, ref, run=None, options=None, force=None): + def test_require(self, ref, run=None, options=None, force=None, git=None): """ Represent a testing framework like gtest @@ -592,13 +594,13 @@ def test_require(self, ref, run=None, options=None, force=None): # libs = True => We need to link with it # headers = True => We need to include it req = Requirement(ref, headers=True, libs=True, build=False, run=run, visible=False, - test=True, package_id_mode=None, options=options, force=force) + test=True, package_id_mode=None, options=options, force=force, git=git) if self._requires.get(req): raise ConanException("Duplicated requirement: {}".format(ref)) self._requires[req] = req def tool_require(self, ref, raise_if_duplicated=True, package_id_mode=None, visible=False, - run=True, options=None, override=None): + run=True, options=None, override=None, git=None): """ Represent a build tool like "cmake". @@ -612,7 +614,8 @@ def tool_require(self, ref, raise_if_duplicated=True, package_id_mode=None, visi # FIXME: This raise_if_duplicated is ugly, possibly remove ref = RecipeReference.loads(ref) req = Requirement(ref, headers=False, libs=False, build=True, run=run, visible=visible, - package_id_mode=package_id_mode, options=options, override=override) + package_id_mode=package_id_mode, options=options, override=override, + git=git) if raise_if_duplicated and self._requires.get(req): raise ConanException("Duplicated requirement: {}".format(ref)) self._requires[req] = req diff --git a/conan/test/assets/genconanfile.py b/conan/test/assets/genconanfile.py index 53e23d38040..6e8900359ed 100644 --- a/conan/test/assets/genconanfile.py +++ b/conan/test/assets/genconanfile.py @@ -494,7 +494,8 @@ def __repr__(self): ): if member == "requirements": # FIXME: This seems exclusive, but we could mix them? - v = self._requirements or self._tool_requirements or self._build_requirements + v = (self._requirements or self._tool_requirements + or self._build_requirements or self._test_requirements) else: v = getattr(self, "_{}".format(member), None) if v is not None: diff --git a/conan/test/utils/scm.py b/conan/test/utils/scm.py index 537e35289b0..68bc48135f5 100644 --- a/conan/test/utils/scm.py +++ b/conan/test/utils/scm.py @@ -20,7 +20,7 @@ def create_local_git_repo(files=None, branch=None, submodules=None, folder=None, tags=None, origin_url=None, main_branch="master"): tmp = folder or temp_folder() if files: - save_files(tmp, files) + save_files(tmp, {k: str(v) for k, v in files.items()}) def _run(cmd, p): with chdir(p): diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py index 7959c2ce74f..c0c75f06c95 100644 --- a/test/functional/graph/test_git_remotes.py +++ b/test/functional/graph/test_git_remotes.py @@ -1,5 +1,3 @@ -import textwrap - import pytest from conan.test.assets.genconanfile import GenConanfile @@ -7,69 +5,37 @@ from conan.test.utils.tools import TestClient -def _header_lib(name, version): - """Generate a header-only conanfile (no binary required)""" - return str(GenConanfile(name, version).with_package_type("header-library")) - - @pytest.mark.tool("git") class TestGitRemotesBasic: def test_basic_resolution_from_git(self): - """Package not in cache: profile git_remote entry clones and exports it""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("zlib", "1.2.11")} - ) + """Package not in cache: git= on require clones and exports it""" + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.2.11: {repo_url} - """) - c.save({ - "profile": profile, - "conanfile.py": str(GenConanfile().with_requires("zlib/1.2.11")), - }) - c.run("install . -pr=profile --build=missing") + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.run("install . --build=missing") assert "resolving from git remote" in c.out assert "zlib/1.2.11" in c.out def test_cache_first_no_reclone_on_second_run(self): """Second install reuses cache — no re-clone""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("zlib", "1.2.11")} - ) + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.2.11: {repo_url} - """) - c.save({ - "profile": profile, - "conanfile.py": str(GenConanfile().with_requires("zlib/1.2.11")), - }) - c.run("install . -pr=profile --build=missing") + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.run("install . --build=missing") assert "resolving from git remote" in c.out - c.run("install . -pr=profile --build=missing") + c.run("install . --build=missing") assert "Found in cache (configured via git remote" in c.out assert "Cloning" not in c.out def test_update_flag_forces_reclone(self): """--update forces a re-clone from git""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("zlib", "1.2.11")} - ) + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.2.11: {repo_url} - """) - c.save({ - "profile": profile, - "conanfile.py": str(GenConanfile().with_requires("zlib/1.2.11")), - }) - c.run("install . -pr=profile --build=missing") - c.run("install . -pr=profile --build=missing --update") + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.run("install . --build=missing") + c.run("install . --build=missing --update") assert "Updating from git remote" in c.out assert "Cloning" in c.out @@ -78,162 +44,54 @@ def test_update_flag_forces_reclone(self): class TestGitRemotesRef: def test_branch_ref(self): - """Profile entry with @branch clones and checks out that branch""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("mypkg", "1.0")}, - branch="dev", - ) + """git= with @branch clones and checks out that branch""" + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "1.0")}, + branch="dev") c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - mypkg/1.0: {repo_url}@dev - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("mypkg/1.0"), - }) - c.run("install . -pr=profile --build=missing") + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", git=f"{repo_url}@dev")}) + c.run("install . --build=missing") assert "mypkg/1.0" in c.out assert "git ref: dev" in c.out def test_tag_ref(self): - """Profile entry with @tag clones and checks out that tag""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("mypkg", "2.0")}, - tags=["v2.0"], - ) + """git= with @tag clones and checks out that tag""" + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "2.0")}, + tags=["v2.0"]) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - mypkg/2.0: {repo_url}@v2.0 - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("mypkg/2.0"), - }) - c.run("install . -pr=profile --build=missing") + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/2.0", + git=f"{repo_url}@v2.0")}) + c.run("install . --build=missing") assert "mypkg/2.0" in c.out assert "git ref: v2.0" in c.out def test_commit_ref(self): - """Profile entry with @ checks out that exact commit""" - repo_url, commit = create_local_git_repo( - files={"conanfile.py": _header_lib("mypkg", "3.0")}, - ) + """git= with @ checks out that exact commit""" + repo_url, commit = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "3.0")}) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - mypkg/3.0: {repo_url}@{commit} - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("mypkg/3.0"), - }) - c.run("install . -pr=profile --build=missing") + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/3.0", + git=f"{repo_url}@{commit}")}) + c.run("install . --build=missing") assert "mypkg/3.0" in c.out assert f"git ref: {commit}" in c.out @pytest.mark.tool("git") -class TestGitRemotesVersionRange: - - def test_version_range_resolved_via_git_remotes(self): - """Version range resolves to a version defined in [git_remotes]""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("zlib", "1.3.0")} - ) +class TestGitRemotesRequireTypes: + + @pytest.mark.parametrize("method", [ + "with_requirement", # self.requires(git=) — host dependency + "with_tool_requirement", # self.tool_requires(git=) — build-context tool + "with_test_requirement", # self.test_requires(git=) — test-only host dependency + ]) + def test_git_resolution_by_require_type(self, method): + """git= works for requires, tool_requires and test_requires""" + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "1.0")}) + consumer = getattr(GenConanfile(), method)("mypkg/1.0", git=repo_url) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.3.0: {repo_url} - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("zlib/[>=1.0 <2.0]"), - }) - c.run("install . -pr=profile --build=missing") - assert "zlib/1.3.0" in c.out + c.save({"conanfile.py": str(consumer)}) + c.run("install . --build=missing") assert "resolving from git remote" in c.out - - def test_no_match_falls_through(self): - """Non-matching package is not handled by git_remotes""" - repo_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("pkga", "1.0")} - ) - c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - pkga/1.0: {repo_url} - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("pkgb/1.0"), - }) - c.run("install . -pr=profile --build=missing", assert_error=True) - assert "pkgb/1.0" in c.out - # pkga git_remote must not have been invoked - assert "Resolving from git remote" not in c.out - - -@pytest.mark.tool("git") -class TestGitRemotesProfileComposition: - - def test_profile_composition_last_wins(self): - """When two profiles define the same key, the last profile's URL wins""" - repo_url1, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("zlib", "1.2.11")} - ) - repo_url2, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("zlib", "1.2.11")} - ) - c = TestClient(light=True) - profile1 = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.2.11: {repo_url1} - """) - profile2 = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.2.11: {repo_url2} - """) - c.save({ - "profile1": profile1, - "profile2": profile2, - "conanfile.py": GenConanfile().with_requires("zlib/1.2.11"), - }) - c.run("install . -pr=profile1 -pr=profile2 --build=missing") - assert "zlib/1.2.11" in c.out - assert repo_url2 in c.out - - def test_profile_composition_additive(self): - """Two profiles with different keys: both entries are available""" - repo_url_a, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("pkga", "1.0")} - ) - repo_url_b, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("pkgb", "2.0")} - ) - c = TestClient(light=True) - profile1 = textwrap.dedent(f"""\ - [git_remotes] - pkga/1.0: {repo_url_a} - """) - profile2 = textwrap.dedent(f"""\ - [git_remotes] - pkgb/2.0: {repo_url_b} - """) - conanfile = textwrap.dedent("""\ - from conan import ConanFile - class Consumer(ConanFile): - requires = "pkga/1.0", "pkgb/2.0" - """) - c.save({ - "profile1": profile1, - "profile2": profile2, - "conanfile.py": conanfile, - }) - c.run("install . -pr=profile1 -pr=profile2 --build=missing") - assert "pkga/1.0" in c.out - assert "pkgb/2.0" in c.out + assert "mypkg/1.0" in c.out @pytest.mark.tool("git") @@ -241,68 +99,55 @@ class TestGitRemotesErrors: def test_missing_conanfile_in_repo(self): """Repo without conanfile.py gives a clear error message""" - repo_url, _ = create_local_git_repo( - files={"README.md": "# hello"} - ) + repo_url, _ = create_local_git_repo(files={"README.md": "# hello"}) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - zlib/1.2.11: {repo_url} - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("zlib/1.2.11"), - }) - c.run("install . -pr=profile --build=missing", assert_error=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.run("install . --build=missing", assert_error=True) assert "conanfile.py not found" in c.out -@pytest.mark.tool("git") -class TestGitRemotesProfileShow: - - def test_profile_show_displays_git_remotes_section(self): - """conan profile show includes the [git_remotes] section""" - c = TestClient(light=True) - profile = textwrap.dedent("""\ - [git_remotes] - zlib/1.2.11: https://github.com/example/zlib.git@main - """) - c.save({"myprofile": profile}) - c.run("profile show -pr=myprofile") - assert "[git_remotes]" in c.out - assert "zlib/1.2.11: https://github.com/example/zlib.git@main" in c.out - - @pytest.mark.tool("git") class TestGitRemotesTransitive: - def test_transitive_deps_both_from_git_remotes(self): - """Pkg A from git_remotes requires pkg B, which also has a git_remotes entry""" - repo_b_url, _ = create_local_git_repo( - files={"conanfile.py": _header_lib("pkgb", "1.0")} - ) - conanfile_a = textwrap.dedent("""\ - from conan import ConanFile - class PkgA(ConanFile): - name = "pkga" - version = "1.0" - package_type = "header-library" - requires = "pkgb/1.0" - """) + def test_transitive_deps_both_from_git(self): + """Pkg A from git= requires pkg B, which also has a git= entry in its conanfile""" + repo_b_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("pkgb", "1.0")}) repo_a_url, _ = create_local_git_repo( - files={"conanfile.py": conanfile_a} + files={"conanfile.py": GenConanfile("pkga", "1.0").with_requirement("pkgb/1.0", + git=repo_b_url)} ) c = TestClient(light=True) - profile = textwrap.dedent(f"""\ - [git_remotes] - pkga/1.0: {repo_a_url} - pkgb/1.0: {repo_b_url} - """) - c.save({ - "profile": profile, - "conanfile.py": GenConanfile().with_requires("pkga/1.0"), - }) - c.run("install . -pr=profile --build=missing") + c.save({"conanfile.py": GenConanfile().with_requirement("pkga/1.0", git=repo_a_url)}) + c.run("install . --build=missing") assert "pkga/1.0" in c.out assert "pkgb/1.0" in c.out assert c.out.count("resolving from git remote") == 2 + + def test_diamond_all_from_git(self): + """Diamond: consumer->pkga->pkgc and consumer->pkgb->pkgc, all in separate git repos. + pkgc is cloned once (first encounter); the second encounter finds it already in cache.""" + # pkgc: leaf, no dependencies + repo_c_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("pkgc", "1.0")}) + # pkga and pkgb both depend on pkgc/1.0 via git= + repo_a_url, _ = create_local_git_repo( + files={"conanfile.py": GenConanfile("pkga", "1.0").with_requirement("pkgc/1.0", + git=repo_c_url)} + ) + repo_b_url, _ = create_local_git_repo( + files={"conanfile.py": GenConanfile("pkgb", "1.0").with_requirement("pkgc/1.0", + git=repo_c_url)} + ) + # consumer depends on both pkga and pkgb + c = TestClient(light=True) + c.save({"conanfile.py": str( + GenConanfile() + .with_requirement("pkga/1.0", git=repo_a_url) + .with_requirement("pkgb/1.0", git=repo_b_url))}) + c.run("install . --build=missing") + assert "pkga/1.0" in c.out + assert "pkgb/1.0" in c.out + assert "pkgc/1.0" in c.out + # pkga, pkgb, pkgc each cloned once from git (3 fresh resolutions) + assert c.out.count("resolving from git remote") == 3 + # pkgc is found in cache the second time (via pkgb's requires, after pkga already exported it) + assert c.out.count("Found in cache (configured via git remote") == 1 From 404b8047d9fe2d8458d33dc245a3f4cfbbaf5c35 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 8 May 2026 16:16:36 +0200 Subject: [PATCH 05/10] wip --- conan/internal/graph/graph_builder.py | 19 +++++-- test/functional/graph/test_git_remotes.py | 64 +++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 6084c4536ad..8d67ed15899 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -322,10 +322,12 @@ def _prefetch_git_remote(self, require): else: try: if ref.revision: - self._cache.recipe_layout(ref) + layout = self._cache.recipe_layout(ref) else: - self._cache.recipe_layout_latest(ref) + layout = self._cache.recipe_layout_latest(ref) output.info(f"Found in cache (configured via git remote '{url}')") + # Stash cached revision so _create_new_node can detect a lockfile mismatch + require._git_exported_revision = layout.reference.revision return # Already in cache, nothing to do except ConanException: pass # Not in cache — proceed with clone+export @@ -334,8 +336,10 @@ def _prefetch_git_remote(self, require): if git_ref: output.info(f" git ref: {git_ref}") - GitRemotesResolver(self._cache).clone_and_export(ref, url, git_ref, self._loader, - force_clone=force_clone) + exported_ref, _ = GitRemotesResolver(self._cache).clone_and_export( + ref, url, git_ref, self._loader, force_clone=force_clone) + # Stash the exported revision so _create_new_node can detect a lockfile mismatch + require._git_exported_revision = exported_ref.revision @staticmethod def _resolved_system(node, require, profile_build, profile_host, resolve_prereleases): @@ -439,6 +443,13 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr if graph_lock is not None: # Here is when the ranges and revisions are resolved graph_lock.resolve_locked(node, require, self._resolve_prereleases) + git_rev = getattr(require, "_git_exported_revision", None) + if git_rev is not None and require.ref.revision is not None \ + and require.ref.revision != git_rev: + raise ConanException( + f"Lockfile revision '{require.ref.revision}' for '{require.ref.name}/" + f"{require.ref.version}' does not match the revision '{git_rev}' " + f"exported from git. The lockfile is out of date with the git source.") if resolved is None: try: diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py index c0c75f06c95..f944c49a127 100644 --- a/test/functional/graph/test_git_remotes.py +++ b/test/functional/graph/test_git_remotes.py @@ -1,3 +1,6 @@ +import json +import re + import pytest from conan.test.assets.genconanfile import GenConanfile @@ -151,3 +154,64 @@ def test_diamond_all_from_git(self): assert c.out.count("resolving from git remote") == 3 # pkgc is found in cache the second time (via pkgb's requires, after pkga already exported it) assert c.out.count("Found in cache (configured via git remote") == 1 + + +@pytest.mark.tool("git") +class TestGitRemotesLockfile: + + def test_lockfile_happy_path(self): + """First install with --lockfile-out captures the git-exported revision. + Second install with --lockfile reuses cache via the locked revision.""" + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + + # First install: clone from git, export to cache, write lockfile + c.run("install . --build=missing --lockfile-out=conan.lock") + assert "resolving from git remote" in c.out + assert "zlib/1.2.11" in c.out + + # Lockfile must contain zlib/1.2.11 with a recipe revision + lock = json.loads(c.load("conan.lock")) + requires = lock["requires"] + assert len(requires) == 1 + locked_ref = requires[0] + assert locked_ref.startswith("zlib/1.2.11#") + revision = locked_ref.split("#")[1] + assert revision # non-empty revision hash + + # Second install with lockfile: recipe already in cache → no clone + c.run("install . --lockfile=conan.lock") + assert "Found in cache (configured via git remote" in c.out + assert "Cloning" not in c.out + assert "zlib/1.2.11" in c.out + + # Now removing the cache one + c.run("remove * -c") + c.run("install . --lockfile=conan.lock --build=missing") + assert "Cloning" not in c.out # it reuses the previous clone + assert "zlib/1.2.11" in c.out + + def test_lockfile_revision_mismatch_fails(self): + """If the lockfile contains a recipe revision that differs from what git exports, + Conan must raise an error because the locked revision is not present in cache.""" + repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + + # First install: populate cache and generate a valid lockfile + c.run("install . --build=missing --lockfile-out=conan.lock") + assert "resolving from git remote" in c.out + + # Tamper the lockfile: replace the real revision with a fake one + raw = c.load("conan.lock") + tampered = re.sub(r"(zlib/1\.2\.11#)[0-9a-f]+", r"\1deadbeef00000000000000000000000", raw) + c.save({"conan.lock": tampered}) + + # Second install with tampered lockfile: cached revision X, lockfile expects Y → clear error + c.run("install . --lockfile=conan.lock --build=missing", assert_error=True) + print(c.out) + assert "zlib/1.2.11" in c.out + assert "does not match the revision" in c.out + assert "deadbeef" in c.out + assert "The lockfile is out of date with the git source" in c.out From 401e545c16753162a81db4598587fa674a24b1dc Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 8 May 2026 17:24:25 +0200 Subject: [PATCH 06/10] wip --- conan/internal/graph/graph_builder.py | 17 ++++--------- test/functional/graph/test_git_remotes.py | 29 +++++++++++++++++------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 8d67ed15899..2aacc5ed8b7 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -322,12 +322,12 @@ def _prefetch_git_remote(self, require): else: try: if ref.revision: - layout = self._cache.recipe_layout(ref) + self._cache.recipe_layout(ref) else: layout = self._cache.recipe_layout_latest(ref) + # annotate revision to compare with lockfile one later + require.ref.revision = layout.reference.revision output.info(f"Found in cache (configured via git remote '{url}')") - # Stash cached revision so _create_new_node can detect a lockfile mismatch - require._git_exported_revision = layout.reference.revision return # Already in cache, nothing to do except ConanException: pass # Not in cache — proceed with clone+export @@ -338,8 +338,8 @@ def _prefetch_git_remote(self, require): exported_ref, _ = GitRemotesResolver(self._cache).clone_and_export( ref, url, git_ref, self._loader, force_clone=force_clone) - # Stash the exported revision so _create_new_node can detect a lockfile mismatch - require._git_exported_revision = exported_ref.revision + # Get the recipe revision from the export, to annotate it and checke later with lockfile + require.ref.revision = exported_ref.revision @staticmethod def _resolved_system(node, require, profile_build, profile_host, resolve_prereleases): @@ -443,13 +443,6 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr if graph_lock is not None: # Here is when the ranges and revisions are resolved graph_lock.resolve_locked(node, require, self._resolve_prereleases) - git_rev = getattr(require, "_git_exported_revision", None) - if git_rev is not None and require.ref.revision is not None \ - and require.ref.revision != git_rev: - raise ConanException( - f"Lockfile revision '{require.ref.revision}' for '{require.ref.name}/" - f"{require.ref.version}' does not match the revision '{git_rev}' " - f"exported from git. The lockfile is out of date with the git source.") if resolved is None: try: diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py index f944c49a127..1db3ee87f8f 100644 --- a/test/functional/graph/test_git_remotes.py +++ b/test/functional/graph/test_git_remotes.py @@ -1,10 +1,12 @@ import json +import os import re import pytest +from conan.internal.util.files import save from conan.test.assets.genconanfile import GenConanfile -from conan.test.utils.scm import create_local_git_repo +from conan.test.utils.scm import create_local_git_repo, git_add_changes_commit from conan.test.utils.tools import TestClient @@ -206,12 +208,25 @@ def test_lockfile_revision_mismatch_fails(self): # Tamper the lockfile: replace the real revision with a fake one raw = c.load("conan.lock") tampered = re.sub(r"(zlib/1\.2\.11#)[0-9a-f]+", r"\1deadbeef00000000000000000000000", raw) - c.save({"conan.lock": tampered}) + c.save({"conan2.lock": tampered}) # Second install with tampered lockfile: cached revision X, lockfile expects Y → clear error - c.run("install . --lockfile=conan.lock --build=missing", assert_error=True) - print(c.out) + c.run("install . --lockfile=conan2.lock", assert_error=True) assert "zlib/1.2.11" in c.out - assert "does not match the revision" in c.out - assert "deadbeef" in c.out - assert "The lockfile is out of date with the git source" in c.out + assert ("Requirement 'zlib/1.2.11#20823ba3fead87d6e797bd33010ca88a' " + "not in lockfile 'requires'") in c.out + + # Now removing the cache one + c.run("remove * -c") + c.run("install . --lockfile=conan.lock --build=missing") + assert "Cloning" not in c.out # it reuses the previous clone + assert "zlib/1.2.11" in c.out + + # It it updates a new commit, and the lockfile pins previous recipe-revision, it will fail + save(os.path.join(repo_url, "conanfile.py"), + str(GenConanfile("zlib", "1.2.11").with_class_attribute("somevar=3"))) + git_add_changes_commit(repo_url) + c.run("install . --lockfile=conan.lock --build=missing --update", assert_error=True) + assert "Cloning" in c.out + assert ("Requirement 'zlib/1.2.11#6a02546722ab83f3926c350c001c9c4d' " + "not in lockfile 'requires'") in c.out From cd6fbc1548669c888ab10d13ccdfe5d7a9f8aa4a Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 14 Jul 2026 13:04:35 +0200 Subject: [PATCH 07/10] wip --- conan/internal/graph/git_remotes_resolver.py | 8 +- conan/internal/graph/graph_builder.py | 10 +- conan/internal/model/requires.py | 2 +- test/functional/graph/test_git_remotes.py | 139 +++++++++++-------- 4 files changed, 93 insertions(+), 66 deletions(-) diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py index fc5b752acff..4bac84ae70c 100644 --- a/conan/internal/graph/git_remotes_resolver.py +++ b/conan/internal/graph/git_remotes_resolver.py @@ -14,7 +14,13 @@ def __init__(self, cache): self._cache = cache self._clones_base = os.path.join(cache.store, "git_clones") - def clone_and_export(self, ref, url, git_ref, loader, force_clone=False): + @staticmethod + def get_url(repo): + # Maybe we need to extend this to gitlab too, we could check a "gl:org/repo" format + return f"https://github.com/{repo}.git" + + def clone_and_export(self, ref, repo, git_ref, loader, force_clone=False): + url = self.get_url(repo) clone_folder = self._clone_folder(url, git_ref) if force_clone and os.path.exists(clone_folder): rmdir(clone_folder) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 97fe4fde16a..4b786ecd9d6 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -310,10 +310,12 @@ def _prefetch_git_remote(self, require): ref = require.ref if ref.revision: raise ConanException(f"Ref {ref.revision} cannot be specified with git") - git = require.git # raw string: "url" or "url@ref" + git = require.git # raw string: "org/repo" or "org/repo@ref" idx = git.rsplit("@", 1) - url, git_ref = idx if len(idx) == 2 else (idx[0], None) + repo, git_ref = idx if len(idx) == 2 else (idx[0], None) + git_resolver = GitRemotesResolver(self._cache) + url = git_resolver.get_url(repo) output = ConanOutput(scope=str(ref)) force_clone = should_update_reference(ref, self._update) @@ -336,8 +338,8 @@ def _prefetch_git_remote(self, require): if git_ref: output.info(f" git ref: {git_ref}") - exported_ref, _ = GitRemotesResolver(self._cache).clone_and_export( - ref, url, git_ref, self._loader, force_clone=force_clone) + exported_ref, _ = git_resolver.clone_and_export(ref, repo, git_ref, self._loader, + force_clone=force_clone) # Get the recipe revision from the export, to annotate it and checke later with lockfile require.ref.revision = exported_ref.revision diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index 26ccc639f07..e0de4559304 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -40,7 +40,7 @@ def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visib self.skip = False self.required_nodes = set() # store which intermediate nodes are required, to compute "Skip" self.no_skip = no_skip - # git source: raw string "url" or "url@ref"; parsed at use time in DepsGraphBuilder + # git source: raw string for https://github.com public open source repositories self.git = git # computed ones, not default ones self.consistent_policy_new = False diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py index 1db3ee87f8f..cecb21bf6d8 100644 --- a/test/functional/graph/test_git_remotes.py +++ b/test/functional/graph/test_git_remotes.py @@ -1,6 +1,7 @@ import json import os import re +from unittest import mock import pytest @@ -10,23 +11,44 @@ from conan.test.utils.tools import TestClient +@pytest.fixture +def git_repos(): + """Patches GitRemotesResolver.get_url to serve local repos by org/repo slug. + Yields a register(slug, files, **kwargs) helper that creates a local git repo + and maps slug -> local path for the duration of the test.""" + mapping = {} + + def register(slug, files=None, **kwargs): + path, commit = create_local_git_repo(files, **kwargs) + mapping[slug] = path + return path, commit + + with mock.patch( + "conan.internal.graph.git_remotes_resolver.GitRemotesResolver.get_url", + side_effect=lambda repo: mapping[repo], + ): + yield register + + @pytest.mark.tool("git") class TestGitRemotesBasic: - def test_basic_resolution_from_git(self): + def test_basic_resolution_from_git(self, git_repos): """Package not in cache: git= on require clones and exports it""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) c.run("install . --build=missing") assert "resolving from git remote" in c.out assert "zlib/1.2.11" in c.out - def test_cache_first_no_reclone_on_second_run(self): + def test_cache_first_no_reclone_on_second_run(self, git_repos): """Second install reuses cache — no re-clone""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) c.run("install . --build=missing") assert "resolving from git remote" in c.out @@ -34,11 +56,12 @@ def test_cache_first_no_reclone_on_second_run(self): assert "Found in cache (configured via git remote" in c.out assert "Cloning" not in c.out - def test_update_flag_forces_reclone(self): + def test_update_flag_forces_reclone(self, git_repos): """--update forces a re-clone from git""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) c.run("install . --build=missing") c.run("install . --build=missing --update") assert "Updating from git remote" in c.out @@ -48,33 +71,32 @@ def test_update_flag_forces_reclone(self): @pytest.mark.tool("git") class TestGitRemotesRef: - def test_branch_ref(self): + def test_branch_ref(self, git_repos): """git= with @branch clones and checks out that branch""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "1.0")}, - branch="dev") + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}, branch="dev") c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", git=f"{repo_url}@dev")}) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg@dev")}) c.run("install . --build=missing") assert "mypkg/1.0" in c.out assert "git ref: dev" in c.out - def test_tag_ref(self): + def test_tag_ref(self, git_repos): """git= with @tag clones and checks out that tag""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "2.0")}, - tags=["v2.0"]) + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "2.0")}, tags=["v2.0"]) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/2.0", - git=f"{repo_url}@v2.0")}) + git="myorg/mypkg@v2.0")}) c.run("install . --build=missing") assert "mypkg/2.0" in c.out assert "git ref: v2.0" in c.out - def test_commit_ref(self): + def test_commit_ref(self, git_repos): """git= with @ checks out that exact commit""" - repo_url, commit = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "3.0")}) + _, commit = git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "3.0")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/3.0", - git=f"{repo_url}@{commit}")}) + git=f"myorg/mypkg@{commit}")}) c.run("install . --build=missing") assert "mypkg/3.0" in c.out assert f"git ref: {commit}" in c.out @@ -84,14 +106,14 @@ def test_commit_ref(self): class TestGitRemotesRequireTypes: @pytest.mark.parametrize("method", [ - "with_requirement", # self.requires(git=) — host dependency + "with_requirement", # self.requires(git=) — host dependency "with_tool_requirement", # self.tool_requires(git=) — build-context tool "with_test_requirement", # self.test_requires(git=) — test-only host dependency ]) - def test_git_resolution_by_require_type(self, method): + def test_git_resolution_by_require_type(self, method, git_repos): """git= works for requires, tool_requires and test_requires""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("mypkg", "1.0")}) - consumer = getattr(GenConanfile(), method)("mypkg/1.0", git=repo_url) + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + consumer = getattr(GenConanfile(), method)("mypkg/1.0", git="myorg/mypkg") c = TestClient(light=True) c.save({"conanfile.py": str(consumer)}) c.run("install . --build=missing") @@ -102,11 +124,12 @@ def test_git_resolution_by_require_type(self, method): @pytest.mark.tool("git") class TestGitRemotesErrors: - def test_missing_conanfile_in_repo(self): + def test_missing_conanfile_in_repo(self, git_repos): """Repo without conanfile.py gives a clear error message""" - repo_url, _ = create_local_git_repo(files={"README.md": "# hello"}) + git_repos("myorg/myrepo", {"README.md": "# hello"}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="myorg/myrepo")}) c.run("install . --build=missing", assert_error=True) assert "conanfile.py not found" in c.out @@ -114,40 +137,34 @@ def test_missing_conanfile_in_repo(self): @pytest.mark.tool("git") class TestGitRemotesTransitive: - def test_transitive_deps_both_from_git(self): + def test_transitive_deps_both_from_git(self, git_repos): """Pkg A from git= requires pkg B, which also has a git= entry in its conanfile""" - repo_b_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("pkgb", "1.0")}) - repo_a_url, _ = create_local_git_repo( - files={"conanfile.py": GenConanfile("pkga", "1.0").with_requirement("pkgb/1.0", - git=repo_b_url)} - ) + git_repos("myorg/pkgb", {"conanfile.py": GenConanfile("pkgb", "1.0")}) + git_repos("myorg/pkga", + {"conanfile.py": GenConanfile("pkga", "1.0") + .with_requirement("pkgb/1.0", git="myorg/pkgb")}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("pkga/1.0", git=repo_a_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("pkga/1.0", git="myorg/pkga")}) c.run("install . --build=missing") assert "pkga/1.0" in c.out assert "pkgb/1.0" in c.out assert c.out.count("resolving from git remote") == 2 - def test_diamond_all_from_git(self): + def test_diamond_all_from_git(self, git_repos): """Diamond: consumer->pkga->pkgc and consumer->pkgb->pkgc, all in separate git repos. pkgc is cloned once (first encounter); the second encounter finds it already in cache.""" - # pkgc: leaf, no dependencies - repo_c_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("pkgc", "1.0")}) - # pkga and pkgb both depend on pkgc/1.0 via git= - repo_a_url, _ = create_local_git_repo( - files={"conanfile.py": GenConanfile("pkga", "1.0").with_requirement("pkgc/1.0", - git=repo_c_url)} - ) - repo_b_url, _ = create_local_git_repo( - files={"conanfile.py": GenConanfile("pkgb", "1.0").with_requirement("pkgc/1.0", - git=repo_c_url)} - ) - # consumer depends on both pkga and pkgb + git_repos("myorg/pkgc", {"conanfile.py": GenConanfile("pkgc", "1.0")}) + git_repos("myorg/pkga", + {"conanfile.py": GenConanfile("pkga", "1.0") + .with_requirement("pkgc/1.0", git="myorg/pkgc")}) + git_repos("myorg/pkgb", + {"conanfile.py": GenConanfile("pkgb", "1.0") + .with_requirement("pkgc/1.0", git="myorg/pkgc")}) c = TestClient(light=True) c.save({"conanfile.py": str( GenConanfile() - .with_requirement("pkga/1.0", git=repo_a_url) - .with_requirement("pkgb/1.0", git=repo_b_url))}) + .with_requirement("pkga/1.0", git="myorg/pkga") + .with_requirement("pkgb/1.0", git="myorg/pkgb"))}) c.run("install . --build=missing") assert "pkga/1.0" in c.out assert "pkgb/1.0" in c.out @@ -161,12 +178,13 @@ def test_diamond_all_from_git(self): @pytest.mark.tool("git") class TestGitRemotesLockfile: - def test_lockfile_happy_path(self): + def test_lockfile_happy_path(self, git_repos): """First install with --lockfile-out captures the git-exported revision. Second install with --lockfile reuses cache via the locked revision.""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) # First install: clone from git, export to cache, write lockfile c.run("install . --build=missing --lockfile-out=conan.lock") @@ -179,8 +197,7 @@ def test_lockfile_happy_path(self): assert len(requires) == 1 locked_ref = requires[0] assert locked_ref.startswith("zlib/1.2.11#") - revision = locked_ref.split("#")[1] - assert revision # non-empty revision hash + assert locked_ref.split("#")[1] # non-empty revision hash # Second install with lockfile: recipe already in cache → no clone c.run("install . --lockfile=conan.lock") @@ -194,12 +211,14 @@ def test_lockfile_happy_path(self): assert "Cloning" not in c.out # it reuses the previous clone assert "zlib/1.2.11" in c.out - def test_lockfile_revision_mismatch_fails(self): + def test_lockfile_revision_mismatch_fails(self, git_repos): """If the lockfile contains a recipe revision that differs from what git exports, Conan must raise an error because the locked revision is not present in cache.""" - repo_url, _ = create_local_git_repo({"conanfile.py": GenConanfile("zlib", "1.2.11")}) + repo_path, _ = git_repos("conan-io/zlib", + {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) - c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git=repo_url)}) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) # First install: populate cache and generate a valid lockfile c.run("install . --build=missing --lockfile-out=conan.lock") @@ -222,10 +241,10 @@ def test_lockfile_revision_mismatch_fails(self): assert "Cloning" not in c.out # it reuses the previous clone assert "zlib/1.2.11" in c.out - # It it updates a new commit, and the lockfile pins previous recipe-revision, it will fail - save(os.path.join(repo_url, "conanfile.py"), + # If the repo gets a new commit and the lockfile pins the old recipe-revision, it fails + save(os.path.join(repo_path, "conanfile.py"), str(GenConanfile("zlib", "1.2.11").with_class_attribute("somevar=3"))) - git_add_changes_commit(repo_url) + git_add_changes_commit(repo_path) c.run("install . --lockfile=conan.lock --build=missing --update", assert_error=True) assert "Cloning" in c.out assert ("Requirement 'zlib/1.2.11#6a02546722ab83f3926c350c001c9c4d' " From 1085efdd6b7571bde1f4c9983a015d265276e618 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 22 Jul 2026 17:52:47 +0200 Subject: [PATCH 08/10] wip --- conan/internal/graph/graph_builder.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 8a9481c20a1..677d6d25e03 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -241,7 +241,8 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h if not resolved: self._resolve_alias(node, require, alias, graph) self._resolve_replace_requires(node, require, profile_build, profile_host, graph) - self._prefetch_git_remote(require) + if require.git is not None: + self._prefetch_git_remote(require) if graph_lock: graph_lock.resolve_overrides(require, node.context) node.transitive_deps[require] = TransitiveRequirement(require, node=None) @@ -300,9 +301,6 @@ def _prefetch_git_remote(self, require): before range resolution and proxy lookup run. This mirrors the _resolve_replace_requires pattern — acting early in _initialize_requires so the rest of graph resolution is transparent (proxy just finds the recipe in cache as usual).""" - if require.git is None: - return - from conan.api.output import ConanOutput from conan.internal.graph.git_remotes_resolver import GitRemotesResolver from conan.internal.graph.proxy import should_update_reference @@ -323,12 +321,9 @@ def _prefetch_git_remote(self, require): output.info(f"Updating from git remote '{url}'...") else: try: - if ref.revision: - self._cache.recipe_layout(ref) - else: - layout = self._cache.recipe_layout_latest(ref) - # annotate revision to compare with lockfile one later - require.ref.revision = layout.reference.revision + layout = self._cache.recipe_layout_latest(ref) + # annotate revision to compare with lockfile one later + require.ref.revision = layout.reference.revision output.info(f"Found in cache (configured via git remote '{url}')") return # Already in cache, nothing to do except ConanException: From fa2c057fc7705084bcef258dded2e1c17927c549 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 30 Jul 2026 13:42:01 +0200 Subject: [PATCH 09/10] wip --- conan/api/subapi/cache.py | 1 + conan/internal/api/export.py | 7 +- conan/internal/cache/cache.py | 4 + conan/internal/graph/git_remotes_resolver.py | 163 ++++++++-- conan/internal/graph/graph_builder.py | 47 +-- conan/internal/model/requires.py | 7 + test/functional/graph/test_git_remotes.py | 318 ++++++++++++++++--- 7 files changed, 432 insertions(+), 115 deletions(-) diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index c95796ce4f0..6ccda39534c 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -271,6 +271,7 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf) if temp: rmdir(cache.temp_folder) + rmdir(cache.git_clones_folder) # git-source clones are scratch space too # Clean those build folders that didn't succeed to create a package and wont be in DB builds_folder = cache.builds_folder if os.path.isdir(builds_folder): diff --git a/conan/internal/api/export.py b/conan/internal/api/export.py index 0e55711f90d..f1d7475440c 100644 --- a/conan/internal/api/export.py +++ b/conan/internal/api/export.py @@ -14,13 +14,18 @@ def cmd_export(loader, cache, hook_manager, global_conf, conanfile_path, name, version, user, channel, - graph_lock=None, remotes=None): + graph_lock=None, remotes=None, revision_mode_scm=False): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py + param revision_mode_scm: force revision_mode='scm' regardless of what the recipe + declares. Used by git= sources so the recipe revision IS the + git commit — see GitRemotesResolver. """ conanfile = loader.load_export(conanfile_path, name, version, user, channel, graph_lock, remotes=remotes) + if revision_mode_scm: + conanfile.revision_mode = "scm" ref = RecipeReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) ref.validate_ref(allow_uppercase=global_conf.get("core:allow_uppercase_pkg_names", diff --git a/conan/internal/cache/cache.py b/conan/internal/cache/cache.py index 27b6c336705..34d72552d90 100644 --- a/conan/internal/cache/cache.py +++ b/conan/internal/cache/cache.py @@ -50,6 +50,10 @@ def temp_folder(self): def builds_folder(self): return os.path.join(self._base_folder, "b") + @property + def git_clones_folder(self): + return os.path.join(self._base_folder, "git_clones") + def _create_path(self, relative_path, remove_contents=True): path = self._full_path(relative_path) if os.path.exists(path) and remove_contents: diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py index 4bac84ae70c..755aa2032f7 100644 --- a/conan/internal/graph/git_remotes_resolver.py +++ b/conan/internal/graph/git_remotes_resolver.py @@ -1,27 +1,137 @@ import hashlib import os +import subprocess from conan.api.output import ConanOutput from conan.errors import ConanException from conan.internal.api.export import cmd_export -from conan.internal.util.files import rmdir -from conan.internal.util.runners import detect_runner +from conan.internal.graph.proxy import should_update_reference +from conan.internal.util.files import remove_if_dirty, rmdir, set_dirty_context_manager class GitRemotesResolver: def __init__(self, cache): self._cache = cache - self._clones_base = os.path.join(cache.store, "git_clones") + self._clones_base = cache.git_clones_folder @staticmethod - def get_url(repo): + def _get_url(repo): # Maybe we need to extend this to gitlab too, we could check a "gl:org/repo" format return f"https://github.com/{repo}.git" - def clone_and_export(self, ref, repo, git_ref, loader, force_clone=False): - url = self.get_url(repo) + def prefetch(self, node, require, update, loader, editable_packages, lockfile=None): + """If the requirement declares a git= source, clone and export it into the local + cache before range resolution and proxy lookup run. Mirrors the + _resolve_replace_requires pattern — invoked early in _initialize_requires so the + rest of graph resolution is transparent (proxy just finds the recipe in cache). + + Semantics: + - The recipe revision IS the git commit SHA (revision_mode='scm' is forced + during export). A lockfile captured today pins the exact commit, so a + later reinstall from that lockfile can reproduce the build even if a + branch tip has moved in the meantime. + - Version ranges are allowed. The clone runs first, the recipe declares its + own version, and the resolver validates that the declared version is + within the require's range. + """ + ref = require.ref + # replace_requires ran and matched: _resolve_replace_requires sets + # _required_ref = ref.copy() BEFORE mutating ref. If they are no longer the same + # object, the ref was renamed and the hardcoded git= source cannot apply — + # replace_requires wins. + output = ConanOutput(scope=str(node)) + if require._required_ref is not ref: # noqa + output.warning(f"Ignoring git={require.git!r}: 'replace_requires' matched " + f"and took precedence over the git= source.") + return + + # Editable takes precedence over git=: a local editable is a stronger + # override than a hardcoded remote source. + if editable_packages is not None and editable_packages.get(ref) is not None: + output.info(f"Ignoring git={require.git!r}: package is in editable mode.") + return + + if ref.revision: + raise ConanException( + f"Requirement '{ref}' with an explicit revision cannot use a 'git=' source") + + git = require.git # raw string: "org/repo" or "org/repo@ref" + # split on the FIRST '@' — org/repo cannot contain '@' (GitHub disallows it), + # so anything after is the ref, even if the ref itself contains '@' + idx = git.split("@", 1) + if len(idx) == 2 and not idx[1]: + raise ConanException( + f"Requirement '{ref}': git={git!r} has a trailing '@' with no ref. " + f"Drop the '@' to use the default branch, or specify a branch/tag/commit.") + repo, git_ref = idx if len(idx) == 2 else (idx[0], None) + + # Lockfile-driven checkout: if a matching entry is locked with a revision + # (git commit SHA under our revision_mode='scm' contract), use it as the + # checkout target so reinstalls reproduce the exact commit even if the + # branch has moved upstream. Peek only — do NOT let resolve_locked mutate + # require.ref to share identity with the lockfile's internal ref (that + # would cause our later revision assignment to poison the lockfile). + if lockfile is not None: + saved_ref = require.ref + try: + lockfile.resolve_locked(node, require, resolve_prereleases=None) + locked_rev = require.ref.revision + except ConanException: + locked_rev = None + finally: + require.ref = saved_ref + if locked_rev: + git_ref = locked_rev + + url = self._get_url(repo) + force_clone = should_update_reference(ref, update) + version_range = require.version_range + + output = ConanOutput(scope=str(ref)) + if force_clone: + output.info(f"Updating from git remote '{url}'...") + elif not version_range: + # Cache-first shortcut only makes sense for a fully-resolved ref. + # With a range, we need to re-resolve which concrete version applies. + try: + layout = self._cache.recipe_layout_latest(ref) + require.ref.revision = layout.reference.revision + output.info(f"Found in cache (configured via git remote '{url}')") + return + except ConanException: + pass # Not in cache — proceed with clone+export + output.info(f"Not found in local cache, resolving from git remote '{url}'") + + if git_ref: + output.info(f" git ref: {git_ref}") + + # With a version range, let the recipe declare its own version and validate + # it after export. Without a range, pass the exact version so cmd_export + # enforces the recipe hardcodes match (existing behavior). + version = None if version_range else str(ref.version) + exported_ref, _ = self._clone_and_export(ref, repo, git_ref, loader, + force_clone, version=version) + + if version_range: + resolved_version = exported_ref.version + resolve_prereleases = None # let VersionRange default apply + if not version_range.contains(resolved_version, resolve_prereleases): + raise ConanException( + f"Requirement '{ref}' with range '{version_range}' does not accept " + f"the version '{resolved_version}' declared by the recipe at " + f"git remote '{url}'") + output.info(f" resolved version: {resolved_version} (in range {version_range})") + require.ref.version = resolved_version + + # Recipe revision is the git commit SHA (revision_mode='scm' forced during export) + require.ref.revision = exported_ref.revision + + def _clone_and_export(self, ref, repo, git_ref, loader, force_clone, version=None): + url = self._get_url(repo) clone_folder = self._clone_folder(url, git_ref) + # Leftover from a previous run interrupted mid-clone/checkout: discard it + remove_if_dirty(clone_folder) if force_clone and os.path.exists(clone_folder): rmdir(clone_folder) if not os.path.exists(clone_folder): @@ -31,13 +141,20 @@ def clone_and_export(self, ref, repo, git_ref, loader, force_clone=False): raise ConanException( f"conanfile.py not found at root of git repo '{url}'") + # Hooks are intentionally skipped: git= is aimed at open-source / + # community workflows that pull recipes straight from public github.com + # repos, not at organizations that rely on pre/post_export hooks for + # policy, signing or scanning. class _NoopHooks: def execute(self, *a, **kw): pass from conan.internal.model.conf import ConfDefinition + # Force revision_mode='scm' → recipe revision = git commit SHA. Enables + # lockfile reproducibility even against moving branches. + # no remotes, no lockfile, as python-requires are not supported now return cmd_export(loader, self._cache, _NoopHooks(), ConfDefinition(), - conanfile_path, ref.name, str(ref.version), - ref.user, ref.channel, graph_lock=None, remotes=None) + conanfile_path, ref.name, version, + ref.user, ref.channel, revision_mode_scm=True) def _clone_folder(self, url, git_ref): key = f"{url}:{git_ref or ''}" @@ -45,17 +162,25 @@ def _clone_folder(self, url, git_ref): return os.path.join(self._clones_base, h) @staticmethod - def _do_clone(url, git_ref, clone_folder): + def _run_git(argv): + # argv-form; never shell=True — keeps refs/URLs with metachars intact + proc = subprocess.run(argv, capture_output=True, text=True) + return proc.returncode, (proc.stdout or "") + (proc.stderr or "") + + @classmethod + def _do_clone(cls, url, git_ref, clone_folder): output = ConanOutput() os.makedirs(clone_folder, exist_ok=True) - output.info(f"Cloning git repository '{url}'...") - ret, out = detect_runner(f'git clone "{url}" "{clone_folder}"') - if ret != 0: - rmdir(clone_folder) - raise ConanException(f"git clone failed for '{url}':\n{out}") - if git_ref: - output.info(f"Checking out git ref '{git_ref}'...") - ret, out = detect_runner(f'git -C "{clone_folder}" checkout {git_ref}') + # dirty marker: if the process is interrupted mid-clone/checkout, the + # next run detects the marker via remove_if_dirty and starts fresh + with set_dirty_context_manager(clone_folder): + output.info(f"Cloning git repository '{url}'...") + ret, out = cls._run_git(["git", "clone", url, clone_folder]) if ret != 0: - raise ConanException( - f"git checkout '{git_ref}' failed for '{url}':\n{out}") + raise ConanException(f"git clone failed for '{url}':\n{out}") + if git_ref: + output.info(f"Checking out git ref '{git_ref}'...") + ret, out = cls._run_git(["git", "-C", clone_folder, "checkout", git_ref]) + if ret != 0: + raise ConanException( + f"git checkout '{git_ref}' failed for '{url}':\n{out}") diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 677d6d25e03..894766714a3 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -242,7 +242,10 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h self._resolve_alias(node, require, alias, graph) self._resolve_replace_requires(node, require, profile_build, profile_host, graph) if require.git is not None: - self._prefetch_git_remote(require) + from conan.internal.graph.git_remotes_resolver import GitRemotesResolver + GitRemotesResolver(self._cache).prefetch(node, require, self._update, self._loader, + self._proxy._editable_packages, # noqa + graph_lock) if graph_lock: graph_lock.resolve_overrides(require, node.context) node.transitive_deps[require] = TransitiveRequirement(require, node=None) @@ -296,48 +299,6 @@ def _resolve_recipe(self, ref, graph_lock): check_update=self._check_update) return layout, dep_conanfile, recipe_status, remote - def _prefetch_git_remote(self, require): - """If the requirement declares a git= source, clone and export it into the local cache - before range resolution and proxy lookup run. This mirrors the _resolve_replace_requires - pattern — acting early in _initialize_requires so the rest of graph resolution is - transparent (proxy just finds the recipe in cache as usual).""" - from conan.api.output import ConanOutput - from conan.internal.graph.git_remotes_resolver import GitRemotesResolver - from conan.internal.graph.proxy import should_update_reference - - ref = require.ref - if ref.revision: - raise ConanException(f"Ref {ref.revision} cannot be specified with git") - git = require.git # raw string: "org/repo" or "org/repo@ref" - idx = git.rsplit("@", 1) - repo, git_ref = idx if len(idx) == 2 else (idx[0], None) - - git_resolver = GitRemotesResolver(self._cache) - url = git_resolver.get_url(repo) - output = ConanOutput(scope=str(ref)) - force_clone = should_update_reference(ref, self._update) - - if force_clone: - output.info(f"Updating from git remote '{url}'...") - else: - try: - layout = self._cache.recipe_layout_latest(ref) - # annotate revision to compare with lockfile one later - require.ref.revision = layout.reference.revision - output.info(f"Found in cache (configured via git remote '{url}')") - return # Already in cache, nothing to do - except ConanException: - pass # Not in cache — proceed with clone+export - output.info(f"Not found in local cache, resolving from git remote '{url}'") - - if git_ref: - output.info(f" git ref: {git_ref}") - - exported_ref, _ = git_resolver.clone_and_export(ref, repo, git_ref, self._loader, - force_clone=force_clone) - # Get the recipe revision from the export, to annotate it and checke later with lockfile - require.ref.revision = exported_ref.revision - @staticmethod def _resolved_system(node, require, profile_build, profile_host, resolve_prereleases): profile = profile_build if node.context == CONTEXT_BUILD else profile_host diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index b65a81cd3f8..a340b99e17b 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -527,6 +527,13 @@ def values(self): return self._requires.values() def __call__(self, str_ref, **kwargs): + """Add a regular requirement (as in ``self.requires("zlib/1.2.11")``). + + Any keyword accepted by ``Requirement`` may be passed through kwargs, including + ``git="org/repo[@ref]"`` to source the recipe from a public GitHub repository — + symmetric with ``self.test_requires`` and ``self.tool_requires``, which declare + ``git=`` in their own signatures. + """ if str_ref is None: return assert isinstance(str_ref, str) diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py index cecb21bf6d8..620d71f4657 100644 --- a/test/functional/graph/test_git_remotes.py +++ b/test/functional/graph/test_git_remotes.py @@ -1,6 +1,7 @@ import json import os import re +import subprocess from unittest import mock import pytest @@ -13,9 +14,9 @@ @pytest.fixture def git_repos(): - """Patches GitRemotesResolver.get_url to serve local repos by org/repo slug. + """Patches GitRemotesResolver._get_url to serve local repos by 'org/repo' slug. Yields a register(slug, files, **kwargs) helper that creates a local git repo - and maps slug -> local path for the duration of the test.""" + and maps the slug -> local path for the duration of the test.""" mapping = {} def register(slug, files=None, **kwargs): @@ -24,7 +25,7 @@ def register(slug, files=None, **kwargs): return path, commit with mock.patch( - "conan.internal.graph.git_remotes_resolver.GitRemotesResolver.get_url", + "conan.internal.graph.git_remotes_resolver.GitRemotesResolver._get_url", side_effect=lambda repo: mapping[repo], ): yield register @@ -32,9 +33,10 @@ def register(slug, files=None, **kwargs): @pytest.mark.tool("git") class TestGitRemotesBasic: + """First-run resolution, cache reuse, --update behavior, and support across + self.requires / self.tool_requires / self.test_requires.""" def test_basic_resolution_from_git(self, git_repos): - """Package not in cache: git= on require clones and exports it""" git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", @@ -44,7 +46,6 @@ def test_basic_resolution_from_git(self, git_repos): assert "zlib/1.2.11" in c.out def test_cache_first_no_reclone_on_second_run(self, git_repos): - """Second install reuses cache — no re-clone""" git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", @@ -57,7 +58,6 @@ def test_cache_first_no_reclone_on_second_run(self, git_repos): assert "Cloning" not in c.out def test_update_flag_forces_reclone(self, git_repos): - """--update forces a re-clone from git""" git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", @@ -67,12 +67,27 @@ def test_update_flag_forces_reclone(self, git_repos): assert "Updating from git remote" in c.out assert "Cloning" in c.out + @pytest.mark.parametrize("method", [ + "with_requirement", # self.requires(git=) — host dependency + "with_tool_requirement", # self.tool_requires(git=) — build-context tool + "with_test_requirement", # self.test_requires(git=) — test-only host dependency + ]) + def test_resolution_by_require_type(self, method, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + consumer = getattr(GenConanfile(), method)("mypkg/1.0", git="myorg/mypkg") + c = TestClient(light=True) + c.save({"conanfile.py": str(consumer)}) + c.run("install . --build=missing") + assert "resolving from git remote" in c.out + assert "mypkg/1.0" in c.out + @pytest.mark.tool("git") class TestGitRemotesRef: + """The ``git="org/repo[@ref]"`` mini-DSL: branch/tag/commit refs, parsing + corner cases, and refs containing shell-special or ambiguous characters.""" def test_branch_ref(self, git_repos): - """git= with @branch clones and checks out that branch""" git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}, branch="dev") c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", @@ -82,7 +97,6 @@ def test_branch_ref(self, git_repos): assert "git ref: dev" in c.out def test_tag_ref(self, git_repos): - """git= with @tag clones and checks out that tag""" git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "2.0")}, tags=["v2.0"]) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/2.0", @@ -92,7 +106,6 @@ def test_tag_ref(self, git_repos): assert "git ref: v2.0" in c.out def test_commit_ref(self, git_repos): - """git= with @ checks out that exact commit""" _, commit = git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "3.0")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/3.0", @@ -101,31 +114,76 @@ def test_commit_ref(self, git_repos): assert "mypkg/3.0" in c.out assert f"git ref: {commit}" in c.out + def test_at_in_ref_name_resolves(self, git_repos): + """Refs may contain '@' (git allows it; only '@{' is forbidden). We split + on the FIRST '@' — GitHub org/repo names can't contain '@' — so anything + after is the ref, even if the ref itself has further '@'s. + """ + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}, + branch="foo@bar") + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg@foo@bar")}) + c.run("install . --build=missing") + assert "git ref: foo@bar" in c.out + assert "mypkg/1.0" in c.out -@pytest.mark.tool("git") -class TestGitRemotesRequireTypes: + def test_ref_with_shell_metacharacter_resolves(self, git_repos): + """Refs with shell metacharacters (like '&') are legal in git but would + break if the resolver interpolated the ref into a shell string. The + resolver must invoke git via argv, not shell=True.""" + repo_path, _ = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + tag = "v1&hotfix" + subprocess.run(["git", "-C", repo_path, "tag", tag], check=True, capture_output=True) - @pytest.mark.parametrize("method", [ - "with_requirement", # self.requires(git=) — host dependency - "with_tool_requirement", # self.tool_requires(git=) — build-context tool - "with_test_requirement", # self.test_requires(git=) — test-only host dependency - ]) - def test_git_resolution_by_require_type(self, method, git_repos): - """git= works for requires, tool_requires and test_requires""" - git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) - consumer = getattr(GenConanfile(), method)("mypkg/1.0", git="myorg/mypkg") c = TestClient(light=True) - c.save({"conanfile.py": str(consumer)}) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git=f"myorg/mypkg@{tag}")}) c.run("install . --build=missing") - assert "resolving from git remote" in c.out + assert f"git ref: {tag}" in c.out assert "mypkg/1.0" in c.out + def test_trailing_at_is_error(self, git_repos): + """A trailing '@' with an empty ref is almost always a typo. Reject it + rather than silently falling back to the default branch.""" + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", git="myorg/mypkg@")}) + c.run("install . --build=missing", assert_error=True) + assert "trailing '@'" in c.out + assert "myorg/mypkg@" in c.out + assert "Cloning git repository" not in c.out + + def test_revision_with_git_is_error(self): + """git= plus an explicit recipe revision (#hash) is inconsistent — the + revision is now the git commit SHA, so pinning both is nonsense. Error + message must include the full ref so the user can locate the require.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement( + "zlib/1.2.11#" + "a" * 32, git="conan-io/zlib")}) + c.run("install . --build=missing", assert_error=True) + assert "zlib/1.2.11" in c.out + assert "'git='" in c.out or "git= source" in c.out.lower() + @pytest.mark.tool("git") -class TestGitRemotesErrors: +class TestGitRemotesContract: + """The require declares (name, version) and git= points at a source repo; + these tests pin how the two are reconciled. + + Contract: + - If the recipe hardcodes name/version, they MUST match the require + (enforced by cmd_export -> load_named). Mismatch → hard error. + - If the recipe does NOT declare name/version, the require's values are + used verbatim. git= is 'trust the URL' — the repo can host arbitrary + content and be labeled with any (name, version) the consumer picks. + - Version ranges: clone first; the recipe's declared version is validated + against the range and, if it passes, becomes the resolved version. + Out-of-range → hard error. + """ def test_missing_conanfile_in_repo(self, git_repos): - """Repo without conanfile.py gives a clear error message""" git_repos("myorg/myrepo", {"README.md": "# hello"}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", @@ -133,12 +191,56 @@ def test_missing_conanfile_in_repo(self, git_repos): c.run("install . --build=missing", assert_error=True) assert "conanfile.py not found" in c.out + def test_repo_hardcodes_different_name_errors(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("wrongname", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing", assert_error=True) + assert "Package recipe with name mypkg!=wrongname" in c.out + + def test_repo_hardcodes_different_version_errors(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "9.9.9")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing", assert_error=True) + assert "Package recipe with version 1.0!=9.9.9" in c.out + + def test_repo_declares_nothing_git_is_authoritative(self, git_repos): + """No name/version declared by the recipe → the require's values win.""" + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile()}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + + def test_version_range_resolves_from_repo(self, git_repos): + """Version range + git=: the recipe's declared version is checked + against the range and, if it fits, becomes the resolved version.""" + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/[>=1.0]", + git="myorg/mypkg")}) + c.run("install . --build=missing") + assert "resolved version: 1.2.11" in c.out + assert "mypkg/1.2.11" in c.out + + def test_version_range_out_of_range_errors(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "0.5")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/[>=1.0]", + git="myorg/mypkg")}) + c.run("install . --build=missing", assert_error=True) + assert "does not accept the version '0.5'" in c.out + @pytest.mark.tool("git") class TestGitRemotesTransitive: + """Chains and diamonds where multiple nodes use git= to source their recipes.""" def test_transitive_deps_both_from_git(self, git_repos): - """Pkg A from git= requires pkg B, which also has a git= entry in its conanfile""" git_repos("myorg/pkgb", {"conanfile.py": GenConanfile("pkgb", "1.0")}) git_repos("myorg/pkga", {"conanfile.py": GenConanfile("pkga", "1.0") @@ -151,8 +253,8 @@ def test_transitive_deps_both_from_git(self, git_repos): assert c.out.count("resolving from git remote") == 2 def test_diamond_all_from_git(self, git_repos): - """Diamond: consumer->pkga->pkgc and consumer->pkgb->pkgc, all in separate git repos. - pkgc is cloned once (first encounter); the second encounter finds it already in cache.""" + """Diamond: consumer->pkga->pkgc and consumer->pkgb->pkgc, all from git. + pkgc is cloned once; the second encounter finds it in cache.""" git_repos("myorg/pkgc", {"conanfile.py": GenConanfile("pkgc", "1.0")}) git_repos("myorg/pkga", {"conanfile.py": GenConanfile("pkga", "1.0") @@ -169,18 +271,16 @@ def test_diamond_all_from_git(self, git_repos): assert "pkga/1.0" in c.out assert "pkgb/1.0" in c.out assert "pkgc/1.0" in c.out - # pkga, pkgb, pkgc each cloned once from git (3 fresh resolutions) assert c.out.count("resolving from git remote") == 3 - # pkgc is found in cache the second time (via pkgb's requires, after pkga already exported it) assert c.out.count("Found in cache (configured via git remote") == 1 @pytest.mark.tool("git") class TestGitRemotesLockfile: + """Lockfile capture/reuse. Under revision_mode='scm' the recipe revision IS + the git commit SHA, which makes lockfiles reproducible across branch drift.""" def test_lockfile_happy_path(self, git_repos): - """First install with --lockfile-out captures the git-exported revision. - Second install with --lockfile reuses cache via the locked revision.""" git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", @@ -191,13 +291,11 @@ def test_lockfile_happy_path(self, git_repos): assert "resolving from git remote" in c.out assert "zlib/1.2.11" in c.out - # Lockfile must contain zlib/1.2.11 with a recipe revision + # Lockfile carries zlib/1.2.11 with a recipe revision lock = json.loads(c.load("conan.lock")) - requires = lock["requires"] - assert len(requires) == 1 - locked_ref = requires[0] + (locked_ref,) = lock["requires"] assert locked_ref.startswith("zlib/1.2.11#") - assert locked_ref.split("#")[1] # non-empty revision hash + assert locked_ref.split("#")[1] # Second install with lockfile: recipe already in cache → no clone c.run("install . --lockfile=conan.lock") @@ -205,47 +303,163 @@ def test_lockfile_happy_path(self, git_repos): assert "Cloning" not in c.out assert "zlib/1.2.11" in c.out - # Now removing the cache one + # Cache purged: lockfile drives a checkout of the locked commit (fresh + # clone, folder keyed by the locked SHA). Install still succeeds. c.run("remove * -c") c.run("install . --lockfile=conan.lock --build=missing") - assert "Cloning" not in c.out # it reuses the previous clone assert "zlib/1.2.11" in c.out + def test_recipe_revision_is_git_commit(self, git_repos): + """The revision annotated on the require (and stored in the lockfile) + equals the HEAD commit SHA of the cloned repo.""" + _, expected_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing --lockfile-out=conan.lock") + + lock = json.loads(c.load("conan.lock")) + (locked_ref,) = lock["requires"] + # Lockfile encodes 'ref#revision%timestamp'; keep only the revision + m = re.fullmatch(r"mypkg/1\.0#([0-9a-f]+)(?:%.*)?", locked_ref) + assert m and m.group(1) == expected_sha, (locked_ref, expected_sha) + + def test_lockfile_reproduces_after_branch_advances(self, git_repos): + """Capture a lockfile pointing to a branch; advance the branch upstream; + reinstall from the lockfile on a cold cache — the OLD commit is checked + out because the lockfile pins the SHA.""" + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing --lockfile-out=conan.lock") + + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + c.run("remove * -c") + c.run("cache clean --temp") + + c.run("install . --lockfile=conan.lock --build=missing") + assert f"mypkg/1.0#{first_sha}" in c.out + def test_lockfile_revision_mismatch_fails(self, git_repos): - """If the lockfile contains a recipe revision that differs from what git exports, - Conan must raise an error because the locked revision is not present in cache.""" + """Tampered lockfile (revision replaced with a bogus one) is caught by + the graph_lock check: the exported/cached revision does not match the + locked revision → clear error.""" repo_path, _ = git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git="conan-io/zlib")}) - # First install: populate cache and generate a valid lockfile c.run("install . --build=missing --lockfile-out=conan.lock") assert "resolving from git remote" in c.out - # Tamper the lockfile: replace the real revision with a fake one raw = c.load("conan.lock") tampered = re.sub(r"(zlib/1\.2\.11#)[0-9a-f]+", r"\1deadbeef00000000000000000000000", raw) c.save({"conan2.lock": tampered}) - # Second install with tampered lockfile: cached revision X, lockfile expects Y → clear error c.run("install . --lockfile=conan2.lock", assert_error=True) assert "zlib/1.2.11" in c.out - assert ("Requirement 'zlib/1.2.11#20823ba3fead87d6e797bd33010ca88a' " - "not in lockfile 'requires'") in c.out + assert re.search(r"Requirement 'zlib/1\.2\.11#[0-9a-f]+' not in lockfile 'requires'", + c.out) - # Now removing the cache one + # Cache purged: lockfile still drives checkout of the locked commit c.run("remove * -c") c.run("install . --lockfile=conan.lock --build=missing") - assert "Cloning" not in c.out # it reuses the previous clone assert "zlib/1.2.11" in c.out - # If the repo gets a new commit and the lockfile pins the old recipe-revision, it fails + # Branch advances upstream. Lockfile still pins the old SHA, so the + # reinstall reproduces the old commit — no revision mismatch. save(os.path.join(repo_path, "conanfile.py"), str(GenConanfile("zlib", "1.2.11").with_class_attribute("somevar=3"))) git_add_changes_commit(repo_path) - c.run("install . --lockfile=conan.lock --build=missing --update", assert_error=True) - assert "Cloning" in c.out - assert ("Requirement 'zlib/1.2.11#6a02546722ab83f3926c350c001c9c4d' " - "not in lockfile 'requires'") in c.out + c.run("remove * -c") + c.run("install . --lockfile=conan.lock --build=missing") + assert "zlib/1.2.11" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesPrecedence: + """git= is a source hint, not an override. Other mechanisms that redirect + or override a requirement (profile [replace_requires], editable packages) + take precedence — the git prefetch is skipped, with visible feedback.""" + + def test_replace_requires_wins_over_git(self, git_repos): + git_repos("myorg/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"replacement/conanfile.py": GenConanfile("my_zlib", "1.0")}) + c.run("export replacement") + + profile = "[replace_requires]\nzlib/*: my_zlib/1.0" + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git="myorg/zlib"), + "myprofile": profile}) + c.run("install . -pr=myprofile --build=missing") + assert "Ignoring git=" in c.out + assert "my_zlib/1.0" in c.out + assert "Cloning git repository" not in c.out + + def test_editable_wins_over_git(self, git_repos): + """An editable registration for the same ref short-circuits git=. + The git URL is intentionally bad (no conanfile.py) so a stray clone + would blow up — proving the editable path took over.""" + git_repos("myorg/mypkg", {"README.md": "not a conanfile"}) + c = TestClient(light=True) + c.save({"editable/conanfile.py": GenConanfile("mypkg", "1.0")}) + c.run("editable add editable") + + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + assert "Cloning git repository" not in c.out + assert "conanfile.py not found" not in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesLifecycle: + """Lifecycle of the on-disk git_clones/ folder: cleanup on demand, and + auto-recovery from a half-clone left by a previous interrupted run.""" + + def test_cache_clean_removes_git_clones(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", git="myorg/mypkg")}) + c.run("install . --build=missing") + + clones_root = c.cache.git_clones_folder + assert os.path.isdir(clones_root) and os.listdir(clones_root) + + c.run("cache clean --temp") + assert not os.path.exists(clones_root) or not os.listdir(clones_root) + + def test_dirty_clone_auto_recovers_on_next_run(self, git_repos): + """Interrupted clone/checkout leaves a .dirty marker; the next run + against the same ref detects it, re-clones from scratch, and succeeds + without --update or manual cleanup.""" + repo_path, _ = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + + # First attempt: tag doesn't exist yet → checkout fails → dirty stays + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg@v1.0")}) + c.run("install . --build=missing", assert_error=True) + + clones_root = c.cache.git_clones_folder + subdirs = os.listdir(clones_root) + assert any(name.endswith(".dirty") for name in subdirs) + + # Upstream fixed: the tag now exists in the remote + subprocess.run(["git", "-C", repo_path, "tag", "v1.0"], + check=True, capture_output=True) + + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + subdirs = os.listdir(clones_root) + assert not any(name.endswith(".dirty") for name in subdirs), subdirs From 855b74e778cbc362e1c52abcf95422280391d2a1 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 3 Aug 2026 13:41:25 +0200 Subject: [PATCH 10/10] review --- conan/internal/graph/git_remotes_resolver.py | 54 ++++++++--- test/functional/graph/test_git_remotes.py | 98 ++++++++++++++++++-- 2 files changed, 132 insertions(+), 20 deletions(-) diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py index 755aa2032f7..64a09fbaa48 100644 --- a/conan/internal/graph/git_remotes_resolver.py +++ b/conan/internal/graph/git_remotes_resolver.py @@ -36,15 +36,22 @@ def prefetch(self, node, require, update, loader, editable_packages, lockfile=No within the require's range. """ ref = require.ref - # replace_requires ran and matched: _resolve_replace_requires sets - # _required_ref = ref.copy() BEFORE mutating ref. If they are no longer the same - # object, the ref was renamed and the hardcoded git= source cannot apply — - # replace_requires wins. + # replace_requires ran: _resolve_replace_requires copies the original ref into + # _required_ref BEFORE mutating ref. If (name, version, user, channel) changed, + # the require was RENAMED and git= (which hardcodes a specific package identity) + # cannot follow — replace_requires wins. If only the revision was added, the + # replacement is compatible with git= and its revision drives the checkout. output = ConanOutput(scope=str(node)) - if require._required_ref is not ref: # noqa - output.warning(f"Ignoring git={require.git!r}: 'replace_requires' matched " - f"and took precedence over the git= source.") - return + orig_ref = require._required_ref # noqa + if orig_ref is not ref: + if (ref.name != orig_ref.name or ref.version != orig_ref.version + or ref.user != orig_ref.user or ref.channel != orig_ref.channel): + output.warning(f"Ignoring git={require.git!r}: 'replace_requires' " + f"renamed the require ({orig_ref} → {ref}) and took precedence " + f"over the git= source.") + return + # Otherwise: replacement only added a revision — fall through, and the + # revision-driven checkout logic below will pick it up. # Editable takes precedence over git=: a local editable is a stronger # override than a hardcoded remote source. @@ -52,10 +59,6 @@ def prefetch(self, node, require, update, loader, editable_packages, lockfile=No output.info(f"Ignoring git={require.git!r}: package is in editable mode.") return - if ref.revision: - raise ConanException( - f"Requirement '{ref}' with an explicit revision cannot use a 'git=' source") - git = require.git # raw string: "org/repo" or "org/repo@ref" # split on the FIRST '@' — org/repo cannot contain '@' (GitHub disallows it), # so anything after is the ref, even if the ref itself contains '@' @@ -66,6 +69,24 @@ def prefetch(self, node, require, update, loader, editable_packages, lockfile=No f"Drop the '@' to use the default branch, or specify a branch/tag/commit.") repo, git_ref = idx if len(idx) == 2 else (idx[0], None) + # Reconcile the two ways to pin a commit: an explicit recipe revision on the + # require ('pkg/version#') vs a ref in git= ('org/repo@'). Under + # revision_mode='scm' the recipe revision IS the git commit. + # + # Rule: only the RECIPE-authored combination (both #revision and @ref written + # by the recipe author) is a contradiction. If the revision was added from + # outside the recipe (replace_requires, later lockfile peek), it wins over + # git=@ref silently — those are explicit out-of-recipe overrides and Conan + # cannot tell from ref shape whether @ref is mutable, so it just trusts them. + # _required_ref carries the pre-replace state; if it has a revision, the + # recipe itself declared one. + if orig_ref.revision and git_ref: + raise ConanException( + f"Requirement '{ref}' pins a revision and git={git!r} also pins " + f"a ref — use only one") + if ref.revision: + git_ref = ref.revision # recipe-authored (git_ref is None here) or override + # Lockfile-driven checkout: if a matching entry is locked with a revision # (git commit SHA under our revision_mode='scm' contract), use it as the # checkout target so reinstalls reproduce the exact commit even if the @@ -95,8 +116,13 @@ def prefetch(self, node, require, update, loader, editable_packages, lockfile=No # Cache-first shortcut only makes sense for a fully-resolved ref. # With a range, we need to re-resolve which concrete version applies. try: - layout = self._cache.recipe_layout_latest(ref) - require.ref.revision = layout.reference.revision + if ref.revision: + # Specific revision pinned (recipe / replace_requires / lockfile) — + # only accept an exact match in the cache + _ = self._cache.recipe_layout(ref) + else: + layout = self._cache.recipe_layout_latest(ref) + require.ref.revision = layout.reference.revision output.info(f"Found in cache (configured via git remote '{url}')") return except ConanException: diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py index 620d71f4657..1a49c975e15 100644 --- a/test/functional/graph/test_git_remotes.py +++ b/test/functional/graph/test_git_remotes.py @@ -155,16 +155,17 @@ def test_trailing_at_is_error(self, git_repos): assert "myorg/mypkg@" in c.out assert "Cloning git repository" not in c.out - def test_revision_with_git_is_error(self): - """git= plus an explicit recipe revision (#hash) is inconsistent — the - revision is now the git commit SHA, so pinning both is nonsense. Error - message must include the full ref so the user can locate the require.""" + def test_revision_and_git_ref_together_error(self): + """A require may pin a commit either by recipe revision ('pkg/1.0#') + OR by ref in git= ('org/repo@') — not both. Under revision_mode='scm' + those two pins refer to the same identity, so specifying both from the + recipe is ambiguous. Missing conflict is reported clearly.""" c = TestClient(light=True) c.save({"conanfile.py": GenConanfile().with_requirement( - "zlib/1.2.11#" + "a" * 32, git="conan-io/zlib")}) + "zlib/1.2.11#" + "a" * 32, git="conan-io/zlib@main")}) c.run("install . --build=missing", assert_error=True) assert "zlib/1.2.11" in c.out - assert "'git='" in c.out or "git= source" in c.out.lower() + assert "use only one" in c.out @pytest.mark.tool("git") @@ -404,6 +405,91 @@ def test_replace_requires_wins_over_git(self, git_repos): assert "my_zlib/1.0" in c.out assert "Cloning git repository" not in c.out + def test_revision_on_require_pins_git_checkout(self, git_repos): + """A require of the form ``pkg/version#`` combined with a + commit-less ``git="org/repo"`` — under revision_mode='scm' the recipe + revision IS the git SHA, so the two annotations refer to the same + identity and could reasonably drive a git checkout of that commit. + NOT supported today: prefetch rejects revision + git= as inconsistent. + Companion to test_revision_with_git_is_error (which pins the current + rejection behavior). + """ + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + # Branch advances upstream — the require should still resolve to first_sha + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement( + f"mypkg/1.0#{first_sha}", git="myorg/mypkg")}) + c.run("install . --build=missing") + + assert f"mypkg/1.0#{first_sha}" in c.out + assert second_sha not in c.out + + def test_replace_requires_pins_commit_over_recipe_branch(self, git_repos): + """Same reproducibility pattern as test_replace_requires_pins_commit_from_profile, + but the recipe already carries a branch ref: ``git="myorg/mypkg@main"``. + The profile [replace_requires] adds a specific revision on top. Intent: + the profile-supplied commit should override the recipe's branch pin — + replace_requires is an explicit, out-of-recipe knob and its revision + should take precedence. + + Currently FAILS: with both @ref (from git=) and #revision (from + replace_requires) present, prefetch raises "use only one". + """ + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement( + "mypkg/1.0", git="myorg/mypkg@main")}) + + # Branch advances upstream + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + profile = f"[replace_requires]\nmypkg/1.0: mypkg/1.0#{first_sha}" + c.save({"myprofile": profile}) + c.run("install . -pr=myprofile --build=missing") + + assert f"mypkg/1.0#{first_sha}" in c.out + assert second_sha not in c.out + + def test_replace_requires_pins_commit_from_profile(self, git_repos): + """A user tries to pin a specific commit via profile [replace_requires] + (name/version unchanged, revision added), so builds reproduce even after + the branch tip has moved — analogous to the lockfile-driven flow but + on-demand from a profile. Currently NOT supported: [replace_requires] + takes precedence and skips the git prefetch entirely, so the resolver + looks for the pinned revision in the cache (miss) rather than driving + a git checkout of that commit. + """ + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + + # Branch advances upstream after we recorded 'first_sha' + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + # Profile pins the earlier commit as the revision to install + profile = f"[replace_requires]\nmypkg/1.0: mypkg/1.0#{first_sha}" + c.save({"myprofile": profile}) + c.run("install . -pr=myprofile --build=missing") + + # Reproducibility win: first commit is checked out, not the new tip + assert f"mypkg/1.0#{first_sha}" in c.out + assert second_sha not in c.out + def test_editable_wins_over_git(self, git_repos): """An editable registration for the same ref short-circuits git=. The git URL is intentionally bad (no conanfile.py) so a stray clone