diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py index c95796ce4f0..6ccda39534c 100644 --- a/conan/api/subapi/cache.py +++ b/conan/api/subapi/cache.py @@ -271,6 +271,7 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True, cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf) if temp: rmdir(cache.temp_folder) + rmdir(cache.git_clones_folder) # git-source clones are scratch space too # Clean those build folders that didn't succeed to create a package and wont be in DB builds_folder = cache.builds_folder if os.path.isdir(builds_folder): diff --git a/conan/internal/api/export.py b/conan/internal/api/export.py index 0e55711f90d..f1d7475440c 100644 --- a/conan/internal/api/export.py +++ b/conan/internal/api/export.py @@ -14,13 +14,18 @@ def cmd_export(loader, cache, hook_manager, global_conf, conanfile_path, name, version, user, channel, - graph_lock=None, remotes=None): + graph_lock=None, remotes=None, revision_mode_scm=False): """ Export the recipe param conanfile_path: the original source directory of the user containing a conanfile.py + param revision_mode_scm: force revision_mode='scm' regardless of what the recipe + declares. Used by git= sources so the recipe revision IS the + git commit — see GitRemotesResolver. """ conanfile = loader.load_export(conanfile_path, name, version, user, channel, graph_lock, remotes=remotes) + if revision_mode_scm: + conanfile.revision_mode = "scm" ref = RecipeReference(conanfile.name, conanfile.version, conanfile.user, conanfile.channel) ref.validate_ref(allow_uppercase=global_conf.get("core:allow_uppercase_pkg_names", diff --git a/conan/internal/cache/cache.py b/conan/internal/cache/cache.py index 27b6c336705..34d72552d90 100644 --- a/conan/internal/cache/cache.py +++ b/conan/internal/cache/cache.py @@ -50,6 +50,10 @@ def temp_folder(self): def builds_folder(self): return os.path.join(self._base_folder, "b") + @property + def git_clones_folder(self): + return os.path.join(self._base_folder, "git_clones") + def _create_path(self, relative_path, remove_contents=True): path = self._full_path(relative_path) if os.path.exists(path) and remove_contents: diff --git a/conan/internal/graph/git_remotes_resolver.py b/conan/internal/graph/git_remotes_resolver.py new file mode 100644 index 00000000000..64a09fbaa48 --- /dev/null +++ b/conan/internal/graph/git_remotes_resolver.py @@ -0,0 +1,212 @@ +import hashlib +import os +import subprocess + +from conan.api.output import ConanOutput +from conan.errors import ConanException +from conan.internal.api.export import cmd_export +from conan.internal.graph.proxy import should_update_reference +from conan.internal.util.files import remove_if_dirty, rmdir, set_dirty_context_manager + + +class GitRemotesResolver: + + def __init__(self, cache): + self._cache = cache + self._clones_base = cache.git_clones_folder + + @staticmethod + def _get_url(repo): + # Maybe we need to extend this to gitlab too, we could check a "gl:org/repo" format + return f"https://github.com/{repo}.git" + + def prefetch(self, node, require, update, loader, editable_packages, lockfile=None): + """If the requirement declares a git= source, clone and export it into the local + cache before range resolution and proxy lookup run. Mirrors the + _resolve_replace_requires pattern — invoked early in _initialize_requires so the + rest of graph resolution is transparent (proxy just finds the recipe in cache). + + Semantics: + - The recipe revision IS the git commit SHA (revision_mode='scm' is forced + during export). A lockfile captured today pins the exact commit, so a + later reinstall from that lockfile can reproduce the build even if a + branch tip has moved in the meantime. + - Version ranges are allowed. The clone runs first, the recipe declares its + own version, and the resolver validates that the declared version is + within the require's range. + """ + ref = require.ref + # replace_requires ran: _resolve_replace_requires copies the original ref into + # _required_ref BEFORE mutating ref. If (name, version, user, channel) changed, + # the require was RENAMED and git= (which hardcodes a specific package identity) + # cannot follow — replace_requires wins. If only the revision was added, the + # replacement is compatible with git= and its revision drives the checkout. + output = ConanOutput(scope=str(node)) + orig_ref = require._required_ref # noqa + if orig_ref is not ref: + if (ref.name != orig_ref.name or ref.version != orig_ref.version + or ref.user != orig_ref.user or ref.channel != orig_ref.channel): + output.warning(f"Ignoring git={require.git!r}: 'replace_requires' " + f"renamed the require ({orig_ref} → {ref}) and took precedence " + f"over the git= source.") + return + # Otherwise: replacement only added a revision — fall through, and the + # revision-driven checkout logic below will pick it up. + + # Editable takes precedence over git=: a local editable is a stronger + # override than a hardcoded remote source. + if editable_packages is not None and editable_packages.get(ref) is not None: + output.info(f"Ignoring git={require.git!r}: package is in editable mode.") + return + + git = require.git # raw string: "org/repo" or "org/repo@ref" + # split on the FIRST '@' — org/repo cannot contain '@' (GitHub disallows it), + # so anything after is the ref, even if the ref itself contains '@' + idx = git.split("@", 1) + if len(idx) == 2 and not idx[1]: + raise ConanException( + f"Requirement '{ref}': git={git!r} has a trailing '@' with no ref. " + f"Drop the '@' to use the default branch, or specify a branch/tag/commit.") + repo, git_ref = idx if len(idx) == 2 else (idx[0], None) + + # Reconcile the two ways to pin a commit: an explicit recipe revision on the + # require ('pkg/version#') vs a ref in git= ('org/repo@'). Under + # revision_mode='scm' the recipe revision IS the git commit. + # + # Rule: only the RECIPE-authored combination (both #revision and @ref written + # by the recipe author) is a contradiction. If the revision was added from + # outside the recipe (replace_requires, later lockfile peek), it wins over + # git=@ref silently — those are explicit out-of-recipe overrides and Conan + # cannot tell from ref shape whether @ref is mutable, so it just trusts them. + # _required_ref carries the pre-replace state; if it has a revision, the + # recipe itself declared one. + if orig_ref.revision and git_ref: + raise ConanException( + f"Requirement '{ref}' pins a revision and git={git!r} also pins " + f"a ref — use only one") + if ref.revision: + git_ref = ref.revision # recipe-authored (git_ref is None here) or override + + # Lockfile-driven checkout: if a matching entry is locked with a revision + # (git commit SHA under our revision_mode='scm' contract), use it as the + # checkout target so reinstalls reproduce the exact commit even if the + # branch has moved upstream. Peek only — do NOT let resolve_locked mutate + # require.ref to share identity with the lockfile's internal ref (that + # would cause our later revision assignment to poison the lockfile). + if lockfile is not None: + saved_ref = require.ref + try: + lockfile.resolve_locked(node, require, resolve_prereleases=None) + locked_rev = require.ref.revision + except ConanException: + locked_rev = None + finally: + require.ref = saved_ref + if locked_rev: + git_ref = locked_rev + + url = self._get_url(repo) + force_clone = should_update_reference(ref, update) + version_range = require.version_range + + output = ConanOutput(scope=str(ref)) + if force_clone: + output.info(f"Updating from git remote '{url}'...") + elif not version_range: + # Cache-first shortcut only makes sense for a fully-resolved ref. + # With a range, we need to re-resolve which concrete version applies. + try: + if ref.revision: + # Specific revision pinned (recipe / replace_requires / lockfile) — + # only accept an exact match in the cache + _ = self._cache.recipe_layout(ref) + else: + layout = self._cache.recipe_layout_latest(ref) + require.ref.revision = layout.reference.revision + output.info(f"Found in cache (configured via git remote '{url}')") + return + except ConanException: + pass # Not in cache — proceed with clone+export + output.info(f"Not found in local cache, resolving from git remote '{url}'") + + if git_ref: + output.info(f" git ref: {git_ref}") + + # With a version range, let the recipe declare its own version and validate + # it after export. Without a range, pass the exact version so cmd_export + # enforces the recipe hardcodes match (existing behavior). + version = None if version_range else str(ref.version) + exported_ref, _ = self._clone_and_export(ref, repo, git_ref, loader, + force_clone, version=version) + + if version_range: + resolved_version = exported_ref.version + resolve_prereleases = None # let VersionRange default apply + if not version_range.contains(resolved_version, resolve_prereleases): + raise ConanException( + f"Requirement '{ref}' with range '{version_range}' does not accept " + f"the version '{resolved_version}' declared by the recipe at " + f"git remote '{url}'") + output.info(f" resolved version: {resolved_version} (in range {version_range})") + require.ref.version = resolved_version + + # Recipe revision is the git commit SHA (revision_mode='scm' forced during export) + require.ref.revision = exported_ref.revision + + def _clone_and_export(self, ref, repo, git_ref, loader, force_clone, version=None): + url = self._get_url(repo) + clone_folder = self._clone_folder(url, git_ref) + # Leftover from a previous run interrupted mid-clone/checkout: discard it + remove_if_dirty(clone_folder) + if force_clone and os.path.exists(clone_folder): + rmdir(clone_folder) + if not os.path.exists(clone_folder): + self._do_clone(url, git_ref, clone_folder) + conanfile_path = os.path.join(clone_folder, "conanfile.py") + if not os.path.exists(conanfile_path): + raise ConanException( + f"conanfile.py not found at root of git repo '{url}'") + + # Hooks are intentionally skipped: git= is aimed at open-source / + # community workflows that pull recipes straight from public github.com + # repos, not at organizations that rely on pre/post_export hooks for + # policy, signing or scanning. + class _NoopHooks: + def execute(self, *a, **kw): pass + + from conan.internal.model.conf import ConfDefinition + # Force revision_mode='scm' → recipe revision = git commit SHA. Enables + # lockfile reproducibility even against moving branches. + # no remotes, no lockfile, as python-requires are not supported now + return cmd_export(loader, self._cache, _NoopHooks(), ConfDefinition(), + conanfile_path, ref.name, version, + ref.user, ref.channel, revision_mode_scm=True) + + def _clone_folder(self, url, git_ref): + key = f"{url}:{git_ref or ''}" + h = hashlib.md5(key.encode()).hexdigest()[:12] + return os.path.join(self._clones_base, h) + + @staticmethod + def _run_git(argv): + # argv-form; never shell=True — keeps refs/URLs with metachars intact + proc = subprocess.run(argv, capture_output=True, text=True) + return proc.returncode, (proc.stdout or "") + (proc.stderr or "") + + @classmethod + def _do_clone(cls, url, git_ref, clone_folder): + output = ConanOutput() + os.makedirs(clone_folder, exist_ok=True) + # dirty marker: if the process is interrupted mid-clone/checkout, the + # next run detects the marker via remove_if_dirty and starts fresh + with set_dirty_context_manager(clone_folder): + output.info(f"Cloning git repository '{url}'...") + ret, out = cls._run_git(["git", "clone", url, clone_folder]) + if ret != 0: + raise ConanException(f"git clone failed for '{url}':\n{out}") + if git_ref: + output.info(f"Checking out git ref '{git_ref}'...") + ret, out = cls._run_git(["git", "-C", clone_folder, "checkout", git_ref]) + if ret != 0: + raise ConanException( + f"git checkout '{git_ref}' failed for '{url}':\n{out}") diff --git a/conan/internal/graph/graph_builder.py b/conan/internal/graph/graph_builder.py index 1c9ab6f1adc..894766714a3 100644 --- a/conan/internal/graph/graph_builder.py +++ b/conan/internal/graph/graph_builder.py @@ -241,6 +241,11 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h if not resolved: self._resolve_alias(node, require, alias, graph) self._resolve_replace_requires(node, require, profile_build, profile_host, graph) + if require.git is not None: + from conan.internal.graph.git_remotes_resolver import GitRemotesResolver + GitRemotesResolver(self._cache).prefetch(node, require, self._update, self._loader, + self._proxy._editable_packages, # noqa + graph_lock) if graph_lock: graph_lock.resolve_overrides(require, node.context) node.transitive_deps[require] = TransitiveRequirement(require, node=None) diff --git a/conan/internal/model/requires.py b/conan/internal/model/requires.py index 4f30269a52d..a340b99e17b 100644 --- a/conan/internal/model/requires.py +++ b/conan/internal/model/requires.py @@ -10,7 +10,7 @@ class Requirement: def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visible=None, transitive_headers=None, transitive_libs=None, test=None, package_id_mode=None, force=None, override=None, direct=None, options=None, no_skip=False, - consistent=None): + consistent=None, git=None): # * prevents the usage of more positional parameters, always ref + **kwargs # By default this is a generic library requirement self.ref = ref @@ -40,6 +40,8 @@ def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visib self.skip = False self.required_nodes = set() # store which intermediate nodes are required, to compute "Skip" self.no_skip = no_skip + # git source: raw string for https://github.com public open source repositories + self.git = git # computed ones, not default ones self.consistent_policy_new = False if self.visible and not self.consistent: @@ -443,10 +445,10 @@ def __init__(self, requires): self._requires = requires def __call__(self, ref, package_id_mode=None, visible=False, run=True, options=None, - override=None): + override=None, git=None): # TODO: Check which arguments could be user-defined self._requires.tool_require(ref, package_id_mode=package_id_mode, visible=visible, run=run, - options=options, override=override) + options=options, override=override, git=git) class TestRequirements: @@ -454,8 +456,8 @@ class TestRequirements: def __init__(self, requires): self._requires = requires - def __call__(self, ref, run=None, options=None, force=None): - self._requires.test_require(ref, run=run, options=options, force=force) + def __call__(self, ref, run=None, options=None, force=None, git=None): + self._requires.test_require(ref, run=run, options=options, force=force, git=git) class Requirements: @@ -525,6 +527,13 @@ def values(self): return self._requires.values() def __call__(self, str_ref, **kwargs): + """Add a regular requirement (as in ``self.requires("zlib/1.2.11")``). + + Any keyword accepted by ``Requirement`` may be passed through kwargs, including + ``git="org/repo[@ref]"`` to source the recipe from a public GitHub repository — + symmetric with ``self.test_requires`` and ``self.tool_requires``, which declare + ``git=`` in their own signatures. + """ if str_ref is None: return assert isinstance(str_ref, str) @@ -557,7 +566,7 @@ def build_require(self, ref, raise_if_duplicated=True, package_id_mode=None, vis raise ConanException("Duplicated requirement: {}".format(ref)) self._requires[req] = req - def test_require(self, ref, run=None, options=None, force=None): + def test_require(self, ref, run=None, options=None, force=None, git=None): """ Represent a testing framework like gtest @@ -573,13 +582,13 @@ def test_require(self, ref, run=None, options=None, force=None): # libs = True => We need to link with it # headers = True => We need to include it req = Requirement(ref, headers=True, libs=True, build=False, run=run, visible=False, - test=True, package_id_mode=None, options=options, force=force) + test=True, package_id_mode=None, options=options, force=force, git=git) if self._requires.get(req): raise ConanException("Duplicated requirement: {}".format(ref)) self._requires[req] = req def tool_require(self, ref, raise_if_duplicated=True, package_id_mode=None, visible=False, - run=True, options=None, override=None): + run=True, options=None, override=None, git=None): """ Represent a build tool like "cmake". @@ -593,7 +602,8 @@ def tool_require(self, ref, raise_if_duplicated=True, package_id_mode=None, visi # FIXME: This raise_if_duplicated is ugly, possibly remove ref = RecipeReference.loads(ref) req = Requirement(ref, headers=False, libs=False, build=True, run=run, visible=visible, - package_id_mode=package_id_mode, options=options, override=override) + package_id_mode=package_id_mode, options=options, override=override, + git=git) if raise_if_duplicated and self._requires.get(req): raise ConanException("Duplicated requirement: {}".format(ref)) self._requires[req] = req diff --git a/test/functional/graph/test_git_remotes.py b/test/functional/graph/test_git_remotes.py new file mode 100644 index 00000000000..1a49c975e15 --- /dev/null +++ b/test/functional/graph/test_git_remotes.py @@ -0,0 +1,551 @@ +import json +import os +import re +import subprocess +from unittest import mock + +import pytest + +from conan.internal.util.files import save +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.scm import create_local_git_repo, git_add_changes_commit +from conan.test.utils.tools import TestClient + + +@pytest.fixture +def git_repos(): + """Patches GitRemotesResolver._get_url to serve local repos by 'org/repo' slug. + Yields a register(slug, files, **kwargs) helper that creates a local git repo + and maps the slug -> local path for the duration of the test.""" + mapping = {} + + def register(slug, files=None, **kwargs): + path, commit = create_local_git_repo(files, **kwargs) + mapping[slug] = path + return path, commit + + with mock.patch( + "conan.internal.graph.git_remotes_resolver.GitRemotesResolver._get_url", + side_effect=lambda repo: mapping[repo], + ): + yield register + + +@pytest.mark.tool("git") +class TestGitRemotesBasic: + """First-run resolution, cache reuse, --update behavior, and support across + self.requires / self.tool_requires / self.test_requires.""" + + def test_basic_resolution_from_git(self, git_repos): + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) + c.run("install . --build=missing") + assert "resolving from git remote" in c.out + assert "zlib/1.2.11" in c.out + + def test_cache_first_no_reclone_on_second_run(self, git_repos): + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) + c.run("install . --build=missing") + assert "resolving from git remote" in c.out + + c.run("install . --build=missing") + assert "Found in cache (configured via git remote" in c.out + assert "Cloning" not in c.out + + def test_update_flag_forces_reclone(self, git_repos): + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) + c.run("install . --build=missing") + c.run("install . --build=missing --update") + assert "Updating from git remote" in c.out + assert "Cloning" in c.out + + @pytest.mark.parametrize("method", [ + "with_requirement", # self.requires(git=) — host dependency + "with_tool_requirement", # self.tool_requires(git=) — build-context tool + "with_test_requirement", # self.test_requires(git=) — test-only host dependency + ]) + def test_resolution_by_require_type(self, method, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + consumer = getattr(GenConanfile(), method)("mypkg/1.0", git="myorg/mypkg") + c = TestClient(light=True) + c.save({"conanfile.py": str(consumer)}) + c.run("install . --build=missing") + assert "resolving from git remote" in c.out + assert "mypkg/1.0" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesRef: + """The ``git="org/repo[@ref]"`` mini-DSL: branch/tag/commit refs, parsing + corner cases, and refs containing shell-special or ambiguous characters.""" + + def test_branch_ref(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}, branch="dev") + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg@dev")}) + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + assert "git ref: dev" in c.out + + def test_tag_ref(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "2.0")}, tags=["v2.0"]) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/2.0", + git="myorg/mypkg@v2.0")}) + c.run("install . --build=missing") + assert "mypkg/2.0" in c.out + assert "git ref: v2.0" in c.out + + def test_commit_ref(self, git_repos): + _, commit = git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "3.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/3.0", + git=f"myorg/mypkg@{commit}")}) + c.run("install . --build=missing") + assert "mypkg/3.0" in c.out + assert f"git ref: {commit}" in c.out + + def test_at_in_ref_name_resolves(self, git_repos): + """Refs may contain '@' (git allows it; only '@{' is forbidden). We split + on the FIRST '@' — GitHub org/repo names can't contain '@' — so anything + after is the ref, even if the ref itself has further '@'s. + """ + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}, + branch="foo@bar") + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg@foo@bar")}) + c.run("install . --build=missing") + assert "git ref: foo@bar" in c.out + assert "mypkg/1.0" in c.out + + def test_ref_with_shell_metacharacter_resolves(self, git_repos): + """Refs with shell metacharacters (like '&') are legal in git but would + break if the resolver interpolated the ref into a shell string. The + resolver must invoke git via argv, not shell=True.""" + repo_path, _ = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + tag = "v1&hotfix" + subprocess.run(["git", "-C", repo_path, "tag", tag], check=True, capture_output=True) + + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git=f"myorg/mypkg@{tag}")}) + c.run("install . --build=missing") + assert f"git ref: {tag}" in c.out + assert "mypkg/1.0" in c.out + + def test_trailing_at_is_error(self, git_repos): + """A trailing '@' with an empty ref is almost always a typo. Reject it + rather than silently falling back to the default branch.""" + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", git="myorg/mypkg@")}) + c.run("install . --build=missing", assert_error=True) + assert "trailing '@'" in c.out + assert "myorg/mypkg@" in c.out + assert "Cloning git repository" not in c.out + + def test_revision_and_git_ref_together_error(self): + """A require may pin a commit either by recipe revision ('pkg/1.0#') + OR by ref in git= ('org/repo@') — not both. Under revision_mode='scm' + those two pins refer to the same identity, so specifying both from the + recipe is ambiguous. Missing conflict is reported clearly.""" + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement( + "zlib/1.2.11#" + "a" * 32, git="conan-io/zlib@main")}) + c.run("install . --build=missing", assert_error=True) + assert "zlib/1.2.11" in c.out + assert "use only one" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesContract: + """The require declares (name, version) and git= points at a source repo; + these tests pin how the two are reconciled. + + Contract: + - If the recipe hardcodes name/version, they MUST match the require + (enforced by cmd_export -> load_named). Mismatch → hard error. + - If the recipe does NOT declare name/version, the require's values are + used verbatim. git= is 'trust the URL' — the repo can host arbitrary + content and be labeled with any (name, version) the consumer picks. + - Version ranges: clone first; the recipe's declared version is validated + against the range and, if it passes, becomes the resolved version. + Out-of-range → hard error. + """ + + def test_missing_conanfile_in_repo(self, git_repos): + git_repos("myorg/myrepo", {"README.md": "# hello"}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="myorg/myrepo")}) + c.run("install . --build=missing", assert_error=True) + assert "conanfile.py not found" in c.out + + def test_repo_hardcodes_different_name_errors(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("wrongname", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing", assert_error=True) + assert "Package recipe with name mypkg!=wrongname" in c.out + + def test_repo_hardcodes_different_version_errors(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "9.9.9")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing", assert_error=True) + assert "Package recipe with version 1.0!=9.9.9" in c.out + + def test_repo_declares_nothing_git_is_authoritative(self, git_repos): + """No name/version declared by the recipe → the require's values win.""" + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile()}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + + def test_version_range_resolves_from_repo(self, git_repos): + """Version range + git=: the recipe's declared version is checked + against the range and, if it fits, becomes the resolved version.""" + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/[>=1.0]", + git="myorg/mypkg")}) + c.run("install . --build=missing") + assert "resolved version: 1.2.11" in c.out + assert "mypkg/1.2.11" in c.out + + def test_version_range_out_of_range_errors(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "0.5")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/[>=1.0]", + git="myorg/mypkg")}) + c.run("install . --build=missing", assert_error=True) + assert "does not accept the version '0.5'" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesTransitive: + """Chains and diamonds where multiple nodes use git= to source their recipes.""" + + def test_transitive_deps_both_from_git(self, git_repos): + git_repos("myorg/pkgb", {"conanfile.py": GenConanfile("pkgb", "1.0")}) + git_repos("myorg/pkga", + {"conanfile.py": GenConanfile("pkga", "1.0") + .with_requirement("pkgb/1.0", git="myorg/pkgb")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("pkga/1.0", git="myorg/pkga")}) + c.run("install . --build=missing") + assert "pkga/1.0" in c.out + assert "pkgb/1.0" in c.out + assert c.out.count("resolving from git remote") == 2 + + def test_diamond_all_from_git(self, git_repos): + """Diamond: consumer->pkga->pkgc and consumer->pkgb->pkgc, all from git. + pkgc is cloned once; the second encounter finds it in cache.""" + git_repos("myorg/pkgc", {"conanfile.py": GenConanfile("pkgc", "1.0")}) + git_repos("myorg/pkga", + {"conanfile.py": GenConanfile("pkga", "1.0") + .with_requirement("pkgc/1.0", git="myorg/pkgc")}) + git_repos("myorg/pkgb", + {"conanfile.py": GenConanfile("pkgb", "1.0") + .with_requirement("pkgc/1.0", git="myorg/pkgc")}) + c = TestClient(light=True) + c.save({"conanfile.py": str( + GenConanfile() + .with_requirement("pkga/1.0", git="myorg/pkga") + .with_requirement("pkgb/1.0", git="myorg/pkgb"))}) + c.run("install . --build=missing") + assert "pkga/1.0" in c.out + assert "pkgb/1.0" in c.out + assert "pkgc/1.0" in c.out + assert c.out.count("resolving from git remote") == 3 + assert c.out.count("Found in cache (configured via git remote") == 1 + + +@pytest.mark.tool("git") +class TestGitRemotesLockfile: + """Lockfile capture/reuse. Under revision_mode='scm' the recipe revision IS + the git commit SHA, which makes lockfiles reproducible across branch drift.""" + + def test_lockfile_happy_path(self, git_repos): + git_repos("conan-io/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) + + # First install: clone from git, export to cache, write lockfile + c.run("install . --build=missing --lockfile-out=conan.lock") + assert "resolving from git remote" in c.out + assert "zlib/1.2.11" in c.out + + # Lockfile carries zlib/1.2.11 with a recipe revision + lock = json.loads(c.load("conan.lock")) + (locked_ref,) = lock["requires"] + assert locked_ref.startswith("zlib/1.2.11#") + assert locked_ref.split("#")[1] + + # Second install with lockfile: recipe already in cache → no clone + c.run("install . --lockfile=conan.lock") + assert "Found in cache (configured via git remote" in c.out + assert "Cloning" not in c.out + assert "zlib/1.2.11" in c.out + + # Cache purged: lockfile drives a checkout of the locked commit (fresh + # clone, folder keyed by the locked SHA). Install still succeeds. + c.run("remove * -c") + c.run("install . --lockfile=conan.lock --build=missing") + assert "zlib/1.2.11" in c.out + + def test_recipe_revision_is_git_commit(self, git_repos): + """The revision annotated on the require (and stored in the lockfile) + equals the HEAD commit SHA of the cloned repo.""" + _, expected_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing --lockfile-out=conan.lock") + + lock = json.loads(c.load("conan.lock")) + (locked_ref,) = lock["requires"] + # Lockfile encodes 'ref#revision%timestamp'; keep only the revision + m = re.fullmatch(r"mypkg/1\.0#([0-9a-f]+)(?:%.*)?", locked_ref) + assert m and m.group(1) == expected_sha, (locked_ref, expected_sha) + + def test_lockfile_reproduces_after_branch_advances(self, git_repos): + """Capture a lockfile pointing to a branch; advance the branch upstream; + reinstall from the lockfile on a cold cache — the OLD commit is checked + out because the lockfile pins the SHA.""" + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing --lockfile-out=conan.lock") + + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + c.run("remove * -c") + c.run("cache clean --temp") + + c.run("install . --lockfile=conan.lock --build=missing") + assert f"mypkg/1.0#{first_sha}" in c.out + + def test_lockfile_revision_mismatch_fails(self, git_repos): + """Tampered lockfile (revision replaced with a bogus one) is caught by + the graph_lock check: the exported/cached revision does not match the + locked revision → clear error.""" + repo_path, _ = git_repos("conan-io/zlib", + {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", + git="conan-io/zlib")}) + + c.run("install . --build=missing --lockfile-out=conan.lock") + assert "resolving from git remote" in c.out + + raw = c.load("conan.lock") + tampered = re.sub(r"(zlib/1\.2\.11#)[0-9a-f]+", r"\1deadbeef00000000000000000000000", raw) + c.save({"conan2.lock": tampered}) + + c.run("install . --lockfile=conan2.lock", assert_error=True) + assert "zlib/1.2.11" in c.out + assert re.search(r"Requirement 'zlib/1\.2\.11#[0-9a-f]+' not in lockfile 'requires'", + c.out) + + # Cache purged: lockfile still drives checkout of the locked commit + c.run("remove * -c") + c.run("install . --lockfile=conan.lock --build=missing") + assert "zlib/1.2.11" in c.out + + # Branch advances upstream. Lockfile still pins the old SHA, so the + # reinstall reproduces the old commit — no revision mismatch. + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("zlib", "1.2.11").with_class_attribute("somevar=3"))) + git_add_changes_commit(repo_path) + c.run("remove * -c") + c.run("install . --lockfile=conan.lock --build=missing") + assert "zlib/1.2.11" in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesPrecedence: + """git= is a source hint, not an override. Other mechanisms that redirect + or override a requirement (profile [replace_requires], editable packages) + take precedence — the git prefetch is skipped, with visible feedback.""" + + def test_replace_requires_wins_over_git(self, git_repos): + git_repos("myorg/zlib", {"conanfile.py": GenConanfile("zlib", "1.2.11")}) + c = TestClient(light=True) + c.save({"replacement/conanfile.py": GenConanfile("my_zlib", "1.0")}) + c.run("export replacement") + + profile = "[replace_requires]\nzlib/*: my_zlib/1.0" + c.save({"conanfile.py": GenConanfile().with_requirement("zlib/1.2.11", git="myorg/zlib"), + "myprofile": profile}) + c.run("install . -pr=myprofile --build=missing") + assert "Ignoring git=" in c.out + assert "my_zlib/1.0" in c.out + assert "Cloning git repository" not in c.out + + def test_revision_on_require_pins_git_checkout(self, git_repos): + """A require of the form ``pkg/version#`` combined with a + commit-less ``git="org/repo"`` — under revision_mode='scm' the recipe + revision IS the git SHA, so the two annotations refer to the same + identity and could reasonably drive a git checkout of that commit. + NOT supported today: prefetch rejects revision + git= as inconsistent. + Companion to test_revision_with_git_is_error (which pins the current + rejection behavior). + """ + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + # Branch advances upstream — the require should still resolve to first_sha + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement( + f"mypkg/1.0#{first_sha}", git="myorg/mypkg")}) + c.run("install . --build=missing") + + assert f"mypkg/1.0#{first_sha}" in c.out + assert second_sha not in c.out + + def test_replace_requires_pins_commit_over_recipe_branch(self, git_repos): + """Same reproducibility pattern as test_replace_requires_pins_commit_from_profile, + but the recipe already carries a branch ref: ``git="myorg/mypkg@main"``. + The profile [replace_requires] adds a specific revision on top. Intent: + the profile-supplied commit should override the recipe's branch pin — + replace_requires is an explicit, out-of-recipe knob and its revision + should take precedence. + + Currently FAILS: with both @ref (from git=) and #revision (from + replace_requires) present, prefetch raises "use only one". + """ + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement( + "mypkg/1.0", git="myorg/mypkg@main")}) + + # Branch advances upstream + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + profile = f"[replace_requires]\nmypkg/1.0: mypkg/1.0#{first_sha}" + c.save({"myprofile": profile}) + c.run("install . -pr=myprofile --build=missing") + + assert f"mypkg/1.0#{first_sha}" in c.out + assert second_sha not in c.out + + def test_replace_requires_pins_commit_from_profile(self, git_repos): + """A user tries to pin a specific commit via profile [replace_requires] + (name/version unchanged, revision added), so builds reproduce even after + the branch tip has moved — analogous to the lockfile-driven flow but + on-demand from a profile. Currently NOT supported: [replace_requires] + takes precedence and skips the git prefetch entirely, so the resolver + looks for the pinned revision in the cache (miss) rather than driving + a git checkout of that commit. + """ + repo_path, first_sha = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + + # Branch advances upstream after we recorded 'first_sha' + save(os.path.join(repo_path, "conanfile.py"), + str(GenConanfile("mypkg", "1.0").with_class_attribute("marker='v2'"))) + second_sha = git_add_changes_commit(repo_path) + assert second_sha != first_sha + + # Profile pins the earlier commit as the revision to install + profile = f"[replace_requires]\nmypkg/1.0: mypkg/1.0#{first_sha}" + c.save({"myprofile": profile}) + c.run("install . -pr=myprofile --build=missing") + + # Reproducibility win: first commit is checked out, not the new tip + assert f"mypkg/1.0#{first_sha}" in c.out + assert second_sha not in c.out + + def test_editable_wins_over_git(self, git_repos): + """An editable registration for the same ref short-circuits git=. + The git URL is intentionally bad (no conanfile.py) so a stray clone + would blow up — proving the editable path took over.""" + git_repos("myorg/mypkg", {"README.md": "not a conanfile"}) + c = TestClient(light=True) + c.save({"editable/conanfile.py": GenConanfile("mypkg", "1.0")}) + c.run("editable add editable") + + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg")}) + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + assert "Cloning git repository" not in c.out + assert "conanfile.py not found" not in c.out + + +@pytest.mark.tool("git") +class TestGitRemotesLifecycle: + """Lifecycle of the on-disk git_clones/ folder: cleanup on demand, and + auto-recovery from a half-clone left by a previous interrupted run.""" + + def test_cache_clean_removes_git_clones(self, git_repos): + git_repos("myorg/mypkg", {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", git="myorg/mypkg")}) + c.run("install . --build=missing") + + clones_root = c.cache.git_clones_folder + assert os.path.isdir(clones_root) and os.listdir(clones_root) + + c.run("cache clean --temp") + assert not os.path.exists(clones_root) or not os.listdir(clones_root) + + def test_dirty_clone_auto_recovers_on_next_run(self, git_repos): + """Interrupted clone/checkout leaves a .dirty marker; the next run + against the same ref detects it, re-clones from scratch, and succeeds + without --update or manual cleanup.""" + repo_path, _ = git_repos("myorg/mypkg", + {"conanfile.py": GenConanfile("mypkg", "1.0")}) + c = TestClient(light=True) + + # First attempt: tag doesn't exist yet → checkout fails → dirty stays + c.save({"conanfile.py": GenConanfile().with_requirement("mypkg/1.0", + git="myorg/mypkg@v1.0")}) + c.run("install . --build=missing", assert_error=True) + + clones_root = c.cache.git_clones_folder + subdirs = os.listdir(clones_root) + assert any(name.endswith(".dirty") for name in subdirs) + + # Upstream fixed: the tag now exists in the remote + subprocess.run(["git", "-C", repo_path, "tag", "v1.0"], + check=True, capture_output=True) + + c.run("install . --build=missing") + assert "mypkg/1.0" in c.out + subdirs = os.listdir(clones_root) + assert not any(name.endswith(".dirty") for name in subdirs), subdirs