Feature/git remote

conan/api/subapi/cache.py

@@ -271,6 +271,7 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True,
         cache = PkgCache(self._conan_api.cache_folder, self._api_helpers.global_conf)
         if temp:
             rmdir(cache.temp_folder)
+            rmdir(cache.git_clones_folder)  # git-source clones are scratch space too
             # Clean those build folders that didn't succeed to create a package and wont be in DB
             builds_folder = cache.builds_folder
             if os.path.isdir(builds_folder):

conan/internal/api/export.py

@@ -14,13 +14,18 @@
 
 def cmd_export(loader, cache, hook_manager, global_conf, conanfile_path,
                name, version, user, channel,
-               graph_lock=None, remotes=None):
+               graph_lock=None, remotes=None, revision_mode_scm=False):
     """ Export the recipe
     param conanfile_path: the original source directory of the user containing a
                        conanfile.py
+    param revision_mode_scm: force revision_mode='scm' regardless of what the recipe
+                       declares. Used by git= sources so the recipe revision IS the
+                       git commit — see GitRemotesResolver.
     """
     conanfile = loader.load_export(conanfile_path, name, version, user, channel, graph_lock,
                                    remotes=remotes)
+    if revision_mode_scm:
+        conanfile.revision_mode = "scm"
 
     ref = RecipeReference(conanfile.name, conanfile.version,  conanfile.user, conanfile.channel)
     ref.validate_ref(allow_uppercase=global_conf.get("core:allow_uppercase_pkg_names",

conan/internal/cache/cache.py

@@ -50,6 +50,10 @@ def temp_folder(self):
     def builds_folder(self):
         return os.path.join(self._base_folder, "b")
 
+    @property
+    def git_clones_folder(self):
+        return os.path.join(self._base_folder, "git_clones")
+
     def _create_path(self, relative_path, remove_contents=True):
         path = self._full_path(relative_path)
         if os.path.exists(path) and remove_contents:

conan/internal/graph/git_remotes_resolver.py

@@ -0,0 +1,212 @@
+import hashlib
+import os
+import subprocess
+
+from conan.api.output import ConanOutput
+from conan.errors import ConanException
+from conan.internal.api.export import cmd_export
+from conan.internal.graph.proxy import should_update_reference
+from conan.internal.util.files import remove_if_dirty, rmdir, set_dirty_context_manager
+
+
+class GitRemotesResolver:
+
+    def __init__(self, cache):
+        self._cache = cache
+        self._clones_base = cache.git_clones_folder
+
+    @staticmethod
+    def _get_url(repo):
+        # Maybe we need to extend this to gitlab too, we could check a "gl:org/repo" format
+        return f"https://github.com/{repo}.git"
+
+    def prefetch(self, node, require, update, loader, editable_packages, lockfile=None):
+        """If the requirement declares a git= source, clone and export it into the local
+        cache before range resolution and proxy lookup run. Mirrors the
+        _resolve_replace_requires pattern — invoked early in _initialize_requires so the
+        rest of graph resolution is transparent (proxy just finds the recipe in cache).
+
+        Semantics:
+          - The recipe revision IS the git commit SHA (revision_mode='scm' is forced
+            during export). A lockfile captured today pins the exact commit, so a
+            later reinstall from that lockfile can reproduce the build even if a
+            branch tip has moved in the meantime.
+          - Version ranges are allowed. The clone runs first, the recipe declares its
+            own version, and the resolver validates that the declared version is
+            within the require's range.
+        """
+        ref = require.ref
+        # replace_requires ran: _resolve_replace_requires copies the original ref into
+        # _required_ref BEFORE mutating ref. If (name, version, user, channel) changed,
+        # the require was RENAMED and git= (which hardcodes a specific package identity)
+        # cannot follow — replace_requires wins. If only the revision was added, the
+        # replacement is compatible with git= and its revision drives the checkout.
+        output = ConanOutput(scope=str(node))
+        orig_ref = require._required_ref  # noqa
+        if orig_ref is not ref:
+            if (ref.name != orig_ref.name or ref.version != orig_ref.version
+                    or ref.user != orig_ref.user or ref.channel != orig_ref.channel):
+                output.warning(f"Ignoring git={require.git!r}: 'replace_requires' "
+                               f"renamed the require ({orig_ref} → {ref}) and took precedence "
+                               f"over the git= source.")
+                return
+            # Otherwise: replacement only added a revision — fall through, and the
+            # revision-driven checkout logic below will pick it up.
+
+        # Editable takes precedence over git=: a local editable is a stronger
+        # override than a hardcoded remote source.
+        if editable_packages is not None and editable_packages.get(ref) is not None:
+            output.info(f"Ignoring git={require.git!r}: package is in editable mode.")
+            return
+
+        git = require.git  # raw string: "org/repo" or "org/repo@ref"
+        # split on the FIRST '@' — org/repo cannot contain '@' (GitHub disallows it),
+        # so anything after is the ref, even if the ref itself contains '@'
+        idx = git.split("@", 1)
+        if len(idx) == 2 and not idx[1]:
+            raise ConanException(
+                f"Requirement '{ref}': git={git!r} has a trailing '@' with no ref. "
+                f"Drop the '@' to use the default branch, or specify a branch/tag/commit.")
+        repo, git_ref = idx if len(idx) == 2 else (idx[0], None)
+
+        # Reconcile the two ways to pin a commit: an explicit recipe revision on the
+        # require ('pkg/version#<sha>') vs a ref in git= ('org/repo@<sha>'). Under
+        # revision_mode='scm' the recipe revision IS the git commit.
+        #
+        # Rule: only the RECIPE-authored combination (both #revision and @ref written
+        # by the recipe author) is a contradiction. If the revision was added from
+        # outside the recipe (replace_requires, later lockfile peek), it wins over
+        # git=@ref silently — those are explicit out-of-recipe overrides and Conan
+        # cannot tell from ref shape whether @ref is mutable, so it just trusts them.
+        # _required_ref carries the pre-replace state; if it has a revision, the
+        # recipe itself declared one.
+        if orig_ref.revision and git_ref:
+            raise ConanException(
+                f"Requirement '{ref}' pins a revision and git={git!r} also pins "
+                f"a ref — use only one")
+        if ref.revision:
+            git_ref = ref.revision  # recipe-authored (git_ref is None here) or override
+
+        # Lockfile-driven checkout: if a matching entry is locked with a revision
+        # (git commit SHA under our revision_mode='scm' contract), use it as the
+        # checkout target so reinstalls reproduce the exact commit even if the
+        # branch has moved upstream. Peek only — do NOT let resolve_locked mutate
+        # require.ref to share identity with the lockfile's internal ref (that
+        # would cause our later revision assignment to poison the lockfile).
+        if lockfile is not None:
+            saved_ref = require.ref
+            try:
+                lockfile.resolve_locked(node, require, resolve_prereleases=None)
+                locked_rev = require.ref.revision
+            except ConanException:
+                locked_rev = None
+            finally:
+                require.ref = saved_ref
+            if locked_rev:
+                git_ref = locked_rev
+
+        url = self._get_url(repo)
+        force_clone = should_update_reference(ref, update)
+        version_range = require.version_range
+
+        output = ConanOutput(scope=str(ref))
+        if force_clone:
+            output.info(f"Updating from git remote '{url}'...")
+        elif not version_range:
+            # Cache-first shortcut only makes sense for a fully-resolved ref.
+            # With a range, we need to re-resolve which concrete version applies.
+            try:
+                if ref.revision:
+                    # Specific revision pinned (recipe / replace_requires / lockfile) —
+                    # only accept an exact match in the cache
+                    _ = self._cache.recipe_layout(ref)
+                else:
+                    layout = self._cache.recipe_layout_latest(ref)
+                    require.ref.revision = layout.reference.revision
+                output.info(f"Found in cache (configured via git remote '{url}')")
+                return
+            except ConanException:
+                pass  # Not in cache — proceed with clone+export
+            output.info(f"Not found in local cache, resolving from git remote '{url}'")
+
+        if git_ref:
+            output.info(f"  git ref: {git_ref}")
+
+        # With a version range, let the recipe declare its own version and validate
+        # it after export. Without a range, pass the exact version so cmd_export
+        # enforces the recipe hardcodes match (existing behavior).
+        version = None if version_range else str(ref.version)
+        exported_ref, _ = self._clone_and_export(ref, repo, git_ref, loader,
+                                                 force_clone, version=version)
+
+        if version_range:
+            resolved_version = exported_ref.version
+            resolve_prereleases = None  # let VersionRange default apply
+            if not version_range.contains(resolved_version, resolve_prereleases):
+                raise ConanException(
+                    f"Requirement '{ref}' with range '{version_range}' does not accept "
+                    f"the version '{resolved_version}' declared by the recipe at "
+                    f"git remote '{url}'")
+            output.info(f"  resolved version: {resolved_version} (in range {version_range})")
+            require.ref.version = resolved_version
+
+        # Recipe revision is the git commit SHA (revision_mode='scm' forced during export)
+        require.ref.revision = exported_ref.revision
+
+    def _clone_and_export(self, ref, repo, git_ref, loader, force_clone, version=None):
+        url = self._get_url(repo)
+        clone_folder = self._clone_folder(url, git_ref)
+        # Leftover from a previous run interrupted mid-clone/checkout: discard it
+        remove_if_dirty(clone_folder)
+        if force_clone and os.path.exists(clone_folder):
+            rmdir(clone_folder)
+        if not os.path.exists(clone_folder):
+            self._do_clone(url, git_ref, clone_folder)
+        conanfile_path = os.path.join(clone_folder, "conanfile.py")
+        if not os.path.exists(conanfile_path):
+            raise ConanException(
+                f"conanfile.py not found at root of git repo '{url}'")
+
+        # Hooks are intentionally skipped: git= is aimed at open-source /
+        # community workflows that pull recipes straight from public github.com
+        # repos, not at organizations that rely on pre/post_export hooks for
+        # policy, signing or scanning.
+        class _NoopHooks:
+            def execute(self, *a, **kw): pass
+
+        from conan.internal.model.conf import ConfDefinition
+        # Force revision_mode='scm' → recipe revision = git commit SHA. Enables
+        # lockfile reproducibility even against moving branches.
+        # no remotes, no lockfile, as python-requires are not supported now
+        return cmd_export(loader, self._cache, _NoopHooks(), ConfDefinition(),
+                          conanfile_path, ref.name, version,
+                          ref.user, ref.channel, revision_mode_scm=True)
+
+    def _clone_folder(self, url, git_ref):
+        key = f"{url}:{git_ref or ''}"
+        h = hashlib.md5(key.encode()).hexdigest()[:12]
+        return os.path.join(self._clones_base, h)
+
+    @staticmethod
+    def _run_git(argv):
+        # argv-form; never shell=True — keeps refs/URLs with metachars intact
+        proc = subprocess.run(argv, capture_output=True, text=True)
+        return proc.returncode, (proc.stdout or "") + (proc.stderr or "")
+
+    @classmethod
+    def _do_clone(cls, url, git_ref, clone_folder):
+        output = ConanOutput()
+        os.makedirs(clone_folder, exist_ok=True)
+        # dirty marker: if the process is interrupted mid-clone/checkout, the
+        # next run detects the marker via remove_if_dirty and starts fresh
+        with set_dirty_context_manager(clone_folder):
+            output.info(f"Cloning git repository '{url}'...")
+            ret, out = cls._run_git(["git", "clone", url, clone_folder])
+            if ret != 0:
+                raise ConanException(f"git clone failed for '{url}':\n{out}")
+            if git_ref:
+                output.info(f"Checking out git ref '{git_ref}'...")
+                ret, out = cls._run_git(["git", "-C", clone_folder, "checkout", git_ref])
+                if ret != 0:
+                    raise ConanException(
+                        f"git checkout '{git_ref}' failed for '{url}':\n{out}")

conan/internal/graph/graph_builder.py

@@ -241,6 +241,11 @@ def _initialize_requires(self, node, graph, graph_lock, profile_build, profile_h
                 if not resolved:
                     self._resolve_alias(node, require, alias, graph)
             self._resolve_replace_requires(node, require, profile_build, profile_host, graph)
+            if require.git is not None:
+                from conan.internal.graph.git_remotes_resolver import GitRemotesResolver
+                GitRemotesResolver(self._cache).prefetch(node, require, self._update, self._loader,
+                                                         self._proxy._editable_packages, # noqa
+                                                         graph_lock)
             if graph_lock:
                 graph_lock.resolve_overrides(require, node.context)
             node.transitive_deps[require] = TransitiveRequirement(require, node=None)

conan/internal/model/requires.py

@@ -10,7 +10,7 @@ class Requirement:
     def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visible=None,
                  transitive_headers=None, transitive_libs=None, test=None, package_id_mode=None,
                  force=None, override=None, direct=None, options=None, no_skip=False,
-                 consistent=None):
+                 consistent=None, git=None):
         # * prevents the usage of more positional parameters, always ref + **kwargs
         # By default this is a generic library requirement
         self.ref = ref
@@ -40,6 +40,8 @@ def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visib
         self.skip = False
         self.required_nodes = set()  # store which intermediate nodes are required, to compute "Skip"
         self.no_skip = no_skip
+        # git source: raw string for https://github.com public open source repositories
+        self.git = git
         # computed ones, not default ones
         self.consistent_policy_new = False
         if self.visible and not self.consistent:
@@ -443,19 +445,19 @@ def __init__(self, requires):
         self._requires = requires
 
     def __call__(self, ref, package_id_mode=None, visible=False, run=True, options=None,
-                 override=None):
+                 override=None, git=None):
         # TODO: Check which arguments could be user-defined
         self._requires.tool_require(ref, package_id_mode=package_id_mode, visible=visible, run=run,
-                                    options=options, override=override)
+                                    options=options, override=override, git=git)
 
 
 class TestRequirements:
     # Just a wrapper around requires for backwards compatibility with self.build_requires() syntax
     def __init__(self, requires):
         self._requires = requires
 
-    def __call__(self, ref, run=None, options=None, force=None):
-        self._requires.test_require(ref, run=run, options=options, force=force)
+    def __call__(self, ref, run=None, options=None, force=None, git=None):
+        self._requires.test_require(ref, run=run, options=options, force=force, git=git)
 
 
 class Requirements:
@@ -525,6 +527,13 @@ def values(self):
         return self._requires.values()
 
     def __call__(self, str_ref, **kwargs):
+        """Add a regular requirement (as in ``self.requires("zlib/1.2.11")``).
+
+        Any keyword accepted by ``Requirement`` may be passed through kwargs, including
+        ``git="org/repo[@ref]"`` to source the recipe from a public GitHub repository —
+        symmetric with ``self.test_requires`` and ``self.tool_requires``, which declare
+        ``git=`` in their own signatures.
+        """
         if str_ref is None:
             return
         assert isinstance(str_ref, str)
@@ -557,7 +566,7 @@ def build_require(self, ref, raise_if_duplicated=True, package_id_mode=None, vis
             raise ConanException("Duplicated requirement: {}".format(ref))
         self._requires[req] = req
 
-    def test_require(self, ref, run=None, options=None, force=None):
+    def test_require(self, ref, run=None, options=None, force=None, git=None):
         """
              Represent a testing framework like gtest
 
@@ -573,13 +582,13 @@ def test_require(self, ref, run=None, options=None, force=None):
         # libs = True => We need to link with it
         # headers = True => We need to include it
         req = Requirement(ref, headers=True, libs=True, build=False, run=run, visible=False,
-                          test=True, package_id_mode=None, options=options, force=force)
+                          test=True, package_id_mode=None, options=options, force=force, git=git)
         if self._requires.get(req):
             raise ConanException("Duplicated requirement: {}".format(ref))
         self._requires[req] = req
 
     def tool_require(self, ref, raise_if_duplicated=True, package_id_mode=None, visible=False,
-                     run=True, options=None, override=None):
+                     run=True, options=None, override=None, git=None):
         """
          Represent a build tool like "cmake".
 
@@ -593,7 +602,8 @@ def tool_require(self, ref, raise_if_duplicated=True, package_id_mode=None, visi
         # FIXME: This raise_if_duplicated is ugly, possibly remove
         ref = RecipeReference.loads(ref)
         req = Requirement(ref, headers=False, libs=False, build=True, run=run, visible=visible,
-                          package_id_mode=package_id_mode, options=options, override=override)
+                          package_id_mode=package_id_mode, options=options, override=override,
+                          git=git)
         if raise_if_duplicated and self._requires.get(req):
             raise ConanException("Duplicated requirement: {}".format(ref))
         self._requires[req] = req