caching generated sources

conan/internal/rest/caching_file_downloader.py

@@ -26,17 +26,47 @@ def __init__(self, conanfile):
         self._output = conanfile.output
         self._conanfile = conanfile
 
+    def save(self, filepath, url, identifier):
+        """Store a locally-produced file into the sources download cache under
+        `identifier`, recording `url` in the backup-sources JSON metadata so a
+        later `conan cache backup-upload` pushes it to `core.sources:upload_url`.
+        Falls back to the default cache folder when `core.sources:download_cache`
+        is not configured — the identifier feature works out of the box.
+        """
+        download_cache_folder = (self._global_conf.get("core.sources:download_cache")
+                                 or HomePaths(self._home_folder).default_sources_backup_folder)
+        if not os.path.isabs(download_cache_folder):
+            raise ConanException("core.sources:download_cache must be an absolute path")
+        download_cache = DownloadCache(download_cache_folder)
+        target = download_cache.source_path(identifier)
+        with download_cache.lock(identifier):
+            remove_if_dirty(target)  # clear any leftover from a prior failed download
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            with set_dirty_context_manager(target):
+                shutil.copy2(filepath, target)
+            download_cache.update_backup_sources_json(target, self._conanfile, url)
+
     def download(self, urls, file_path,
-                 retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256):
+                 retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256,
+                 identifier=None):
+        # Normalize: `identifier` is the blob name (cache path / mirror URL suffix),
+        # `sha256` (if any) is for content verification. In identifier mode content is
+        # NOT verified; in classic backup-sources mode identifier defaults to sha256.
+        identifier_mode = identifier is not None
+        if identifier:
+            md5 = sha1 = sha256 = None
+        else:
+            identifier = sha256
         download_cache_folder = self._global_conf.get("core.sources:download_cache")
         source_origins = self._global_conf.get("core.sources:download_urls", check_type=list)
-        if source_origins and not download_cache_folder:
-            # If backups are defined, but the download cache is not defined, use a default one
+        # Auto-default the cache folder when a backup URL is defined (classic backup-sources)
+        # OR when the caller uses identifier-mode (so identifier works out of the box).
+        if not download_cache_folder and (source_origins or identifier_mode):
             download_cache_folder = HomePaths(self._home_folder).default_sources_backup_folder
         if download_cache_folder and not os.path.isabs(download_cache_folder):
             raise ConanException("core.sources:download_cache must be an absolute path")
         source_origins = source_origins or ["origin"]
-        if download_cache_folder and not sha256:
+        if download_cache_folder and not identifier:
             self._output.warning("Cannot cache download() without sha256 checksum")
             download_cache_folder = None  # Cannot cache
             source_origins = ["origin"]
@@ -47,9 +77,9 @@ def download(self, urls, file_path,
         # First, see if it is already in the download cache
         if download_cache_folder:
             download_cache = DownloadCache(download_cache_folder)
-            download_path = download_cache.source_path(sha256)
+            download_path = download_cache.source_path(identifier)
 
-            with download_cache.lock(sha256):
+            with download_cache.lock(identifier):
                 remove_if_dirty(download_path)
 
                 in_cache = os.path.exists(download_path)
@@ -73,7 +103,7 @@ def download(self, urls, file_path,
                 if need_download:
                     with set_dirty_context_manager(download_path):
                         self._do_download(source_origins, urls, download_path, retry, retry_wait,
-                                          verify_ssl, auth, headers, md5, sha1, sha256)
+                                          verify_ssl, auth, headers, md5, sha1, sha256, identifier)
 
                 # copy it to the package "source" folder
                 os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -83,11 +113,11 @@ def download(self, urls, file_path,
             # Not in local cache, check origins from core.sources:download_urls
             # This doesn't need to be dirty-protected, as the full "source" folder is protected
             self._do_download(source_origins, urls, file_path, retry, retry_wait, verify_ssl, auth,
-                              headers, md5, sha1, sha256)
+                              headers, md5, sha1, sha256, identifier)
 
     def _do_download(self, source_origins, urls, download_path, retry, retry_wait, verify_ssl,
-                     auth, headers, md5, sha1, sha256):
-        # iterates the origins until one works
+                     auth, headers, md5, sha1, sha256, identifier):
+        # `identifier`: blob name on the mirror. `sha256`: verify hash (None in identifier mode).
         for backup_url in source_origins:
             if backup_url == "origin":  # download from the internet
                 try:
@@ -103,9 +133,9 @@ def _do_download(self, source_origins, urls, download_path, retry, retry_wait, v
                     self._output.info(f"Checking backup: {backup_url}")
                     backup_url = backup_url if backup_url.endswith("/") else backup_url + "/"
                     # The download happens to the user download folder, not to the download cache
-                    self._file_downloader.download(backup_url + sha256, download_path,
+                    self._file_downloader.download(backup_url + identifier, download_path,
                                                    sha256=sha256, overwrite=True)
-                    self._file_downloader.download(backup_url + sha256 + ".json",
+                    self._file_downloader.download(backup_url + identifier + ".json",
                                                    download_path + ".json", overwrite=True)
                     self._output.info(f"Sources for {urls} found in remote backup {backup_url}")
                     return

conan/tools/files/__init__.py

@@ -1,6 +1,6 @@
 from conan.tools.files.files import load, save, mkdir, rmdir, rm, ftp_download, download, get, \
     rename, chdir, unzip, replace_in_file, collect_libs, check_md5, check_sha1, check_sha256, \
-    move_folder_contents, chmod
+    move_folder_contents, chmod, save_backup_source
 
 from conan.tools.files.patches import patch, apply_conandata_patches, export_conandata_patches
 from conan.tools.files.symlinks import symlinks

conan/tools/files/files.py

@@ -178,7 +178,7 @@ def ftp_download(conanfile, host, filename, login='', password='', secure=False)
 
 
 def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None,
-             auth=None, headers=None, md5=None, sha1=None, sha256=None):
+             auth=None, headers=None, md5=None, sha1=None, sha256=None, identifier=None):
     """
     Retrieves a file from a given URL into a file with a given filename. It uses certificates from
     a list of known verifiers for https downloads, but this can be optionally disabled.
@@ -203,6 +203,10 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None,
     :param md5: MD5 hash code to check the downloaded file
     :param sha1: SHA-1 hash code to check the downloaded file
     :param sha256: SHA-256 hash code to check the downloaded file
+    :param identifier: Opaque key used in place of sha256 to name the file in the
+           local download cache and in the backup URL (``<backup_url>/<identifier>``).
+           Content is NOT verified. Intended for input-hashed generated blobs whose
+           final sha256 is not known ahead of time.
     """
     config = conanfile.conf
 
@@ -214,7 +218,22 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None,
 
     filename = os.path.abspath(filename)
     downloader = SourcesCachingDownloader(conanfile)
-    downloader.download(url, filename, retry, retry_wait, verify, auth, headers, md5, sha1, sha256)
+    downloader.download(url, filename, retry, retry_wait, verify, auth, headers, md5, sha1, sha256,
+                        identifier=identifier)
+
+
+def save_backup_source(conanfile, filepath, url, identifier):
+    """
+    Save a locally-produced file into the sources download cache under `identifier`
+    so subsequent ``download(..., identifier=identifier)`` calls hit it from cache,
+    and so that ``conan cache backup-upload`` pushes it to ``core.sources:upload_url``.
+
+    :param conanfile: The current recipe object. Always use ``self``.
+    :param filepath: Absolute path to the local file to store as a backup source.
+    :param url: URL recorded in the backup-sources JSON metadata for this entry.
+    :param identifier: Opaque key identifying the entry in the cache and on the mirror.
+    """
+    SourcesCachingDownloader(conanfile).save(filepath, url, identifier)
 
 
 def rename(conanfile, src, dst):

test/integration/cache/test_generate_cache_backup_poc.py

@@ -0,0 +1,147 @@
+import os
+import textwrap
+
+import pytest
+
+from conan.internal.cache.home_paths import HomePaths
+from conan.test.assets.genconanfile import GenConanfile
+from conan.test.utils.file_server import TestFileServer
+from conan.test.utils.tools import TestClient
+
+
+HEAVY_CONANFILE = textwrap.dedent("""
+    import hashlib
+    from conan import ConanFile
+    from conan.errors import ConanException
+    from conan.tools.files import download, save_backup_source, save
+
+    class Heavy(ConanFile):
+        name = "heavy"
+        version = "0.1"
+        requires = "dep/0.1"
+
+        def generate(self):
+            h = hashlib.sha256()
+            for dep in self.dependencies.host.values():
+                h.update(dep.ref.repr_notime().encode())
+            key = h.hexdigest()
+            url = f"gen-conan://{key}"
+            out = "generated.txt"
+            try:
+                download(self, url, out, identifier=key, retry=0)
+                self.output.info("GENCACHE hit")
+            except ConanException:
+                self.output.info("GENCACHE miss")
+                save(self, out, "expensive output")
+                save_backup_source(self, out, url, identifier=key)
+""")
+
+
+@pytest.fixture()
+def producer():
+    """Producer: creates dep + heavy, then uploads recipes to the Conan remote and
+    the generated blob to the backup-sources mirror."""
+    file_server = TestFileServer()
+    backup_url = file_server.fake_url + "/genbackup/"
+
+    c = TestClient(default_server_user=True, light=True)
+    c.servers["file_server"] = file_server
+    c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"),
+            "heavy/conanfile.py": HEAVY_CONANFILE})
+    c.save_home({"global.conf": f"core.sources:upload_url={backup_url}\n"})
+    c.run("create dep")
+    c.run("create heavy")
+    assert "GENCACHE miss" in c.out
+
+    c.run("upload * -r=default -c")     # recipes/binaries to the Conan remote
+    c.run("cache backup-upload")        # generated blob to the backup mirror
+
+    # Sanity: one blob (+ its .json) on the mirror, named by the identifier
+    mirror_dir = os.path.join(file_server.store, "genbackup")
+    blobs = [f for f in os.listdir(mirror_dir) if not f.endswith(".json")]
+    assert len(blobs) == 1 and len(blobs[0]) == 64  # sha256 hex
+
+    return c, file_server, backup_url
+
+
+def _consumer(producer):
+    """Fresh consumer sharing the producer's Conan remote and backup mirror."""
+    producer_c, file_server, backup_url = producer
+    conan_remotes = {k: v for k, v in producer_c.servers.items() if k != "file_server"}
+    c = TestClient(servers=conan_remotes, inputs=["admin", "password"], light=True)
+    c.servers["file_server"] = file_server
+    c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"),
+            "heavy/conanfile.py": HEAVY_CONANFILE})
+    c.save_home({"global.conf": f"core.sources:download_urls=['{backup_url}']\n"})
+    return c
+
+
+def test_conan_create_reuses_uploaded_cache(producer):
+    """Fresh `conan create` finds the generated blob on the mirror."""
+    c2 = _consumer(producer)
+    c2.run("create dep")
+    c2.run("create heavy")
+    assert "GENCACHE hit" in c2.out
+    assert "GENCACHE miss" not in c2.out
+
+
+def test_conan_install_local_reuses_uploaded_cache(producer):
+    """Developer workflow: `conan install <path>` on the local heavy recipe."""
+    c2 = _consumer(producer)
+    c2.run("create dep")
+    c2.run("install heavy")
+    assert "GENCACHE hit" in c2.out
+    assert "GENCACHE miss" not in c2.out
+
+
+def test_no_download_urls_does_not_use_mirror(producer):
+    """Without `core.sources:download_urls`, the consumer never contacts the mirror,
+    even when the producer has uploaded there — so a fresh consumer misses. This
+    isolates the role of `download_urls` as the wire-up between recipe and mirror."""
+    producer_c, file_server, _ = producer
+    conan_remotes = {k: v for k, v in producer_c.servers.items() if k != "file_server"}
+    c = TestClient(servers=conan_remotes, inputs=["admin", "password"], light=True)
+    c.servers["file_server"] = file_server
+    c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"),
+            "heavy/conanfile.py": HEAVY_CONANFILE})
+    c.run("create dep")
+    c.run("create heavy")
+    assert "GENCACHE miss" in c.out
+    assert "GENCACHE hit" not in c.out
+
+
+def test_default_cache_folder_used_when_not_configured():
+    """`identifier` works out of the box: with no `core.sources:*` configured, the
+    default sources backup folder is used, so a second `create heavy` hits the
+    local cache the first run seeded."""
+    c = TestClient(light=True)
+    c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"),
+            "heavy/conanfile.py": HEAVY_CONANFILE})
+    c.run("create dep")
+    c.run("create heavy")
+    assert "GENCACHE miss" in c.out
+
+    # Second create: local (default) cache holds the entry saved by the first run.
+    c.run("create heavy")
+    assert "GENCACHE hit" in c.out
+    assert "GENCACHE miss" not in c.out
+
+
+def test_dep_rrev_change_produces_new_key():
+    """Bumping dep's rrev changes the identifier → new miss + new cache entry."""
+    c = TestClient(light=True)
+    c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"),
+            "heavy/conanfile.py": HEAVY_CONANFILE})
+    c.run("create dep")
+    c.run("create heavy")
+    assert "GENCACHE miss" in c.out
+
+    c.save({"dep/conanfile.py": GenConanfile("dep", "0.1").with_import("import os")})
+    c.run("create dep")
+    c.run("create heavy")
+    assert "GENCACHE miss" in c.out
+
+    dl_cache = HomePaths(c.cache_folder).default_sources_backup_folder
+    entries = os.listdir(os.path.join(dl_cache, "s"))
+    blobs = [e for e in entries if not e.endswith(".json") and not e.endswith(".dirty")]
+    assert len(blobs) == 2