diff --git a/conan/internal/rest/caching_file_downloader.py b/conan/internal/rest/caching_file_downloader.py index 9cd20c039f9..713cd9754ef 100644 --- a/conan/internal/rest/caching_file_downloader.py +++ b/conan/internal/rest/caching_file_downloader.py @@ -26,17 +26,47 @@ def __init__(self, conanfile): self._output = conanfile.output self._conanfile = conanfile + def save(self, filepath, url, identifier): + """Store a locally-produced file into the sources download cache under + `identifier`, recording `url` in the backup-sources JSON metadata so a + later `conan cache backup-upload` pushes it to `core.sources:upload_url`. + Falls back to the default cache folder when `core.sources:download_cache` + is not configured — the identifier feature works out of the box. + """ + download_cache_folder = (self._global_conf.get("core.sources:download_cache") + or HomePaths(self._home_folder).default_sources_backup_folder) + if not os.path.isabs(download_cache_folder): + raise ConanException("core.sources:download_cache must be an absolute path") + download_cache = DownloadCache(download_cache_folder) + target = download_cache.source_path(identifier) + with download_cache.lock(identifier): + remove_if_dirty(target) # clear any leftover from a prior failed download + os.makedirs(os.path.dirname(target), exist_ok=True) + with set_dirty_context_manager(target): + shutil.copy2(filepath, target) + download_cache.update_backup_sources_json(target, self._conanfile, url) + def download(self, urls, file_path, - retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256): + retry, retry_wait, verify_ssl, auth, headers, md5, sha1, sha256, + identifier=None): + # Normalize: `identifier` is the blob name (cache path / mirror URL suffix), + # `sha256` (if any) is for content verification. In identifier mode content is + # NOT verified; in classic backup-sources mode identifier defaults to sha256. + identifier_mode = identifier is not None + if identifier: + md5 = sha1 = sha256 = None + else: + identifier = sha256 download_cache_folder = self._global_conf.get("core.sources:download_cache") source_origins = self._global_conf.get("core.sources:download_urls", check_type=list) - if source_origins and not download_cache_folder: - # If backups are defined, but the download cache is not defined, use a default one + # Auto-default the cache folder when a backup URL is defined (classic backup-sources) + # OR when the caller uses identifier-mode (so identifier works out of the box). + if not download_cache_folder and (source_origins or identifier_mode): download_cache_folder = HomePaths(self._home_folder).default_sources_backup_folder if download_cache_folder and not os.path.isabs(download_cache_folder): raise ConanException("core.sources:download_cache must be an absolute path") source_origins = source_origins or ["origin"] - if download_cache_folder and not sha256: + if download_cache_folder and not identifier: self._output.warning("Cannot cache download() without sha256 checksum") download_cache_folder = None # Cannot cache source_origins = ["origin"] @@ -47,9 +77,9 @@ def download(self, urls, file_path, # First, see if it is already in the download cache if download_cache_folder: download_cache = DownloadCache(download_cache_folder) - download_path = download_cache.source_path(sha256) + download_path = download_cache.source_path(identifier) - with download_cache.lock(sha256): + with download_cache.lock(identifier): remove_if_dirty(download_path) in_cache = os.path.exists(download_path) @@ -73,7 +103,7 @@ def download(self, urls, file_path, if need_download: with set_dirty_context_manager(download_path): self._do_download(source_origins, urls, download_path, retry, retry_wait, - verify_ssl, auth, headers, md5, sha1, sha256) + verify_ssl, auth, headers, md5, sha1, sha256, identifier) # copy it to the package "source" folder os.makedirs(os.path.dirname(file_path), exist_ok=True) @@ -83,11 +113,11 @@ def download(self, urls, file_path, # Not in local cache, check origins from core.sources:download_urls # This doesn't need to be dirty-protected, as the full "source" folder is protected self._do_download(source_origins, urls, file_path, retry, retry_wait, verify_ssl, auth, - headers, md5, sha1, sha256) + headers, md5, sha1, sha256, identifier) def _do_download(self, source_origins, urls, download_path, retry, retry_wait, verify_ssl, - auth, headers, md5, sha1, sha256): - # iterates the origins until one works + auth, headers, md5, sha1, sha256, identifier): + # `identifier`: blob name on the mirror. `sha256`: verify hash (None in identifier mode). for backup_url in source_origins: if backup_url == "origin": # download from the internet try: @@ -103,9 +133,9 @@ def _do_download(self, source_origins, urls, download_path, retry, retry_wait, v self._output.info(f"Checking backup: {backup_url}") backup_url = backup_url if backup_url.endswith("/") else backup_url + "/" # The download happens to the user download folder, not to the download cache - self._file_downloader.download(backup_url + sha256, download_path, + self._file_downloader.download(backup_url + identifier, download_path, sha256=sha256, overwrite=True) - self._file_downloader.download(backup_url + sha256 + ".json", + self._file_downloader.download(backup_url + identifier + ".json", download_path + ".json", overwrite=True) self._output.info(f"Sources for {urls} found in remote backup {backup_url}") return diff --git a/conan/tools/files/__init__.py b/conan/tools/files/__init__.py index 1e5f2aeeec0..74879b19ac0 100644 --- a/conan/tools/files/__init__.py +++ b/conan/tools/files/__init__.py @@ -1,6 +1,6 @@ from conan.tools.files.files import load, save, mkdir, rmdir, rm, ftp_download, download, get, \ rename, chdir, unzip, replace_in_file, collect_libs, check_md5, check_sha1, check_sha256, \ - move_folder_contents, chmod + move_folder_contents, chmod, save_backup_source from conan.tools.files.patches import patch, apply_conandata_patches, export_conandata_patches from conan.tools.files.symlinks import symlinks diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index d8e4227b96f..ec8588a470a 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -178,7 +178,7 @@ def ftp_download(conanfile, host, filename, login='', password='', secure=False) def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, - auth=None, headers=None, md5=None, sha1=None, sha256=None): + auth=None, headers=None, md5=None, sha1=None, sha256=None, identifier=None): """ Retrieves a file from a given URL into a file with a given filename. It uses certificates from a list of known verifiers for https downloads, but this can be optionally disabled. @@ -203,6 +203,10 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, :param md5: MD5 hash code to check the downloaded file :param sha1: SHA-1 hash code to check the downloaded file :param sha256: SHA-256 hash code to check the downloaded file + :param identifier: Opaque key used in place of sha256 to name the file in the + local download cache and in the backup URL (``/``). + Content is NOT verified. Intended for input-hashed generated blobs whose + final sha256 is not known ahead of time. """ config = conanfile.conf @@ -214,7 +218,22 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, filename = os.path.abspath(filename) downloader = SourcesCachingDownloader(conanfile) - downloader.download(url, filename, retry, retry_wait, verify, auth, headers, md5, sha1, sha256) + downloader.download(url, filename, retry, retry_wait, verify, auth, headers, md5, sha1, sha256, + identifier=identifier) + + +def save_backup_source(conanfile, filepath, url, identifier): + """ + Save a locally-produced file into the sources download cache under `identifier` + so subsequent ``download(..., identifier=identifier)`` calls hit it from cache, + and so that ``conan cache backup-upload`` pushes it to ``core.sources:upload_url``. + + :param conanfile: The current recipe object. Always use ``self``. + :param filepath: Absolute path to the local file to store as a backup source. + :param url: URL recorded in the backup-sources JSON metadata for this entry. + :param identifier: Opaque key identifying the entry in the cache and on the mirror. + """ + SourcesCachingDownloader(conanfile).save(filepath, url, identifier) def rename(conanfile, src, dst): diff --git a/test/integration/cache/test_generate_cache_backup_poc.py b/test/integration/cache/test_generate_cache_backup_poc.py new file mode 100644 index 00000000000..8410373a856 --- /dev/null +++ b/test/integration/cache/test_generate_cache_backup_poc.py @@ -0,0 +1,147 @@ +import os +import textwrap + +import pytest + +from conan.internal.cache.home_paths import HomePaths +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.file_server import TestFileServer +from conan.test.utils.tools import TestClient + + +HEAVY_CONANFILE = textwrap.dedent(""" + import hashlib + from conan import ConanFile + from conan.errors import ConanException + from conan.tools.files import download, save_backup_source, save + + class Heavy(ConanFile): + name = "heavy" + version = "0.1" + requires = "dep/0.1" + + def generate(self): + h = hashlib.sha256() + for dep in self.dependencies.host.values(): + h.update(dep.ref.repr_notime().encode()) + key = h.hexdigest() + url = f"gen-conan://{key}" + out = "generated.txt" + try: + download(self, url, out, identifier=key, retry=0) + self.output.info("GENCACHE hit") + except ConanException: + self.output.info("GENCACHE miss") + save(self, out, "expensive output") + save_backup_source(self, out, url, identifier=key) +""") + + +@pytest.fixture() +def producer(): + """Producer: creates dep + heavy, then uploads recipes to the Conan remote and + the generated blob to the backup-sources mirror.""" + file_server = TestFileServer() + backup_url = file_server.fake_url + "/genbackup/" + + c = TestClient(default_server_user=True, light=True) + c.servers["file_server"] = file_server + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "heavy/conanfile.py": HEAVY_CONANFILE}) + c.save_home({"global.conf": f"core.sources:upload_url={backup_url}\n"}) + c.run("create dep") + c.run("create heavy") + assert "GENCACHE miss" in c.out + + c.run("upload * -r=default -c") # recipes/binaries to the Conan remote + c.run("cache backup-upload") # generated blob to the backup mirror + + # Sanity: one blob (+ its .json) on the mirror, named by the identifier + mirror_dir = os.path.join(file_server.store, "genbackup") + blobs = [f for f in os.listdir(mirror_dir) if not f.endswith(".json")] + assert len(blobs) == 1 and len(blobs[0]) == 64 # sha256 hex + + return c, file_server, backup_url + + +def _consumer(producer): + """Fresh consumer sharing the producer's Conan remote and backup mirror.""" + producer_c, file_server, backup_url = producer + conan_remotes = {k: v for k, v in producer_c.servers.items() if k != "file_server"} + c = TestClient(servers=conan_remotes, inputs=["admin", "password"], light=True) + c.servers["file_server"] = file_server + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "heavy/conanfile.py": HEAVY_CONANFILE}) + c.save_home({"global.conf": f"core.sources:download_urls=['{backup_url}']\n"}) + return c + + +def test_conan_create_reuses_uploaded_cache(producer): + """Fresh `conan create` finds the generated blob on the mirror.""" + c2 = _consumer(producer) + c2.run("create dep") + c2.run("create heavy") + assert "GENCACHE hit" in c2.out + assert "GENCACHE miss" not in c2.out + + +def test_conan_install_local_reuses_uploaded_cache(producer): + """Developer workflow: `conan install ` on the local heavy recipe.""" + c2 = _consumer(producer) + c2.run("create dep") + c2.run("install heavy") + assert "GENCACHE hit" in c2.out + assert "GENCACHE miss" not in c2.out + + +def test_no_download_urls_does_not_use_mirror(producer): + """Without `core.sources:download_urls`, the consumer never contacts the mirror, + even when the producer has uploaded there — so a fresh consumer misses. This + isolates the role of `download_urls` as the wire-up between recipe and mirror.""" + producer_c, file_server, _ = producer + conan_remotes = {k: v for k, v in producer_c.servers.items() if k != "file_server"} + c = TestClient(servers=conan_remotes, inputs=["admin", "password"], light=True) + c.servers["file_server"] = file_server + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "heavy/conanfile.py": HEAVY_CONANFILE}) + c.run("create dep") + c.run("create heavy") + assert "GENCACHE miss" in c.out + assert "GENCACHE hit" not in c.out + + +def test_default_cache_folder_used_when_not_configured(): + """`identifier` works out of the box: with no `core.sources:*` configured, the + default sources backup folder is used, so a second `create heavy` hits the + local cache the first run seeded.""" + c = TestClient(light=True) + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "heavy/conanfile.py": HEAVY_CONANFILE}) + c.run("create dep") + c.run("create heavy") + assert "GENCACHE miss" in c.out + + # Second create: local (default) cache holds the entry saved by the first run. + c.run("create heavy") + assert "GENCACHE hit" in c.out + assert "GENCACHE miss" not in c.out + + +def test_dep_rrev_change_produces_new_key(): + """Bumping dep's rrev changes the identifier → new miss + new cache entry.""" + c = TestClient(light=True) + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"), + "heavy/conanfile.py": HEAVY_CONANFILE}) + c.run("create dep") + c.run("create heavy") + assert "GENCACHE miss" in c.out + + c.save({"dep/conanfile.py": GenConanfile("dep", "0.1").with_import("import os")}) + c.run("create dep") + c.run("create heavy") + assert "GENCACHE miss" in c.out + + dl_cache = HomePaths(c.cache_folder).default_sources_backup_folder + entries = os.listdir(os.path.join(dl_cache, "s")) + blobs = [e for e in entries if not e.endswith(".json") and not e.endswith(".dirty")] + assert len(blobs) == 2