From 608635e96ed64a7c38a014deb5d6b2daf70b93a7 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 30 Jun 2026 11:33:56 +0200 Subject: [PATCH 1/2] proposal for internal files --- conan/internal/api/uploader.py | 18 +++++- .../internal/cache/conan_reference_layout.py | 4 ++ conan/internal/paths.py | 1 + conan/internal/rest/remote_manager.py | 10 +++- conan/internal/rest/rest_client_v2.py | 5 +- conan/test/utils/tools.py | 2 + test/integration/conan_internal/__init__.py | 0 .../test_conan_internal_files.py | 56 +++++++++++++++++++ 8 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 test/integration/conan_internal/__init__.py create mode 100644 test/integration/conan_internal/test_conan_internal_files.py diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 4f46738a2dd..66d65a26e7c 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -11,7 +11,7 @@ from conan.internal.errors import NotFoundException from conan.errors import ConanException from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \ - EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME + EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME, CONAN_INTERNAL_FILE_NAME from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) @@ -134,6 +134,22 @@ def prepare(self, pkg_list, enabled_remotes, metadata, force=False): if bundle.get("upload") or force: self._prepare_recipe(recipe_layout, ref, bundle, conanfile, enabled_remotes) + # Conan-internal files always travel with the recipe — pack and add unconditionally + internal_folder = recipe_layout.conan_internal() + internal_files, _ = gather_files(internal_folder) + if internal_files: + download_export_folder = recipe_layout.download_export() + mkdir(download_export_folder) + # Always recompress — discard any stale archive left by a previous download + for ext in COMPRESSIONS: + stale = os.path.join(download_export_folder, CONAN_INTERNAL_FILE_NAME + ext) + if os.path.isfile(stale): + os.remove(stale) + comp = self._compressed_file(CONAN_INTERNAL_FILE_NAME, internal_files, + download_export_folder, ref) + bundle.setdefault("files", {})[comp] = os.path.join(download_export_folder, comp) + bundle["upload"] = True + # Package metadata files too if metadata != [""] and (metadata or bundle.get("upload")): metadata_folder = recipe_layout.metadata() diff --git a/conan/internal/cache/conan_reference_layout.py b/conan/internal/cache/conan_reference_layout.py index 37ffe9b9dca..79e4d9aab61 100644 --- a/conan/internal/cache/conan_reference_layout.py +++ b/conan/internal/cache/conan_reference_layout.py @@ -15,6 +15,7 @@ EXPORT_SRC_FOLDER = "es" DOWNLOAD_EXPORT_FOLDER = "d" METADATA = "metadata" +CONAN_INTERNAL_FOLDER = "ci" class LayoutBase: @@ -66,6 +67,9 @@ def metadata(self): def download_export(self): return os.path.join(self._base_folder, DOWNLOAD_EXPORT_FOLDER) + def conan_internal(self): + return os.path.join(self._base_folder, CONAN_INTERNAL_FOLDER) + def source(self): return os.path.join(self._base_folder, SRC_FOLDER) diff --git a/conan/internal/paths.py b/conan/internal/paths.py index adc7d0dfa6f..6dda18deade 100644 --- a/conan/internal/paths.py +++ b/conan/internal/paths.py @@ -89,5 +89,6 @@ def _user_home_from_conanrc_file(): PACKAGE_FILE_NAME = "conan_package.t" EXPORT_FILE_NAME = "conan_export.t" EXPORT_SOURCES_FILE_NAME = "conan_sources.t" +CONAN_INTERNAL_FILE_NAME = "conan_internal.t" COMPRESSIONS = "gz", "xz", "zst" DATA_YML = "conandata.yml" diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index fce3be27d2c..f984e0f70d8 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -8,7 +8,8 @@ from requests.exceptions import ConnectionError from conan.api.model import LOCAL_RECIPES_INDEX -from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME +from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \ + CONAN_INTERNAL_FILE_NAME from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex from conan.api.model import Remote from conan.api.output import ConanOutput @@ -97,10 +98,17 @@ def _download_recipe(self, layout, ref, remote, metadata): export_folder = layout.export() export_file = next((f for f in zipped_files if f.startswith(EXPORT_FILE_NAME)), None) tgz_file = zipped_files.pop(export_file, None) + internal_file = next((f for f in zipped_files if f.startswith(CONAN_INTERNAL_FILE_NAME)), + None) + internal_tgz = zipped_files.pop(internal_file, None) if tgz_file: uncompress_file(tgz_file, export_folder, scope=str(ref)) mkdir(export_folder) + if internal_tgz: + internal_folder = layout.conan_internal() + mkdir(internal_folder) + uncompress_file(internal_tgz, internal_folder, scope=str(ref)) for file_name, file_path in zipped_files.items(): # copy CONANFILE shutil.move(file_path, os.path.join(export_folder, file_name)) diff --git a/conan/internal/rest/rest_client_v2.py b/conan/internal/rest/rest_client_v2.py index 9033e6d9189..742b8d73fa6 100644 --- a/conan/internal/rest/rest_client_v2.py +++ b/conan/internal/rest/rest_client_v2.py @@ -10,7 +10,7 @@ from conan.api.output import ConanOutput from conan.internal.paths import EXPORT_SOURCES_FILE_NAME, CONANINFO, CONAN_MANIFEST, \ - EXPORT_FILE_NAME, PACKAGE_FILE_NAME + EXPORT_FILE_NAME, PACKAGE_FILE_NAME, CONAN_INTERNAL_FILE_NAME from conan.internal.rest.caching_file_downloader import ConanInternalCacheDownloader from conan.internal.rest import response_to_str from conan.internal.rest.client_routes import ClientV2Router @@ -217,6 +217,9 @@ def get_recipe(self, ref, dest_folder, metadata, only_metadata): export_file = self._find_compressed_file(ref, server_files, EXPORT_FILE_NAME) if export_file is not None: files.append(export_file) + internal_file = self._find_compressed_file(ref, server_files, CONAN_INTERNAL_FILE_NAME) + if internal_file is not None: + files.append(internal_file) # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} self._download_and_save_files(urls, dest_folder, files, parallel=True) diff --git a/conan/test/utils/tools.py b/conan/test/utils/tools.py index 64fa06c0af7..bdba8990766 100644 --- a/conan/test/utils/tools.py +++ b/conan/test/utils/tools.py @@ -714,6 +714,8 @@ def get_latest_pkg_layout(self, pref: PkgReference) -> PackageLayout: def get_latest_ref_layout(self, ref) -> RecipeLayout: """Get the latest RecipeLayout given a file reference""" + if isinstance(ref, str): + ref = RecipeReference.loads(ref) if not ref.revision: ref = self.cache.get_latest_recipe_revision(ref) ref_layout = self.cache.recipe_layout(ref) diff --git a/test/integration/conan_internal/__init__.py b/test/integration/conan_internal/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/conan_internal/test_conan_internal_files.py b/test/integration/conan_internal/test_conan_internal_files.py new file mode 100644 index 00000000000..00cd63010e6 --- /dev/null +++ b/test/integration/conan_internal/test_conan_internal_files.py @@ -0,0 +1,56 @@ +import os + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient +from conan.internal.util.files import save, load +from conan.internal.cache.conan_reference_layout import CONAN_INTERNAL_FOLDER + + +class TestConanInternalFiles: + + def test_upload_download_roundtrip(self): + """Full round-trip: + 1. Create recipe, write internal file, upload. + 2. Install in fresh client, verify internal file downloaded (not in manifest). + 3. Update internal file, re-upload. + 4. Install in third client, verify updated file is present. + """ + c1 = TestClient(default_server_user=True, light=True) + c1.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c1.run("create .") + + # Write an internal file into ci/ for the recipe + layout1 = c1.exported_layout() + internal_folder1 = layout1.conan_internal() + os.makedirs(internal_folder1, exist_ok=True) + save(os.path.join(internal_folder1, "data.json"), '{"version": 1}') + + # Upload — internal file travels with recipe, no extra flag + c1.run("upload pkg/0.1 -c -r=default") + + # Install in a clean second client + c2 = TestClient(servers=c1.servers, inputs=["admin", "password"], light=True) + c2.run("install --requires=pkg/0.1") + + # Internal file must be present in ci/ + layout2 = c2.get_latest_ref_layout("pkg/0.1") + downloaded_file = os.path.join(layout2.conan_internal(), "data.json") + assert os.path.isfile(downloaded_file) + assert load(downloaded_file) == '{"version": 1}' + + # Internal file must NOT appear in conanmanifest.txt + manifest_content = load(os.path.join(layout2.export(), "conanmanifest.txt")) + assert "data.json" not in manifest_content + assert CONAN_INTERNAL_FOLDER not in manifest_content + + # Update the internal file in c2 and re-upload + save(downloaded_file, '{"version": 2}') + c2.run("upload pkg/0.1 -c -r=default") + + # Third client installs and sees the updated internal file + c3 = TestClient(servers=c1.servers, inputs=["admin", "password"], light=True) + c3.run("install --requires=pkg/0.1") + layout3 = c3.get_latest_ref_layout("pkg/0.1") + updated_file = os.path.join(layout3.conan_internal(), "data.json") + assert os.path.isfile(updated_file) + assert load(updated_file) == '{"version": 2}' From 9a92db26063305e4af1681661b4172e907e0e0fd Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 30 Jun 2026 15:16:30 +0200 Subject: [PATCH 2/2] wip --- conan/internal/graph/installer.py | 2 + conan/internal/model/conan_file.py | 1 + .../test_recipe_internal_folder.py | 74 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 test/integration/conan_internal/test_recipe_internal_folder.py diff --git a/conan/internal/graph/installer.py b/conan/internal/graph/installer.py index e3d13f23b7d..8d259d3b703 100644 --- a/conan/internal/graph/installer.py +++ b/conan/internal/graph/installer.py @@ -145,6 +145,7 @@ def build_package(self, node, recipe_layout, package_layout): # In local cache, generators folder always in build_folder conanfile.folders.set_base_generators(base_build) conanfile.folders.set_base_pkg_metadata(package_layout.metadata()) + conanfile.recipe_internal_folder = recipe_layout.conan_internal() if not skip_build: conanfile.output.info('Building your package in %s' % base_build) @@ -194,6 +195,7 @@ def _install_source(self, node, remotes, need_conf=False): conanfile.folders.set_base_source(source_folder) conanfile.folders.set_base_export_sources(source_folder) conanfile.folders.set_base_recipe_metadata(recipe_layout.metadata()) + conanfile.recipe_internal_folder = recipe_layout.conan_internal() config_source(export_source_folder, conanfile, self._hook_manager) return recipe_layout diff --git a/conan/internal/model/conan_file.py b/conan/internal/model/conan_file.py index 1267c82ad49..a260b742465 100644 --- a/conan/internal/model/conan_file.py +++ b/conan/internal/model/conan_file.py @@ -70,6 +70,7 @@ class ConanFile: no_copy_source = False recipe_folder = None + recipe_internal_folder = None # Package information cpp = None diff --git a/test/integration/conan_internal/test_recipe_internal_folder.py b/test/integration/conan_internal/test_recipe_internal_folder.py new file mode 100644 index 00000000000..a5c78827920 --- /dev/null +++ b/test/integration/conan_internal/test_recipe_internal_folder.py @@ -0,0 +1,74 @@ +import os +import textwrap + +from conan.test.utils.tools import TestClient +from conan.internal.util.files import load + + +class TestRecipeInternalFolderCodegen: + """Emulates the issue #20118 use case: a recipe runs codegen in generate() that is + slow and profile-independent. Results are cached in recipe_internal_folder so that + subsequent builds (different profiles) skip the generation entirely.""" + + def test_codegen_roundtrip(self): + # A recipe whose generate() writes a "generated" file from its sources. + # The first call runs the (simulated) codegen; subsequent calls reuse the cache. + conanfile = textwrap.dedent("""\ + import os + from conan import ConanFile + from conan.tools.files import copy, save, load + + class MyIDLLib(ConanFile): + name = "myidl" + version = "0.1" + exports = "schema.idl" + + def generate(self): + cache_dir = self.recipe_internal_folder + generated_h = os.path.join(cache_dir, "generated.h") + + if not os.path.isfile(generated_h): + self.output.info("Running codegen (first time)") + idl = load(self, os.path.join(self.recipe_folder, "schema.idl")) + save(self, generated_h, f"// generated from: {idl.strip()}") + else: + self.output.info("Reusing cached generated files") + + copy(self, "generated.h", cache_dir, self.build_folder) + """) + + c = TestClient(default_server_user=True, light=True) + c.save({"conanfile.py": conanfile, "schema.idl": "struct Foo { int x; };"}) + c.run("export .") + + # --- First install (profile A) --- + c.run("install --requires=myidl/0.1 --build=myidl/0.1") + assert "Running codegen (first time)" in c.out + + layout = c.get_latest_ref_layout("myidl/0.1") + cached_h = os.path.join(layout.conan_internal(), "generated.h") + assert os.path.isfile(cached_h) + assert "struct Foo" in load(cached_h) + + # --- Second install (simulates a different profile) --- + # Cache is populated; codegen must NOT run again + c.run("install --requires=myidl/0.1 --build=myidl/0.1") + assert "Reusing cached generated files" in c.out + assert "Running codegen (first time)" not in c.out + + # --- Upload and install on a fresh client --- + c.run("upload myidl/0.1 -c -r=default") + + c2 = TestClient(servers=c.servers, inputs=["admin", "password"], light=True) + c2.run("install --requires=myidl/0.1") + + layout2 = c2.get_latest_ref_layout("myidl/0.1") + downloaded_h = os.path.join(layout2.conan_internal(), "generated.h") + assert os.path.isfile(downloaded_h) + assert "struct Foo" in load(downloaded_h) + + # On the fresh client the cache is already populated from the download, + # so a build also skips codegen + c2.run("install --requires=myidl/0.1 --build=myidl/0.1") + assert "Reusing cached generated files" in c2.out + assert "Running codegen (first time)" not in c2.out