proposal for internal files

conan/internal/api/uploader.py

@@ -11,7 +11,7 @@
 from conan.internal.errors import NotFoundException
 from conan.errors import ConanException
 from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \
-    EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME
+    EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME, CONAN_INTERNAL_FILE_NAME
 from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
                                        set_dirty_context_manager, mkdir, human_size)
 
@@ -134,6 +134,22 @@ def prepare(self, pkg_list, enabled_remotes, metadata, force=False):
             if bundle.get("upload") or force:
                 self._prepare_recipe(recipe_layout, ref, bundle, conanfile, enabled_remotes)
 
+            # Conan-internal files always travel with the recipe — pack and add unconditionally
+            internal_folder = recipe_layout.conan_internal()
+            internal_files, _ = gather_files(internal_folder)
+            if internal_files:
+                download_export_folder = recipe_layout.download_export()
+                mkdir(download_export_folder)
+                # Always recompress — discard any stale archive left by a previous download
+                for ext in COMPRESSIONS:
+                    stale = os.path.join(download_export_folder, CONAN_INTERNAL_FILE_NAME + ext)
+                    if os.path.isfile(stale):
+                        os.remove(stale)
+                comp = self._compressed_file(CONAN_INTERNAL_FILE_NAME, internal_files,
+                                             download_export_folder, ref)
+                bundle.setdefault("files", {})[comp] = os.path.join(download_export_folder, comp)
+                bundle["upload"] = True
+
             # Package metadata files too
             if metadata != [""] and (metadata or bundle.get("upload")):
                 metadata_folder = recipe_layout.metadata()

conan/internal/cache/conan_reference_layout.py

@@ -15,6 +15,7 @@
 EXPORT_SRC_FOLDER = "es"
 DOWNLOAD_EXPORT_FOLDER = "d"
 METADATA = "metadata"
+CONAN_INTERNAL_FOLDER = "ci"
 
 
 class LayoutBase:
@@ -66,6 +67,9 @@ def metadata(self):
     def download_export(self):
         return os.path.join(self._base_folder, DOWNLOAD_EXPORT_FOLDER)
 
+    def conan_internal(self):
+        return os.path.join(self._base_folder, CONAN_INTERNAL_FOLDER)
+
     def source(self):
         return os.path.join(self._base_folder, SRC_FOLDER)
 

conan/internal/graph/installer.py

@@ -145,6 +145,7 @@ def build_package(self, node, recipe_layout, package_layout):
                 # In local cache, generators folder always in build_folder
                 conanfile.folders.set_base_generators(base_build)
                 conanfile.folders.set_base_pkg_metadata(package_layout.metadata())
+                conanfile.recipe_internal_folder = recipe_layout.conan_internal()
 
                 if not skip_build:
                     conanfile.output.info('Building your package in %s' % base_build)
@@ -194,6 +195,7 @@ def _install_source(self, node, remotes, need_conf=False):
         conanfile.folders.set_base_source(source_folder)
         conanfile.folders.set_base_export_sources(source_folder)
         conanfile.folders.set_base_recipe_metadata(recipe_layout.metadata())
+        conanfile.recipe_internal_folder = recipe_layout.conan_internal()
         config_source(export_source_folder, conanfile, self._hook_manager)
         return recipe_layout
 

conan/internal/model/conan_file.py

@@ -70,6 +70,7 @@ class ConanFile:
 
     no_copy_source = False
     recipe_folder = None
+    recipe_internal_folder = None
 
     # Package information
     cpp = None

conan/internal/paths.py

@@ -89,5 +89,6 @@ def _user_home_from_conanrc_file():
 PACKAGE_FILE_NAME = "conan_package.t"
 EXPORT_FILE_NAME = "conan_export.t"
 EXPORT_SOURCES_FILE_NAME = "conan_sources.t"
+CONAN_INTERNAL_FILE_NAME = "conan_internal.t"
 COMPRESSIONS = "gz", "xz", "zst"
 DATA_YML = "conandata.yml"

conan/internal/rest/remote_manager.py

@@ -8,7 +8,8 @@
 from requests.exceptions import ConnectionError
 
 from conan.api.model import LOCAL_RECIPES_INDEX
-from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME
+from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \
+    CONAN_INTERNAL_FILE_NAME
 from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex
 from conan.api.model import Remote
 from conan.api.output import ConanOutput
@@ -97,10 +98,17 @@ def _download_recipe(self, layout, ref, remote, metadata):
         export_folder = layout.export()
         export_file = next((f for f in zipped_files if f.startswith(EXPORT_FILE_NAME)), None)
         tgz_file = zipped_files.pop(export_file, None)
+        internal_file = next((f for f in zipped_files if f.startswith(CONAN_INTERNAL_FILE_NAME)),
+                             None)
+        internal_tgz = zipped_files.pop(internal_file, None)
 
         if tgz_file:
             uncompress_file(tgz_file, export_folder, scope=str(ref))
         mkdir(export_folder)
+        if internal_tgz:
+            internal_folder = layout.conan_internal()
+            mkdir(internal_folder)
+            uncompress_file(internal_tgz, internal_folder, scope=str(ref))
         for file_name, file_path in zipped_files.items():  # copy CONANFILE
             shutil.move(file_path, os.path.join(export_folder, file_name))
 

conan/internal/rest/rest_client_v2.py

@@ -10,7 +10,7 @@
 
 from conan.api.output import ConanOutput
 from conan.internal.paths import EXPORT_SOURCES_FILE_NAME, CONANINFO, CONAN_MANIFEST, \
-    EXPORT_FILE_NAME, PACKAGE_FILE_NAME
+    EXPORT_FILE_NAME, PACKAGE_FILE_NAME, CONAN_INTERNAL_FILE_NAME
 from conan.internal.rest.caching_file_downloader import ConanInternalCacheDownloader
 from conan.internal.rest import response_to_str
 from conan.internal.rest.client_routes import ClientV2Router
@@ -217,6 +217,9 @@ def get_recipe(self, ref, dest_folder, metadata, only_metadata):
             export_file = self._find_compressed_file(ref, server_files, EXPORT_FILE_NAME)
             if export_file is not None:
                 files.append(export_file)
+            internal_file = self._find_compressed_file(ref, server_files, CONAN_INTERNAL_FILE_NAME)
+            if internal_file is not None:
+                files.append(internal_file)
             # If we didn't indicated reference, server got the latest, use absolute now, it's safer
             urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
             self._download_and_save_files(urls, dest_folder, files, parallel=True)

conan/test/utils/tools.py

@@ -714,6 +714,8 @@ def get_latest_pkg_layout(self, pref: PkgReference) -> PackageLayout:
 
     def get_latest_ref_layout(self, ref) -> RecipeLayout:
         """Get the latest RecipeLayout given a file reference"""
+        if isinstance(ref, str):
+            ref = RecipeReference.loads(ref)
         if not ref.revision:
             ref = self.cache.get_latest_recipe_revision(ref)
         ref_layout = self.cache.recipe_layout(ref)

test/integration/conan_internal/test_conan_internal_files.py

@@ -0,0 +1,56 @@
+import os
+
+from conan.test.assets.genconanfile import GenConanfile
+from conan.test.utils.tools import TestClient
+from conan.internal.util.files import save, load
+from conan.internal.cache.conan_reference_layout import CONAN_INTERNAL_FOLDER
+
+
+class TestConanInternalFiles:
+
+    def test_upload_download_roundtrip(self):
+        """Full round-trip:
+        1. Create recipe, write internal file, upload.
+        2. Install in fresh client, verify internal file downloaded (not in manifest).
+        3. Update internal file, re-upload.
+        4. Install in third client, verify updated file is present.
+        """
+        c1 = TestClient(default_server_user=True, light=True)
+        c1.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c1.run("create .")
+
+        # Write an internal file into ci/ for the recipe
+        layout1 = c1.exported_layout()
+        internal_folder1 = layout1.conan_internal()
+        os.makedirs(internal_folder1, exist_ok=True)
+        save(os.path.join(internal_folder1, "data.json"), '{"version": 1}')
+
+        # Upload — internal file travels with recipe, no extra flag
+        c1.run("upload pkg/0.1 -c -r=default")
+
+        # Install in a clean second client
+        c2 = TestClient(servers=c1.servers, inputs=["admin", "password"], light=True)
+        c2.run("install --requires=pkg/0.1")
+
+        # Internal file must be present in ci/
+        layout2 = c2.get_latest_ref_layout("pkg/0.1")
+        downloaded_file = os.path.join(layout2.conan_internal(), "data.json")
+        assert os.path.isfile(downloaded_file)
+        assert load(downloaded_file) == '{"version": 1}'
+
+        # Internal file must NOT appear in conanmanifest.txt
+        manifest_content = load(os.path.join(layout2.export(), "conanmanifest.txt"))
+        assert "data.json" not in manifest_content
+        assert CONAN_INTERNAL_FOLDER not in manifest_content
+
+        # Update the internal file in c2 and re-upload
+        save(downloaded_file, '{"version": 2}')
+        c2.run("upload pkg/0.1 -c -r=default")
+
+        # Third client installs and sees the updated internal file
+        c3 = TestClient(servers=c1.servers, inputs=["admin", "password"], light=True)
+        c3.run("install --requires=pkg/0.1")
+        layout3 = c3.get_latest_ref_layout("pkg/0.1")
+        updated_file = os.path.join(layout3.conan_internal(), "data.json")
+        assert os.path.isfile(updated_file)
+        assert load(updated_file) == '{"version": 2}'

test/integration/conan_internal/test_recipe_internal_folder.py

@@ -0,0 +1,74 @@
+import os
+import textwrap
+
+from conan.test.utils.tools import TestClient
+from conan.internal.util.files import load
+
+
+class TestRecipeInternalFolderCodegen:
+    """Emulates the issue #20118 use case: a recipe runs codegen in generate() that is
+    slow and profile-independent. Results are cached in recipe_internal_folder so that
+    subsequent builds (different profiles) skip the generation entirely."""
+
+    def test_codegen_roundtrip(self):
+        # A recipe whose generate() writes a "generated" file from its sources.
+        # The first call runs the (simulated) codegen; subsequent calls reuse the cache.
+        conanfile = textwrap.dedent("""\
+            import os
+            from conan import ConanFile
+            from conan.tools.files import copy, save, load
+
+            class MyIDLLib(ConanFile):
+                name = "myidl"
+                version = "0.1"
+                exports = "schema.idl"
+
+                def generate(self):
+                    cache_dir = self.recipe_internal_folder
+                    generated_h = os.path.join(cache_dir, "generated.h")
+
+                    if not os.path.isfile(generated_h):
+                        self.output.info("Running codegen (first time)")
+                        idl = load(self, os.path.join(self.recipe_folder, "schema.idl"))
+                        save(self, generated_h, f"// generated from: {idl.strip()}")
+                    else:
+                        self.output.info("Reusing cached generated files")
+
+                    copy(self, "generated.h", cache_dir, self.build_folder)
+        """)
+
+        c = TestClient(default_server_user=True, light=True)
+        c.save({"conanfile.py": conanfile, "schema.idl": "struct Foo { int x; };"})
+        c.run("export .")
+
+        # --- First install (profile A) ---
+        c.run("install --requires=myidl/0.1 --build=myidl/0.1")
+        assert "Running codegen (first time)" in c.out
+
+        layout = c.get_latest_ref_layout("myidl/0.1")
+        cached_h = os.path.join(layout.conan_internal(), "generated.h")
+        assert os.path.isfile(cached_h)
+        assert "struct Foo" in load(cached_h)
+
+        # --- Second install (simulates a different profile) ---
+        # Cache is populated; codegen must NOT run again
+        c.run("install --requires=myidl/0.1 --build=myidl/0.1")
+        assert "Reusing cached generated files" in c.out
+        assert "Running codegen (first time)" not in c.out
+
+        # --- Upload and install on a fresh client ---
+        c.run("upload myidl/0.1 -c -r=default")
+
+        c2 = TestClient(servers=c.servers, inputs=["admin", "password"], light=True)
+        c2.run("install --requires=myidl/0.1")
+
+        layout2 = c2.get_latest_ref_layout("myidl/0.1")
+        downloaded_h = os.path.join(layout2.conan_internal(), "generated.h")
+        assert os.path.isfile(downloaded_h)
+        assert "struct Foo" in load(downloaded_h)
+
+        # On the fresh client the cache is already populated from the download,
+        # so a build also skips codegen
+        c2.run("install --requires=myidl/0.1 --build=myidl/0.1")
+        assert "Reusing cached generated files" in c2.out
+        assert "Running codegen (first time)" not in c2.out