Compression plugin

conan/api/subapi/cache.py

@@ -1,7 +1,6 @@
 import json
 import os
 import shutil
-import tarfile
 import tempfile
 
 from conan.api.model import PackagesList
@@ -17,7 +16,7 @@
 from conan.api.model import PkgReference
 from conan.api.model import RecipeReference
 from conan.internal.util.dates import revision_timestamp_now
-from conan.internal.util.files import rmdir, mkdir, remove, save
+from conan.internal.util.files import rmdir, mkdir, remove, save, tar_extract
 
 
 class CacheAPI:
@@ -131,7 +130,6 @@ def save(self, package_list, tgz_path, no_source=False):
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
         out = ConanOutput()
         mkdir(os.path.dirname(tgz_path))
-        compresslevel = global_conf.get("core.gzip:compresslevel", check_type=int)
         tar_files: dict[str,str] = {} # {path_in_tar: abs_path}
 
         for ref, ref_bundle in package_list.refs().items():
@@ -172,7 +170,8 @@ def save(self, package_list, tgz_path, no_source=False):
         pkglist_path = os.path.join(tempfile.gettempdir(), "pkglist.json")
         save(pkglist_path, serialized)
         tar_files["pkglist.json"] = pkglist_path
-        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), compresslevel, recursive=True)
+        compress_files(tar_files, os.path.basename(tgz_path), os.path.dirname(tgz_path), conf=self.conan_api.config,
+                       recursive=True, ref=None, compression_plugin=self.conan_api.config.compression_plugin)
         remove(pkglist_path)
 
     def restore(self, path):
@@ -182,13 +181,20 @@ def restore(self, path):
         cache = PkgCache(self.conan_api.cache_folder, self.conan_api.config.global_conf)
         cache_folder = cache.store  # Note, this is not the home, but the actual package cache
 
-        with open(path, mode='rb') as file_handler:
-            the_tar = tarfile.open(fileobj=file_handler)
-            fileobj = the_tar.extractfile("pkglist.json")
-            pkglist = fileobj.read()
-            the_tar.extraction_filter = (lambda member, _: member)  # fully_trusted (Py 3.14)
-            the_tar.extractall(path=cache_folder)
-            the_tar.close()
+        with open(path, mode="rb") as file_handler:
+            tar_extract(
+                fileobj=file_handler,
+                destination_dir=cache_folder,
+                compression_plugin=self.conan_api.config.compression_plugin,
+                conf=self.conan_api.config.global_conf,
+            )
+
+        # Retrieve the package list from the already extracted archive
+        pkglist_path = os.path.join(cache_folder, "pkglist.json")
+        with open(pkglist_path) as file_handler:
+            pkglist = file_handler.read()
+        # Delete the pkglist.json file to keep cache clean
+        remove(pkglist_path)
 
         # After unzipping the files, we need to update the DB that references these files
         out = ConanOutput()

conan/api/subapi/config.py

@@ -3,6 +3,7 @@
 import platform
 import textwrap
 import yaml
+from conan.internal.loader import load_python_file
 from jinja2 import Environment, FileSystemLoader
 
 from conan import conan_version
@@ -31,6 +32,7 @@ def __init__(self, conan_api):
         self._new_config = None
         self._cli_core_confs = None
         self.hook_manager = HookManager(HomePaths(conan_api.home_folder).hooks_path)
+        self._compression_plugin = None
 
     def home(self):
         return self.conan_api.cache_folder
@@ -236,4 +238,18 @@ def reinit(self):
         if self._new_config is not None:
             self._new_config.clear()
             self._populate_global_conf()
+            self._compression_plugin = None
         self.hook_manager = HookManager(HomePaths(self.conan_api.home_folder).hooks_path)
+
+    @property
+    def compression_plugin(self):
+        if self._compression_plugin is None:
+            compression_plugin_path = HomePaths(self.conan_api.home_folder).compression_plugin_path
+            if not os.path.exists(compression_plugin_path):
+                self._compression_plugin = False # Avoid FS re-check
+                return None
+            mod, _ = load_python_file(compression_plugin_path)
+            if not hasattr(mod, "tar_extract") or not hasattr(mod, "tar_compress"):
+                raise ConanException("The 'compression.py' plugin does not contain required `tar_extract` or `tar_compress` functions")
+            self._compression_plugin = mod
+        return self._compression_plugin or None

conan/internal/api/uploader.py

@@ -12,7 +12,7 @@
 from conan.errors import ConanException
 from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
                                   EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
-from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
+from conan.internal.util.files import (COMPRESSED_PLUGIN_TAR_NAME, clean_dirty, is_dirty, gather_files, remove,
                                        set_dirty_context_manager, mkdir, human_size)
 
 UPLOAD_POLICY_FORCE = "force-upload"
@@ -155,9 +155,9 @@ def add_tgz(tgz_name, tgz_files):
             if os.path.isfile(tgz):
                 result[tgz_name] = tgz
             elif tgz_files:
-                compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
                 tgz = compress_files(tgz_files, tgz_name, download_export_folder,
-                                     compresslevel=compresslevel, ref=ref)
+                                     conf=self._global_conf, ref=ref,
+                                     compression_plugin=self._app.conan_api.config.compression_plugin)
                 result[tgz_name] = tgz
 
         add_tgz(EXPORT_TGZ_NAME, files)
@@ -202,9 +202,9 @@ def _compress_package_files(self, layout, pref):
 
         if not os.path.isfile(package_tgz):
             tgz_files = {f: path for f, path in files.items()}
-            compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
             tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel, ref=pref)
+                                      conf=self._global_conf, ref=pref,
+                                      compression_plugin=self._app.conan_api.config.compression_plugin)
             assert tgz_path == package_tgz
             assert os.path.exists(package_tgz)
 
@@ -271,13 +271,16 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+def compress_files(files, name, dest_dir, conf=None, ref=None, recursive=False, compression_plugin=None):
+    if compression_plugin:
+        return _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin)
+
+    tgz_path = os.path.join(dest_dir, name)
     t1 = time.time()
     # FIXME, better write to disk sequentially and not keep tgz contents in memory
-    tgz_path = os.path.join(dest_dir, name)
-    if ref:
-        ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}")
+    ConanOutput(scope=str(ref or "")).info(f"Compressing {name}")
     with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
+        compresslevel = conf.get("core.gzip:compresslevel", check_type=int) if conf else None
         tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
         for filename, abs_path in sorted(files.items()):
             # recursive is False by default in case it is a symlink to a folder
@@ -288,6 +291,31 @@ def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursiv
     ConanOutput().debug(f"{name} compressed in {duration} time")
     return tgz_path
 
+def _compress_files_with_plugin(files, name, dest_dir, conf, ref, recursive, compression_plugin):
+    t1 = time.time()
+    abs_path_without_extension = os.path.join(dest_dir, COMPRESSED_PLUGIN_TAR_NAME)
+    ConanOutput(scope=str(ref or "")).info(f"Compressing {name} using compression plugin")
+    compressed_extension = compression_plugin.tar_compress(
+        archive_path=abs_path_without_extension,
+        files=files,
+        recursive=recursive,
+        conf=conf,
+        ref=ref,
+    )
+    ConanOutput().debug(f"Compressed in {time.time() - t1} time")
+    if not compressed_extension or not compressed_extension.startswith("."):
+        raise ConanException("The 'compression.py' did not return the compressed extension.")
+
+    compressed_path = abs_path_without_extension + compressed_extension
+    t1 = time.time()
+    tgz_path = os.path.join(dest_dir, name)
+    with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
+        tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=0)
+        tgz.add(compressed_path, arcname=os.path.basename(compressed_path), recursive=recursive)
+        tgz.close()
+    ConanOutput().debug(f"{name} wrapped in {time.time() - t1} time")
+    remove(compressed_path)
+    return tgz_path
 
 def _total_size(cache_files):
     total_size = 0

conan/internal/cache/home_paths.py

@@ -90,3 +90,7 @@ def settings_path_user(self):
     @property
     def config_version_path(self):
         return os.path.join(self._home, "config_version.json")
+
+    @property
+    def compression_plugin_path(self):
+        return os.path.join(self._home, _EXTENSIONS_FOLDER, _PLUGINS, "compression.py")

conan/internal/conan_app.py

@@ -52,7 +52,7 @@ def __init__(self, conan_api):
         localdb = LocalDB(cache_folder)
         auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, global_conf)
         # Handle remote connections
-        self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder)
+        self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder, self.conan_api.config)
         global_editables = conan_api.local.editable_packages
         ws_editables = conan_api.workspace.packages()
         self.editable_packages = global_editables.update_copy(ws_editables)
@@ -81,10 +81,10 @@ class LocalRecipesIndexApp:
     - loader (for the export phase of local-recipes-index)
     The others are internally use by other collaborators
     """
-    def __init__(self, cache_folder):
+    def __init__(self, cache_folder, config_api):
         self.global_conf = ConfDefinition()
         self.cache = PkgCache(cache_folder, self.global_conf)
-        self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder)
+        self.remote_manager = RemoteManager(self.cache, auth_manager=None, home_folder=cache_folder, config_api=config_api)
         editable_packages = EditablePackages()
         self.proxy = ConanProxy(self, editable_packages)
         self.range_resolver = RangeResolver(self, self.global_conf, editable_packages)

conan/internal/rest/remote_manager.py

@@ -22,15 +22,16 @@
 
 class RemoteManager:
     """ Will handle the remotes to get recipes, packages etc """
-    def __init__(self, cache, auth_manager, home_folder):
+    def __init__(self, cache, auth_manager, home_folder, config_api):
         self._cache = cache
         self._auth_manager = auth_manager
         self._signer = PkgSignaturesPlugin(cache, home_folder)
         self._home_folder = home_folder
+        self._config_api = config_api
 
     def _local_folder_remote(self, remote):
         if remote.remote_type == LOCAL_RECIPES_INDEX:
-            return RestApiClientLocalRecipesIndex(remote, self._home_folder)
+            return RestApiClientLocalRecipesIndex(remote, self._home_folder, self._config_api)
 
     def check_credentials(self, remote, force_auth=False):
         self._call_remote(remote, "check_credentials", force_auth)
@@ -83,7 +84,7 @@ def get_recipe(self, ref, remote, metadata=None):
         tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
 
         if tgz_file:
-            uncompress_file(tgz_file, export_folder, scope=str(ref))
+            uncompress_file(tgz_file, export_folder, scope=str(ref), config_api=self._config_api)
         mkdir(export_folder)
         for file_name, file_path in zipped_files.items():  # copy CONANFILE
             shutil.move(file_path, os.path.join(export_folder, file_name))
@@ -125,7 +126,7 @@ def get_recipe_sources(self, ref, layout, remote):
 
         self._signer.verify(ref, download_folder, files=zipped_files)
         tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
-        uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
+        uncompress_file(tgz_file, export_sources_folder, scope=str(ref), config_api=self._config_api)
 
     def get_package(self, pref, remote, metadata=None):
         output = ConanOutput(scope=str(pref.ref))
@@ -173,7 +174,7 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
 
             tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
             package_folder = layout.package()
-            uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
+            uncompress_file(tgz_file, package_folder, scope=str(pref.ref), config_api=self._config_api)
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
             for file_name, file_path in zipped_files.items():  # copy CONANINFO and CONANMANIFEST
                 shutil.move(file_path, os.path.join(package_folder, file_name))
@@ -295,15 +296,21 @@ def _call_remote(self, remote, method, *args, **kwargs):
             raise ConanException(exc, remote=remote)
 
 
-def uncompress_file(src_path, dest_folder, scope=None):
+def uncompress_file(src_path, dest_folder, scope="", config_api=None):
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB
         if big_file:
             hs = human_size(filesize)
             ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}")
+
         with open(src_path, mode='rb') as file_handler:
-            tar_extract(file_handler, dest_folder)
+            tar_extract(
+                fileobj=file_handler,
+                destination_dir=dest_folder,
+                compression_plugin=config_api.compression_plugin if config_api and config_api.compression_plugin else None,
+                conf=config_api.global_conf if config_api else None
+            )
     except Exception as e:
         error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\
                     % (src_path, dest_folder, str(e))

conan/internal/rest/rest_client_local_recipe_index.py

@@ -58,14 +58,14 @@ class RestApiClientLocalRecipesIndex:
     a local folder assuming the conan-center-index repo layout
     """
 
-    def __init__(self, remote, home_folder):
+    def __init__(self, remote, home_folder, config_api):
         self._remote = remote
         local_recipes_index_path = HomePaths(home_folder).local_recipes_index_path
         local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name, ".conan")
         repo_folder = self._remote.url
 
         from conan.internal.conan_app import LocalRecipesIndexApp
-        self._app = LocalRecipesIndexApp(local_recipes_index_path)
+        self._app = LocalRecipesIndexApp(local_recipes_index_path, config_api)
         self._hook_manager = HookManager(HomePaths(local_recipes_index_path).hooks_path)
         self._layout = _LocalRecipesIndexLayout(repo_folder)
 

conan/internal/util/files.py

@@ -1,4 +1,6 @@
 import errno
+from pathlib import Path
+import tempfile
 import gzip
 import hashlib
 import os
@@ -11,10 +13,13 @@
 
 from contextlib import contextmanager
 
+from conan.api.output import ConanOutput
 from conan.errors import ConanException
 
 _DIRTY_FOLDER = ".dirty"
 
+# Name (without extension) of the tar file to be created by the compression plugin
+COMPRESSED_PLUGIN_TAR_NAME = "__conan_plugin_compressed_contents__"
 
 def set_dirty(folder):
     dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER
@@ -256,15 +261,46 @@ def mkdir(path):
     os.makedirs(path)
 
 
-def tar_extract(fileobj, destination_dir):
+def tar_extract(fileobj, destination_dir, compression_plugin=None, conf=None):
+    if compression_plugin:
+        _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf)
+        return
+
     the_tar = tarfile.open(fileobj=fileobj)
     # NOTE: The errorlevel=2 has been removed because it was failing in Win10, it didn't allow to
     # "could not change modification time", with time=0
     # the_tar.errorlevel = 2  # raise exception if any error
     the_tar.extraction_filter = (lambda member, path: member)  # fully_trusted, avoid Py3.14 break
     the_tar.extractall(path=destination_dir)
     the_tar.close()
-
+    if list(Path(destination_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")):
+        raise ConanException(f"Error while extracting {os.path.basename(fileobj.name)}.\n"
+                             "This file has been compressed using a `compression` plugin.\n"
+                             "If your organization uses this plugin, ensure it is correctly installed on your environment.")
+
+def _tar_extract_with_plugin(fileobj, destination_dir, compression_plugin, conf):
+    """First remove tar.gz wrapper and then call the plugin to extract"""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        t1 = time.time()
+        the_tar = tarfile.open(fileobj=fileobj)
+        the_tar.extraction_filter = (lambda member, path: member)  # fully_trusted, avoid Py3.14 break
+        the_tar.extractall(path=temp_dir)
+        extracted_file = the_tar.getnames()[0]
+        the_tar.close()
+        # Check if the tar was compressed with the compression plugin by checking the existence of
+        # our constant COMPRESSED_PLUGIN_TAR_NAME (without extension as extension is added by the plugin)
+        if list(Path(temp_dir).glob(f"{COMPRESSED_PLUGIN_TAR_NAME}.*")):
+            # Get the only extracted file: the plugin tar
+            plugin_tar_path = os.path.join(temp_dir, extracted_file)
+            ConanOutput().debug(f"Unwrapped in {time.time() - t1} time")
+            t1 = time.time()
+            compression_plugin.tar_extract(archive_path=plugin_tar_path, dest_dir=destination_dir, conf=conf)
+            ConanOutput().debug(f"Extracted in {time.time() - t1} time on plugin")
+        else:
+            # The tar was not compressed using the plugin, copy files to destination
+            from conan.tools.files import copy
+            ConanOutput().debug(f"Extracted in {time.time() - t1} time built in")
+            copy(None, pattern="*", src=temp_dir, dst=destination_dir)
 
 def merge_directories(src, dst):
     from conan.tools.files import copy

test/integration/command/cache/test_cache_save_restore.py

@@ -158,9 +158,13 @@ def test_cache_save_excluded_folders():
 
     # exclude source
     c.run("cache save * --no-source")
+    # Check default compression function is being used and not compression.py plugin one
+    assert "Compressing conan_cache_save.tgz\n" in c.out
     c3 = TestClient()
     shutil.copy2(cache_path, c3.current_folder)
     c3.run("cache restore conan_cache_save.tgz")
+    # Default decompress does not have any output
+    assert "Decompressing conan_cache_save.tgz" not in c3.out
     ref_layout = c3.get_latest_ref_layout(ref)
     assert not os.path.exists(os.path.join(ref_layout.source(), "mysrc.c"))