Add support for zstd compression

conans/client/cmd/uploader.py

@@ -1,13 +1,15 @@
 import os
 import shutil
+import tarfile
 import time
+import zstandard
 
 from conan.internal.conan_app import ConanApp
 from conan.api.output import ConanOutput
 from conans.client.source import retrieve_exports_sources
 from conans.errors import ConanException, NotFoundException
 from conans.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
-                          EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
+                          EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME, CONANINFO)
 from conans.util.files import (clean_dirty, is_dirty, gather_files,
                                gzopen_without_timestamps, set_dirty_context_manager, mkdir)
 
@@ -165,11 +167,22 @@ def _prepare_package(self, pref, prev_bundle):
 
     def _compress_package_files(self, layout, pref):
         download_pkg_folder = layout.download_package()
-        package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
-        if is_dirty(package_tgz):
-            self._output.warning("%s: Removing %s, marked as dirty" % (str(pref), PACKAGE_TGZ_NAME))
-            os.remove(package_tgz)
-            clean_dirty(package_tgz)
+        compression_format = self._app.cache.new_config.get("core.upload:compression_format") or "gzip"
+        if compression_format == "gzip":
+            compress_level_config = "core.gzip:compresslevel"
+            package_file_name = PACKAGE_TGZ_NAME
+            package_file = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
+        elif compression_format == "zstd":
+            compress_level_config = "core.zstd:compresslevel"
+            package_file_name = PACKAGE_TZSTD_NAME
+            package_file = os.path.join(download_pkg_folder, PACKAGE_TZSTD_NAME)
+        else:
+            raise ConanException(f"Unsupported compression level '{compression_format}'")
+
+        if is_dirty(package_file):
+            self._output.warning("%s: Removing %s, marked as dirty" % (str(pref), package_file_name))
+            os.remove(package_file)
+            clean_dirty(package_file)
 
         # Get all the files in that directory
         # existing package, will use short paths if defined
@@ -190,17 +203,21 @@ def _compress_package_files(self, layout, pref):
         files.pop(CONANINFO)
         files.pop(CONAN_MANIFEST)
 
-        if not os.path.isfile(package_tgz):
+        if os.path.isfile(package_file):
+            self._output.info(f"Not writing '{package_file}' because it already exists.")
+        else:
             if self._output and not self._output.is_terminal:
                 self._output.info("Compressing package...")
-            tgz_files = {f: path for f, path in files.items()}
-            compresslevel = self._app.cache.new_config.get("core.gzip:compresslevel", check_type=int)
-            tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel)
-            assert tgz_path == package_tgz
-            assert os.path.exists(package_tgz)
-
-        return {PACKAGE_TGZ_NAME: package_tgz,
+
+            source_files = {f: path for f, path in files.items()}
+            compresslevel = self._app.cache.new_config.get(compress_level_config, check_type=int)
+            compressed_path = compress_files(source_files, package_file_name, download_pkg_folder,
+                                             compresslevel=compresslevel, compressformat=compression_format)
+
+            assert compressed_path == package_file
+            assert os.path.exists(package_file)
+
+        return {package_file_name: package_file,
                 CONANINFO: os.path.join(download_pkg_folder, CONANINFO),
                 CONAN_MANIFEST: os.path.join(download_pkg_folder, CONAN_MANIFEST)}
 
@@ -246,21 +263,48 @@ def upload_package(self, pref, prev_bundle, remote):
         self._output.debug(f"Upload {pref} in {duration} time")
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None):
+def compress_files(files, name, dest_dir, compressformat=None, compresslevel=None, ref=None):
     t1 = time.time()
-    # FIXME, better write to disk sequentially and not keep tgz contents in memory
-    tgz_path = os.path.join(dest_dir, name)
-    if name in (PACKAGE_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME) and len(files) > 100:
-        ref_name = f"{ref}:" or ""
+    tar_path = os.path.join(dest_dir, name)
+    if name in (PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME, EXPORT_SOURCES_TGZ_NAME) and len(files) > 100:
+        ref_name = f"{ref}:" if ref else ""
         ConanOutput().info(f"Compressing {ref_name}{name}")
-    with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
-        tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle,
-                                        compresslevel=compresslevel)
-        for filename, abs_path in sorted(files.items()):
-            # recursive is False in case it is a symlink to a folder
-            tgz.add(abs_path, filename, recursive=False)
-        tgz.close()
+
+    if compressformat == "zstd":
+        with open(tar_path, "wb") as tarfile_obj:
+            def reset_tarinfo(tarinfo):
+                """
+                Resets mtime in the tarinfo for consistency with
+                gzopen_without_timestamps()
+                """
+                tarinfo.mtime = 0
+                return tarinfo
+
+            # Only provide level if it was overridden by config.
+            zstd_kwargs = {}
+            if compresslevel is not None:
+                zstd_kwargs["level"] = compresslevel
+
+            dctx = zstandard.ZstdCompressor(**zstd_kwargs)
+
+            # Create a zstd stream writer so tarfile writes uncompressed data to
+            # the zstd stream writer, which in turn writes compressed data to the
+            # output tar.zst file.
+            with dctx.stream_writer(tarfile_obj) as stream_writer:
+                with tarfile.open(mode="w|", fileobj=stream_writer,
+                                  format=tarfile.PAX_FORMAT) as tar:
+                    for filename, abs_path in sorted(files.items()):
+                        tar.add(abs_path, filename, recursive=False, filter=reset_tarinfo)
+    else:
+        # FIXME, better write to disk sequentially and not keep tgz contents in memory
+        with set_dirty_context_manager(tar_path), open(tar_path, "wb") as tgz_handle:
+            tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle,
+                                            compresslevel=compresslevel)
+            for filename, abs_path in sorted(files.items()):
+                # recursive is False in case it is a symlink to a folder
+                tgz.add(abs_path, filename, recursive=False)
+            tgz.close()
 
     duration = time.time() - t1
     ConanOutput().debug(f"{name} compressed in {duration} time")
-    return tgz_path
+    return tar_path

conans/client/remote_manager.py

@@ -1,5 +1,8 @@
 import os
 import shutil
+import tarfile
+import time
+import zstandard
 from typing import List
 
 from requests.exceptions import ConnectionError
@@ -14,7 +17,7 @@
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
 from conans.util.files import rmdir, human_size
-from conans.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
+from conans.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME
 from conans.util.files import mkdir, tar_extract
 
 
@@ -148,14 +151,21 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
                                              metadata, only_metadata=False)
             zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
             # quick server package integrity check:
-            for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"):
+            for f in ("conaninfo.txt", "conanmanifest.txt"):
                 if f not in zipped_files:
                     raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {f}")
+            accepted_package_files = [PACKAGE_TZSTD_NAME, PACKAGE_TGZ_NAME]
+            package_file = next((f for f in zipped_files if f in accepted_package_files), None)
+            if not package_file:
+                raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {accepted_package_files} found")
             self._signer.verify(pref, download_pkg_folder, zipped_files)
 
-            tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
+            package_file = zipped_files.pop(package_file, None)
             package_folder = layout.package()
-            uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
+            t1 = time.time()
+            uncompress_file(package_file, package_folder, scope=str(pref.ref))
+            duration = time.time() - t1
+            scoped_output.debug(f"Decompressed {package_file} in {duration} seconds")
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
             for file_name, file_path in zipped_files.items():  # copy CONANINFO and CONANMANIFEST
                 shutil.move(file_path, os.path.join(package_folder, file_name))
@@ -255,8 +265,15 @@ def uncompress_file(src_path, dest_folder, scope=None):
         if big_file:
             hs = human_size(filesize)
             ConanOutput(scope=scope).info(f"Decompressing {hs} {os.path.basename(src_path)}")
+
         with open(src_path, mode='rb') as file_handler:
-            tar_extract(file_handler, dest_folder)
+            if src_path.endswith(".tar.zst"):
+                dctx = zstandard.ZstdDecompressor()
+                stream_reader = dctx.stream_reader(file_handler)
+                with tarfile.open(fileobj=stream_reader, mode='r|') as the_tar:
+                    the_tar.extractall(dest_folder)
+            else:
+                tar_extract(file_handler, dest_folder)
     except Exception as e:
         error_msg = "Error while extracting downloaded file '%s' to %s\n%s\n"\
                     % (src_path, dest_folder, str(e))

conans/client/rest/rest_client_v2.py

@@ -11,7 +11,7 @@
 from conans.errors import ConanException, NotFoundException, PackageNotFoundException, \
     RecipeNotFoundException, AuthenticationException, ForbiddenException
 from conans.model.package_ref import PkgReference
-from conans.paths import EXPORT_SOURCES_TGZ_NAME
+from conans.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME
 from conans.util.dates import from_iso8601_to_timestamp
 from conans.util.thread import ExceptionThread
 
@@ -81,8 +81,12 @@ def get_package(self, pref, dest_folder, metadata, only_metadata):
         result = {}
         # Download only known files, but not metadata (except sign)
         if not only_metadata:  # Retrieve package first, then metadata
-            accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt",
-                              "metadata/sign"]
+            accepted_package_files = [PACKAGE_TZSTD_NAME, PACKAGE_TGZ_NAME]
+            accepted_files = ["conaninfo.txt", "conanmanifest.txt", "metadata/sign"]
+            for f in accepted_package_files:
+                if f in server_files:
+                    accepted_files = [f] + accepted_files
+                    break
             files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)]
             # If we didn't indicated reference, server got the latest, use absolute now, it's safer
             urls = {fn: self.router.package_file(pref, fn) for fn in files}

conans/model/conf.py

@@ -42,8 +42,11 @@
     "core.net.http:cacert_path": "Path containing a custom Cacert file",
     "core.net.http:client_cert": "Path or tuple of files containing a client cert (and key)",
     "core.net.http:clean_system_proxy": "If defined, the proxies system env-vars will be discarded",
-    # Gzip compression
+    # Compression for `conan upload`
+    "core.upload:compression_format": "The compression format used when uploading Conan packages. "
+                                      "Possible values: 'zstd', 'gzip' (default=gzip)",
     "core.gzip:compresslevel": "The Gzip compresion level for Conan artifacts (default=9)",
+    "core.zstd:compresslevel": "The zstd compression level for Conan artifacts",
     # Tools
     "tools.android:ndk_path": "Argument for the CMAKE_ANDROID_NDK",
     "tools.android:cmake_legacy_toolchain": "Define to explicitly pass ANDROID_USE_LEGACY_TOOLCHAIN_FILE in CMake toolchain",

conans/model/manifest.py

@@ -1,7 +1,7 @@
 import os
 from collections import defaultdict
 
-from conans.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
+from conans.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME
 from conans.util.dates import timestamp_now, timestamp_to_str
 from conans.util.files import load, md5, md5sum, save, gather_files
 
@@ -91,7 +91,7 @@ def create(cls, folder, exports_sources_folder=None):
         """
         files, _ = gather_files(folder)
         # The folders symlinks are discarded for the manifest
-        for f in (PACKAGE_TGZ_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME):
+        for f in (PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME):
             files.pop(f, None)
 
         file_dict = {}

conans/paths/__init__.py

@@ -58,6 +58,7 @@ def _user_home_from_conanrc_file():
 CONAN_MANIFEST = "conanmanifest.txt"
 CONANINFO = "conaninfo.txt"
 PACKAGE_TGZ_NAME = "conan_package.tgz"
+PACKAGE_TZSTD_NAME = "conan_package.tar.zst"
 EXPORT_TGZ_NAME = "conan_export.tgz"
 EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz"
 DEFAULT_PROFILE_NAME = "default"

conans/requirements.txt

@@ -7,3 +7,4 @@ fasteners>=0.15
 distro>=1.4.0, <=1.8.0; sys_platform == 'linux' or sys_platform == 'linux2'
 Jinja2>=3.0, <4.0.0
 python-dateutil>=2.8.0, <3
+zstandard>=0.20, <= 0.21

conans/test/unittests/client/remote_manager_test.py

@@ -2,14 +2,15 @@
 import unittest
 
 from conans.client.cmd.uploader import compress_files
-from conans.paths import PACKAGE_TGZ_NAME
+from conans.client.remote_manager import uncompress_file
+from conans.paths import PACKAGE_TGZ_NAME, PACKAGE_TZSTD_NAME
 from conans.test.utils.test_files import temp_folder
 from conans.util.files import save
 
 
 class RemoteManagerTest(unittest.TestCase):
 
-    def test_compress_files(self):
+    def test_compress_files_tgz(self):
         folder = temp_folder()
         save(os.path.join(folder, "one_file.txt"), "The contents")
         save(os.path.join(folder, "Two_file.txt"), "Two contents")
@@ -23,3 +24,30 @@ def test_compress_files(self):
         self.assertTrue(os.path.exists(path))
         expected_path = os.path.join(folder, PACKAGE_TGZ_NAME)
         self.assertEqual(path, expected_path)
+
+    def test_compress_and_uncompress_zst_files(self):
+        folder = temp_folder()
+        save(os.path.join(folder, "one_file.txt"), "The contents")
+        save(os.path.join(folder, "Two_file.txt"), "Two contents")
+
+        files = {
+            "one_file.txt": os.path.join(folder, "one_file.txt"),
+            "Two_file.txt": os.path.join(folder, "Two_file.txt"),
+        }
+
+        path = compress_files(files, PACKAGE_TZSTD_NAME, dest_dir=folder, compressformat='zstd')
+        self.assertTrue(os.path.exists(path))
+        expected_path = os.path.join(folder, PACKAGE_TZSTD_NAME)
+        self.assertEqual(path, expected_path)
+
+        extract_dir = os.path.join(folder, 'extracted')
+        uncompress_file(path, extract_dir)
+
+        extract_files = list(sorted(os.listdir(extract_dir)))
+        expected_files = sorted(files.keys())
+        self.assertEqual(extract_files, expected_files)
+
+        for name, path in sorted(files.items()):
+            extract_path = os.path.join(extract_dir, name)
+            with open(path, 'r') as f1, open(extract_path, 'r') as f2:
+                self.assertEqual(f1.read(), f2.read())