diff --git a/.github/workflows/on_push.yml b/.github/workflows/on_push.yml index acad105b..f5cf898b 100644 --- a/.github/workflows/on_push.yml +++ b/.github/workflows/on_push.yml @@ -262,6 +262,48 @@ jobs: COMPOSE_PROJECT_NAME: clickhouse-connect-ci run: docker compose -f docker-compose.yml down --volumes --remove-orphans + no-lz4-test: + runs-on: ubuntu-26.04 + needs: [bare-import-test, clickhouse-versions] + name: Local Tests Py=3.14 without lz4 + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Start ClickHouse (${{ needs.clickhouse-versions.outputs.default }}) in Docker + env: + CLICKHOUSE_CONNECT_TEST_CH_VERSION: ${{ needs.clickhouse-versions.outputs.default }} + COMPOSE_PROJECT_NAME: clickhouse-connect-ci + run: docker compose -f docker-compose.yml up -d --wait + - name: Set up Python 3.14 + uses: actions/setup-python@v7 + with: + python-version: '3.14' + - name: Install pip + run: python -m pip install --upgrade pip + - name: Install Test Dependencies without lz4 + run: | + grep -v '^lz4' tests/test_requirements.txt > /tmp/test_requirements.txt + pip install -r /tmp/test_requirements.txt + python -c "import importlib.util, sys; sys.exit(importlib.util.find_spec('lz4') is not None)" + - name: Build cython extensions + run: python setup.py build_ext --inplace + - name: "Add distribution info" # This lets SQLAlchemy find entry points + run: pip install -e . --no-deps + - name: Add ClickHouse TLS instance to /etc/hosts + run: | + sudo echo "127.0.0.1 server1.clickhouse.test" | sudo tee -a /etc/hosts + - name: Run tests + env: + CLICKHOUSE_CONNECT_TEST_TLS: 1 + CLICKHOUSE_CONNECT_TEST_DOCKER: 'False' + SQLALCHEMY_SILENCE_UBER_WARNING: 1 + run: pytest -n 4 tests + - name: Stop ClickHouse + if: ${{ always() && hashFiles('docker-compose.yml') != '' }} + env: + COMPOSE_PROJECT_NAME: clickhouse-connect-ci + run: docker compose -f docker-compose.yml down --volumes --remove-orphans + sqlalchemy-1x-compat-test: runs-on: ubuntu-26.04 needs: [bare-import-test, clickhouse-versions] @@ -308,7 +350,7 @@ jobs: check-secret: runs-on: ubuntu-26.04 - needs: [bare-import-test, tests, pandas-3x-compat-test, sqlalchemy-1x-compat-test] + needs: [bare-import-test, tests, pandas-3x-compat-test, no-lz4-test, sqlalchemy-1x-compat-test] outputs: has_secrets: ${{ steps.has_secrets.outputs.HAS_SECRETS }} steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index b764df62..cfb05047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ This changelog covers the `clickhouse-connect` driver. The independently version ## UNRELEASED +### Improvements + +- On Python 3.14 and later, `lz4` is no longer installed by default. It publishes no wheels for Python 3.14+ free-threaded builds, CPython 3.15 or musllinux, and zstd is in the standard library from 3.14. Without `lz4`, `compress=True` compresses inserts with zstd. Query responses already use zstd because the server prefers it. Install `clickhouse-connect[lz4]` to use `compress="lz4"` on 3.14 and later. Nothing changes below 3.14. + ### Bug Fixes - With the Python codec and extended pandas dtypes, `query_df` preserves `Nullable(UInt64)` values above `2**53` without rounding, keeps timezone-aware nullable datetime dtypes across all-NULL blocks, and reads valid `Nullable(DateTime64)` values beyond the nanosecond range on pandas 2. diff --git a/clickhouse_connect/driver/__init__.py b/clickhouse_connect/driver/__init__.py index 77e74df3..356b6cf7 100644 --- a/clickhouse_connect/driver/__init__.py +++ b/clickhouse_connect/driver/__init__.py @@ -290,8 +290,9 @@ def create_client( :param kwargs: Recognized keyword arguments (used by the HTTP client), see below - :param compress: Enable compression for ClickHouse HTTP inserts and query results. True will select the preferred - compression method (lz4). A str of 'lz4', 'zstd', 'br', or 'gzip' can be used to use a specific compression type + :param compress: Enable compression for ClickHouse HTTP inserts and query results. True advertises every available + method and compresses inserts with lz4, or zstd when lz4 is not installed. A str of 'lz4', 'zstd', 'br', or 'gzip' + can be used to use a specific compression type :param query_limit: Default LIMIT on returned rows. 0 means no limit :param connect_timeout: Timeout in seconds for establishing a new HTTP connection, excluding pool wait time. :param send_receive_timeout: Read timeout in seconds for http connection @@ -460,8 +461,9 @@ async def create_async_client( connector arguments take precedence over DSN and generic_args values; generic_args take precedence over the DSN. :param kwargs: Recognized keyword arguments (used by the async HTTP client), see below - :param compress: Enable compression for ClickHouse HTTP inserts and query results. True will select the preferred - compression method (lz4). A str of 'lz4', 'zstd', 'br', or 'gzip' can be used to use a specific compression type + :param compress: Enable compression for ClickHouse HTTP inserts and query results. True advertises every available + method and compresses inserts with lz4, or zstd when lz4 is not installed. A str of 'lz4', 'zstd', 'br', or 'gzip' + can be used to use a specific compression type :param query_limit: Default LIMIT on returned rows. 0 means no limit :param connect_timeout: Timeout in seconds for establishing a new HTTP connection, excluding pool wait time. :param send_receive_timeout: Read timeout in seconds for http connection diff --git a/clickhouse_connect/driver/_backend/chdb_backend.py b/clickhouse_connect/driver/_backend/chdb_backend.py index 33bfdb6e..22cf5dd3 100644 --- a/clickhouse_connect/driver/_backend/chdb_backend.py +++ b/clickhouse_connect/driver/_backend/chdb_backend.py @@ -180,9 +180,9 @@ def _write_block_to_file(block: Any, file: Any) -> None: def _decompress(data: bytes, encoding: str) -> bytes: if encoding == "lz4": - import lz4.frame + from clickhouse_connect.driver.compression import _require_lz4 - return lz4.frame.decompress(data) + return _require_lz4().decompress(data) if encoding == "zstd": from clickhouse_connect.driver.compression import _zstd_decompress diff --git a/clickhouse_connect/driver/_backend/httpcommon.py b/clickhouse_connect/driver/_backend/httpcommon.py index 27881d66..0e40e259 100644 --- a/clickhouse_connect/driver/_backend/httpcommon.py +++ b/clickhouse_connect/driver/_backend/httpcommon.py @@ -18,8 +18,6 @@ from importlib.metadata import version as dist_version from typing import TYPE_CHECKING, Any, Protocol -import lz4.frame - if TYPE_CHECKING: from clickhouse_connect.driver.client import Client from clickhouse_connect.driver.external import ExternalData @@ -30,7 +28,7 @@ from clickhouse_connect.driver._backend.models import QueryRuntime from clickhouse_connect.driver.binding import _query_has_trailing_limit_zero, _query_is_read_only, quote_identifier, use_form_encoding from clickhouse_connect.driver.common import ShowClickHouseErrors, coerce_bool, dict_copy -from clickhouse_connect.driver.compression import _zstd_decompress, available_compression +from clickhouse_connect.driver.compression import LZ4_MISSING_MESSAGE, _lz4_decompressor, _zstd_decompress, available_compression from clickhouse_connect.driver.exceptions import ( GENERIC_CLICKHOUSE_ERROR, DatabaseError, @@ -181,6 +179,8 @@ def negotiate_compression(compress: bool | str) -> tuple[str | None, str | None] return ",".join(available_compression), available_compression[0] if compress and compress not in ("False", "false", "0"): if compress not in available_compression: + if compress == "lz4": + raise ProgrammingError(LZ4_MISSING_MESSAGE) raise ProgrammingError(f"Unsupported compression method {compress}") return compress, compress return None, None @@ -192,7 +192,7 @@ def decompress_response(data: bytes, encoding: str | None) -> bytes: return data if encoding == "lz4": - lz4_decom = lz4.frame.LZ4FrameDecompressor() + lz4_decom = _lz4_decompressor() return lz4_decom.decompress(data, len(data)) if encoding == "zstd": return _zstd_decompress(data) diff --git a/clickhouse_connect/driver/compression.py b/clickhouse_connect/driver/compression.py index 51f11c54..7c0019ae 100644 --- a/clickhouse_connect/driver/compression.py +++ b/clickhouse_connect/driver/compression.py @@ -1,8 +1,10 @@ import sys import zlib -import lz4 -import lz4.frame +try: + import lz4.frame as _lz4_frame +except ImportError: + _lz4_frame = None try: if sys.version_info >= (3, 14): @@ -35,6 +37,20 @@ def _require_zstd(): return _zstd +LZ4_MISSING_MESSAGE = "lz4 compression requires the lz4 package. Install it with: pip install 'clickhouse-connect[lz4]'" + + +def _require_lz4(): + if _lz4_frame is None: + raise ImportError(LZ4_MISSING_MESSAGE) + return _lz4_frame + + +def _lz4_decompressor(): + """Returns an LZ4FrameDecompressor for incremental decompression.""" + return _require_lz4().LZ4FrameDecompressor() + + def _zstd_compress(data: bytes) -> bytes: """One-shot compression.""" return _require_zstd().compress(data) @@ -50,7 +66,9 @@ def _zstd_decompressor(): return _require_zstd().ZstdDecompressor() -available_compression = ["lz4"] +available_compression = [] +if _lz4_frame is not None: + available_compression.append("lz4") if _zstd is not None: available_compression.append("zstd") if brotli: @@ -84,7 +102,7 @@ def flush(self): class Lz4Compressor(Compressor, tag="lz4", thread_safe=False): def __init__(self): - self.comp = lz4.frame.LZ4FrameCompressor() + self.comp = _require_lz4().LZ4FrameCompressor() def compress_block(self, block): output = self.comp.begin(len(block)) diff --git a/clickhouse_connect/driver/httputil.py b/clickhouse_connect/driver/httputil.py index 754e2e40..250bbd52 100644 --- a/clickhouse_connect/driver/httputil.py +++ b/clickhouse_connect/driver/httputil.py @@ -12,14 +12,13 @@ from typing import Any import certifi -import lz4.frame import urllib3 from urllib3.exceptions import HTTPError from urllib3.poolmanager import PoolManager, ProxyManager from urllib3.response import HTTPResponse from clickhouse_connect import common -from clickhouse_connect.driver.compression import _zstd_decompress, _zstd_decompressor, _ZstdError +from clickhouse_connect.driver.compression import _lz4_decompressor, _zstd_decompress, _zstd_decompressor, _ZstdError from clickhouse_connect.driver.exceptions import OperationalError, ProgrammingError logger = logging.getLogger(__name__) @@ -170,7 +169,7 @@ def get_response_data(response: HTTPResponse) -> bytes: except _ZstdError: pass if encoding == "lz4": - lz4_decom = lz4.frame.LZ4FrameDecompressor() + lz4_decom = _lz4_decompressor() return lz4_decom.decompress(response.data, len(response.data)) return response.data @@ -254,7 +253,7 @@ def zstd_decompress(c: deque) -> tuple[bytes, int]: decompress = zstd_decompress elif compression == "lz4": - lz4_decom = lz4.frame.LZ4FrameDecompressor() + lz4_decom = _lz4_decompressor() def lz_decompress(c: deque) -> tuple[bytes | None, int]: read_amt = 0 diff --git a/clickhouse_connect/driver/streaming.py b/clickhouse_connect/driver/streaming.py index 763cbeb9..20533116 100644 --- a/clickhouse_connect/driver/streaming.py +++ b/clickhouse_connect/driver/streaming.py @@ -7,11 +7,9 @@ from collections.abc import Callable, Iterable, Iterator from typing import cast -import lz4.frame - from clickhouse_connect.driver.asyncqueue import EOF_SENTINEL, AsyncSyncQueue from clickhouse_connect.driver.asyncqueue import Full as AsyncQueueFull -from clickhouse_connect.driver.compression import _zstd_decompressor, available_compression +from clickhouse_connect.driver.compression import _lz4_decompressor, _zstd_decompressor, available_compression from clickhouse_connect.driver.exceptions import Error, OperationalError from clickhouse_connect.driver.transform import Transform from clickhouse_connect.driver.types import ByteSource, Closable @@ -217,7 +215,7 @@ def _create_decompressor(encoding: str): return _zstd_decompressor() if encoding == "lz4": - return lz4.frame.LZ4FrameDecompressor() + return _lz4_decompressor() raise ValueError(f"Unsupported compression encoding: {encoding}") diff --git a/docs/additional-options.mdx b/docs/additional-options.mdx index a08fa4e7..c8383740 100644 --- a/docs/additional-options.mdx +++ b/docs/additional-options.mdx @@ -51,11 +51,13 @@ ClickHouse Connect supports lz4, zstd, brotli, gzip, and deflate response compre To receive compressed data, the ClickHouse server `enable_http_compression` must be set to 1, or the user must have permission to change the setting on a "per query" basis. -Compression is controlled by the `compress` argument to `get_client` and `get_async_client`. The default, `True`, advertises every available response encoding and compresses Native insert blocks with lz4. Set `compress=False` to disable compression or pass one of `"lz4"`, `"zstd"`, `"br"`, or `"gzip"` to request a specific method. +Compression is controlled by the `compress` argument to `get_client` and `get_async_client`. The default, `True`, advertises every available response encoding and compresses Native insert blocks with lz4, or with zstd when lz4 is not installed. Set `compress=False` to disable compression or pass one of `"lz4"`, `"zstd"`, `"br"`, or `"gzip"` to request a specific method. The raw client methods don't use the client-level `compress` setting. `raw_query` and `raw_stream` return uncompressed data, and `raw_insert` takes its own `compression` argument describing compression already applied to the payload. -lz4 and zstd support are installed with ClickHouse Connect. On Python 3.14 and later, zstd uses the standard library `compression.zstd` module. Python 3.10 through 3.13 use `backports.zstd`. A custom CPython 3.14+ interpreter built without zstd support still imports; zstd is dropped from the available methods and an error is raised only when zstd is explicitly requested. Brotli is optional and must be installed separately before using `compress="br"`. +zstd support is installed with ClickHouse Connect. On Python 3.14 and later, zstd uses the standard library `compression.zstd` module. Python 3.10 through 3.13 use `backports.zstd`. A custom CPython 3.14+ interpreter built without zstd support still imports; zstd is dropped from the available methods and an error is raised only when zstd is explicitly requested. Brotli is optional and must be installed separately before using `compress="br"`. + +lz4 is installed by default on Python 3.10 through 3.13. On Python 3.14 and later it is optional. Install it with `pip install "clickhouse-connect[lz4]"`. Without it, lz4 is dropped from the available methods and `compress="lz4"` raises an error. gzip is generally slower than lz4 or zstd for ClickHouse workloads. diff --git a/docs/index.mdx b/docs/index.mdx index d26be673..781ad152 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -40,7 +40,7 @@ The standard ClickHouse Connect clients use the HTTP interface. This supports HT | aiohttp | 3.9 or later | | Platforms | Linux, macOS, and Windows on the wheel architectures published for each Python version | -The package includes compiled wheels where available and falls back to a pure Python implementation when the Cython extensions cannot be built. PyArrow is supported on Python 3.10 through 3.15. Python 3.14 requires PyArrow 22 or later, and Python 3.15 requires PyArrow 26 or later. `lz4` does not publish CPython 3.15 or free-threaded wheels yet, so installing on Python 3.15 or free-threaded Python builds `lz4` from source and requires a C compiler. +The package includes compiled wheels where available and falls back to a pure Python implementation when the Cython extensions cannot be built. PyArrow is supported on Python 3.10 through 3.15. Python 3.14 requires PyArrow 22 or later, and Python 3.15 requires PyArrow 26 or later. On Python 3.14 and later, `lz4` is an optional extra and the default compression uses the standard library zstd module. `lz4` does not publish CPython 3.15 or Python 3.14+ free-threaded wheels yet, so installing the `lz4` extra on those versions builds it from source and requires a C compiler. ## Installation {#installation} @@ -62,6 +62,7 @@ pip install "clickhouse-connect[sqlalchemy-async]" # Async SQLAlchemy dialect pip install "clickhouse-connect[alembic]" # SQLAlchemy and Alembic pip install "clickhouse-connect[chdb]" # Embedded chDB backend pip install "clickhouse-connect[rust]" # Experimental Rust codec +pip install "clickhouse-connect[lz4]" # lz4 compression on Python 3.14+ pip install "clickhouse-connect[tzdata]" # IANA time zones on minimal systems ``` diff --git a/setup.py b/setup.py index f617c05f..2a241b2c 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,6 @@ def run_setup(): 'tzdata; sys_platform == "win32"', 'backports.zstd>=1.3.0; python_version<"3.14"', 'lz4; python_version<"3.14"', - 'lz4>=4.4.5; python_version>="3.14"', ], extras_require={ "sqlalchemy": ["sqlalchemy>=1.4.40,<3.0"], @@ -107,6 +106,7 @@ def run_setup(): "tzlocal": ["tzlocal>=4.0"], "tzdata": ["tzdata"], "async": ["aiohttp>=3.9.0"], + "lz4": ["lz4>=4.4.5"], "chdb": ["chdb>=4.1.7"], "rust": ["clickhouse-connect-core>=0.2.1,<0.3"], }, diff --git a/tests/integration_tests/test_streaming.py b/tests/integration_tests/test_streaming.py index c4b2adfa..0f4fd720 100644 --- a/tests/integration_tests/test_streaming.py +++ b/tests/integration_tests/test_streaming.py @@ -5,6 +5,7 @@ import pytest from clickhouse_connect import common +from clickhouse_connect.driver.compression import _lz4_frame from clickhouse_connect.driver.exceptions import ProgrammingError, StreamClosedError, StreamFailureError from clickhouse_connect.driver.httputil import ResponseSource from clickhouse_connect.driver.streaming import StreamingResponseSource @@ -66,7 +67,9 @@ def stream(*stream_args, **stream_kwargs): @pytest.mark.parametrize("native_codec", ["python", "rust_strict"]) -@pytest.mark.parametrize("compress", [False, "lz4"]) +@pytest.mark.parametrize( + "compress", [False, pytest.param("lz4", marks=pytest.mark.skipif(_lz4_frame is None, reason="lz4 is not installed"))] +) def test_early_stream_close_reuses_client(client_factory, call, consume_stream, client_mode, response_reads, native_codec, compress): if native_codec == "rust_strict": pytest.importorskip("_ch_core") diff --git a/tests/unit_tests/test_driver/test_compression.py b/tests/unit_tests/test_driver/test_compression.py index b890b6dd..90f5fb3d 100644 --- a/tests/unit_tests/test_driver/test_compression.py +++ b/tests/unit_tests/test_driver/test_compression.py @@ -1,7 +1,13 @@ +import importlib +import sys + import pytest +import clickhouse_connect from clickhouse_connect.driver import compression +from clickhouse_connect.driver._backend import httpcommon from clickhouse_connect.driver.compression import _zstd_compress, _zstd_decompress, _zstd_decompressor +from clickhouse_connect.driver.exceptions import ProgrammingError def test_zstd_round_trip(): @@ -25,3 +31,28 @@ def test_zstd_unavailable(monkeypatch): compression._zstd_decompress(b"data") with pytest.raises(ImportError, match="zstd support is unavailable"): compression._zstd_decompressor() + + +def test_lz4_unavailable(monkeypatch): + monkeypatch.setattr(compression, "_lz4_frame", None) + with pytest.raises(ImportError, match=r"clickhouse-connect\[lz4\]"): + compression._lz4_decompressor() + with pytest.raises(ImportError, match=r"clickhouse-connect\[lz4\]"): + compression.Lz4Compressor() + + +def test_explicit_lz4_without_package(monkeypatch): + monkeypatch.setattr(httpcommon, "available_compression", ["zstd", "gzip", "deflate"]) + with pytest.raises(ProgrammingError, match=r"clickhouse-connect\[lz4\]"): + clickhouse_connect.get_client(host="localhost", compress="lz4") + + +def test_lz4_missing_at_import(monkeypatch): + monkeypatch.setitem(sys.modules, "lz4", None) + monkeypatch.setitem(sys.modules, "lz4.frame", None) + reloaded = importlib.reload(compression) + try: + assert "lz4" not in reloaded.available_compression + finally: + monkeypatch.undo() + importlib.reload(compression) diff --git a/tests/unit_tests/test_driver/test_httputil.py b/tests/unit_tests/test_driver/test_httputil.py index 84fadef2..88d49702 100644 --- a/tests/unit_tests/test_driver/test_httputil.py +++ b/tests/unit_tests/test_driver/test_httputil.py @@ -11,7 +11,7 @@ import pytest from clickhouse_connect.driver import httputil -from clickhouse_connect.driver.compression import _zstd_compress +from clickhouse_connect.driver.compression import _lz4_frame, _zstd_compress from clickhouse_connect.driver.exceptions import OperationalError from clickhouse_connect.driver.httputil import ResponseSource @@ -442,6 +442,7 @@ def empty_stream(chunk_size, decompress): assert len(chunks) == 0 + @pytest.mark.skipif(_lz4_frame is None, reason="lz4 is not installed") def test_network_error_with_compressed_response(self): """Test network error handling with compressed (lz4) response""" mock_response = Mock() diff --git a/tests/unit_tests/test_streaming_source.py b/tests/unit_tests/test_streaming_source.py index 9736e9f7..ae49918f 100644 --- a/tests/unit_tests/test_streaming_source.py +++ b/tests/unit_tests/test_streaming_source.py @@ -11,11 +11,10 @@ from contextlib import suppress from unittest.mock import AsyncMock, Mock -import lz4.frame import pytest from clickhouse_connect.driver._backend.http_async import SessionLease, _one_shot -from clickhouse_connect.driver.compression import _zstd_compress +from clickhouse_connect.driver.compression import _lz4_frame, _zstd_compress from clickhouse_connect.driver.exceptions import NotSupportedError, OperationalError from clickhouse_connect.driver.streaming import ( ReadAheadSource, @@ -172,11 +171,12 @@ def consume(): assert decompressed == original_data +@pytest.mark.skipif(_lz4_frame is None, reason="lz4 is not installed") @pytest.mark.asyncio async def test_streaming_with_lz4_compression(): """Test streaming with lz4 decompression.""" original_data = b"lz4 test data " * 500 - compressed = lz4.frame.compress(original_data) + compressed = _lz4_frame.compress(original_data) chunks = [compressed[i : i + 50] for i in range(0, len(compressed), 50)]