Keep DateTime64 precision in pandas.Timestamp binds and inserts and in Tuple, Nested, Variant and Map binds

CHANGELOG.md

@@ -17,6 +17,10 @@ This changelog covers the `clickhouse-connect` driver. The independently version
 - Tuple serialization now reports non-sequence rows with `DataError` and a row number instead of a raw `TypeError` or `AttributeError`, including `None` rows in dictionary input for a non-nullable named tuple.
 - `Decimal256(S)` now resolves to the supported precision of 76 instead of 79. Explicit Decimal256 type names now produce valid Native insert headers, and SQLAlchemy Decimal256 types report the correct precision.
 - Sync and async clients no longer automatically use form encoding when the final SQL exceeds 128 KiB, avoiding `HTML Form Exception: Field value too long` for large queries with large bound parameters. Explicit form encoding and queries with external data retain their existing limits. Closes [#1105](https://github.com/ClickHouse/clickhouse-connect/issues/1105).
+- `pandas.Timestamp` values keep their nanoseconds when bound to a `DateTime64(9)` query parameter, including typed SQLAlchemy bind parameters and values wrapped in `DT64Param`, and when inserted as rows into `DateTime64(7)` through `DateTime64(9)` columns with the Python codec. Previously both paths truncated them to microseconds, so `2024-01-02 03:04:05.123456789` arrived as `2024-01-02 03:04:05.123456000`. Python codec row inserts of a `pandas.Timestamp` whose sub-microsecond part rounds up, such as `03:04:05.999999999`, also no longer store a value one second late in any `DateTime64` column. Related to [#429](https://github.com/ClickHouse/clickhouse-connect/issues/429).
+- Server-side query parameters typed as a named `Tuple`, `Nested`, or `Map` now keep the sub-second precision of `datetime` values bound to their `DateTime64` elements, and each element uses its declared timezone. `Map` parameters require `dict_parameter_format="map"`. A `Variant` keeps precision and timezone only when its sole datetime alternative is a `DateTime64`. Enum labels and field names containing `datetime` don't count as datetime types. Variants with several datetime alternatives are unchanged.
+
+
 - Enum names that contain a backslash or a control character such as a tab now parse correctly from server type names and are escaped correctly in Native insert headers. Previously such names read back escaped, for example `c\\d` instead of `c\d`. Inserting them stored `0`, which became the wrong label when `0` was a declared member and an unreadable value otherwise. See [#1117](https://github.com/ClickHouse/clickhouse-connect/pull/1117).
 - The asynchronous client now raises `OperationalError` when the connection drops while `raw_query`, `command`, or a `LIMIT 0` metadata query reads the response body, matching the synchronous client. Previously it leaked `aiohttp.ClientPayloadError`.
 - When the connection drops partway through a streamed query result, both sync and async clients now raise `StreamFailureError: Stream failed during read (connection closed by server)` and preserve the transport error as the cause. Previously, with `show_clickhouse_errors=True`, they could report buffered result bytes as `unrecognized data found in stream: <hex>` and suppress the cause. Recognizable ClickHouse errors written into the response before the drop are still reported.

clickhouse_connect/cc_sqlalchemy/datatypes/base.py

@@ -21,14 +21,15 @@
 class _DateTime64QueryParam(DT64Param):
     def format(self, tz: tzinfo | None, top_level: bool) -> str:
         # Preserve SQL-text datetime timezone rules, including legacy naive UTC values.
+        if self.tz is not None:
+            tz = self.tz
         value = self.value
         if _legacy_naive_datetime_binding():
             if value.tzinfo is not None or not tzutil.is_utc_timezone(tz):
                 value = value.astimezone(tz)
         elif _datetime_is_aware(value):
             value = value.astimezone(tz)
-        rendered = value.strftime("%Y-%m-%d %H:%M:%S.%f")
-        return rendered if top_level else f"'{rendered}'"
+        return self._render(value, top_level)
 
 
 def _non_temporal_query_shape(ch_type: ClickHouseType) -> _DateTime64QuerySignature:

clickhouse_connect/datatypes/temporal.py

@@ -335,6 +335,12 @@ def _read_binary_naive(self, column: Sequence):
 
     def _datetime64_ticks(self, value: datetime, active_tz: tzinfo | None = None) -> int:
         timestamp = _localized_timestamp(value, active_tz) if active_tz is not None else value.timestamp()
+        if type(value) is not datetime:
+            # pandas.Timestamp carries nanoseconds, and its timestamp() rounds to microseconds,
+            # which can carry into the next second
+            nanos = value.microsecond * 1000 + getattr(value, "nanosecond", 0)
+            seconds = round(timestamp - nanos / 1000000000)
+            return ((seconds * 1000000000 + nanos) * self.prec) // 1000000000
         seconds = floor(timestamp)
         return ((seconds * 1000000 + value.microsecond) * self.prec) // 1000000
 

clickhouse_connect/driver/binding.py

@@ -1,18 +1,20 @@
+import copy
 import ipaddress
 import re
 import uuid
 import zoneinfo
 from collections.abc import Collection, Sequence
 from datetime import date, datetime, time, timedelta, timezone, tzinfo
 from enum import Enum
+from functools import lru_cache
 from typing import Any
 from urllib.parse import quote, urlencode
 
 from clickhouse_connect import common
 from clickhouse_connect.driver import tzutil
 from clickhouse_connect.driver.common import dict_copy
 from clickhouse_connect.driver.exceptions import ProgrammingError
-from clickhouse_connect.driver.parser import parse_callable
+from clickhouse_connect.driver.parser import parse_callable, parse_columns
 from clickhouse_connect.json_impl import any_to_json
 
 BS = "\\"
@@ -684,20 +686,30 @@ def _legacy_naive_datetime_binding() -> bool:
 
 
 class DT64Param:
-    def __init__(self, value: datetime):
+    tz: tzinfo | None = None
+
+    def __init__(self, value: datetime, tz: tzinfo | None = None):
         self.value = value
+        self.tz = tz
 
     def format(self, tz: tzinfo | None, top_level: bool) -> str:
+        if self.tz is not None:
+            tz = self.tz
         value = self.value
         if _legacy_naive_datetime_binding():
             if tz:
                 value = value.astimezone(tz)
         elif tz is not None and _datetime_is_aware(value):
             value = value.astimezone(tz)
-        s = value.strftime("%Y-%m-%d %H:%M:%S.%f")
-        if top_level:
-            return s
-        return f"'{s}'"
+        return self._render(value, top_level)
+
+    @staticmethod
+    def _render(value: datetime, top_level: bool) -> str:
+        rendered = value.strftime("%Y-%m-%d %H:%M:%S.%f")
+        nanosecond = getattr(value, "nanosecond", 0)
+        if nanosecond:
+            rendered += f"{nanosecond:03d}"
+        return rendered if top_level else f"'{rendered}'"
 
 
 def quote_identifier(identifier: str) -> str:
@@ -750,6 +762,8 @@ def finalize_query(query: str, parameters: Sequence | dict[str, Any] | None, ser
     return query % tuple(format_query_value(v, server_tz) for v in parameters)
 
 
+# Cached because containers unwrap the same element type for every value
+@lru_cache(maxsize=256)
 def _unwrap_outer(type_str: str) -> tuple[str, tuple]:
     """Strip LowCardinality/Nullable wrappers and return (base_name, args)"""
     base = type_str.strip()
@@ -761,6 +775,27 @@ def _unwrap_outer(type_str: str) -> tuple[str, tuple]:
     return base_name, values
 
 
+@lru_cache(maxsize=256)
+def _container_element_types(type_str: str) -> tuple[str, ...]:
+    """Parse Tuple or Nested element types."""
+    _, values = _unwrap_outer(type_str)
+    return parse_columns(f"({', '.join(str(t).strip() for t in values)})")[1]
+
+
+@lru_cache(maxsize=256)
+def _mentions_datetime(type_str: str) -> bool:
+    """Find datetime types without counting field names or enum labels."""
+    base_name, values = _unwrap_outer(type_str)
+    base_name = base_name.lower()
+    if base_name in ("datetime", "datetime32", "datetime64"):
+        return True
+    if base_name in ("tuple", "nested"):
+        return any(_mentions_datetime(t) for t in _container_element_types(type_str))
+    if base_name in ("array", "map", "variant", "nullable", "lowcardinality"):
+        return any(_mentions_datetime(str(t)) for t in values)
+    return False
+
+
 def _extract_tz_from_type(type_str: str) -> tzinfo | None:
     """Resolve the timezone named in a ClickHouse type hint."""
     try:
@@ -786,20 +821,46 @@ def _extract_tz_from_type(type_str: str) -> tzinfo | None:
         return None
 
 
+@lru_cache(maxsize=256)
+def _datetime64_type_tz(type_str: str) -> tzinfo | None:
+    """Cached _extract_tz_from_type, since containers resolve the same element type for every value."""
+    return _extract_tz_from_type(type_str)
+
+
 def _promote_datetime64(type_str: str, value):
     """Wrap values bound to a DateTime64 hint in DT64Param to preserve precision."""
     if value is None or "datetime64" not in type_str.lower():
         return value
     try:
         base_name, values = _unwrap_outer(type_str)
         base_name = base_name.lower()
+        if base_name == "variant" and isinstance(value, (datetime, DT64Param)):
+            datetime_alts = [str(t) for t in values if _mentions_datetime(str(t))]
+            if len(datetime_alts) == 1 and _unwrap_outer(datetime_alts[0])[0].lower() == "datetime64":
+                return _promote_datetime64(datetime_alts[0], value)
         if base_name == "datetime64":
-            return DT64Param(value) if isinstance(value, datetime) else value
+            tz = _datetime64_type_tz(type_str)
+            if isinstance(value, datetime):
+                return DT64Param(value, tz)
+            if isinstance(value, DT64Param) and value.tz is None and tz is not None:
+                promoted = copy.copy(value)
+                promoted.tz = tz
+                return promoted
+            return value
         if base_name == "array" and values and isinstance(value, (list, tuple)):
             inner = str(values[0])
             return type(value)(_promote_datetime64(inner, x) for x in value)
-        if base_name == "tuple" and isinstance(value, tuple) and len(values) == len(value):
-            return tuple(_promote_datetime64(str(t), x) for t, x in zip(values, value))
+        if base_name == "tuple" and isinstance(value, tuple):
+            element_types = _container_element_types(type_str)
+            if len(element_types) == len(value):
+                return tuple(_promote_datetime64(t, x) for t, x in zip(element_types, value))
+        if base_name == "nested" and isinstance(value, (list, tuple)):
+            element_types = _container_element_types(type_str)
+            element_type = f"Tuple({', '.join(element_types)})"
+            return type(value)(_promote_datetime64(element_type, x) for x in value)
+        if base_name == "map" and isinstance(value, dict) and len(values) == 2 and common.get_setting("dict_parameter_format") == "map":
+            key_type, value_type = (str(t) for t in values)
+            return {_promote_datetime64(key_type, k): _promote_datetime64(value_type, v) for k, v in value.items()}
         return value
     except Exception:
         return value

docs/driver-api.mdx

@@ -401,7 +401,7 @@ For temporary compatibility with older host-local conversion, set `common.set_se
 
 Native inserts into `Date` and `Date32` columns use a Python `datetime` value's own calendar date without timezone conversion. When the same calendar date is needed for both an insert and a query parameter, pass `value.date()` explicitly. See [Date and Date32 values](/integrations/language-clients/python/advanced-inserting#date-and-date32-values).
 
-For a server-side `{value:DateTime64(precision)}` placeholder, the declared type preserves sub-second precision automatically, including inside `Array` and `Tuple` hints.
+For a server-side `{value:DateTime64(precision)}` placeholder, the declared type preserves sub-second precision automatically, including inside `Array`, named or unnamed `Tuple`, `Nested`, and `Map` hints. `Map` parameters require `dict_parameter_format="map"`. A `Variant` keeps `DateTime64` precision and timezone only when its sole datetime alternative is a `DateTime64`. Variants with several datetime alternatives behave as before.
 
 Client-side `%s` binding has no declared type. Wrap a `datetime` in `DT64Param` when it must render with sub-second precision:
 

docs/sqlalchemy.mdx

@@ -229,6 +229,8 @@ When SQLAlchemy inlines a bound value through `literal_binds` or `literal_execut
 
 Python `datetime` values with a ClickHouse `DateTime64` SQLAlchemy type retain their microseconds in client-side parameters and inline literals, including nullable values and values nested in arrays and tuples. ClickHouse applies the declared precision. Python `datetime` provides up to six fractional digits. Plain `DateTime` values retain whole-second formatting. For a `text()` statement, supply the type explicitly with `bindparam("ts", type_=DateTime64(6))` to preserve fractional seconds.
 
+A `pandas.Timestamp` value bound with `DateTime64(9)` retains its nanoseconds in parameters and inline literals.
+
 SQLAlchemy column types must match the server schema. Declaring `DateTime64` over a server `DateTime` column renders fractional seconds and can raise conversion errors on insert and in `IN` comparisons.
 
 On SQLAlchemy 2.x, inline literals of generic `sqlalchemy.ARRAY` types containing ClickHouse `Tuple` items need `dimensions=1`, or the appropriate higher dimension count for nested arrays, so SQLAlchemy treats each tuple as one item. SQLAlchemy 1.4 does not support inline literals for generic `ARRAY` types.

tests/integration_tests/test_pandas.py

@@ -663,6 +663,29 @@ def test_pandas_time_resolution(test_config: TestConfig, param_client: Client, c
         assert _dtype_unit(df_res["t"].dtype) == "s"
 
 
+PD_TIMESTAMP_NS = "2024-01-02 03:04:05.123456789"
+
+
+def test_pandas_timestamp_datetime64_bind_nanoseconds(param_client: Client, call):
+    ts = pd.Timestamp(PD_TIMESTAMP_NS)
+    result = call(param_client.query, "SELECT toString({d:DateTime64(9)})", parameters={"d": ts})
+    assert result.result_rows[0][0] == PD_TIMESTAMP_NS
+
+
+def test_pandas_timestamp_datetime64_insert_nanoseconds(param_client: Client, call, table_context: Callable, test_config):
+    if test_config.native_codec != "python":
+        pytest.skip("Nanosecond Timestamp row inserts require the Python codec")
+    # The last two values round up to the next second in pandas.Timestamp.timestamp()
+    values = [PD_TIMESTAMP_NS, "2024-01-02 03:04:05.999999999", "1960-01-02 03:04:05.999999999"]
+    columns = ["key UInt8", "v DateTime64(9)", "v6 DateTime64(6)", "vn Nullable(DateTime64(9, 'UTC'))"]
+    with table_context("test_pd_timestamp_ns", columns):
+        data = [[key, pd.Timestamp(value), pd.Timestamp(value), pd.Timestamp(value, tz="UTC")] for key, value in enumerate(values)]
+        call(param_client.insert, "test_pd_timestamp_ns", data)
+        query = "SELECT toString(v), toString(v6), toString(vn) FROM test_pd_timestamp_ns ORDER BY key"
+        result = call(param_client.query, query)
+        assert result.result_rows == [(value, value[:-3], value) for value in values]
+
+
 def test_pandas_map_values(param_client: Client, call, test_config):
     """Map columns in query_df match the decoded keys and values across blocks, including NULL values."""
     if test_config.native_codec != "python":

tests/integration_tests/test_params.py

@@ -1,7 +1,7 @@
 import os
 import time
 from collections.abc import Callable
-from datetime import date, datetime
+from datetime import date, datetime, timezone
 
 import pytest
 
@@ -122,6 +122,61 @@ def test_datetime_64_params(param_client: Client, call):
     assert result[0] == dt_values
 
 
+@pytest.mark.parametrize(
+    "type_hint, value, expected",
+    [
+        ("Tuple(a DateTime64(6), b Int32)", (datetime(2024, 1, 2, 3, 4, 5, 123456), 1), "('2024-01-02 03:04:05.123456',1)"),
+        (
+            "Tuple(a DateTime64(6, 'Asia/Tokyo'))",
+            (datetime(2024, 1, 2, 3, 4, 5, 123456, tzinfo=timezone.utc),),
+            "('2024-01-02 12:04:05.123456')",
+        ),
+        (
+            "Tuple(a DateTime64(6, 'Asia/Tokyo'))",
+            (DT64Param(datetime(2024, 1, 2, 3, 4, 5, 123456, tzinfo=timezone.utc)),),
+            "('2024-01-02 12:04:05.123456')",
+        ),
+        (
+            "Tuple(a DateTime64(6, 'Asia/Tokyo'), b DateTime64(6, 'UTC'))",
+            (datetime(2024, 1, 2, 3, 4, 5, 123456, tzinfo=timezone.utc),) * 2,
+            "('2024-01-02 12:04:05.123456','2024-01-02 03:04:05.123456')",
+        ),
+        ("Nested(a DateTime64(6))", [(datetime(2024, 1, 2, 3, 4, 5, 123456),)], "[('2024-01-02 03:04:05.123456')]"),
+        ("Variant(DateTime64(6), String)", datetime(2024, 1, 2, 3, 4, 5, 123456), "2024-01-02 03:04:05.123456"),
+    ],
+)
+def test_datetime64_params_in_containers(param_client: Client, call, type_hint, value, expected):
+    result = call(param_client.query, f"SELECT toString({{p:{type_hint}}})", parameters={"p": value})
+    assert result.first_row[0] == expected
+
+
+@pytest.mark.parametrize("reverse", [False, True])
+@pytest.mark.parametrize("other_type", ["String", "Enum8('datetime' = 1)", "Tuple(last_datetime String)"])
+def test_datetime64_variant_single_alternative(param_client: Client, call, reverse, other_type):
+    alternatives = ["DateTime64(6, 'Asia/Tokyo')", other_type]
+    if reverse:
+        alternatives.reverse()
+    type_hint = f"Variant({', '.join(alternatives)})"
+    value = datetime(2024, 1, 2, 3, 4, 5, 123456, tzinfo=timezone.utc)
+    query = (
+        "SELECT variantType(v), toUnixTimestamp64Micro(variantElement(v, 'DateTime64(6, \\'Asia/Tokyo\\')')) "
+        f"FROM (SELECT {{p:{type_hint}}} AS v)"
+    )
+    result = call(param_client.query, query, parameters={"p": value})
+    assert result.first_row == ("DateTime64(6, 'Asia/Tokyo')", 1704164645123456)
+
+
+def test_datetime64_params_in_map(param_client: Client, call):
+    original = common.get_setting("dict_parameter_format")
+    common.set_setting("dict_parameter_format", "map")
+    try:
+        value = {"k": datetime(2024, 1, 2, 3, 4, 5, 123456)}
+        result = call(param_client.query, "SELECT toString({p:Map(String, DateTime64(6))})", parameters={"p": value})
+    finally:
+        common.set_setting("dict_parameter_format", original)
+    assert result.first_row[0] == "{'k':'2024-01-02 03:04:05.123456'}"
+
+
 def test_pyformat_64_datetime_params(param_client: Client, call):
     parameters = {f"stamp_{index}": datetime(2026, 1, 1, 12, 0, index % 60, 250306) for index in range(1, 65)}
     query = "SELECT " + ", ".join(f"%({name})s" for name in parameters)

tests/integration_tests/test_sqlalchemy/test_datetime64.py

@@ -101,6 +101,18 @@ def test_datetime64_typed_shapes(datetime_engine, type_, value, expected):
         assert conn.exec_driver_sql(compiled).scalar_one() == expected
 
 
+@pytest.mark.parametrize("literal_execute", [False, True])
+def test_pandas_timestamp_datetime64_nanoseconds(datetime_engine, literal_execute):
+    pd = pytest.importorskip("pandas")
+    value = pd.Timestamp("2024-01-02 03:04:05.123456789", tz="UTC")
+    stamp = sa.cast(sa.bindparam("stamp", value, DateTime64(9, "UTC"), literal_execute=literal_execute), DateTime64(9, "UTC"))
+    statement = sa.select(sa.func.toString(stamp))
+    with datetime_engine.connect() as conn:
+        assert conn.execute(statement).scalar_one() == "2024-01-02 03:04:05.123456789"
+        compiled = str(statement.compile(datetime_engine, compile_kwargs={"literal_binds": True}))
+        assert conn.exec_driver_sql(compiled).scalar_one() == "2024-01-02 03:04:05.123456789"
+
+
 def test_datetime64_expanding_and_literal_execute(datetime_engine):
     value = sa.cast(sa.literal(STAMP, DateTime64(6, "UTC")), DateTime64(6, "UTC"))
     with datetime_engine.connect() as conn: