diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.py index 50074ae2a73..de2ba96b540 100644 --- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py +++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py @@ -380,9 +380,9 @@ def _download_as(self, args: DownloadAsArgs) -> DownloadAsResponse: json=JsonOptions( ensure_ascii=self._download_json_ensure_ascii ), + locale=args.locale, ), filename=bound_filename, - locale=args.locale, ) except InvalidExportLocaleError as error: return DownloadAsResponse(error=str(error)) diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.py index ea559c1ffd6..28b80e85d82 100644 --- a/marimo/_plugins/ui/_impl/table.py +++ b/marimo/_plugins/ui/_impl/table.py @@ -59,6 +59,7 @@ ) from marimo._plugins.ui._impl.tables.utils import get_table_manager from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, ListOrTuple, TableData, download_as, @@ -1105,7 +1106,7 @@ def _download_as(self, args: DownloadAsArgs) -> DownloadAsResponse: args.format, drop_marimo_index=True, filename=bound_filename, - locale=args.locale, + options=DownloadOptions(locale=args.locale), ) except InvalidExportLocaleError as error: return DownloadAsResponse(error=str(error)) diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py index 53717de39cd..4318ef865c2 100644 --- a/marimo/_plugins/ui/_impl/tables/pandas_table.py +++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py @@ -4,6 +4,7 @@ import functools import io import json +import math from functools import cached_property from typing import TYPE_CHECKING, Any @@ -30,8 +31,11 @@ TableManager, TableManagerFactory, ) -from marimo._utils.delimited import DelimitedDialect -from marimo._utils.narwhals_utils import dataframe_to_csv +from marimo._utils.delimited import ( + DelimitedDialect, + format_delimited_number, + is_delimited_number, +) if TYPE_CHECKING: import pandas as pd @@ -204,6 +208,55 @@ def package_name() -> str: def create() -> type[TableManager[Any]]: import pandas as pd + def prepare_delimited_data( + data: pd.DataFrame, decimal_separator: str + ) -> pd.DataFrame: + """Prepare values pandas does not localize itself.""" + + def localize_number( + value: object, *, stringify_unchanged: bool + ) -> object: + if not is_delimited_number(value): + return value + formatted = format_delimited_number(value, decimal_separator) + if ( + not stringify_unchanged + and formatted == str(value) + and not ( + isinstance(value, float) and not math.isfinite(value) + ) + ): + return value + return formatted + + localized_data: pd.DataFrame | None = None + for position, dtype in enumerate(data.dtypes): + column = data.iloc[:, position] + force_numeric_strings = bool( + pd.api.types.is_float_dtype(dtype) + and not pd.api.types.is_extension_array_dtype(dtype) + and column.isna().any() + ) + if not force_numeric_strings and ( + decimal_separator == "." + or not pd.api.types.is_object_dtype(dtype) + ): + continue + + localized_column = column.map( + functools.partial( + localize_number, + stringify_unchanged=force_numeric_strings, + ) + ) + if localized_column.equals(column): + continue + if localized_data is None: + localized_data = data.copy() + localized_data.isetitem(position, localized_column.array) + + return localized_data if localized_data is not None else data + class PandasTableManager(NarwhalsTableManager[pd.DataFrame, Any]): type = "pandas" @@ -273,17 +326,48 @@ def to_csv_str( sep=resolved_separator, ) - # Include a non-trivial pandas index, then reuse the Narwhals - # writer so decimal formatting lives in one place. def to_delimited_str( self, dialect: DelimitedDialect, format_mapping: FormatMapping | None = None, ) -> str: manager = self.apply_formatting(format_mapping) - if len(self.get_row_headers()) > 0: - manager = manager.with_index_as_columns() - return dataframe_to_csv(manager.as_frame(), dialect=dialect) + # pandas applies `decimal` only to numeric dtypes. Prepare + # Decimal and mixed object columns before using its native + # writer for the full frame. + data = prepare_delimited_data( + manager._original_data, dialect.decimal_separator + ) + include_index = len(self.get_row_headers()) > 0 + if include_index: + index_frame = data.index.to_frame(index=False) + localized_index = prepare_delimited_data( + index_frame, dialect.decimal_separator + ) + if localized_index is not index_frame: + if data is manager._original_data: + data = data.copy() + if isinstance(data.index, pd.MultiIndex): + data.index = pd.MultiIndex.from_frame( + localized_index, names=data.index.names + ) + else: + data.index = pd.Index( + localized_index.iloc[:, 0].array, + name=data.index.name, + ) + + # Default exports historically use pandas' platform newline. + is_default_dialect = ( + dialect.field_separator == "," + and dialect.decimal_separator == "." + ) + return data.to_csv( + index=include_index, + sep=dialect.field_separator, + decimal=dialect.decimal_separator, + lineterminator=None if is_default_dialect else "\n", + ) def to_json_str( self, diff --git a/marimo/_plugins/ui/_impl/tables/polars_table.py b/marimo/_plugins/ui/_impl/tables/polars_table.py index d34c5e8aedf..ae3e2bbc463 100644 --- a/marimo/_plugins/ui/_impl/tables/polars_table.py +++ b/marimo/_plugins/ui/_impl/tables/polars_table.py @@ -2,6 +2,7 @@ from __future__ import annotations import functools +import inspect import io import json from functools import cached_property @@ -30,6 +31,14 @@ LOGGER = _loggers.marimo_logger() +def _supports_decimal_comma() -> bool: + import polars as pl + + return ( + "decimal_comma" in inspect.signature(pl.DataFrame.write_csv).parameters + ) + + class PolarsTableManagerFactory(TableManagerFactory): @staticmethod def package_name() -> str: @@ -40,6 +49,8 @@ def package_name() -> str: def create() -> type[TableManager[Any]]: import polars as pl + supports_decimal_comma = _supports_decimal_comma() + def serialize_sequence_column( column: pl.Series, dtype: pl.List | pl.Array ) -> pl.Series: @@ -127,6 +138,25 @@ def to_delimited_str( result = self._cast_object_to_string(result, column) elif isinstance(dtype, pl.Duration): result = self._convert_time_to_string(result, column) + + # Native Polars uses different temporal and binary text + # representations, and writes NaN as an empty field. + requires_compatibility_writer = any( + column.dtype.is_temporal() + or column.dtype == pl.Binary + or (column.dtype.is_float() and column.is_nan().any()) + for column in result.get_columns() + ) + if requires_compatibility_writer: + return dataframe_to_csv(result, dialect=dialect) + + if dialect.decimal_separator == ".": + return result.write_csv(separator=dialect.field_separator) + if dialect.decimal_separator == "," and supports_decimal_comma: + return result.write_csv( + separator=dialect.field_separator, + decimal_comma=True, + ) return dataframe_to_csv(result, dialect=dialect) def to_json_str( diff --git a/marimo/_plugins/ui/_impl/utils/dataframe.py b/marimo/_plugins/ui/_impl/utils/dataframe.py index ccab33b95a1..7bf1ebc9f41 100644 --- a/marimo/_plugins/ui/_impl/utils/dataframe.py +++ b/marimo/_plugins/ui/_impl/utils/dataframe.py @@ -1,7 +1,7 @@ # Copyright 2026 Marimo. All rights reserved. from __future__ import annotations -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union from narwhals.typing import IntoDataFrame, IntoLazyFrame @@ -177,7 +177,6 @@ def download_as( drop_marimo_index: bool = False, options: DownloadOptions | None = None, filename: str | None = None, - locale: ResolvedExportLocale | None = None, ) -> tuple[str, str]: """Download the table data in the specified format. @@ -192,9 +191,6 @@ def download_as( Defaults to each format's defaults. filename (str | None, optional): The filename to use for the downloaded file. Defaults to None, which uses a random filename. - locale (ResolvedExportLocale | None, optional): Browser-resolved - locale for CSV and TSV numeric decimals. Ignored for JSON and - Parquet. Defaults to None, which keeps locale-neutral output. Returns: tuple: (url, user-facing filename with extension) for the downloaded file. @@ -203,8 +199,6 @@ def download_as( ValueError: If unrecognized format. """ options = options or DownloadOptions() - if locale is not None: - options = replace(options, locale=locale) if drop_marimo_index: # Remove the selection column if exists manager = manager.drop_columns([INDEX_COLUMN_NAME]) diff --git a/marimo/_utils/narwhals_utils.py b/marimo/_utils/narwhals_utils.py index 514fd0f402d..8d58cb84bfc 100644 --- a/marimo/_utils/narwhals_utils.py +++ b/marimo/_utils/narwhals_utils.py @@ -97,7 +97,6 @@ def assert_can_narwhalify(obj: Any) -> TypeGuard[IntoFrame]: def dataframe_to_csv( df: IntoFrame, - separator: str | None = None, dialect: DelimitedDialect | None = None, ) -> str: """ @@ -108,31 +107,28 @@ def dataframe_to_csv( assert_can_narwhalify(df) df = nw.from_native(df, pass_through=False) df = upgrade_narwhals_df(df) - field_separator = ( - dialect.field_separator if dialect is not None else separator or "," - ) - decimal_separator = ( - dialect.decimal_separator if dialect is not None else "." - ) + dialect = dialect or DelimitedDialect(",", ".") frame = df.collect() if is_narwhals_lazyframe(df) else df - if field_separator == "," and decimal_separator == ".": + if dialect.field_separator == "," and dialect.decimal_separator == ".": return frame.write_csv() # Narwhals inputs can map to different backends, and # write_csv(separator=...) is not consistently reliable across them. # For non-default dialects, use Python's csv writer for stable behavior. buffer = io.StringIO() - writer = csv.writer(buffer, delimiter=field_separator, lineterminator="\n") + writer = csv.writer( + buffer, delimiter=dialect.field_separator, lineterminator="\n" + ) writer.writerow(frame.columns) rows = frame.iter_rows() - if decimal_separator == ".": + if dialect.decimal_separator == ".": writer.writerows(rows) return buffer.getvalue() writer.writerows( tuple( - format_delimited_number(value, decimal_separator) + format_delimited_number(value, dialect.decimal_separator) if is_delimited_number(value) else value for value in row diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py index 32c6d2676e6..b7d632515e1 100644 --- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py +++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py @@ -262,6 +262,42 @@ def test_to_delimited_str_pt_br(self) -> None: == 'integer;fraction;text;null\n1234;1234,567890123456;"value.1,2;3";\n' ) + def test_to_delimited_str_uses_native_writer(self) -> None: + df = pd.DataFrame({"value": [1.5]}) + manager = PandasTableManagerFactory.create()(df) + + with patch.object(df, "to_csv", wraps=df.to_csv) as writer: + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "value\n1,5\n" + ) + + writer.assert_called_once_with( + index=False, + sep=";", + decimal=",", + lineterminator="\n", + ) + + def test_to_delimited_str_uses_platform_newline_for_default(self) -> None: + df = pd.DataFrame({"value": [1.5]}) + manager = PandasTableManagerFactory.create()(df) + + with ( + patch("os.linesep", "\r\n"), + patch.object(df, "to_csv", wraps=df.to_csv) as writer, + ): + assert ( + manager.to_delimited_str(DelimitedDialect(",", ".")) + == "value\r\n1.5\r\n" + ) + + writer.assert_called_once_with( + index=False, + sep=",", + decimal=".", + lineterminator=None, + ) + def test_to_delimited_str_decimal_and_exponent(self) -> None: manager = PandasTableManagerFactory.create()( pd.DataFrame( @@ -274,6 +310,22 @@ def test_to_delimited_str_decimal_and_exponent(self) -> None: result = manager.to_delimited_str(DelimitedDialect(";", ",")) assert result == "decimal;exponent\n1234,567890123456;1,23e-10\n" + def test_to_delimited_str_localizes_mixed_object_numbers(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame( + { + "mixed": pd.Series( + [1.25, decimal.Decimal("2.5")], dtype=object + ), + "text": ["1.25", "unchanged"], + } + ) + ) + + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + + assert result == "mixed;text\n1,25;1.25\n2,5;unchanged\n" + def test_to_delimited_str_non_finite(self) -> None: manager = PandasTableManagerFactory.create()( pd.DataFrame( @@ -283,6 +335,15 @@ def test_to_delimited_str_non_finite(self) -> None: result = manager.to_delimited_str(DelimitedDialect(";", ",")) assert result == "value\nnan\ninf\n-inf\n" + def test_to_delimited_str_localizes_float_column_with_nan(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame({"value": [1.5, float("nan")]}) + ) + + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + + assert result == "value\n1,5\nnan\n" + def test_to_delimited_str_unicode_decimal(self) -> None: manager = PandasTableManagerFactory.create()( pd.DataFrame({"value": [12.5]}) @@ -302,6 +363,38 @@ def test_to_csv_str_preserves_unnamed_index_header(self) -> None: assert manager.to_csv_str() == df.to_csv(index=True) + def test_to_delimited_str_preserves_unnamed_index_header(self) -> None: + df = pd.DataFrame({"value": [1.5]}, index=[10]) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + ";value\n10;1,5\n" + ) + + def test_to_delimited_str_localizes_decimal_index(self) -> None: + df = pd.DataFrame( + {"value": [decimal.Decimal("2.5")]}, + index=[decimal.Decimal("1.5")], + ) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + ";value\n1,5;2,5\n" + ) + + def test_to_delimited_str_preserves_unnamed_multi_index_headers( + self, + ) -> None: + df = pd.DataFrame( + {"value": [1.5]}, + index=pd.MultiIndex.from_tuples([("a", 1)]), + ) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + ";;value\na;1;1,5\n" + ) + def test_to_csv_datetime(self) -> None: D = pd.to_datetime("2024-12-17", errors="coerce") diff --git a/tests/_plugins/ui/_impl/tables/test_polars_table.py b/tests/_plugins/ui/_impl/tables/test_polars_table.py index d176dea5c24..af413b89e5b 100644 --- a/tests/_plugins/ui/_impl/tables/test_polars_table.py +++ b/tests/_plugins/ui/_impl/tables/test_polars_table.py @@ -8,6 +8,7 @@ from enum import Enum from math import isnan from typing import Any +from unittest import mock import narwhals.stable.v2 as nw import pytest @@ -20,6 +21,7 @@ from marimo._plugins.ui._impl.tables.format import FormatMapping from marimo._plugins.ui._impl.tables.polars_table import ( PolarsTableManagerFactory, + _supports_decimal_comma, ) from marimo._plugins.ui._impl.tables.table_manager import TableManager from marimo._utils.platform import is_windows @@ -190,6 +192,65 @@ def test_to_delimited_str_pt_br(self) -> None: == 'integer;fraction;text;null\n1234;1234,567890123456;"value.1,2;3";\n' ) + def test_to_delimited_str_uses_native_decimal_comma_writer(self) -> None: + import polars as pl + + if not _supports_decimal_comma(): + pytest.skip("Polars does not support native decimal commas") + + manager = self.factory.create()(pl.DataFrame({"value": [1.5]})) + with mock.patch( + "marimo._plugins.ui._impl.tables.polars_table.dataframe_to_csv" + ) as fallback: + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "value\n1,5\n" + ) + fallback.assert_not_called() + + def test_to_delimited_str_preserves_temporal_formatting(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame( + { + "datetime": [ + datetime.datetime(2026, 1, 2, 3, 4, 5, 123456) + ], + "date": [datetime.date(2026, 1, 2)], + "time": [datetime.time(3, 4, 5, 123456)], + } + ) + ) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "datetime;date;time\n" + "2026-01-02 03:04:05.123456;2026-01-02;" + "03:04:05.123456\n" + ) + + def test_to_delimited_str_preserves_binary_formatting(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame({"binary": [b"abc", bytes([0, 255])]}) + ) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "binary\nb'abc'\nb'\\x00\\xff'\n" + ) + + def test_to_delimited_str_uses_native_tsv_writer(self) -> None: + import polars as pl + + manager = self.factory.create()(pl.DataFrame({"value": [1.5]})) + with mock.patch( + "marimo._plugins.ui._impl.tables.polars_table.dataframe_to_csv" + ) as fallback: + assert manager.to_delimited_str(DelimitedDialect("\t", ".")) == ( + "value\n1.5\n" + ) + fallback.assert_not_called() + def test_to_delimited_str_normalizes_nested_values(self) -> None: import polars as pl diff --git a/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py b/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py index 68747d1ee68..91177d39a3b 100644 --- a/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py +++ b/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py @@ -148,11 +148,16 @@ def test_download_as_pt_br_csv() -> None: DefaultTableManager, ) from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale - from marimo._plugins.ui._impl.utils.dataframe import download_as + from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, + download_as, + ) manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) url, _ = download_as( - manager, "csv", locale=ResolvedExportLocale("pt-BR", ",") + manager, + "csv", + options=DownloadOptions(locale=ResolvedExportLocale("pt-BR", ",")), ) assert _download_text(url) == "value;text\n1234,56;unchanged.1\n" @@ -162,11 +167,16 @@ def test_download_as_pt_br_tsv() -> None: DefaultTableManager, ) from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale - from marimo._plugins.ui._impl.utils.dataframe import download_as + from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, + download_as, + ) manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) url, _ = download_as( - manager, "tsv", locale=ResolvedExportLocale("pt-BR", ",") + manager, + "tsv", + options=DownloadOptions(locale=ResolvedExportLocale("pt-BR", ",")), ) assert _download_text(url) == "value\ttext\n1234,56\tunchanged.1\n" @@ -197,8 +207,10 @@ def test_download_as_explicit_csv_separator_wins() -> None: url, _ = download_as( manager, "csv", - locale=ResolvedExportLocale("pt-BR", ","), - options=DownloadOptions(delimited=DelimitedOptions(separator="|")), + options=DownloadOptions( + delimited=DelimitedOptions(separator="|"), + locale=ResolvedExportLocale("pt-BR", ","), + ), ) assert _download_text(url) == "value|text\n1234,56|unchanged.1\n" @@ -208,11 +220,16 @@ def test_download_as_json_stays_locale_neutral() -> None: DefaultTableManager, ) from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale - from marimo._plugins.ui._impl.utils.dataframe import download_as + from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, + download_as, + ) manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) url, _ = download_as( - manager, "json", locale=ResolvedExportLocale("pt-BR", ",") + manager, + "json", + options=DownloadOptions(locale=ResolvedExportLocale("pt-BR", ",")), ) text = _download_text(url) assert "1234.56" in text diff --git a/tests/_utils/test_narwhals_utils.py b/tests/_utils/test_narwhals_utils.py index 17a63277826..d1c94f4abee 100644 --- a/tests/_utils/test_narwhals_utils.py +++ b/tests/_utils/test_narwhals_utils.py @@ -119,7 +119,7 @@ def test_dataframe_to_csv(df: IntoDataFrame) -> None: @pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") def test_dataframe_to_csv_with_separator(df: IntoDataFrame) -> None: df_wrapped = nw.from_native(df) - csv = dataframe_to_csv(df_wrapped, separator=";") + csv = dataframe_to_csv(df_wrapped, dialect=DelimitedDialect(";", ".")) assert "a;b" in csv assert "1;x" in csv assert "2;y" in csv