diff --git a/frontend/plugins.openapi.yaml b/frontend/plugins.openapi.yaml index e0e7042e011..47e75bbf492 100644 --- a/frontend/plugins.openapi.yaml +++ b/frontend/plugins.openapi.yaml @@ -3140,6 +3140,16 @@ components: - json - parquet - tsv + locale: + type: object + properties: + tag: + type: string + decimal_separator: + type: string + required: + - tag + - decimal_separator required: - format marimo-dataframe.download_as.output: @@ -4577,6 +4587,16 @@ components: - json - parquet - tsv + locale: + type: object + properties: + tag: + type: string + decimal_separator: + type: string + required: + - tag + - decimal_separator required: - format marimo-table.download_as.output: diff --git a/frontend/src/components/data-table/__tests__/export-actions.test.tsx b/frontend/src/components/data-table/__tests__/export-actions.test.tsx new file mode 100644 index 00000000000..cd8a5f89ada --- /dev/null +++ b/frontend/src/components/data-table/__tests__/export-actions.test.tsx @@ -0,0 +1,121 @@ +/* Copyright 2026 Marimo. All rights reserved. */ + +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import type React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { toast } from "@/components/ui/use-toast"; +import { downloadByURL } from "@/utils/download"; +import { + buildDownloadAsRequest, + ExportMenu, + sourceFormatForCopy, +} from "../export-actions"; + +vi.mock("@/components/ui/use-toast", () => ({ + toast: vi.fn(() => ({ dismiss: vi.fn(), update: vi.fn() })), +})); + +vi.mock("@/components/ui/tooltip", () => ({ + Tooltip: ({ children }: { children: React.ReactNode }) => children, + TooltipProvider: ({ children }: { children: React.ReactNode }) => children, +})); + +vi.mock("@/utils/download", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, downloadByURL: vi.fn() }; +}); + +const PT_BR_LOCALE = { tag: "pt-BR", decimal_separator: "," }; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("ExportMenu", () => { + it("stops a download and shows a standalone export error", async () => { + const downloadAs = vi.fn().mockResolvedValue({ + url: "", + filename: "", + error: "The export locale is invalid.", + }); + render( + + + , + ); + + fireEvent.keyDown(screen.getByTestId("export-button"), { + key: "ArrowDown", + }); + fireEvent.click( + await screen.findByRole("menuitem", { + name: "Download CSV: Comma-separated values", + }), + ); + + await waitFor(() => { + expect(toast).toHaveBeenCalledWith({ + title: "Export failed", + description: "The export locale is invalid.", + variant: "danger", + }); + }); + expect(downloadByURL).not.toHaveBeenCalled(); + }); +}); + +describe("buildDownloadAsRequest", () => { + it("includes locale for CSV and TSV", () => { + expect(buildDownloadAsRequest("csv", "pt-BR")).toEqual({ + format: "csv", + locale: PT_BR_LOCALE, + }); + expect(buildDownloadAsRequest("tsv", "pt-BR")).toEqual({ + format: "tsv", + locale: PT_BR_LOCALE, + }); + }); + + it("omits locale for JSON and Parquet", () => { + expect(buildDownloadAsRequest("json", "pt-BR")).toEqual({ format: "json" }); + expect(buildDownloadAsRequest("parquet", "pt-BR")).toEqual({ + format: "parquet", + }); + }); +}); + +describe("file and clipboard request parity", () => { + it("sends the same locale for CSV download and clipboard copy", () => { + const download = buildDownloadAsRequest("csv", "pt-BR"); + const clipboard = buildDownloadAsRequest( + sourceFormatForCopy("csv"), + "pt-BR", + ); + expect(download).toEqual({ format: "csv", locale: PT_BR_LOCALE }); + expect(clipboard).toEqual(download); + }); + + it("sends the same locale for TSV download and clipboard copy", () => { + const download = buildDownloadAsRequest("tsv", "pt-BR"); + const clipboard = buildDownloadAsRequest( + sourceFormatForCopy("tsv"), + "pt-BR", + ); + expect(download).toEqual({ format: "tsv", locale: PT_BR_LOCALE }); + expect(clipboard).toEqual(download); + }); + + it("omits locale for JSON, Parquet, and Markdown source requests", () => { + expect(buildDownloadAsRequest("json", "pt-BR")).toEqual({ format: "json" }); + expect(buildDownloadAsRequest("parquet", "pt-BR")).toEqual({ + format: "parquet", + }); + expect( + buildDownloadAsRequest(sourceFormatForCopy("json"), "pt-BR"), + ).toEqual({ format: "json" }); + expect( + buildDownloadAsRequest(sourceFormatForCopy("markdown"), "pt-BR"), + ).toEqual({ format: "json" }); + }); +}); diff --git a/frontend/src/components/data-table/__tests__/export-locale.test.ts b/frontend/src/components/data-table/__tests__/export-locale.test.ts new file mode 100644 index 00000000000..066feaa1f3e --- /dev/null +++ b/frontend/src/components/data-table/__tests__/export-locale.test.ts @@ -0,0 +1,46 @@ +/* Copyright 2026 Marimo. All rights reserved. */ + +import { describe, expect, it } from "vitest"; +import { resolveExportLocale } from "../export-locale"; +import { DownloadAsSchema } from "../schemas"; + +describe("resolveExportLocale", () => { + it("resolves a decimal point for en-US", () => { + expect(resolveExportLocale("en-US")).toEqual({ + tag: "en-US", + decimal_separator: ".", + }); + }); + + it("resolves a decimal comma for pt-BR", () => { + expect(resolveExportLocale("pt-BR").decimal_separator).toBe(","); + }); + + it("preserves a non-comma Unicode decimal separator", () => { + expect(resolveExportLocale("ar-EG").decimal_separator).toBe("٫"); + }); + + it("rejects an invalid locale", () => { + expect(() => resolveExportLocale("not_a_locale")).toThrow(); + }); + + it("rejects an empty locale tag", () => { + expect(() => resolveExportLocale("")).toThrow(); + }); +}); + +describe("DownloadAsSchema", () => { + it("accepts a request with only format", () => { + expect(DownloadAsSchema.input.parse({ format: "csv" })).toEqual({ + format: "csv", + }); + }); + + it("accepts an optional locale value", () => { + const locale = { tag: "pt-BR", decimal_separator: "," }; + expect(DownloadAsSchema.input.parse({ format: "csv", locale })).toEqual({ + format: "csv", + locale, + }); + }); +}); diff --git a/frontend/src/components/data-table/export-actions.tsx b/frontend/src/components/data-table/export-actions.tsx index b44e90a3580..f69777a9f5e 100644 --- a/frontend/src/components/data-table/export-actions.tsx +++ b/frontend/src/components/data-table/export-actions.tsx @@ -9,6 +9,7 @@ import { TableIcon, } from "lucide-react"; import React from "react"; +import { useLocale } from "react-aria"; import { downloadSizeLimitAtom } from "./download-policy/atoms"; import { logNever } from "@/utils/assertNever"; import { cn } from "@/utils/cn"; @@ -32,6 +33,8 @@ import { } from "../ui/dropdown-menu"; import { Tooltip } from "../ui/tooltip"; import { toast } from "../ui/use-toast"; +import { resolveExportLocale } from "./export-locale"; +import type { DownloadAsArgs } from "./schemas"; const FILE_TYPES = { CSV: { @@ -91,13 +94,22 @@ const COPY_SOURCE_FORMAT: Record = { markdown: "json", }; +export function sourceFormatForCopy(format: CopyFormat): DownloadFormat { + return COPY_SOURCE_FORMAT[format]; +} + +export function buildDownloadAsRequest( + format: DownloadFormat, + localeTag: string, +): Parameters[0] { + if (format === "csv" || format === "tsv") { + return { format, locale: resolveExportLocale(localeTag) }; + } + return { format }; +} + export interface ExportActionProps { - downloadAs: (req: { format: DownloadFormat }) => Promise<{ - url: string; - filename: string; - error?: string | null; - missing_packages?: string[] | null; - }>; + downloadAs: DownloadAsArgs; // JSON-serialized size of the currently-rendered data. Used together with // downloadSizeLimitAtom to disable the Export button when a host (e.g., // marimo-lsp inside VS Code) declares a download size cap. Null/undefined @@ -113,6 +125,7 @@ const labelForCopyFormat = (format: CopyFormat): string => export const ExportMenu: React.FC = (props) => { const [downloadMenuOpen, setDownloadMenuOpen] = React.useState(false); + const { locale } = useLocale(); const policy = useAtomValue(downloadSizeLimitAtom); const overLimit = !!( policy && @@ -151,7 +164,8 @@ export const ExportMenu: React.FC = (props) => { } | null> => { let response: Awaited>; try { - response = await props.downloadAs({ format }); + const request = buildDownloadAsRequest(format, locale); + response = await props.downloadAs(request); } catch (error) { toast({ title: "Failed to download", @@ -178,6 +192,15 @@ export const ExportMenu: React.FC = (props) => { return null; } + if (response.error) { + toast({ + title: "Export failed", + description: response.error, + variant: "danger", + }); + return null; + } + return { url: response.url, filename: response.filename, @@ -224,7 +247,7 @@ export const ExportMenu: React.FC = (props) => { await withLoadingToast( `Preparing ${labelForCopyFormat(format)} for clipboard...`, async () => { - const sourceFormat = COPY_SOURCE_FORMAT[format]; + const sourceFormat = sourceFormatForCopy(format); const result = await resolveDownloadUrl(sourceFormat, () => { void handleClipboardCopy(format); }); @@ -284,6 +307,7 @@ export const ExportMenu: React.FC = (props) => { {downloadOptions.map((option) => ( { void handleDownload(option.format); }} @@ -304,6 +328,7 @@ export const ExportMenu: React.FC = (props) => { {copyOptions.map((option) => ( { try { await handleClipboardCopy(option.format); diff --git a/frontend/src/components/data-table/export-locale.ts b/frontend/src/components/data-table/export-locale.ts new file mode 100644 index 00000000000..6cd533ce2b9 --- /dev/null +++ b/frontend/src/components/data-table/export-locale.ts @@ -0,0 +1,33 @@ +/* Copyright 2026 Marimo. All rights reserved. */ + +export interface ResolvedExportLocale { + tag: string; + decimal_separator: string; +} + +const FORBIDDEN_DECIMAL_SEPARATORS = new Set(["\r", "\n", "\0"]); + +export function resolveExportLocale(locale: string): ResolvedExportLocale { + if (locale === "") { + throw new Error("Locale tag must be a non-empty string."); + } + + const decimalSeparator = new Intl.NumberFormat(locale) + .formatToParts(1.1) + .find((part) => part.type === "decimal")?.value; + + if ( + decimalSeparator === undefined || + [...decimalSeparator].length !== 1 || + FORBIDDEN_DECIMAL_SEPARATORS.has(decimalSeparator) + ) { + throw new Error( + `Locale "${locale}" did not resolve a valid decimal separator.`, + ); + } + + return { + tag: locale, + decimal_separator: decimalSeparator, + }; +} diff --git a/frontend/src/components/data-table/schemas.ts b/frontend/src/components/data-table/schemas.ts index d56ce10b149..3c593cc5288 100644 --- a/frontend/src/components/data-table/schemas.ts +++ b/frontend/src/components/data-table/schemas.ts @@ -2,9 +2,11 @@ import z from "zod"; import { rpc } from "@/plugins/core/rpc"; +import type { ResolvedExportLocale } from "./export-locale"; export type DownloadAsArgs = (req: { format: "csv" | "json" | "parquet" | "tsv"; + locale?: ResolvedExportLocale; }) => Promise<{ url: string; filename: string; @@ -16,6 +18,12 @@ export const DownloadAsSchema = rpc .input( z.object({ format: z.enum(["csv", "json", "parquet", "tsv"]), + locale: z + .object({ + tag: z.string(), + decimal_separator: z.string(), + }) + .optional(), }), ) .output( diff --git a/marimo/_plugins/ui/_impl/dataframes/dataframe.py b/marimo/_plugins/ui/_impl/dataframes/dataframe.py index 45c3967db58..de2ba96b540 100644 --- a/marimo/_plugins/ui/_impl/dataframes/dataframe.py +++ b/marimo/_plugins/ui/_impl/dataframes/dataframe.py @@ -33,6 +33,7 @@ SortArgs, TableSearchError, ) +from marimo._plugins.ui._impl.tables.delimited import InvalidExportLocaleError from marimo._plugins.ui._impl.tables.format import FormatMapping from marimo._plugins.ui._impl.tables.table_manager import ( FieldTypes, @@ -367,20 +368,24 @@ def _download_as(self, args: DownloadAsArgs) -> DownloadAsResponse: bound_filename = get_bound_name(self._id) - url, filename = download_as( - manager, - args.format, - options=DownloadOptions( - delimited=DelimitedOptions( - encoding=self._download_csv_encoding, - separator=self._download_csv_separator, - ), - json=JsonOptions( - ensure_ascii=self._download_json_ensure_ascii + try: + url, filename = download_as( + manager, + args.format, + options=DownloadOptions( + delimited=DelimitedOptions( + encoding=self._download_csv_encoding, + separator=self._download_csv_separator, + ), + json=JsonOptions( + ensure_ascii=self._download_json_ensure_ascii + ), + locale=args.locale, ), - ), - filename=bound_filename, - ) + filename=bound_filename, + ) + except InvalidExportLocaleError as error: + return DownloadAsResponse(error=str(error)) return DownloadAsResponse(url=url, filename=filename) def _apply_filters_query_sort( diff --git a/marimo/_plugins/ui/_impl/table.py b/marimo/_plugins/ui/_impl/table.py index 6c207c496e9..28b80e85d82 100644 --- a/marimo/_plugins/ui/_impl/table.py +++ b/marimo/_plugins/ui/_impl/table.py @@ -41,6 +41,10 @@ TransformType, validate_operator_for_dtype, ) +from marimo._plugins.ui._impl.tables.delimited import ( + InvalidExportLocaleError, + ResolvedExportLocale, +) from marimo._plugins.ui._impl.tables.selection import ( INDEX_COLUMN_NAME, add_selection_column, @@ -55,6 +59,7 @@ ) from marimo._plugins.ui._impl.tables.utils import get_table_manager from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, ListOrTuple, TableData, download_as, @@ -94,6 +99,7 @@ def __init__(self, error: str): @dataclass class DownloadAsArgs: format: Literal["csv", "tsv", "json", "parquet"] + locale: ResolvedExportLocale | None = None @dataclass @@ -1094,12 +1100,16 @@ def _download_as(self, args: DownloadAsArgs) -> DownloadAsResponse: if isinstance(manager_candidate, TableManager): bound_filename = get_bound_name(self._id) - url, filename = download_as( - manager_candidate, - args.format, - drop_marimo_index=True, - filename=bound_filename, - ) + try: + url, filename = download_as( + manager_candidate, + args.format, + drop_marimo_index=True, + filename=bound_filename, + options=DownloadOptions(locale=args.locale), + ) + except InvalidExportLocaleError as error: + return DownloadAsResponse(error=str(error)) return DownloadAsResponse(url=url, filename=filename) else: raise NotImplementedError( diff --git a/marimo/_plugins/ui/_impl/tables/default_table.py b/marimo/_plugins/ui/_impl/tables/default_table.py index 6a90b96df36..4fcd9597cd4 100644 --- a/marimo/_plugins/ui/_impl/tables/default_table.py +++ b/marimo/_plugins/ui/_impl/tables/default_table.py @@ -18,6 +18,11 @@ format_column, format_row, ) +from marimo._utils.delimited import ( + DelimitedDialect, + format_delimited_number, + is_delimited_number, +) if TYPE_CHECKING: from marimo._plugins.ui._impl.table import SortArgs @@ -88,6 +93,16 @@ def to_csv_str( self, format_mapping: FormatMapping | None = None, separator: str | None = None, + ) -> str: + return self.to_delimited_str( + DelimitedDialect(separator or ",", "."), + format_mapping, + ) + + def to_delimited_str( + self, + dialect: DelimitedDialect, + format_mapping: FormatMapping | None = None, ) -> str: import csv import io @@ -100,12 +115,12 @@ def to_csv_str( writer = csv.DictWriter( buf, fieldnames=columns, - delimiter=separator or ",", + delimiter=dialect.field_separator, lineterminator="\n", ) writer.writeheader() writer.writerows( - {col: _to_csv_cell(row.get(col)) for col in columns} + {col: _to_delimited_cell(row.get(col), dialect) for col in columns} for row in rows ) return buf.getvalue() @@ -535,3 +550,9 @@ def _to_csv_cell(value: Any) -> str: if isinstance(value, (dict, list, tuple)): return str(encode_json_str(SuperJson(value))) return str(value) + + +def _to_delimited_cell(value: Any, dialect: DelimitedDialect) -> str: + if is_delimited_number(value): + return format_delimited_number(value, dialect.decimal_separator) + return _to_csv_cell(value) diff --git a/marimo/_plugins/ui/_impl/tables/delimited.py b/marimo/_plugins/ui/_impl/tables/delimited.py new file mode 100644 index 00000000000..0fe5a263f42 --- /dev/null +++ b/marimo/_plugins/ui/_impl/tables/delimited.py @@ -0,0 +1,66 @@ +# Copyright 2026 Marimo. All rights reserved. +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +from marimo._utils.delimited import DelimitedDialect + +DownloadFormat = Literal["csv", "tsv"] + +_FORBIDDEN_DECIMAL_SEPARATORS = frozenset({"\r", "\n", "\0"}) + + +class InvalidExportLocaleError(ValueError): + pass + + +@dataclass(frozen=True) +class ResolvedExportLocale: + tag: str + decimal_separator: str + + +def resolve_delimited_dialect( + download_format: DownloadFormat, + locale: ResolvedExportLocale | None, + explicit_separator: str | None = None, +) -> DelimitedDialect: + """Resolve field and decimal separators for CSV or TSV export. + + TSV always uses a tab field separator. An explicit CSV separator wins. + CSV uses `;` when the decimal separator is `,`, otherwise `,`. A missing + locale keeps the current locale-neutral defaults. + """ + if download_format not in ("csv", "tsv"): + raise ValueError(f"Unsupported delimited format: {download_format}") + + if locale is not None: + _validate_locale(locale) + decimal_separator = locale.decimal_separator + else: + decimal_separator = "." + + if download_format == "tsv": + return DelimitedDialect("\t", decimal_separator) + + if explicit_separator is not None: + field_separator = explicit_separator + elif decimal_separator == ",": + field_separator = ";" + else: + field_separator = "," + + return DelimitedDialect(field_separator, decimal_separator) + + +def _validate_locale(locale: ResolvedExportLocale) -> None: + if locale.tag == "": + raise InvalidExportLocaleError( + "Locale tag must be a non-empty string." + ) + separator = locale.decimal_separator + if len(separator) != 1 or separator in _FORBIDDEN_DECIMAL_SEPARATORS: + raise InvalidExportLocaleError( + "Decimal separator must be a single Unicode character." + ) diff --git a/marimo/_plugins/ui/_impl/tables/narwhals_table.py b/marimo/_plugins/ui/_impl/tables/narwhals_table.py index b88fc1cb217..66bf6745971 100644 --- a/marimo/_plugins/ui/_impl/tables/narwhals_table.py +++ b/marimo/_plugins/ui/_impl/tables/narwhals_table.py @@ -35,6 +35,7 @@ TableCoordinate, TableManager, ) +from marimo._utils.delimited import DelimitedDialect from marimo._utils.narwhals_utils import ( can_narwhalify, dataframe_to_csv, @@ -99,8 +100,18 @@ def to_csv_str( format_mapping: FormatMapping | None = None, separator: str | None = None, ) -> str: - _data = self.apply_formatting(format_mapping).as_frame() - return dataframe_to_csv(_data, separator=separator) + return self.to_delimited_str( + DelimitedDialect(separator or ",", "."), + format_mapping, + ) + + def to_delimited_str( + self, + dialect: DelimitedDialect, + format_mapping: FormatMapping | None = None, + ) -> str: + data = self.apply_formatting(format_mapping).as_frame() + return dataframe_to_csv(data, dialect=dialect) def to_json_str( self, diff --git a/marimo/_plugins/ui/_impl/tables/pandas_table.py b/marimo/_plugins/ui/_impl/tables/pandas_table.py index 0b137868141..4318ef865c2 100644 --- a/marimo/_plugins/ui/_impl/tables/pandas_table.py +++ b/marimo/_plugins/ui/_impl/tables/pandas_table.py @@ -4,6 +4,7 @@ import functools import io import json +import math from functools import cached_property from typing import TYPE_CHECKING, Any @@ -30,6 +31,11 @@ TableManager, TableManagerFactory, ) +from marimo._utils.delimited import ( + DelimitedDialect, + format_delimited_number, + is_delimited_number, +) if TYPE_CHECKING: import pandas as pd @@ -202,6 +208,55 @@ def package_name() -> str: def create() -> type[TableManager[Any]]: import pandas as pd + def prepare_delimited_data( + data: pd.DataFrame, decimal_separator: str + ) -> pd.DataFrame: + """Prepare values pandas does not localize itself.""" + + def localize_number( + value: object, *, stringify_unchanged: bool + ) -> object: + if not is_delimited_number(value): + return value + formatted = format_delimited_number(value, decimal_separator) + if ( + not stringify_unchanged + and formatted == str(value) + and not ( + isinstance(value, float) and not math.isfinite(value) + ) + ): + return value + return formatted + + localized_data: pd.DataFrame | None = None + for position, dtype in enumerate(data.dtypes): + column = data.iloc[:, position] + force_numeric_strings = bool( + pd.api.types.is_float_dtype(dtype) + and not pd.api.types.is_extension_array_dtype(dtype) + and column.isna().any() + ) + if not force_numeric_strings and ( + decimal_separator == "." + or not pd.api.types.is_object_dtype(dtype) + ): + continue + + localized_column = column.map( + functools.partial( + localize_number, + stringify_unchanged=force_numeric_strings, + ) + ) + if localized_column.equals(column): + continue + if localized_data is None: + localized_data = data.copy() + localized_data.isetitem(position, localized_column.array) + + return localized_data if localized_data is not None else data + class PandasTableManager(NarwhalsTableManager[pd.DataFrame, Any]): type = "pandas" @@ -263,13 +318,55 @@ def to_csv_str( separator: str | None = None, ) -> str: has_headers = len(self.get_row_headers()) > 0 - resolved_separator = ( - separator if separator is not None else "," - ) + resolved_separator = separator or "," return self.apply_formatting( format_mapping )._original_data.to_csv( - index=has_headers, sep=resolved_separator + index=has_headers, + sep=resolved_separator, + ) + + def to_delimited_str( + self, + dialect: DelimitedDialect, + format_mapping: FormatMapping | None = None, + ) -> str: + manager = self.apply_formatting(format_mapping) + # pandas applies `decimal` only to numeric dtypes. Prepare + # Decimal and mixed object columns before using its native + # writer for the full frame. + data = prepare_delimited_data( + manager._original_data, dialect.decimal_separator + ) + include_index = len(self.get_row_headers()) > 0 + if include_index: + index_frame = data.index.to_frame(index=False) + localized_index = prepare_delimited_data( + index_frame, dialect.decimal_separator + ) + if localized_index is not index_frame: + if data is manager._original_data: + data = data.copy() + if isinstance(data.index, pd.MultiIndex): + data.index = pd.MultiIndex.from_frame( + localized_index, names=data.index.names + ) + else: + data.index = pd.Index( + localized_index.iloc[:, 0].array, + name=data.index.name, + ) + + # Default exports historically use pandas' platform newline. + is_default_dialect = ( + dialect.field_separator == "," + and dialect.decimal_separator == "." + ) + return data.to_csv( + index=include_index, + sep=dialect.field_separator, + decimal=dialect.decimal_separator, + lineterminator=None if is_default_dialect else "\n", ) def to_json_str( diff --git a/marimo/_plugins/ui/_impl/tables/polars_table.py b/marimo/_plugins/ui/_impl/tables/polars_table.py index c9b1f020da2..ae3e2bbc463 100644 --- a/marimo/_plugins/ui/_impl/tables/polars_table.py +++ b/marimo/_plugins/ui/_impl/tables/polars_table.py @@ -2,7 +2,9 @@ from __future__ import annotations import functools +import inspect import io +import json from functools import cached_property from typing import Any @@ -23,10 +25,20 @@ TableManager, TableManagerFactory, ) +from marimo._utils.delimited import DelimitedDialect +from marimo._utils.narwhals_utils import dataframe_to_csv LOGGER = _loggers.marimo_logger() +def _supports_decimal_comma() -> bool: + import polars as pl + + return ( + "decimal_comma" in inspect.signature(pl.DataFrame.write_csv).parameters + ) + + class PolarsTableManagerFactory(TableManagerFactory): @staticmethod def package_name() -> str: @@ -37,6 +49,34 @@ def package_name() -> str: def create() -> type[TableManager[Any]]: import polars as pl + supports_decimal_comma = _supports_decimal_comma() + + def serialize_sequence_column( + column: pl.Series, dtype: pl.List | pl.Array + ) -> pl.Series: + inner = dtype.inner + if not isinstance(inner, (pl.Struct, pl.List, pl.Array)): + try: + if isinstance(dtype, pl.List): + return column.cast(pl.List(pl.Utf8)).list.join(",") + if isinstance(dtype, pl.Array): + return column.cast( + pl.Array(pl.Utf8, shape=dtype.shape) + ).arr.join(",") + except pl.exceptions.PolarsError: + pass + + return pl.Series( + column.name, + [ + None + if value is None + else json.dumps(value, default=str, separators=(",", ":")) + for value in column.to_list() + ], + dtype=pl.Utf8, + ) + class PolarsTableManager( NarwhalsTableManager[pl.DataFrame, pl.LazyFrame] ): @@ -73,41 +113,51 @@ def to_csv_str( format_mapping: FormatMapping | None = None, separator: str | None = None, ) -> str: - resolved_separator = ( - separator if separator is not None else "," + return self.to_delimited_str( + DelimitedDialect(separator or ",", "."), + format_mapping, ) - _data = self.apply_formatting(format_mapping).collect() - try: - return _data.write_csv(separator=resolved_separator) - except pl.exceptions.ComputeError: - # Likely CSV format does not support nested data or objects - # Try to convert columns to json or strings - result = _data - for column in result.get_columns(): - dtype = column.dtype - if isinstance(dtype, pl.Struct): - result = result.with_columns( - column.struct.json_encode() - ) - elif isinstance(dtype, pl.List): - result = result.with_columns( - column.cast(pl.List(pl.Utf8)).list.join(",") - ) - elif isinstance(dtype, pl.Array): - result = result.with_columns( - column.cast( - pl.Array(pl.Utf8, shape=dtype.shape) - ).arr.join(",") - ) - elif isinstance(dtype, pl.Object): - result = self._cast_object_to_string( - result, column - ) - elif isinstance(dtype, pl.Duration): - result = self._convert_time_to_string( - result, column - ) - return result.write_csv(separator=resolved_separator) + + def to_delimited_str( + self, + dialect: DelimitedDialect, + format_mapping: FormatMapping | None = None, + ) -> str: + result = self.apply_formatting(format_mapping).collect() + for column in result.get_columns(): + dtype = column.dtype + if isinstance(dtype, pl.Struct): + result = result.with_columns( + column.struct.json_encode() + ) + elif isinstance(dtype, (pl.List, pl.Array)): + result = result.with_columns( + serialize_sequence_column(column, dtype) + ) + elif isinstance(dtype, pl.Object): + result = self._cast_object_to_string(result, column) + elif isinstance(dtype, pl.Duration): + result = self._convert_time_to_string(result, column) + + # Native Polars uses different temporal and binary text + # representations, and writes NaN as an empty field. + requires_compatibility_writer = any( + column.dtype.is_temporal() + or column.dtype == pl.Binary + or (column.dtype.is_float() and column.is_nan().any()) + for column in result.get_columns() + ) + if requires_compatibility_writer: + return dataframe_to_csv(result, dialect=dialect) + + if dialect.decimal_separator == ".": + return result.write_csv(separator=dialect.field_separator) + if dialect.decimal_separator == "," and supports_decimal_comma: + return result.write_csv( + separator=dialect.field_separator, + decimal_comma=True, + ) + return dataframe_to_csv(result, dialect=dialect) def to_json_str( self, diff --git a/marimo/_plugins/ui/_impl/tables/table_manager.py b/marimo/_plugins/ui/_impl/tables/table_manager.py index 93a5e5fab81..81dd7dd7088 100644 --- a/marimo/_plugins/ui/_impl/tables/table_manager.py +++ b/marimo/_plugins/ui/_impl/tables/table_manager.py @@ -19,6 +19,7 @@ ExternalDataType, ) from marimo._plugins.ui._impl.tables.format import FormatMapping +from marimo._utils.delimited import DelimitedDialect if TYPE_CHECKING: from marimo._plugins.ui._impl.table import SortArgs @@ -104,6 +105,15 @@ def to_csv_str( ) -> str: pass + def to_delimited_str( + self, + dialect: DelimitedDialect, + format_mapping: FormatMapping | None = None, + ) -> str: + return self.to_csv_str( + format_mapping, separator=dialect.field_separator + ) + def to_csv( self, format_mapping: FormatMapping | None = None, diff --git a/marimo/_plugins/ui/_impl/utils/dataframe.py b/marimo/_plugins/ui/_impl/utils/dataframe.py index 2ef12554d8a..7bf1ebc9f41 100644 --- a/marimo/_plugins/ui/_impl/utils/dataframe.py +++ b/marimo/_plugins/ui/_impl/utils/dataframe.py @@ -2,7 +2,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, TypeVar, Union +from typing import TYPE_CHECKING, Any, Literal, TypeVar, Union from narwhals.typing import IntoDataFrame, IntoLazyFrame @@ -10,6 +10,10 @@ from marimo._output.data import data as mo_data from marimo._output.mime import MIME from marimo._plugins.core.web_component import JSONType +from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + resolve_delimited_dialect, +) from marimo._plugins.ui._impl.tables.selection import INDEX_COLUMN_NAME from marimo._plugins.ui._impl.tables.table_manager import TableManager from marimo._runtime.context.types import ( @@ -113,6 +117,7 @@ class DownloadOptions: delimited: DelimitedOptions = field(default_factory=DelimitedOptions) json: JsonOptions = field(default_factory=JsonOptions) + locale: ResolvedExportLocale | None = None @dataclass(frozen=True) @@ -122,18 +127,21 @@ class _ExportFormat: def _serialize_delimited( - separator_override: str | None, + download_format: Literal["csv", "tsv"], ) -> Callable[[TableManager[Any], DownloadOptions], bytes]: def serialize( manager: TableManager[Any], options: DownloadOptions ) -> bytes: encoding = options.delimited.encoding or get_default_csv_encoding() - separator = ( - separator_override - if separator_override is not None - else options.delimited.separator + explicit_separator = options.delimited.separator or None + if download_format == "tsv": + explicit_separator = None + dialect = resolve_delimited_dialect( + download_format, + options.locale, + explicit_separator, ) - return manager.to_csv(encoding=encoding, separator=separator) + return manager.to_delimited_str(dialect).encode(encoding) return serialize @@ -156,8 +164,8 @@ def _serialize_parquet( _EXPORT_FORMATS: dict[str, _ExportFormat] = { - "csv": _ExportFormat("csv", _serialize_delimited(None)), - "tsv": _ExportFormat("tsv", _serialize_delimited("\t")), + "csv": _ExportFormat("csv", _serialize_delimited("csv")), + "tsv": _ExportFormat("tsv", _serialize_delimited("tsv")), "json": _ExportFormat("json", _serialize_json), "parquet": _ExportFormat("parquet", _serialize_parquet), } diff --git a/marimo/_utils/delimited.py b/marimo/_utils/delimited.py new file mode 100644 index 00000000000..d8f242e1245 --- /dev/null +++ b/marimo/_utils/delimited.py @@ -0,0 +1,32 @@ +# Copyright 2026 Marimo. All rights reserved. +from __future__ import annotations + +import math +from dataclasses import dataclass +from decimal import Decimal +from numbers import Integral, Real +from typing import TypeGuard + + +@dataclass(frozen=True) +class DelimitedDialect: + field_separator: str + decimal_separator: str + + +def is_delimited_number(value: object) -> TypeGuard[Real | Decimal]: + if isinstance(value, bool): + return False + return isinstance(value, (Integral, Real, Decimal)) + + +def format_delimited_number( + value: Real | Decimal, decimal_separator: str +) -> str: + """Format a numeric value without grouping.""" + if isinstance(value, float) and not math.isfinite(value): + return str(value) + text = str(value) + if decimal_separator == ".": + return text + return text.replace(".", decimal_separator) diff --git a/marimo/_utils/narwhals_utils.py b/marimo/_utils/narwhals_utils.py index 50ef97d1920..8d58cb84bfc 100644 --- a/marimo/_utils/narwhals_utils.py +++ b/marimo/_utils/narwhals_utils.py @@ -11,6 +11,11 @@ import narwhals.stable.v2 as nw from marimo import _loggers +from marimo._utils.delimited import ( + DelimitedDialect, + format_delimited_number, + is_delimited_number, +) LOGGER = _loggers.marimo_logger() @@ -90,28 +95,46 @@ def assert_can_narwhalify(obj: Any) -> TypeGuard[IntoFrame]: return True -def dataframe_to_csv(df: IntoFrame, separator: str | None = None) -> str: +def dataframe_to_csv( + df: IntoFrame, + dialect: DelimitedDialect | None = None, +) -> str: """ Convert a dataframe to a CSV string. + + `dialect` controls the field and decimal separators. """ assert_can_narwhalify(df) df = nw.from_native(df, pass_through=False) df = upgrade_narwhals_df(df) - resolved_separator = separator if separator is not None else "," + dialect = dialect or DelimitedDialect(",", ".") frame = df.collect() if is_narwhals_lazyframe(df) else df - if resolved_separator == ",": + if dialect.field_separator == "," and dialect.decimal_separator == ".": return frame.write_csv() # Narwhals inputs can map to different backends, and # write_csv(separator=...) is not consistently reliable across them. - # For non-comma separators, use Python's csv writer for stable behavior. + # For non-default dialects, use Python's csv writer for stable behavior. buffer = io.StringIO() writer = csv.writer( - buffer, delimiter=resolved_separator, lineterminator="\n" + buffer, delimiter=dialect.field_separator, lineterminator="\n" ) writer.writerow(frame.columns) - writer.writerows(frame.iter_rows()) + rows = frame.iter_rows() + if dialect.decimal_separator == ".": + writer.writerows(rows) + return buffer.getvalue() + + writer.writerows( + tuple( + format_delimited_number(value, dialect.decimal_separator) + if is_delimited_number(value) + else value + for value in row + ) + for row in rows + ) return buffer.getvalue() diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml index 93149a18a44..76abc1b27cc 100644 --- a/packages/openapi/api.yaml +++ b/packages/openapi/api.yaml @@ -1590,6 +1590,18 @@ components: - label title: DetectedDataSourceOrigin type: object + DiagnosticsConfig: + description: "Configuration options for diagnostics.\n\n **Keys.**\n\n \ + \ - `enabled`: if `True`, diagnostics will be shown in the editor\n -\ + \ `sql_linter`: if `True`, SQL cells will have linting enabled" + properties: + enabled: + type: boolean + sql_linter: + type: boolean + required: [] + title: DiagnosticsConfig + type: object DialectHidesWhen: description: Hide this suggestion when a live SQL engine dialect contains a substring. @@ -1606,18 +1618,6 @@ components: - substrings title: DialectHidesWhen type: object - DiagnosticsConfig: - description: "Configuration options for diagnostics.\n\n **Keys.**\n\n \ - \ - `enabled`: if `True`, diagnostics will be shown in the editor\n -\ - \ `sql_linter`: if `True`, SQL cells will have linting enabled" - properties: - enabled: - type: boolean - sql_linter: - type: boolean - required: [] - title: DiagnosticsConfig - type: object DiscoverDataSourcesCommand: description: "Discover datasource connections from the live kernel environment\ \ and configuration.\n\n Attributes:\n request_id: Unique identifier\ diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts index 69006608a6d..4d4ce7c54bc 100644 --- a/packages/openapi/src/api.ts +++ b/packages/openapi/src/api.ts @@ -4631,15 +4631,6 @@ export interface components { /** @enum {unknown} */ type: "configuration" | "environment"; }; - /** - * DialectHidesWhen - * @description Hide this suggestion when a live SQL engine dialect contains a substring. - */ - DialectHidesWhen: { - /** @enum {unknown} */ - kind: "dialect"; - substrings: string[]; - }; /** * DiagnosticsConfig * @description Configuration options for diagnostics. @@ -4653,6 +4644,15 @@ export interface components { enabled?: boolean; sql_linter?: boolean; }; + /** + * DialectHidesWhen + * @description Hide this suggestion when a live SQL engine dialect contains a substring. + */ + DialectHidesWhen: { + /** @enum {unknown} */ + kind: "dialect"; + substrings: string[]; + }; /** * DiscoverDataSourcesCommand * @description Discover datasource connections from the live kernel environment and configuration. diff --git a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py index fc4ffae4c9e..ce9d87df308 100644 --- a/tests/_plugins/ui/_impl/dataframes/test_dataframe.py +++ b/tests/_plugins/ui/_impl/dataframes/test_dataframe.py @@ -369,6 +369,122 @@ def test_dataframe_download_csv_separator() -> None: assert "A;B" in csv_text assert "1;x" in csv_text + @staticmethod + @pytest.mark.skipif( + not HAS_DEPS, reason="optional dependencies not installed" + ) + def test_dataframe_download_empty_csv_separator_uses_default() -> None: + df = pd.DataFrame({"A": [1], "B": ["x"]}) + subject = ui.dataframe(df, download_csv_separator="") + + csv_url = subject._download_as(DownloadAsArgs(format="csv")).url + csv_text = from_data_uri(csv_url)[1].decode("utf-8") + + assert csv_text == df.to_csv(index=False) + + @staticmethod + @pytest.mark.skipif( + not HAS_DEPS, reason="optional dependencies not installed" + ) + def test_dataframe_download_csv_follows_pt_br() -> None: + from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + ) + + df = pd.DataFrame({"value": [1234.56], "text": ["unchanged.1"]}) + subject = ui.dataframe(df) + csv_text = from_data_uri( + subject._download_as( + DownloadAsArgs( + format="csv", + locale=ResolvedExportLocale("pt-BR", ","), + ) + ).url + )[1].decode("utf-8") + assert csv_text == "value;text\n1234,56;unchanged.1\n" + + @staticmethod + @pytest.mark.skipif( + not HAS_DEPS, reason="optional dependencies not installed" + ) + def test_dataframe_download_tsv_follows_pt_br() -> None: + from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + ) + + df = pd.DataFrame({"value": [1234.56], "text": ["unchanged.1"]}) + subject = ui.dataframe(df, download_csv_separator=";") + tsv_text = from_data_uri( + subject._download_as( + DownloadAsArgs( + format="tsv", + locale=ResolvedExportLocale("pt-BR", ","), + ) + ).url + )[1].decode("utf-8") + assert tsv_text == "value\ttext\n1234,56\tunchanged.1\n" + + @staticmethod + @pytest.mark.skipif( + not HAS_DEPS, reason="optional dependencies not installed" + ) + def test_dataframe_download_reports_malformed_locale() -> None: + from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + ) + + subject = ui.dataframe(pd.DataFrame({"value": [1]})) + + response = subject._download_as( + DownloadAsArgs(format="csv", locale=ResolvedExportLocale("", ",")) + ) + + assert response.url == "" + assert response.filename == "" + assert response.error == "Locale tag must be a non-empty string." + + @staticmethod + @pytest.mark.skipif( + not HAS_DEPS, reason="optional dependencies not installed" + ) + def test_dataframe_download_json_stays_locale_neutral() -> None: + from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + ) + + df = pd.DataFrame({"value": [1234.56], "text": ["unchanged.1"]}) + subject = ui.dataframe(df) + json_text = from_data_uri( + subject._download_as( + DownloadAsArgs( + format="json", + locale=ResolvedExportLocale("pt-BR", ","), + ) + ).url + )[1].decode("utf-8") + assert "1234.56" in json_text + assert "1234,56" not in json_text + + @staticmethod + @pytest.mark.skipif( + not HAS_DEPS, reason="optional dependencies not installed" + ) + def test_dataframe_download_parquet_ignores_locale() -> None: + from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + ) + + df = pd.DataFrame({"value": [1234.56], "text": ["unchanged.1"]}) + subject = ui.dataframe(df) + response = subject._download_as( + DownloadAsArgs( + format="parquet", + locale=ResolvedExportLocale("pt-BR", ","), + ) + ) + assert response.url.startswith("data:") + assert response.error is None + @staticmethod @pytest.mark.skipif( not HAS_DEPS, reason="optional dependencies not installed" diff --git a/tests/_plugins/ui/_impl/tables/test_default_table.py b/tests/_plugins/ui/_impl/tables/test_default_table.py index bd17ce76e5e..117a614e314 100644 --- a/tests/_plugins/ui/_impl/tables/test_default_table.py +++ b/tests/_plugins/ui/_impl/tables/test_default_table.py @@ -5,6 +5,7 @@ import string import unittest from datetime import date +from decimal import Decimal from pathlib import Path from typing import Any @@ -15,6 +16,7 @@ from marimo._output.hypertext import Html from marimo._plugins.ui._impl.table import SortArgs, _validate_header_tooltip from marimo._plugins.ui._impl.tables.default_table import DefaultTableManager +from marimo._plugins.ui._impl.tables.delimited import DelimitedDialect from marimo._plugins.ui._impl.tables.table_manager import ( TableCell, TableCoordinate, @@ -1251,6 +1253,53 @@ def test_validate_header_tooltip_invalid() -> None: _validate_header_tooltip(mapping, columns) +def test_to_delimited_str_formats_numbers_without_grouping() -> None: + manager = DefaultTableManager( + [ + { + "integer": 1234, + "fraction": 1234.56, + "text": "value.1,2;3", + "null": None, + } + ] + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert ( + result == 'integer;fraction;text;null\n1234;1234,56;"value.1,2;3";\n' + ) + + +def test_to_delimited_str_preserves_long_fractional_values() -> None: + manager = DefaultTableManager([{"value": Decimal("1234.567890123456")}]) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "value\n1234,567890123456\n" + + +def test_to_delimited_str_uses_unicode_decimal_separator() -> None: + manager = DefaultTableManager([{"value": 12.5}]) + result = manager.to_delimited_str(DelimitedDialect(",", "٫")) + assert result == "value\n12٫5\n" + + +def test_to_delimited_str_quotes_when_field_equals_decimal() -> None: + manager = DefaultTableManager([{"value": 12.5}]) + result = manager.to_delimited_str(DelimitedDialect(",", ",")) + assert result == 'value\n"12,5"\n' + + +def test_to_delimited_str_leaves_strings_unchanged() -> None: + manager = DefaultTableManager([{"text": '1.2,3;4\t5"6\n7'}]) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == 'text\n"1.2,3;4\t5""6\n7"\n' + + +def test_to_csv_str_remains_locale_neutral() -> None: + manager = DefaultTableManager([{"value": 12.5, "text": "1.2"}]) + assert manager.to_csv_str() == "value,text\n12.5,1.2\n" + assert manager.to_csv(separator="|") == b"value|text\n12.5|1.2\n" + + _column_name = st.text( alphabet=string.ascii_letters + string.digits + "_", min_size=1, diff --git a/tests/_plugins/ui/_impl/tables/test_delimited.py b/tests/_plugins/ui/_impl/tables/test_delimited.py new file mode 100644 index 00000000000..180a8d59d1b --- /dev/null +++ b/tests/_plugins/ui/_impl/tables/test_delimited.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from decimal import Decimal +from numbers import Real + +import pytest + +from marimo._plugins.ui._impl.tables.delimited import ( + ResolvedExportLocale, + resolve_delimited_dialect, +) +from marimo._utils.delimited import ( + DelimitedDialect, + format_delimited_number, + is_delimited_number, +) + + +def locale(tag: str, decimal_separator: str) -> ResolvedExportLocale: + return ResolvedExportLocale(tag=tag, decimal_separator=decimal_separator) + + +def _format_if_delimited_number(value: object) -> str | None: + from typing_extensions import assert_type + + if is_delimited_number(value): + assert_type(value, Real | Decimal) + return format_delimited_number(value, ",") + return None + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (1, "1"), + (1.25, "1,25"), + (Decimal("1.25"), "1,25"), + (True, None), + ("1.25", None), + ], +) +def test_delimited_number_narrowing( + value: object, expected: str | None +) -> None: + assert _format_if_delimited_number(value) == expected + + +def test_resolve_csv_comma_decimal_uses_semicolon_fields() -> None: + assert resolve_delimited_dialect( + "csv", locale("pt-BR", ","), None + ) == DelimitedDialect(";", ",") + + +def test_resolve_csv_explicit_separator_wins() -> None: + assert resolve_delimited_dialect( + "csv", locale("pt-BR", ","), "|" + ) == DelimitedDialect("|", ",") + + +def test_resolve_tsv_ignores_explicit_separator() -> None: + assert resolve_delimited_dialect( + "tsv", locale("pt-BR", ","), "|" + ) == DelimitedDialect("\t", ",") + + +def test_resolve_csv_without_locale_preserves_current_behavior() -> None: + assert resolve_delimited_dialect("csv", None, None) == DelimitedDialect( + ",", "." + ) + + +def test_resolve_tsv_without_locale_uses_tab_and_dot() -> None: + assert resolve_delimited_dialect("tsv", None, None) == DelimitedDialect( + "\t", "." + ) + + +def test_resolve_csv_decimal_point_uses_comma_fields() -> None: + assert resolve_delimited_dialect( + "csv", locale("en-US", "."), None + ) == DelimitedDialect(",", ".") + + +@pytest.mark.parametrize( + "bad_locale", + [ + locale("", ","), + locale("pt-BR", ",,"), + locale("pt-BR", "\r"), + locale("pt-BR", "\n"), + locale("pt-BR", "\0"), + locale("pt-BR", ""), + ], +) +def test_resolve_rejects_malformed_locale( + bad_locale: ResolvedExportLocale, +) -> None: + with pytest.raises(ValueError): + resolve_delimited_dialect("csv", bad_locale, None) diff --git a/tests/_plugins/ui/_impl/tables/test_narwhals_table.py b/tests/_plugins/ui/_impl/tables/test_narwhals_table.py new file mode 100644 index 00000000000..d2b8fcd5537 --- /dev/null +++ b/tests/_plugins/ui/_impl/tables/test_narwhals_table.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from decimal import Decimal + +import pytest + +from marimo._dependencies.dependencies import DependencyManager +from marimo._plugins.ui._impl.tables.delimited import DelimitedDialect +from marimo._plugins.ui._impl.tables.narwhals_table import NarwhalsTableManager + +HAS_DEPS = DependencyManager.polars.has() + + +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_to_delimited_str_pt_br() -> None: + import polars as pl + + manager = NarwhalsTableManager.from_dataframe( + pl.DataFrame( + { + "integer": [1234], + "fraction": [1234.567890123456], + "text": ["value.1,2;3"], + "null": [None], + } + ) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert ( + result + == 'integer;fraction;text;null\n1234;1234,567890123456;"value.1,2;3";\n' + ) + + +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_to_delimited_str_decimal_and_exponent() -> None: + import polars as pl + + manager = NarwhalsTableManager.from_dataframe( + pl.DataFrame( + { + "decimal": [Decimal("1234.567890123456")], + "exponent": [1.23e-10], + } + ) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "decimal;exponent\n1234,567890123456;1,23e-10\n" + + +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_to_delimited_str_non_finite() -> None: + import polars as pl + + manager = NarwhalsTableManager.from_dataframe( + pl.DataFrame({"value": [float("nan"), float("inf"), float("-inf")]}) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "value\nnan\ninf\n-inf\n" + + +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_to_delimited_str_unicode_decimal() -> None: + import polars as pl + + manager = NarwhalsTableManager.from_dataframe( + pl.DataFrame({"value": [12.5]}) + ) + result = manager.to_delimited_str(DelimitedDialect(",", "٫")) + assert result == "value\n12٫5\n" + + +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_to_csv_str_remains_locale_neutral() -> None: + import polars as pl + + manager = NarwhalsTableManager.from_dataframe( + pl.DataFrame({"value": [12.5], "text": ["1.2"]}) + ) + assert manager.to_csv_str() == "value,text\n12.5,1.2\n" diff --git a/tests/_plugins/ui/_impl/tables/test_pandas_table.py b/tests/_plugins/ui/_impl/tables/test_pandas_table.py index aaccc09e16b..b7d632515e1 100644 --- a/tests/_plugins/ui/_impl/tables/test_pandas_table.py +++ b/tests/_plugins/ui/_impl/tables/test_pandas_table.py @@ -19,6 +19,7 @@ from marimo._dependencies.dependencies import DependencyManager from marimo._output.data.data import BIGINT_KEY, sanitize_json_bigint from marimo._plugins.ui._impl.table import SortArgs +from marimo._plugins.ui._impl.tables.delimited import DelimitedDialect from marimo._plugins.ui._impl.tables.format import FormatMapping from marimo._plugins.ui._impl.tables.pandas_table import ( PandasTableManagerFactory, @@ -245,6 +246,155 @@ def test_to_csv(self) -> None: expected_csv = self.data.to_csv(index=False).encode("utf-8") assert self.manager.to_csv() == expected_csv + def test_to_delimited_str_pt_br(self) -> None: + df = pd.DataFrame( + { + "integer": [1234], + "fraction": [1234.567890123456], + "text": ["value.1,2;3"], + "null": [None], + } + ) + manager = PandasTableManagerFactory.create()(df) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert ( + result + == 'integer;fraction;text;null\n1234;1234,567890123456;"value.1,2;3";\n' + ) + + def test_to_delimited_str_uses_native_writer(self) -> None: + df = pd.DataFrame({"value": [1.5]}) + manager = PandasTableManagerFactory.create()(df) + + with patch.object(df, "to_csv", wraps=df.to_csv) as writer: + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "value\n1,5\n" + ) + + writer.assert_called_once_with( + index=False, + sep=";", + decimal=",", + lineterminator="\n", + ) + + def test_to_delimited_str_uses_platform_newline_for_default(self) -> None: + df = pd.DataFrame({"value": [1.5]}) + manager = PandasTableManagerFactory.create()(df) + + with ( + patch("os.linesep", "\r\n"), + patch.object(df, "to_csv", wraps=df.to_csv) as writer, + ): + assert ( + manager.to_delimited_str(DelimitedDialect(",", ".")) + == "value\r\n1.5\r\n" + ) + + writer.assert_called_once_with( + index=False, + sep=",", + decimal=".", + lineterminator=None, + ) + + def test_to_delimited_str_decimal_and_exponent(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame( + { + "decimal": [decimal.Decimal("1234.567890123456")], + "exponent": [1.23e-10], + } + ) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "decimal;exponent\n1234,567890123456;1,23e-10\n" + + def test_to_delimited_str_localizes_mixed_object_numbers(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame( + { + "mixed": pd.Series( + [1.25, decimal.Decimal("2.5")], dtype=object + ), + "text": ["1.25", "unchanged"], + } + ) + ) + + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + + assert result == "mixed;text\n1,25;1.25\n2,5;unchanged\n" + + def test_to_delimited_str_non_finite(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame( + {"value": [float("nan"), float("inf"), float("-inf")]} + ) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "value\nnan\ninf\n-inf\n" + + def test_to_delimited_str_localizes_float_column_with_nan(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame({"value": [1.5, float("nan")]}) + ) + + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + + assert result == "value\n1,5\nnan\n" + + def test_to_delimited_str_unicode_decimal(self) -> None: + manager = PandasTableManagerFactory.create()( + pd.DataFrame({"value": [12.5]}) + ) + result = manager.to_delimited_str(DelimitedDialect(",", "٫")) + assert result == "value\n12٫5\n" + + def test_to_csv_str_remains_locale_neutral(self) -> None: + df = pd.DataFrame({"value": [12.5], "text": ["1.2"]}) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_csv_str() == df.to_csv(index=False) + + def test_to_csv_str_preserves_unnamed_index_header(self) -> None: + df = pd.DataFrame({"value": [1]}, index=[10]) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_csv_str() == df.to_csv(index=True) + + def test_to_delimited_str_preserves_unnamed_index_header(self) -> None: + df = pd.DataFrame({"value": [1.5]}, index=[10]) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + ";value\n10;1,5\n" + ) + + def test_to_delimited_str_localizes_decimal_index(self) -> None: + df = pd.DataFrame( + {"value": [decimal.Decimal("2.5")]}, + index=[decimal.Decimal("1.5")], + ) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + ";value\n1,5;2,5\n" + ) + + def test_to_delimited_str_preserves_unnamed_multi_index_headers( + self, + ) -> None: + df = pd.DataFrame( + {"value": [1.5]}, + index=pd.MultiIndex.from_tuples([("a", 1)]), + ) + manager = PandasTableManagerFactory.create()(df) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + ";;value\na;1;1,5\n" + ) + def test_to_csv_datetime(self) -> None: D = pd.to_datetime("2024-12-17", errors="coerce") diff --git a/tests/_plugins/ui/_impl/tables/test_polars_table.py b/tests/_plugins/ui/_impl/tables/test_polars_table.py index 5e771dcfb26..af413b89e5b 100644 --- a/tests/_plugins/ui/_impl/tables/test_polars_table.py +++ b/tests/_plugins/ui/_impl/tables/test_polars_table.py @@ -8,6 +8,7 @@ from enum import Enum from math import isnan from typing import Any +from unittest import mock import narwhals.stable.v2 as nw import pytest @@ -16,9 +17,11 @@ from marimo._dependencies.dependencies import DependencyManager from marimo._output.data.data import BIGINT_KEY from marimo._plugins.ui._impl.table import SortArgs +from marimo._plugins.ui._impl.tables.delimited import DelimitedDialect from marimo._plugins.ui._impl.tables.format import FormatMapping from marimo._plugins.ui._impl.tables.polars_table import ( PolarsTableManagerFactory, + _supports_decimal_comma, ) from marimo._plugins.ui._impl.tables.table_manager import TableManager from marimo._utils.platform import is_windows @@ -171,6 +174,196 @@ def test_package_name(self) -> None: def test_to_csv(self) -> None: assert isinstance(self.manager.to_csv(), bytes) + def test_to_delimited_str_pt_br(self) -> None: + import polars as pl + + df = pl.DataFrame( + { + "integer": [1234], + "fraction": [1234.567890123456], + "text": ["value.1,2;3"], + "null": [None], + } + ) + manager = self.factory.create()(df) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert ( + result + == 'integer;fraction;text;null\n1234;1234,567890123456;"value.1,2;3";\n' + ) + + def test_to_delimited_str_uses_native_decimal_comma_writer(self) -> None: + import polars as pl + + if not _supports_decimal_comma(): + pytest.skip("Polars does not support native decimal commas") + + manager = self.factory.create()(pl.DataFrame({"value": [1.5]})) + with mock.patch( + "marimo._plugins.ui._impl.tables.polars_table.dataframe_to_csv" + ) as fallback: + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "value\n1,5\n" + ) + fallback.assert_not_called() + + def test_to_delimited_str_preserves_temporal_formatting(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame( + { + "datetime": [ + datetime.datetime(2026, 1, 2, 3, 4, 5, 123456) + ], + "date": [datetime.date(2026, 1, 2)], + "time": [datetime.time(3, 4, 5, 123456)], + } + ) + ) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "datetime;date;time\n" + "2026-01-02 03:04:05.123456;2026-01-02;" + "03:04:05.123456\n" + ) + + def test_to_delimited_str_preserves_binary_formatting(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame({"binary": [b"abc", bytes([0, 255])]}) + ) + + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "binary\nb'abc'\nb'\\x00\\xff'\n" + ) + + def test_to_delimited_str_uses_native_tsv_writer(self) -> None: + import polars as pl + + manager = self.factory.create()(pl.DataFrame({"value": [1.5]})) + with mock.patch( + "marimo._plugins.ui._impl.tables.polars_table.dataframe_to_csv" + ) as fallback: + assert manager.to_delimited_str(DelimitedDialect("\t", ".")) == ( + "value\n1.5\n" + ) + fallback.assert_not_called() + + def test_to_delimited_str_normalizes_nested_values(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame( + { + "struct": [{"value": 1}], + "list": [[1, 2]], + "array": pl.Series([[3, 4]], dtype=pl.Array(pl.Int64, 2)), + "duration": [datetime.timedelta(days=1)], + "object": pl.Series( + "object", [{"value": 2}], dtype=pl.Object + ), + } + ) + ) + + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + + assert ( + result == "struct;list;array;duration;object\n" + '"{""value"":1}";1,2;3,4;1d;{\'value\': 2}\n' + ) + + def test_to_delimited_str_normalizes_nested_containers(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame( + { + "list_struct": pl.Series( + [ + [{"a": 1}, {"a": 2}], + [], + None, + ], + dtype=pl.List(pl.Struct({"a": pl.Int64})), + ), + "list_list": pl.Series( + [ + [[1, 2], [3]], + [], + None, + ], + dtype=pl.List(pl.List(pl.Int64)), + ), + "array_struct": pl.Series( + [ + [{"b": "x"}, {"b": "y"}], + [{"b": "m"}, {"b": "n"}], + None, + ], + dtype=pl.Array(pl.Struct({"b": pl.String}), 2), + ), + } + ) + ) + + assert manager.to_csv_str() == ( + "list_struct,list_list,array_struct\n" + '"[{""a"":1},{""a"":2}]","[[1,2],[3]]",' + '"[{""b"":""x""},{""b"":""y""}]"\n' + '[],[],"[{""b"":""m""},{""b"":""n""}]"\n' + ",,\n" + ) + assert manager.to_delimited_str(DelimitedDialect(";", ",")) == ( + "list_struct;list_list;array_struct\n" + '"[{""a"":1},{""a"":2}]";[[1,2],[3]];' + '"[{""b"":""x""},{""b"":""y""}]"\n' + '[];[];"[{""b"":""m""},{""b"":""n""}]"\n' + ";;\n" + ) + + def test_to_delimited_str_decimal_and_exponent(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame( + { + "decimal": [decimal.Decimal("1234.567890123456")], + "exponent": [1.23e-10], + } + ) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "decimal;exponent\n1234,567890123456;1,23e-10\n" + + def test_to_delimited_str_non_finite(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame( + {"value": [float("nan"), float("inf"), float("-inf")]} + ) + ) + result = manager.to_delimited_str(DelimitedDialect(";", ",")) + assert result == "value\nnan\ninf\n-inf\n" + + def test_to_delimited_str_unicode_decimal(self) -> None: + import polars as pl + + manager = self.factory.create()(pl.DataFrame({"value": [12.5]})) + result = manager.to_delimited_str(DelimitedDialect(",", "٫")) + assert result == "value\n12٫5\n" + + def test_to_csv_str_remains_locale_neutral(self) -> None: + import polars as pl + + manager = self.factory.create()( + pl.DataFrame({"value": [12.5], "text": ["1.2"]}) + ) + assert manager.to_csv_str() == "value,text\n12.5,1.2\n" + @pytest.mark.skipif( is_windows(), reason="Windows doesn't show microseconds unicode properly", diff --git a/tests/_plugins/ui/_impl/test_table.py b/tests/_plugins/ui/_impl/test_table.py index bcbeb04668c..455a4ec8779 100644 --- a/tests/_plugins/ui/_impl/test_table.py +++ b/tests/_plugins/ui/_impl/test_table.py @@ -1700,6 +1700,77 @@ def test_download_as_ignores_cell_selection() -> None: assert int(rows[0]["a"]) == 2 +def test_download_as_args_locale_is_optional() -> None: + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + from marimo._utils.parse_dataclass import parse_raw + + args = parse_raw({"format": "csv"}, DownloadAsArgs) + assert args.format == "csv" + assert args.locale is None + + args = parse_raw( + { + "format": "tsv", + "locale": {"tag": "pt-BR", "decimal_separator": ","}, + }, + DownloadAsArgs, + ) + assert args.locale == ResolvedExportLocale("pt-BR", ",") + + +def test_table_download_csv_follows_pt_br() -> None: + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + + table = ui.table([{"value": 1234.56, "text": "unchanged.1"}]) + url = table._download_as( + DownloadAsArgs(format="csv", locale=ResolvedExportLocale("pt-BR", ",")) + ).url + assert from_data_uri(url)[1].decode("utf-8") == ( + "value;text\n1234,56;unchanged.1\n" + ) + + +def test_table_download_tsv_follows_pt_br() -> None: + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + + table = ui.table([{"value": 1234.56, "text": "unchanged.1"}]) + url = table._download_as( + DownloadAsArgs(format="tsv", locale=ResolvedExportLocale("pt-BR", ",")) + ).url + assert from_data_uri(url)[1].decode("utf-8") == ( + "value\ttext\n1234,56\tunchanged.1\n" + ) + + +def test_table_download_json_stays_locale_neutral() -> None: + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + + table = ui.table([{"value": 1234.56, "text": "unchanged.1"}]) + text = from_data_uri( + table._download_as( + DownloadAsArgs( + format="json", locale=ResolvedExportLocale("pt-BR", ",") + ) + ).url + )[1].decode("utf-8") + assert "1234.56" in text + assert "1234,56" not in text + + +def test_table_download_reports_malformed_locale() -> None: + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + + table = ui.table([{"value": 1}]) + + response = table._download_as( + DownloadAsArgs(format="csv", locale=ResolvedExportLocale("", ",")) + ) + + assert response.url == "" + assert response.filename == "" + assert response.error == "Locale tag must be a non-empty string." + + def test_download_as_parquet_without_libs_reports_missing_packages( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py b/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py index cf189da72ee..91177d39a3b 100644 --- a/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py +++ b/tests/_plugins/ui/_impl/utils/test_dataframe_utils.py @@ -137,6 +137,105 @@ def test_download_as_csv_honors_separator_option() -> None: assert text.splitlines()[0] == "a;b" +def _download_text(url: str) -> str: + from marimo._utils.data_uri import from_data_uri + + return from_data_uri(url)[1].decode("utf-8") + + +def test_download_as_pt_br_csv() -> None: + from marimo._plugins.ui._impl.tables.default_table import ( + DefaultTableManager, + ) + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, + download_as, + ) + + manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) + url, _ = download_as( + manager, + "csv", + options=DownloadOptions(locale=ResolvedExportLocale("pt-BR", ",")), + ) + assert _download_text(url) == "value;text\n1234,56;unchanged.1\n" + + +def test_download_as_pt_br_tsv() -> None: + from marimo._plugins.ui._impl.tables.default_table import ( + DefaultTableManager, + ) + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, + download_as, + ) + + manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) + url, _ = download_as( + manager, + "tsv", + options=DownloadOptions(locale=ResolvedExportLocale("pt-BR", ",")), + ) + assert _download_text(url) == "value\ttext\n1234,56\tunchanged.1\n" + + +def test_download_as_legacy_csv_without_locale() -> None: + from marimo._plugins.ui._impl.tables.default_table import ( + DefaultTableManager, + ) + from marimo._plugins.ui._impl.utils.dataframe import download_as + + manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) + url, _ = download_as(manager, "csv") + assert _download_text(url) == "value,text\n1234.56,unchanged.1\n" + + +def test_download_as_explicit_csv_separator_wins() -> None: + from marimo._plugins.ui._impl.tables.default_table import ( + DefaultTableManager, + ) + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + from marimo._plugins.ui._impl.utils.dataframe import ( + DelimitedOptions, + DownloadOptions, + download_as, + ) + + manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) + url, _ = download_as( + manager, + "csv", + options=DownloadOptions( + delimited=DelimitedOptions(separator="|"), + locale=ResolvedExportLocale("pt-BR", ","), + ), + ) + assert _download_text(url) == "value|text\n1234,56|unchanged.1\n" + + +def test_download_as_json_stays_locale_neutral() -> None: + from marimo._plugins.ui._impl.tables.default_table import ( + DefaultTableManager, + ) + from marimo._plugins.ui._impl.tables.delimited import ResolvedExportLocale + from marimo._plugins.ui._impl.utils.dataframe import ( + DownloadOptions, + download_as, + ) + + manager = DefaultTableManager([{"value": 1234.56, "text": "unchanged.1"}]) + url, _ = download_as( + manager, + "json", + options=DownloadOptions(locale=ResolvedExportLocale("pt-BR", ",")), + ) + text = _download_text(url) + assert "1234.56" in text + assert "1234,56" not in text + + def test_union_tolerates_string_type_aliases() -> None: """Verify that Union[] handles string-valued type aliases (narwhals compat). diff --git a/tests/_utils/test_narwhals_utils.py b/tests/_utils/test_narwhals_utils.py index a4c728abab4..d1c94f4abee 100644 --- a/tests/_utils/test_narwhals_utils.py +++ b/tests/_utils/test_narwhals_utils.py @@ -6,6 +6,7 @@ import pytest from marimo._dependencies.dependencies import DependencyManager +from marimo._utils.delimited import DelimitedDialect from marimo._utils.narwhals_utils import ( assert_narwhals_dataframe_or_lazyframe, assert_narwhals_series, @@ -118,12 +119,41 @@ def test_dataframe_to_csv(df: IntoDataFrame) -> None: @pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") def test_dataframe_to_csv_with_separator(df: IntoDataFrame) -> None: df_wrapped = nw.from_native(df) - csv = dataframe_to_csv(df_wrapped, separator=";") + csv = dataframe_to_csv(df_wrapped, dialect=DelimitedDialect(";", ".")) assert "a;b" in csv assert "1;x" in csv assert "2;y" in csv +@pytest.mark.parametrize( + "df", + create_dataframes( + { + "integer": [1234], + "fraction": [1234.567890123456], + "text": ["value.1,2;3"], + } + ), +) +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_dataframe_to_csv_with_comma_decimal(df: IntoDataFrame) -> None: + csv = dataframe_to_csv( + nw.from_native(df), dialect=DelimitedDialect(";", ",") + ) + assert "integer;fraction;text" in csv + assert "1234;1234,567890123456;" in csv + assert "value.1,2;3" in csv + + +@pytest.mark.parametrize("df", create_dataframes({"value": [12.5]})) +@pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") +def test_dataframe_to_csv_with_unicode_decimal(df: IntoDataFrame) -> None: + csv = dataframe_to_csv( + nw.from_native(df), dialect=DelimitedDialect(",", "٫") + ) + assert csv == "value\n12٫5\n" + + @pytest.mark.skipif(not HAS_DEPS, reason="optional dependencies not installed") def test_narwhals_type_checks(): assert is_narwhals_integer_type(nw.Int64)