diff --git a/src/litdata/utilities/parquet.py b/src/litdata/utilities/parquet.py index f4fcad0ef..44cdc1eff 100644 --- a/src/litdata/utilities/parquet.py +++ b/src/litdata/utilities/parquet.py @@ -78,7 +78,7 @@ def __init__( self, dir_path: str | Dir | None, cache_path: str | None = None, - storage_options: dict | None = {}, + storage_options: dict | None = None, num_workers: int = 4, ): if not _PYARROW_AVAILABLE: @@ -150,7 +150,7 @@ def __init__( for provider in _CLOUD_PROVIDER: if self.dir.url.startswith(provider): # Initialize the cloud filesystem - self.fs = fsspec.filesystem(provider, *self.storage_options) + self.fs = fsspec.filesystem(provider, **self.storage_options) print(f"using provider: {provider}") break @@ -304,7 +304,7 @@ def write_index(self, chunks_info: list[dict[str, Any]], config: dict[str, Any]) def get_parquet_indexer_cls( dir_path: str, cache_path: str | None = None, - storage_options: dict | None = {}, + storage_options: dict | None = None, num_workers: int = 4, ) -> ParquetDir: """Get the appropriate ParquetDir class based on the directory path scheme.