diff --git a/docs/rest.md b/docs/rest.md index 94aecaf..59c2677 100644 --- a/docs/rest.md +++ b/docs/rest.md @@ -111,6 +111,15 @@ reads the Arrow IPC response. Reads therefore do not require DuckDB to open the underlying object-store location directly. Unsupported predicates remain in DuckDB, and an `OFFSET` without a `LIMIT` is also evaluated by DuckDB. +If the namespace server does not implement `query_table`, disable the +`lance_namespace_query_table` setting to scan by opening the underlying +dataset directly (table locations are still resolved through the namespace, +and reads then require direct storage access): + +```sql +SET lance_namespace_query_table = false; +``` + ```sql -- Select all rows SELECT * FROM users; diff --git a/docs/sql.md b/docs/sql.md index d002eb3..39c1522 100644 --- a/docs/sql.md +++ b/docs/sql.md @@ -173,6 +173,9 @@ Scans of REST namespace tables use the Lance Namespace `query_table` API. Projection, supported filters, and `LIMIT`/`OFFSET` pairs are pushed into the request; unsupported filters and standalone `OFFSET` operations remain in DuckDB. Directory namespace scans continue to read the Lance dataset directly. +Set `lance_namespace_query_table = false` to have REST namespace scans open +the underlying dataset directly as well, e.g. when the namespace server does +not implement `query_table`. ## Write datasets diff --git a/src/lance_extension.cpp b/src/lance_extension.cpp index 93fa756..226a344 100644 --- a/src/lance_extension.cpp +++ b/src/lance_extension.cpp @@ -42,6 +42,12 @@ void LanceExtension::Load(ExtensionLoader &loader) { "Enable deferred materialization for heavy columns " "when filter pushdown fails", LogicalType::BOOLEAN, Value::BOOLEAN(true)); + config.AddExtensionOption( + "lance_namespace_query_table", + "Route REST namespace table scans through the Lance Namespace " + "query_table API. Disable to open the underlying dataset directly, " + "e.g. against namespace servers that do not implement query_table", + LogicalType::BOOLEAN, Value::BOOLEAN(true)); RegisterLanceScanOptimizer(config); RegisterLanceStorage(config); RegisterLanceReplacement(config); diff --git a/src/lance_scan.cpp b/src/lance_scan.cpp index 8058a91..890bccf 100644 --- a/src/lance_scan.cpp +++ b/src/lance_scan.cpp @@ -303,6 +303,8 @@ static constexpr column_t LANCE_COLUMN_IDENTIFIER_ROW_ID = static constexpr const char *LANCE_ROW_ID_COLUMN_NAME = "_rowid"; static constexpr const char *LANCE_DEFERRED_SETTING = "lance_deferred_materialization"; +static constexpr const char *LANCE_NAMESPACE_QUERY_TABLE_SETTING = + "lance_namespace_query_table"; static constexpr uint64_t DEFERRED_AVG_BYTES_THRESHOLD = 1024; static bool IsLanceVirtualRowIdColumnId(column_t col_id) { @@ -355,6 +357,17 @@ static bool LanceDeferredMaterializationEnabled(ClientContext &context) { return true; // default on } +// When disabled, REST namespace table scans skip the query_table API and open +// the underlying dataset directly (the pre-query_table behaviour), which is +// required against namespace servers that do not implement query_table. +static bool LanceNamespaceQueryTableEnabled(ClientContext &context) { + Value val; + if (context.TryGetCurrentSetting(LANCE_NAMESPACE_QUERY_TABLE_SETTING, val)) { + return val.GetValue(); + } + return true; // default on +} + static virtual_column_map_t LanceGetVirtualColumns(ClientContext &, optional_ptr) { virtual_column_map_t result; @@ -3843,7 +3856,8 @@ LanceTableEntry::GetScanFunction(ClientContext &context, result->table_entry = this; result->file_path = dataset_uri; - if (IsNamespaceBacked() && NamespaceConfig().IsRest()) { + if (IsNamespaceBacked() && NamespaceConfig().IsRest() && + LanceNamespaceQueryTableEnabled(context)) { result->namespace_query_config = make_uniq(NamespaceConfig()); PopulateNamespaceQueryScanSchema(context, *this, *result); diff --git a/test/sql/namespace_rest_scan_direct.test b/test/sql/namespace_rest_scan_direct.test new file mode 100644 index 0000000..75143a8 --- /dev/null +++ b/test/sql/namespace_rest_scan_direct.test @@ -0,0 +1,50 @@ +# name: test/sql/namespace_rest_scan_direct.test +# description: REST namespace scans fall back to direct dataset reads when lance_namespace_query_table is disabled (requires external service) +# group: [sql] + +require-env LANCE_TEST_NAMESPACE 1 + +test-env LANCE_NAMESPACE_ENDPOINT http://127.0.0.1:2333 + +test-env LANCE_NAMESPACE_ID default + +test-env LANCE_NAMESPACE_TABLE default.table + +require lance + +statement ok +ATTACH '${LANCE_NAMESPACE_ID}' AS ns (TYPE LANCE, ENDPOINT '${LANCE_NAMESPACE_ENDPOINT}'); + +# Default: scans go through the namespace query_table API. +query II +EXPLAIN (FORMAT JSON) +SELECT * FROM ns.main."${LANCE_NAMESPACE_TABLE}" LIMIT 1; +---- +physical_plan :[\s\S]*"Lance Scan Backend": "namespace_query_table"[\s\S]* + +statement ok +SET lance_namespace_query_table = false; + +# Disabled: scans open the underlying dataset directly. +query II +EXPLAIN (FORMAT JSON) +SELECT * FROM ns.main."${LANCE_NAMESPACE_TABLE}" LIMIT 1; +---- +physical_plan :[\s\S]*"Lance Scan Backend": "dataset"[\s\S]* + +query I +SELECT count(*) FROM ns.main."${LANCE_NAMESPACE_TABLE}" +---- +:[0-9]+ + +statement ok +SET lance_namespace_query_table = true; + +query II +EXPLAIN (FORMAT JSON) +SELECT * FROM ns.main."${LANCE_NAMESPACE_TABLE}" LIMIT 1; +---- +physical_plan :[\s\S]*"Lance Scan Backend": "namespace_query_table"[\s\S]* + +statement ok +DETACH ns;