feat(scan): add lance_namespace_query_table setting to opt out of query_table scans

docs/rest.md

@@ -111,6 +111,15 @@ reads the Arrow IPC response. Reads therefore do not require DuckDB to open the
 underlying object-store location directly. Unsupported predicates remain in
 DuckDB, and an `OFFSET` without a `LIMIT` is also evaluated by DuckDB.
 
+If the namespace server does not implement `query_table`, disable the
+`lance_namespace_query_table` setting to scan by opening the underlying
+dataset directly (table locations are still resolved through the namespace,
+and reads then require direct storage access):
+
+```sql
+SET lance_namespace_query_table = false;
+```
+
 ```sql
 -- Select all rows
 SELECT * FROM users;

docs/sql.md

@@ -173,6 +173,9 @@ Scans of REST namespace tables use the Lance Namespace `query_table` API.
 Projection, supported filters, and `LIMIT`/`OFFSET` pairs are pushed into the
 request; unsupported filters and standalone `OFFSET` operations remain in
 DuckDB. Directory namespace scans continue to read the Lance dataset directly.
+Set `lance_namespace_query_table = false` to have REST namespace scans open
+the underlying dataset directly as well, e.g. when the namespace server does
+not implement `query_table`.
 
 ## Write datasets
 

src/lance_extension.cpp

@@ -42,6 +42,12 @@ void LanceExtension::Load(ExtensionLoader &loader) {
                             "Enable deferred materialization for heavy columns "
                             "when filter pushdown fails",
                             LogicalType::BOOLEAN, Value::BOOLEAN(true));
+  config.AddExtensionOption(
+      "lance_namespace_query_table",
+      "Route REST namespace table scans through the Lance Namespace "
+      "query_table API. Disable to open the underlying dataset directly, "
+      "e.g. against namespace servers that do not implement query_table",
+      LogicalType::BOOLEAN, Value::BOOLEAN(true));
   RegisterLanceScanOptimizer(config);
   RegisterLanceStorage(config);
   RegisterLanceReplacement(config);

src/lance_scan.cpp

@@ -303,6 +303,8 @@ static constexpr column_t LANCE_COLUMN_IDENTIFIER_ROW_ID =
 static constexpr const char *LANCE_ROW_ID_COLUMN_NAME = "_rowid";
 static constexpr const char *LANCE_DEFERRED_SETTING =
     "lance_deferred_materialization";
+static constexpr const char *LANCE_NAMESPACE_QUERY_TABLE_SETTING =
+    "lance_namespace_query_table";
 static constexpr uint64_t DEFERRED_AVG_BYTES_THRESHOLD = 1024;
 
 static bool IsLanceVirtualRowIdColumnId(column_t col_id) {
@@ -355,6 +357,17 @@ static bool LanceDeferredMaterializationEnabled(ClientContext &context) {
   return true; // default on
 }
 
+// When disabled, REST namespace table scans skip the query_table API and open
+// the underlying dataset directly (the pre-query_table behaviour), which is
+// required against namespace servers that do not implement query_table.
+static bool LanceNamespaceQueryTableEnabled(ClientContext &context) {
+  Value val;
+  if (context.TryGetCurrentSetting(LANCE_NAMESPACE_QUERY_TABLE_SETTING, val)) {
+    return val.GetValue<bool>();
+  }
+  return true; // default on
+}
+
 static virtual_column_map_t LanceGetVirtualColumns(ClientContext &,
                                                    optional_ptr<FunctionData>) {
   virtual_column_map_t result;
@@ -3843,7 +3856,8 @@ LanceTableEntry::GetScanFunction(ClientContext &context,
   result->table_entry = this;
   result->file_path = dataset_uri;
 
-  if (IsNamespaceBacked() && NamespaceConfig().IsRest()) {
+  if (IsNamespaceBacked() && NamespaceConfig().IsRest() &&
+      LanceNamespaceQueryTableEnabled(context)) {
     result->namespace_query_config =
         make_uniq<LanceNamespaceTableConfig>(NamespaceConfig());
     PopulateNamespaceQueryScanSchema(context, *this, *result);

test/sql/namespace_rest_scan_direct.test

@@ -0,0 +1,50 @@
+# name: test/sql/namespace_rest_scan_direct.test
+# description: REST namespace scans fall back to direct dataset reads when lance_namespace_query_table is disabled (requires external service)
+# group: [sql]
+
+require-env LANCE_TEST_NAMESPACE 1
+
+test-env LANCE_NAMESPACE_ENDPOINT http://127.0.0.1:2333
+
+test-env LANCE_NAMESPACE_ID default
+
+test-env LANCE_NAMESPACE_TABLE default.table
+
+require lance
+
+statement ok
+ATTACH '${LANCE_NAMESPACE_ID}' AS ns (TYPE LANCE, ENDPOINT '${LANCE_NAMESPACE_ENDPOINT}');
+
+# Default: scans go through the namespace query_table API.
+query II
+EXPLAIN (FORMAT JSON)
+SELECT * FROM ns.main."${LANCE_NAMESPACE_TABLE}" LIMIT 1;
+----
+physical_plan	<REGEX>:[\s\S]*"Lance Scan Backend": "namespace_query_table"[\s\S]*
+
+statement ok
+SET lance_namespace_query_table = false;
+
+# Disabled: scans open the underlying dataset directly.
+query II
+EXPLAIN (FORMAT JSON)
+SELECT * FROM ns.main."${LANCE_NAMESPACE_TABLE}" LIMIT 1;
+----
+physical_plan	<REGEX>:[\s\S]*"Lance Scan Backend": "dataset"[\s\S]*
+
+query I
+SELECT count(*) FROM ns.main."${LANCE_NAMESPACE_TABLE}"
+----
+<REGEX>:[0-9]+
+
+statement ok
+SET lance_namespace_query_table = true;
+
+query II
+EXPLAIN (FORMAT JSON)
+SELECT * FROM ns.main."${LANCE_NAMESPACE_TABLE}" LIMIT 1;
+----
+physical_plan	<REGEX>:[\s\S]*"Lance Scan Backend": "namespace_query_table"[\s\S]*
+
+statement ok
+DETACH ns;