From 09a11d5bae9bdf961021e3c7953e3e6b57d46a89 Mon Sep 17 00:00:00 2001 From: arrowbowang Date: Wed, 12 Aug 2026 19:47:30 +0800 Subject: [PATCH 1/3] fix(catalog): refresh schema after table replacement Preserve Lance's zero-row overwrite commit semantics for schema-only CREATE OR REPLACE TABLE operations. After a successful overwrite, invalidate only the replaced directory dataset cache entry and evict its stale attached-catalog entry so lazy discovery reloads the committed schema. Leave cached handles and catalog state unchanged when the writer fails, and preserve cross-query reuse for untouched datasets. Add regression coverage for schema refresh, failed replacements, precise cache invalidation, and untouched cache reuse. --- src/lance_storage.cpp | 41 +++++++++++ test/sql/namespace_replace_schema.test | 97 ++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 test/sql/namespace_replace_schema.test diff --git a/src/lance_storage.cpp b/src/lance_storage.cpp index 4c52802..3fff2e8 100644 --- a/src/lance_storage.cpp +++ b/src/lance_storage.cpp @@ -596,6 +596,12 @@ static string GetDatasetDirName(const string &table_name) { return table_name + ".lance"; } +class LanceSchemaEntry; + +static void EvictLanceTableCatalogEntry(LanceSchemaEntry &schema, + CatalogTransaction transaction, + const string &table_name); + static bool IsSafeDatasetTableName(const string &name) { if (name.empty()) { return false; @@ -885,6 +891,7 @@ class LanceSchemaEntry final : public DuckSchemaEntry { string dataset_path; vector option_keys; vector option_values; + bool refresh_directory_entry = false; if (rest_ns) { unordered_map overrides; @@ -1012,6 +1019,9 @@ class LanceSchemaEntry final : public DuckSchemaEntry { exists) { throw IOException("Lance dataset already exists: " + dataset_path); } + refresh_directory_entry = + exists && + create_info.on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT; option_keys = directory_ns->option_keys; option_values = directory_ns->option_values; @@ -1058,6 +1068,13 @@ class LanceSchemaEntry final : public DuckSchemaEntry { LanceFormatErrorSuffix()); } + if (refresh_directory_entry) { + auto cache_key = LanceBuildResolvedPathDatasetCacheKey( + dataset_path, option_keys, option_values); + LanceInvalidateDatasetCache(context, cache_key); + EvictLanceTableCatalogEntry(*this, transaction, create_info.table); + } + // Best-effort persistence of DuckDB column defaults in Lance field // metadata. Lance itself does not currently expose defaults through // DuckDB's catalog, but we can still use the metadata during UPDATE @@ -1102,6 +1119,30 @@ class LanceSchemaEntry final : public DuckSchemaEntry { DefaultGenerator *table_default_generator = nullptr; }; +static void EvictLanceTableCatalogEntry(LanceSchemaEntry &schema, + CatalogTransaction transaction, + const string &table_name) { + auto &set = schema.GetCatalogSet(CatalogType::TABLE_ENTRY); + auto existing_entry = set.GetEntry(transaction, table_name); + if (!existing_entry) { + return; + } + auto existing_type = existing_entry->type; + if (existing_type != CatalogType::TABLE_ENTRY && + existing_type != CatalogType::VIEW_ENTRY) { + throw InternalException( + "Unexpected catalog entry type for Lance table '%s': %s", table_name, + CatalogTypeToString(existing_type)); + } + auto system_transaction = + CatalogTransaction::GetSystemTransaction(schema.catalog.GetDatabase()); + if (!set.DropEntry(system_transaction, table_name, false, true)) { + throw InternalException("Could not drop catalog entry for Lance table '%s'", + table_name); + } + set.CleanupEntry(*existing_entry); +} + class LanceDuckCatalog final : public DuckCatalog { public: using DuckCatalog::PlanDelete; diff --git a/test/sql/namespace_replace_schema.test b/test/sql/namespace_replace_schema.test new file mode 100644 index 0000000..bde1935 --- /dev/null +++ b/test/sql/namespace_replace_schema.test @@ -0,0 +1,97 @@ +# name: test/sql/namespace_replace_schema.test +# description: CREATE OR REPLACE TABLE replaces schema after INSERT (directory namespace) +# group: [sql] + +require lance + +statement ok +ATTACH 'test/.tmp/nsroot_replace_schema_126' AS ns (TYPE LANCE); + +statement ok +CREATE OR REPLACE TABLE ns.main.replace_schema_t (id BIGINT); + +statement ok +INSERT INTO ns.main.replace_schema_t VALUES (42); + +statement ok +CREATE OR REPLACE TABLE ns.main.untouched_t (id BIGINT); + +statement ok +INSERT INTO ns.main.untouched_t VALUES (7); + +query I +SELECT count(*) FROM ns.main.replace_schema_t +---- +1 + +query II +EXPLAIN (FORMAT JSON) SELECT * FROM ns.main.replace_schema_t; +---- +physical_plan :[\s\S]*"Lance Dataset Cache Hit": "true"[\s\S]* + +query II +EXPLAIN (FORMAT JSON) SELECT * FROM ns.main.untouched_t; +---- +physical_plan :[\s\S]*"Lance Dataset Cache Hit": "false"[\s\S]* + +query II +EXPLAIN (FORMAT JSON) SELECT * FROM ns.main.untouched_t; +---- +physical_plan :[\s\S]*"Lance Dataset Cache Hit": "true"[\s\S]* + +statement error +CREATE OR REPLACE TABLE ns.main.replace_schema_t (id BIGINT, label VARCHAR) + WITH (data_storage_version = 'bad_version'); +---- +IO Error: Failed to open Lance writer: + +query I +SELECT count(*) FROM ns.main.replace_schema_t +---- +1 + +query TT +SELECT column_name, data_type FROM duckdb_columns() +WHERE database_name = 'ns' AND schema_name = 'main' AND table_name = 'replace_schema_t' +ORDER BY column_index +---- +id BIGINT + +statement ok +CREATE OR REPLACE TABLE ns.main.replace_schema_t (id BIGINT, label VARCHAR); + +query II +EXPLAIN (FORMAT JSON) SELECT * FROM ns.main.replace_schema_t; +---- +physical_plan :[\s\S]*"Lance Dataset Cache Hit": "false"[\s\S]* + +query I +SELECT count(*) FROM ns.main.replace_schema_t +---- +0 + +query I +SELECT count(label) FROM ns.main.replace_schema_t +---- +0 + +query II +EXPLAIN (FORMAT JSON) SELECT * FROM ns.main.replace_schema_t; +---- +physical_plan :[\s\S]*"Lance Dataset Cache Hit": "true"[\s\S]* + +query II +EXPLAIN (FORMAT JSON) SELECT * FROM ns.main.untouched_t; +---- +physical_plan :[\s\S]*"Lance Dataset Cache Hit": "true"[\s\S]* + +query TT +SELECT column_name, data_type FROM duckdb_columns() +WHERE database_name = 'ns' AND schema_name = 'main' AND table_name = 'replace_schema_t' +ORDER BY column_index +---- +id BIGINT +label VARCHAR + +statement ok +DETACH ns; From d64beff549fd1be9bd1157da987348c6241ac1f4 Mon Sep 17 00:00:00 2001 From: arrowbowang Date: Thu, 13 Aug 2026 16:23:55 +0800 Subject: [PATCH 2/3] fix(catalog): preserve directory table identity Use the physical identifier returned by case-insensitive namespace discovery when replacing a table. This keeps writes, cache invalidation, and catalog refresh scoped to the existing dataset instead of creating a case-variant sibling. --- src/lance_storage.cpp | 28 ++++++++++--------- test/sql/namespace_replace_schema_case.test | 30 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 test/sql/namespace_replace_schema_case.test diff --git a/src/lance_storage.cpp b/src/lance_storage.cpp index 3fff2e8..4a8f4b6 100644 --- a/src/lance_storage.cpp +++ b/src/lance_storage.cpp @@ -258,16 +258,16 @@ ListRestNamespaceTables(const string &endpoint, const string &namespace_id, return out; } -static bool -DirectoryNamespaceTableExists(const LanceDirectoryNamespaceConfig &ns, - const string &table_name) { +static string +FindDirectoryNamespaceTable(const LanceDirectoryNamespaceConfig &ns, + const string &table_name) { auto tables = ListDirectoryNamespaceTables(ns); for (auto &t : tables) { if (StringUtil::CIEquals(t, table_name)) { - return true; + return t; } } - return false; + return string(); } class LanceDirectoryDefaultGenerator : public DefaultGenerator { @@ -1005,11 +1005,12 @@ class LanceSchemaEntry final : public DuckSchemaEntry { throw InternalException("Lance directory namespace root is empty"); } + auto matched_table = + FindDirectoryNamespaceTable(*directory_ns, create_info.table); + auto exists = !matched_table.empty(); + auto physical_table = exists ? matched_table : create_info.table; dataset_path = JoinNamespacePath(directory_ns->root, - GetDatasetDirName(create_info.table)); - - auto exists = - DirectoryNamespaceTableExists(*directory_ns, create_info.table); + GetDatasetDirName(physical_table)); if (create_info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT && exists) { InvalidateTableDefaults(); @@ -1621,11 +1622,12 @@ class LanceDuckCatalog final : public DuckCatalog { throw InternalException("Lance directory namespace root is empty"); } + auto matched_table = + FindDirectoryNamespaceTable(*directory_ns, create_info.table); + auto exists = !matched_table.empty(); + auto physical_table = exists ? matched_table : create_info.table; auto dataset_path = JoinNamespacePath(directory_ns->root, - GetDatasetDirName(create_info.table)); - - auto exists = - DirectoryNamespaceTableExists(*directory_ns, create_info.table); + GetDatasetDirName(physical_table)); if (create_info.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT && exists) { diff --git a/test/sql/namespace_replace_schema_case.test b/test/sql/namespace_replace_schema_case.test new file mode 100644 index 0000000..ee37dbb --- /dev/null +++ b/test/sql/namespace_replace_schema_case.test @@ -0,0 +1,30 @@ +# name: test/sql/namespace_replace_schema_case.test +# description: Case-variant replacements preserve the discovered physical table identifier +# group: [sql] + +require lance + +statement ok +ATTACH '__TEST_DIR__/nsroot_replace_schema_case' AS ns (TYPE LANCE); + +statement ok +CREATE OR REPLACE TABLE ns.main."Foo" (id BIGINT); + +statement ok +INSERT INTO ns.main."Foo" VALUES (7); + +statement ok +CREATE OR REPLACE TABLE ns.main.foo (id BIGINT, label VARCHAR); + +query I +SELECT count(*) FROM '__TEST_DIR__/nsroot_replace_schema_case/Foo.lance' +---- +0 + +statement error +SELECT count(*) FROM '__TEST_DIR__/nsroot_replace_schema_case/foo.lance'; +---- +IO Error: Failed to open Lance dataset: + +statement ok +DETACH ns; From 9a006d8471b7b06218bf88c2abc9852d659f1b4e Mon Sep 17 00:00:00 2001 From: arrowbowang Date: Thu, 13 Aug 2026 17:37:20 +0800 Subject: [PATCH 3/3] fix(storage): preserve physical table case Resolve directory namespace lookups through the listed table name so case-variant replacements retain the physical dataset identifier. Use that identifier for DROP and add replacement and drop regressions. --- src/lance_storage.cpp | 23 +++++++++++++++------ test/sql/dml_drop_table.test | 16 ++++++++++++++ test/sql/namespace_replace_schema_case.test | 23 +++++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/lance_storage.cpp b/src/lance_storage.cpp index 4a8f4b6..8d14df1 100644 --- a/src/lance_storage.cpp +++ b/src/lance_storage.cpp @@ -287,6 +287,15 @@ class LanceDirectoryDefaultGenerator : public DefaultGenerator { "Unsafe Lance dataset name for directory namespace: " + entry_name); } + // DuckDB resolves catalog identifiers case-insensitively, while directory + // namespace paths are case-sensitive. Resolve the requested spelling back + // to the physical table identifier before opening and materializing the + // catalog entry so an evicted Foo.lance is not rediscovered as foo.lance. + auto physical_table = FindDirectoryNamespaceTable(*ns, entry_name); + if (physical_table.empty()) { + return nullptr; + } + vector key_ptrs; vector value_ptrs; BuildStorageOptionPointerArrays(ns->option_keys, ns->option_values, @@ -294,7 +303,7 @@ class LanceDirectoryDefaultGenerator : public DefaultGenerator { const char *uri_ptr = nullptr; auto *dataset = lance_open_dataset_in_dir_namespace( - ns->root.c_str(), entry_name.c_str(), + ns->root.c_str(), physical_table.c_str(), key_ptrs.empty() ? nullptr : key_ptrs.data(), value_ptrs.empty() ? nullptr : value_ptrs.data(), ns->option_keys.size(), &uri_ptr); @@ -307,7 +316,7 @@ class LanceDirectoryDefaultGenerator : public DefaultGenerator { return nullptr; } - CreateTableInfo info(schema, entry_name); + CreateTableInfo info(schema, physical_table); info.internal = true; info.on_conflict = OnCreateConflict::IGNORE_ON_CONFLICT; vector coerced; @@ -321,12 +330,13 @@ class LanceDirectoryDefaultGenerator : public DefaultGenerator { lance_close_dataset(dataset); if (dataset_uri.empty()) { - dataset_uri = JoinNamespacePath(ns->root, GetDatasetDirName(entry_name)); + dataset_uri = + JoinNamespacePath(ns->root, GetDatasetDirName(physical_table)); } LanceNamespaceTableConfig cfg; cfg.kind = LanceNamespaceKind::Directory; cfg.root = ns->root; - cfg.table_id = entry_name; + cfg.table_id = physical_table; cfg.option_keys = ns->option_keys; cfg.option_values = ns->option_values; cfg.display_uri = std::move(dataset_uri); @@ -830,13 +840,14 @@ class LanceSchemaEntry final : public DuckSchemaEntry { BuildStorageOptionPointerArrays(option_keys, option_values, key_ptrs, value_ptrs); + const auto &physical_table = lance_entry->NamespaceConfig().table_id; auto rc = lance_dir_namespace_drop_table( - root.c_str(), info.name.c_str(), + root.c_str(), physical_table.c_str(), key_ptrs.empty() ? nullptr : key_ptrs.data(), value_ptrs.empty() ? nullptr : value_ptrs.data(), option_keys.size()); if (rc != 0) { throw IOException("Failed to drop Lance dataset: " + root + "/" + - GetDatasetDirName(info.name) + + GetDatasetDirName(physical_table) + LanceFormatErrorSuffix()); } } diff --git a/test/sql/dml_drop_table.test b/test/sql/dml_drop_table.test index 6092e0c..90fd11b 100644 --- a/test/sql/dml_drop_table.test +++ b/test/sql/dml_drop_table.test @@ -45,5 +45,21 @@ SELECT count(*) FROM 'test/.tmp/unsafe.lance'; ---- 1 +statement ok +CREATE TABLE ns.main."DropCase" (id BIGINT); + +statement ok +DROP TABLE ns.main.dropcase; + +statement error +SELECT count(*) FROM 'test/.tmp/nsroot/DropCase.lance'; +---- +IO Error: Failed to open Lance dataset: test/.tmp/nsroot/DropCase.lance (Lance error: dataset open 'test/.tmp/nsroot/DropCase.lance': + +statement error +SELECT count(*) FROM ns.main.dropcase; +---- +Catalog Error: Table with name dropcase does not exist + statement ok DETACH ns; diff --git a/test/sql/namespace_replace_schema_case.test b/test/sql/namespace_replace_schema_case.test index ee37dbb..096f274 100644 --- a/test/sql/namespace_replace_schema_case.test +++ b/test/sql/namespace_replace_schema_case.test @@ -16,6 +16,11 @@ INSERT INTO ns.main."Foo" VALUES (7); statement ok CREATE OR REPLACE TABLE ns.main.foo (id BIGINT, label VARCHAR); +query I +SELECT count(label) FROM ns.main.foo; +---- +0 + query I SELECT count(*) FROM '__TEST_DIR__/nsroot_replace_schema_case/Foo.lance' ---- @@ -26,5 +31,23 @@ SELECT count(*) FROM '__TEST_DIR__/nsroot_replace_schema_case/foo.lance'; ---- IO Error: Failed to open Lance dataset: +statement ok +CREATE OR REPLACE TABLE ns.main.foo AS SELECT 11::BIGINT AS id, 'ctas'::VARCHAR AS label; + +query IT +SELECT id, label FROM ns.main.foo; +---- +11 ctas + +query I +SELECT count(*) FROM '__TEST_DIR__/nsroot_replace_schema_case/Foo.lance' +---- +1 + +statement error +SELECT count(*) FROM '__TEST_DIR__/nsroot_replace_schema_case/foo.lance'; +---- +IO Error: Failed to open Lance dataset: + statement ok DETACH ns;