diff --git a/CMakeLists.txt b/CMakeLists.txt index 16ca85581..35b501123 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -451,6 +451,7 @@ set(DUCKDB_SRC_FILES src/duckdb/ub_src_optimizer_join_order.cpp src/duckdb/ub_src_optimizer_pullup.cpp src/duckdb/ub_src_optimizer_pushdown.cpp + src/duckdb/ub_src_optimizer_relation_statistics.cpp src/duckdb/ub_src_optimizer_rule.cpp src/duckdb/ub_src_optimizer_statistics_expression.cpp src/duckdb/ub_src_optimizer_statistics_operator.cpp diff --git a/src/duckdb/extension/core_functions/aggregate/nested/binned_histogram.cpp b/src/duckdb/extension/core_functions/aggregate/nested/binned_histogram.cpp index 5e35ccfc4..ba173e724 100644 --- a/src/duckdb/extension/core_functions/aggregate/nested/binned_histogram.cpp +++ b/src/duckdb/extension/core_functions/aggregate/nested/binned_histogram.cpp @@ -71,7 +71,7 @@ struct HistogramBinState { } } - counts->resize(bin_list.length + 1); + counts->resize(bin_boundaries->size() + 1); } }; diff --git a/src/duckdb/extension/icu/icu-table-range.cpp b/src/duckdb/extension/icu/icu-table-range.cpp index 43144a6e2..5f69878ef 100644 --- a/src/duckdb/extension/icu/icu-table-range.cpp +++ b/src/duckdb/extension/icu/icu-table-range.cpp @@ -148,7 +148,7 @@ struct ICUTableRange { template static unique_ptr Bind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(context, input.inputs); return_types.push_back(LogicalType::TIMESTAMP_TZ); @@ -229,6 +229,7 @@ struct ICUTableRange { nullptr, Bind, nullptr, RangeDateTimeLocalInit); range_function.in_out_function = ICUTableRangeFunction; range_function.cardinality = Cardinality; + range_function.return_type = TableFunctionReturnType::SET_RETURNING_FUNCTION; range.AddFunction(range_function); loader.RegisterFunction(range); diff --git a/src/duckdb/extension/icu/icu-timezone.cpp b/src/duckdb/extension/icu/icu-timezone.cpp index f485bf6c8..21ded264c 100644 --- a/src/duckdb/extension/icu/icu-timezone.cpp +++ b/src/duckdb/extension/icu/icu-timezone.cpp @@ -23,7 +23,7 @@ struct ICUTimeZoneData : public GlobalTableFunctionState { }; static duckdb::unique_ptr ICUTimeZoneBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("abbrev"); diff --git a/src/duckdb/extension/icu/icu_extension.cpp b/src/duckdb/extension/icu/icu_extension.cpp index 020aac083..188cdaa9b 100644 --- a/src/duckdb/extension/icu/icu_extension.cpp +++ b/src/duckdb/extension/icu/icu_extension.cpp @@ -365,7 +365,7 @@ struct ICUCalendarData : public GlobalTableFunctionState { }; static duckdb::unique_ptr ICUCalendarBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp b/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp index 6a30835e3..6240a9d38 100644 --- a/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +++ b/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp @@ -282,7 +282,7 @@ struct ExecuteSqlTableFunction { }; static unique_ptr Bind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { JSONFunctionLocalState local_state(context); auto alc = local_state.json_allocator->GetYYAlc(); diff --git a/src/duckdb/extension/json/json_functions/json_table_in_out.cpp b/src/duckdb/extension/json/json_functions/json_table_in_out.cpp index a2507276d..f67c50e0e 100644 --- a/src/duckdb/extension/json/json_functions/json_table_in_out.cpp +++ b/src/duckdb/extension/json/json_functions/json_table_in_out.cpp @@ -8,7 +8,7 @@ namespace duckdb { enum class JSONTableInOutType { EACH, TREE }; static unique_ptr JSONTableInOutBind(ClientContext &, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { const child_list_t schema { {"key", LogicalType::VARCHAR}, {"value", LogicalType::JSON()}, {"type", LogicalType::VARCHAR}, {"atom", LogicalType::JSON()}, {"id", LogicalType::UBIGINT}, {"parent", LogicalType::UBIGINT}, diff --git a/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp b/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp index 61fe0f6a4..48d2ae02b 100644 --- a/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp +++ b/src/duckdb/extension/parquet/include/reader/expression_column_reader.hpp @@ -42,9 +42,9 @@ class ExpressionColumnReader : public ColumnReader { static constexpr const PhysicalType TYPE = PhysicalType::INVALID; public: - ExpressionColumnReader(ClientContext &context, vector> child_readers, + ExpressionColumnReader(ClientContext &context_p, vector> child_readers, unique_ptr expr, const ParquetColumnSchema &schema); - ExpressionColumnReader(ClientContext &context, vector> child_readers, + ExpressionColumnReader(ClientContext &context_p, vector> child_readers, unique_ptr expr, unique_ptr owned_schema); //! Reader(s) to produce the input(s) for the expression @@ -61,6 +61,7 @@ class ExpressionColumnReader : public ColumnReader { TProtocol &protocol_p) override; idx_t Read(ColumnReaderInput &input, Vector &result) override; + unique_ptr Stats(idx_t row_group_idx_p, const vector &columns) override; void Select(ColumnReaderInput &input, Vector &result, const SelectionVector &sel, idx_t approved_tuple_count) override; @@ -87,6 +88,7 @@ class ExpressionColumnReader : public ColumnReader { } private: + ClientContext &context; void InitializeChunk(); }; diff --git a/src/duckdb/extension/parquet/parquet_metadata.cpp b/src/duckdb/extension/parquet/parquet_metadata.cpp index 661d9a82a..fcf17b260 100644 --- a/src/duckdb/extension/parquet/parquet_metadata.cpp +++ b/src/duckdb/extension/parquet/parquet_metadata.cpp @@ -118,7 +118,7 @@ class ParquetMetaDataOperator { public: template static unique_ptr Bind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names); + vector &return_types, vector &names); static unique_ptr InitGlobal(ClientContext &context, TableFunctionInitInput &input); template static unique_ptr InitLocal(ExecutionContext &context, TableFunctionInitInput &input, @@ -128,7 +128,7 @@ class ParquetMetaDataOperator { const GlobalTableFunctionState *global_state); template - static void BindSchema(vector &return_types, vector &names); + static void BindSchema(vector &return_types, vector &names); static OperatorPartitionData GetPartitionData(ClientContext &context, TableFunctionGetPartitionInput &input); }; @@ -252,7 +252,7 @@ class ParquetRowGroupMetadataProcessor : public ParquetMetadataFileProcessor { template <> void ParquetMetaDataOperator::BindSchema(vector &return_types, - vector &names) { + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -539,7 +539,7 @@ class ParquetSchemaProcessor : public ParquetMetadataFileProcessor { template <> void ParquetMetaDataOperator::BindSchema(vector &return_types, - vector &names) { + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -688,7 +688,7 @@ class ParquetKeyValueMetadataProcessor : public ParquetMetadataFileProcessor { template <> void ParquetMetaDataOperator::BindSchema( - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -725,7 +725,7 @@ class ParquetFileMetadataProcessor : public ParquetMetadataFileProcessor { template <> void ParquetMetaDataOperator::BindSchema(vector &return_types, - vector &names) { + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -822,7 +822,7 @@ class ParquetBloomProbeProcessor : public ParquetMetadataFileProcessor { template <> void ParquetMetaDataOperator::BindSchema(vector &return_types, - vector &names) { + vector &names) { names.emplace_back("file_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -942,44 +942,44 @@ void FullMetadataProcessor::PopulateMetadata(ParquetMetadataFileProcessor &proce template <> void ParquetMetaDataOperator::BindSchema(vector &return_types, - vector &names) { + vector &names) { names.emplace_back("parquet_file_metadata"); vector file_meta_types; - vector file_meta_names; + vector file_meta_names; ParquetMetaDataOperator::BindSchema(file_meta_types, file_meta_names); child_list_t file_meta_children; for (idx_t i = 0; i < file_meta_types.size(); i++) { - file_meta_children.emplace_back(make_pair(file_meta_names[i], file_meta_types[i])); + file_meta_children.emplace_back(make_pair(file_meta_names[i].GetIdentifierName(), file_meta_types[i])); } return_types.emplace_back(LogicalType::LIST(LogicalType::STRUCT(std::move(file_meta_children)))); names.emplace_back("parquet_metadata"); vector row_group_types; - vector row_group_names; + vector row_group_names; ParquetMetaDataOperator::BindSchema(row_group_types, row_group_names); child_list_t row_group_children; for (idx_t i = 0; i < row_group_types.size(); i++) { - row_group_children.emplace_back(make_pair(row_group_names[i], row_group_types[i])); + row_group_children.emplace_back(make_pair(row_group_names[i].GetIdentifierName(), row_group_types[i])); } return_types.emplace_back(LogicalType::LIST(LogicalType::STRUCT(std::move(row_group_children)))); names.emplace_back("parquet_schema"); vector schema_types; - vector schema_names; + vector schema_names; ParquetMetaDataOperator::BindSchema(schema_types, schema_names); child_list_t schema_children; for (idx_t i = 0; i < schema_types.size(); i++) { - schema_children.emplace_back(make_pair(schema_names[i], schema_types[i])); + schema_children.emplace_back(make_pair(schema_names[i].GetIdentifierName(), schema_types[i])); } return_types.emplace_back(LogicalType::LIST(LogicalType::STRUCT(std::move(schema_children)))); names.emplace_back("parquet_kv_metadata"); vector kv_types; - vector kv_names; + vector kv_names; ParquetMetaDataOperator::BindSchema(kv_types, kv_names); child_list_t kv_children; for (idx_t i = 0; i < kv_types.size(); i++) { - kv_children.emplace_back(make_pair(kv_names[i], kv_types[i])); + kv_children.emplace_back(make_pair(kv_names[i].GetIdentifierName(), kv_types[i])); } return_types.emplace_back(LogicalType::LIST(LogicalType::STRUCT(std::move(kv_children)))); } @@ -1008,7 +1008,7 @@ void FullMetadataProcessor::ReadRow(vector> &output, idx_t row template unique_ptr ParquetMetaDataOperator::Bind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { // Extract file paths from input using MultiFileReader (handles both single files and arrays) auto multi_file_reader = MultiFileReader::CreateDefault("ParquetMetadata"); auto glob_input = FileGlobInput(FileGlobOptions::FALLBACK_GLOB, "parquet"); diff --git a/src/duckdb/extension/parquet/parquet_multi_file_info.cpp b/src/duckdb/extension/parquet/parquet_multi_file_info.cpp index e716bef75..cb0e31469 100644 --- a/src/duckdb/extension/parquet/parquet_multi_file_info.cpp +++ b/src/duckdb/extension/parquet/parquet_multi_file_info.cpp @@ -304,7 +304,7 @@ static unique_ptr ParquetScanDeserialize(Deserializer &deserialize auto &context = deserializer.Get(); auto files = deserializer.ReadProperty>(100, "files"); auto types = deserializer.ReadProperty>(101, "types"); - auto names = deserializer.ReadProperty>(102, "names"); + auto names = StringsToIdentifiers(deserializer.ReadProperty>(102, "names")); auto serialization = deserializer.ReadProperty(103, "parquet_options"); auto table_columns = deserializer.ReadPropertyWithExplicitDefault>(104, "table_columns", vector {}); diff --git a/src/duckdb/extension/parquet/parquet_reader.cpp b/src/duckdb/extension/parquet/parquet_reader.cpp index 76b09a0bb..a1475f8b8 100644 --- a/src/duckdb/extension/parquet/parquet_reader.cpp +++ b/src/duckdb/extension/parquet/parquet_reader.cpp @@ -1677,13 +1677,10 @@ void ParquetReader::PrepareRowGroupBuffer(ClientContext &context, ParquetReaderS has_min_max = group.columns[schema_column_index].meta_data.statistics.__isset.min_value && group.columns[schema_column_index].meta_data.statistics.__isset.max_value; } - if (is_expression) { - // no pruning possible for expressions - prune_result = FilterPropagateResult::NO_PRUNING_POSSIBLE; - } else if (!is_generated_column && has_min_max && - (column_reader.Type().id() == LogicalTypeId::FLOAT || - column_reader.Type().id() == LogicalTypeId::DOUBLE) && - parquet_options.can_have_nan) { + if (!is_expression && !is_generated_column && has_min_max && + (column_reader.Type().id() == LogicalTypeId::FLOAT || + column_reader.Type().id() == LogicalTypeId::DOUBLE) && + parquet_options.can_have_nan) { // floating point columns can have NaN values in addition to the min/max bounds defined in the file // in order to do optimal pruning - we prune based on the [min, max] of the file followed by pruning // based on nan diff --git a/src/duckdb/extension/parquet/reader/expression_column_reader.cpp b/src/duckdb/extension/parquet/reader/expression_column_reader.cpp index da9a48a73..40cebb3bf 100644 --- a/src/duckdb/extension/parquet/reader/expression_column_reader.cpp +++ b/src/duckdb/extension/parquet/reader/expression_column_reader.cpp @@ -5,6 +5,7 @@ #include "parquet_reader.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/planner/filter/expression_filter.hpp" namespace duckdb_apache { namespace thrift { @@ -23,10 +24,11 @@ class ClientContext; //===--------------------------------------------------------------------===// // Expression Column Reader //===--------------------------------------------------------------------===// -ExpressionColumnReader::ExpressionColumnReader(ClientContext &context, vector> child_readers_p, +ExpressionColumnReader::ExpressionColumnReader(ClientContext &context_p, + vector> child_readers_p, unique_ptr expr_p, const ParquetColumnSchema &schema_p) : ColumnReader(child_readers_p[0]->Reader(), schema_p), child_readers(std::move(child_readers_p)), - expr(std::move(expr_p)), executor(context, expr.get()) { + expr(std::move(expr_p)), executor(context_p, expr.get()), context(context_p) { if (child_readers.empty()) { throw InternalException("Can't instantiate an ExpressionColumnReader with 0 children"); } @@ -38,11 +40,13 @@ ExpressionColumnReader::ExpressionColumnReader(ClientContext &context, vector> child_readers_p, +ExpressionColumnReader::ExpressionColumnReader(ClientContext &context_p, + vector> child_readers_p, unique_ptr expr_p, unique_ptr owned_schema_p) : ColumnReader(child_readers_p[0]->Reader(), *owned_schema_p), child_readers(std::move(child_readers_p)), - expr(std::move(expr_p)), executor(context, expr.get()), owned_schema(std::move(owned_schema_p)) { + expr(std::move(expr_p)), executor(context_p, expr.get()), owned_schema(std::move(owned_schema_p)), + context(context_p) { if (child_readers.empty()) { throw InternalException("Can't instantiate an ExpressionColumnReader with 0 children"); } @@ -69,6 +73,23 @@ void ExpressionColumnReader::InitializeRead(idx_t row_group_idx_p, idx_t row_gro } } +unique_ptr ExpressionColumnReader::Stats(idx_t row_group_idx_p, const vector &columns) { + if (Schema().schema_type != ParquetColumnSchemaType::EXPRESSION) { + return ColumnReader::Stats(row_group_idx_p, columns); + } + vector input_stats; + input_stats.reserve(child_readers.size()); + for (auto &child_reader : child_readers) { + auto child_stats = child_reader->Stats(row_group_idx_p, columns); + if (!child_stats) { + return nullptr; + } + input_stats.push_back(child_stats->Copy()); + } + return ExpressionFilter::TryGetExpressionStatistics( + context, *expr, array_ptr(input_stats.data(), input_stats.size())); +} + static void ReverseSelectionVector(const SelectionVector &input, SelectionVector &output, idx_t input_count, idx_t result_count) { //! For an input selection vector: [5, 10], diff --git a/src/duckdb/src/catalog/catalog.cpp b/src/duckdb/src/catalog/catalog.cpp index 906b5abfa..0eab54102 100644 --- a/src/duckdb/src/catalog/catalog.cpp +++ b/src/duckdb/src/catalog/catalog.cpp @@ -591,7 +591,7 @@ vector GetCatalogEntries(CatalogEntryRetriever &retriever, c if (entries.empty()) { auto catalog_entry = Catalog::GetCatalogEntry(context, catalog); if (catalog_entry) { - entries.emplace_back(catalog, Identifier(catalog_entry->GetDefaultSchema())); + entries.emplace_back(catalog, catalog_entry->GetDefaultSchema()); } else { entries.emplace_back(catalog, DEFAULT_SCHEMA); } @@ -1156,8 +1156,8 @@ CatalogEntryLookup Catalog::TryLookupDefaultTable(CatalogEntryRetriever &retriev auto transaction = catalog_by_name->GetCatalogTransaction(retriever.GetContext()); QueryErrorContext context; - string table_schema = catalog_by_name->GetDefaultTableSchema(); - string table_name = catalog_by_name->GetDefaultTable(); + auto table_schema = catalog_by_name->GetDefaultTableSchema(); + auto table_name = catalog_by_name->GetDefaultTable(); optional_ptr at_clause; if (!catalog_by_name->SupportsTimeTravel() && allow_ignore_at_clause) { @@ -1166,10 +1166,8 @@ CatalogEntryLookup Catalog::TryLookupDefaultTable(CatalogEntryRetriever &retriev at_clause = lookup_info.GetAtClause(); } - EntryLookupInfo info( - CatalogType::TABLE_ENTRY, - QualifiedName(catalog_by_name->GetName(), Identifier(table_schema), Identifier(table_name)), at_clause, - context); + EntryLookupInfo info(CatalogType::TABLE_ENTRY, + QualifiedName(catalog_by_name->GetName(), table_schema, table_name), at_clause, context); return catalog_by_name->TryLookupEntryInternal(transaction, info); } @@ -1186,9 +1184,9 @@ CatalogEntryLookup Catalog::TryLookupDefaultSchema(CatalogEntryRetriever &retrie continue; } auto transaction = catalog_entry->GetCatalogTransaction(retriever.GetContext()); - EntryLookupInfo default_schema_lookup(lookup_info, QualifiedName(catalog_entry->GetName(), - Identifier(catalog_entry->GetDefaultSchema()), - lookup_info.GetEntryIdentifier())); + EntryLookupInfo default_schema_lookup(lookup_info, + QualifiedName(catalog_entry->GetName(), catalog_entry->GetDefaultSchema(), + lookup_info.GetEntryIdentifier())); auto result = catalog_entry->TryLookupEntryInternal(transaction, default_schema_lookup); if (result.Found() || result.error.HasError()) { return result; @@ -1470,7 +1468,7 @@ ErrorData Catalog::SupportsCreateTable(BoundCreateTableInfo &info) { return ErrorData(); } -string Catalog::GetDefaultSchema() const { +Identifier Catalog::GetDefaultSchema() const { return DEFAULT_SCHEMA; } @@ -1480,15 +1478,15 @@ bool Catalog::HasDefaultTable() const { } void Catalog::SetDefaultTable(const Identifier &schema, const Identifier &name) { - default_table = name.GetIdentifierName(); - default_table_schema = schema.GetIdentifierName(); + default_table = name; + default_table_schema = schema; } -string Catalog::GetDefaultTable() const { +Identifier Catalog::GetDefaultTable() const { return default_table; } -string Catalog::GetDefaultTableSchema() const { +Identifier Catalog::GetDefaultTableSchema() const { return !default_table_schema.empty() ? default_table_schema : DEFAULT_SCHEMA; } diff --git a/src/duckdb/src/catalog/catalog_search_path.cpp b/src/duckdb/src/catalog/catalog_search_path.cpp index 2ee1d6a76..ebd7602d1 100644 --- a/src/duckdb/src/catalog/catalog_search_path.cpp +++ b/src/duckdb/src/catalog/catalog_search_path.cpp @@ -182,8 +182,7 @@ void CatalogSearchPath::Set(vector new_paths, CatalogSetPath if (path.GetCatalog().empty()) { auto catalog = Catalog::GetCatalogEntry(context, path.GetSchema()); if (catalog) { - auto schema = - catalog->GetSchema(context, Identifier(catalog->GetDefaultSchema()), OnEntryNotFound::RETURN_NULL); + auto schema = catalog->GetSchema(context, catalog->GetDefaultSchema(), OnEntryNotFound::RETURN_NULL); if (schema) { path.SetCatalog(path.GetSchema()); path.SetSchema(schema->name); @@ -250,7 +249,7 @@ Identifier CatalogSearchPath::GetDefaultSchema(ClientContext &context, const Ide } auto catalog_entry = Catalog::GetCatalogEntry(context, catalog); if (catalog_entry) { - return Identifier(catalog_entry->GetDefaultSchema()); + return catalog_entry->GetDefaultSchema(); } return DEFAULT_SCHEMA; } @@ -317,7 +316,7 @@ vector CatalogSearchPath::GetWithPrecedenceSchemas(ClientCon if (!catalog_entry) { continue; } - res.emplace_back(path.GetCatalog(), Identifier(catalog_entry->GetDefaultSchema())); + res.emplace_back(path.GetCatalog(), catalog_entry->GetDefaultSchema()); } else { res.emplace_back(path); } diff --git a/src/duckdb/src/common/enum_util.cpp b/src/duckdb/src/common/enum_util.cpp index 6bae96ccd..a5e54d0b9 100644 --- a/src/duckdb/src/common/enum_util.cpp +++ b/src/duckdb/src/common/enum_util.cpp @@ -165,7 +165,7 @@ #include "duckdb/optimizer/build_probe_side_optimizer.hpp" #include "duckdb/optimizer/compressed_materialization.hpp" #include "duckdb/optimizer/join_order/join_order_operator.hpp" -#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics.hpp" #include "duckdb/optimizer/remove_unused_columns.hpp" #include "duckdb/optimizer/rule/like_optimizations.hpp" #include "duckdb/parallel/async_result.hpp" @@ -1796,6 +1796,25 @@ DeferredRuntimeFilterType EnumUtil::FromString(const return static_cast(StringUtil::StringToEnum(GetDeferredRuntimeFilterTypeValues(), 2, "DeferredRuntimeFilterType", value)); } +const StringUtil::EnumStringLiteral *GetDeleteIdStateValues() { + static constexpr StringUtil::EnumStringLiteral values[] { + { static_cast(DeleteIdState::CONSTANT), "CONSTANT" }, + { static_cast(DeleteIdState::MASKED), "MASKED" }, + { static_cast(DeleteIdState::ARRAY), "ARRAY" } + }; + return values; +} + +template<> +const char* EnumUtil::ToChars(DeleteIdState value) { + return StringUtil::EnumToString(GetDeleteIdStateValues(), 3, "DeleteIdState", static_cast(value)); +} + +template<> +DeleteIdState EnumUtil::FromString(const char *value) { + return static_cast(StringUtil::StringToEnum(GetDeleteIdStateValues(), 3, "DeleteIdState", value)); +} + const StringUtil::EnumStringLiteral *GetDependencyEntryTypeValues() { static constexpr StringUtil::EnumStringLiteral values[] { { static_cast(DependencyEntryType::SUBJECT), "SUBJECT" }, diff --git a/src/duckdb/src/common/multi_file/multi_file_reader.cpp b/src/duckdb/src/common/multi_file/multi_file_reader.cpp index 54d1b81ca..028b45675 100644 --- a/src/duckdb/src/common/multi_file/multi_file_reader.cpp +++ b/src/duckdb/src/common/multi_file/multi_file_reader.cpp @@ -381,7 +381,8 @@ void MultiFileReader::FinalizeBind(MultiFileReaderData &reader_data, const Multi if (not_present_in_file) { // we need to project a column with name \"global_name\" - but it does not exist in the current file // push a NULL value of the specified type - reader_data.constant_map.Add(global_idx, Value(type)); + auto &constant_type = col_id.HasType() ? col_id.GetScanType() : type; + reader_data.constant_map.Add(global_idx, Value(constant_type)); continue; } } diff --git a/src/duckdb/src/execution/column_binding_resolver.cpp b/src/duckdb/src/execution/column_binding_resolver.cpp index cb51f3ea7..7143e0af4 100644 --- a/src/duckdb/src/execution/column_binding_resolver.cpp +++ b/src/duckdb/src/execution/column_binding_resolver.cpp @@ -8,7 +8,6 @@ #include "duckdb/planner/operator/logical_create_index.hpp" #include "duckdb/planner/operator/logical_extension_operator.hpp" #include "duckdb/planner/operator/logical_insert.hpp" -#include "duckdb/planner/operator/logical_recursive_cte.hpp" #include "duckdb/main/settings.hpp" namespace duckdb { @@ -130,6 +129,10 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) { throw InternalException("RIGHT SEMI/ANTI any join not supported yet"); } VisitOperatorExpressions(op); + + // Restore bindings for the caller + bindings = op.GetColumnBindings(); + types = op.types; return; } case LogicalOperatorType::LOGICAL_CREATE_INDEX: { diff --git a/src/duckdb/src/execution/join_hashtable.cpp b/src/duckdb/src/execution/join_hashtable.cpp index 16272ad6b..8a65e1dbe 100644 --- a/src/duckdb/src/execution/join_hashtable.cpp +++ b/src/duckdb/src/execution/join_hashtable.cpp @@ -2878,11 +2878,16 @@ void JoinHashTable::BuildDictionaryArrays(const PhysicalHashJoin &op) { dict_arrays.emplace_back(std::move(dict_entry)); } - // save chain pointers before overwriting NEXT_PTR; GetNextPointer reads them back + // save chain pointers before overwriting NEXT_PTR; GetNextPointer reads them back. + // One extra slot is reserved at the end and left null so that `dead_end` has an index + // that terminates a chain. Without it, `dead_end` is zeroed and therefore resolves to + // index 0, which belongs to a real row, and following it walks an unrelated chain. const auto has_chains = chains_longer_than_one.load(std::memory_order_relaxed); if (has_chains) { - aux_next_ptrs = buffer_manager.GetBufferAllocator().Allocate(build_count * sizeof(data_ptr_t)); + aux_next_ptrs = buffer_manager.GetBufferAllocator().Allocate((build_count + 1) * sizeof(data_ptr_t)); aux_next_ptrs_data = reinterpret_cast(aux_next_ptrs.get()); + aux_next_ptrs_data[build_count] = nullptr; + Store(static_cast(build_count), dead_end.get() + pointer_offset); } // save the original NEXT_PTR into aux_next_ptrs (if chains exist) and embed the dict index diff --git a/src/duckdb/src/execution/operator/join/physical_asof_join.cpp b/src/duckdb/src/execution/operator/join/physical_asof_join.cpp index bf19beb28..95f3692e5 100644 --- a/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +++ b/src/duckdb/src/execution/operator/join/physical_asof_join.cpp @@ -25,13 +25,14 @@ PhysicalAsOfJoin::PhysicalAsOfJoin(PhysicalPlan &physical_plan, LogicalCompariso D_ASSERT(cond.IsComparison()); D_ASSERT(cond.GetLHS().GetReturnType() == cond.GetRHS().GetReturnType()); join_key_types.push_back(cond.GetLHS().GetReturnType()); + const auto join_key_idx = join_key_types.size() - 1; auto left_cond = cond.LeftReference()->Copy(); auto right_cond = cond.RightReference()->Copy(); switch (cond.GetComparisonType()) { case ExpressionType::COMPARE_GREATERTHANOREQUALTO: case ExpressionType::COMPARE_GREATERTHAN: - null_sensitive.emplace_back(lhs_orders.size()); + null_sensitive.emplace_back(join_key_idx); lhs_orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(left_cond)); rhs_orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(right_cond)); comparison_type = cond.GetComparisonType(); @@ -39,13 +40,13 @@ PhysicalAsOfJoin::PhysicalAsOfJoin(PhysicalPlan &physical_plan, LogicalCompariso case ExpressionType::COMPARE_LESSTHANOREQUALTO: case ExpressionType::COMPARE_LESSTHAN: // Always put NULLS LAST so they can be ignored. - null_sensitive.emplace_back(lhs_orders.size()); + null_sensitive.emplace_back(join_key_idx); lhs_orders.emplace_back(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, std::move(left_cond)); rhs_orders.emplace_back(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, std::move(right_cond)); comparison_type = cond.GetComparisonType(); break; case ExpressionType::COMPARE_EQUAL: - null_sensitive.emplace_back(lhs_orders.size()); + null_sensitive.emplace_back(join_key_idx); DUCKDB_EXPLICIT_FALLTHROUGH; case ExpressionType::COMPARE_NOT_DISTINCT_FROM: lhs_partitions.emplace_back(std::move(left_cond)); @@ -730,14 +731,25 @@ struct SortKeyPrefixComparison { auto lhs_width = col.size; auto rhs_width = col.size; int cmp = 1; + const auto has_null = + col.type != SortKeyPrefixComparisonType::NESTED && + (CreateSortKeyHelpers::IsNullSortKey(const_data_ptr_cast(lhs_ptr), modifiers.null_type) || + CreateSortKeyHelpers::IsNullSortKey(const_data_ptr_cast(rhs_ptr), modifiers.null_type)); + if (has_null) { + lhs_width = 1; + rhs_width = 1; + } + switch (col.type) { case SortKeyPrefixComparisonType::FIXED: cmp = memcmp(lhs_ptr, rhs_ptr, lhs_width); break; case SortKeyPrefixComparisonType::VARCHAR: - // Include first null byte. - lhs_width = 1 + strlen(lhs_ptr); - rhs_width = 1 + strlen(rhs_ptr); + if (!has_null) { + // Include first null byte. + lhs_width = 1 + strlen(lhs_ptr); + rhs_width = 1 + strlen(rhs_ptr); + } cmp = memcmp(lhs_ptr, rhs_ptr, MinValue(lhs_width, rhs_width)); break; case SortKeyPrefixComparisonType::NESTED: diff --git a/src/duckdb/src/execution/operator/join/physical_iejoin.cpp b/src/duckdb/src/execution/operator/join/physical_iejoin.cpp index 0766b3607..26d8ecbd9 100644 --- a/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +++ b/src/duckdb/src/execution/operator/join/physical_iejoin.cpp @@ -1359,6 +1359,8 @@ const SelectionVector *IEJoinLocalSourceState::ApplyTailConditions() { auto result_count = lpayload.size(); auto tail_count = result_count; auto match_sel = &true_sel; + left_keys.Reset(); + right_keys.Reset(); for (size_t cmp_idx = 0; cmp_idx < tail_cols; ++cmp_idx) { auto &left = left_keys.data[cmp_idx]; left_executor.ExecuteExpression(cmp_idx, left); diff --git a/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp b/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp index d898290f4..3e00c378a 100644 --- a/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +++ b/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp @@ -726,8 +726,8 @@ struct PartitionedCopyFlushAction { struct PartitionedCopyBatchState { private: void SetValues(vector values_p) { - D_ASSERT(values.empty() || values == values_p); - if (values.empty()) { + D_ASSERT(!values || *values == values_p); + if (!values) { values = std::move(values_p); } } @@ -798,7 +798,8 @@ struct PartitionedCopyBatchState { } const vector &Values() const { - return values; + D_ASSERT(values); + return *values; } idx_t AddCollectionSlot(PartitionedCopyCollectionSchema schema, idx_t row_count) { @@ -814,7 +815,7 @@ struct PartitionedCopyBatchState { PartitionedCopyBatchAction RegisterBatch(vector values_p, idx_t flush_threshold, bool has_delayed_partition) { SetValues(std::move(values_p)); - auto result = PartitionedCopyBatchAction {PartitionedCopyBatchActionType::STORE_COLLECTION, values, nullptr}; + auto result = PartitionedCopyBatchAction {PartitionedCopyBatchActionType::STORE_COLLECTION, Values(), nullptr}; if (mode == PartitionedCopyBatchMode::BUFFERING && count >= flush_threshold && !has_delayed_partition) { StartPreparing(); if (TryReserveWriteInfo()) { @@ -842,7 +843,7 @@ struct PartitionedCopyBatchState { } idx_t FinalizeBatching(idx_t flush_threshold, bool has_delayed_partition) { - D_ASSERT(!values.empty()); + D_ASSERT(values); D_ASSERT(count > 0); if (mode == PartitionedCopyBatchMode::PREPARING) { D_ASSERT(HasWriteInfo()); @@ -881,7 +882,7 @@ struct PartitionedCopyBatchState { PartitionedCopyPrepareAction BeginPrepareTask(idx_t batch_idx) { D_ASSERT(mode == PartitionedCopyBatchMode::PREPARING); PartitionedCopyPrepareAction result; - result.values = values; + result.values = Values(); if (batch_idx == DConstants::INVALID_INDEX) { D_ASSERT(NeedsWriteInfo()); D_ASSERT(write_info_requested); @@ -936,7 +937,7 @@ struct PartitionedCopyBatchState { PartitionedCopyFlushAction TakeFlushAction() { PartitionedCopyFlushAction result; - result.values = values; + result.values = Values(); if (mode == PartitionedCopyBatchMode::DELAYED) { result.type = PartitionedCopyFlushActionType::DELAYED_COLLECTIONS; result.collections = TakeDelayedCollections(); @@ -961,7 +962,7 @@ struct PartitionedCopyBatchState { } private: - vector values; + optional> values; PartitionWriteLease write_lease; vector collections; vector> batches; @@ -2118,6 +2119,10 @@ void PartitionedCopyHashGroup::Mask(const PartitionedCopyTask &task) { // Only compare partition columns (not order columns) const auto key_count = partitioned_copy.op.partition_columns.size(); + if (key_count == 0) { + masked += (task.end_idx - task.begin_idx); + return; + } auto &scan_cols = partitioned_copy.sort_strategy->sort_ids; WindowDeltaScanner(*collection, task.begin_idx, task.end_idx, scan_cols, key_count, @@ -2280,7 +2285,7 @@ void PartitionedCopyHashGroup::Flush(ExecutionContext &execution_context, Interr auto &batch_state = *batch_states[task.thread_idx]; flush_action = batch_state.TakeFlushAction(); } - D_ASSERT(!flush_action.values.empty()); + D_ASSERT(flush_action.values.size() == partitioned_copy.op.partition_columns.size()); if (flush_action.type == PartitionedCopyFlushActionType::DELAYED_COLLECTIONS) { const auto collection_schema = partitioned_copy.GetPartitionCollectionSchema(); @@ -2336,6 +2341,9 @@ bool PartitionedCopyState::ShouldInitiateFlush(const idx_t &local_append_count) if (!exchanged) { return false; // Another thread beat us to it } + if (partitioned_copy.op.partition_columns.empty()) { + return true; // The implicit partition is always dense enough to flush + } // Get counts from the HLL states const auto merged_state = hll.GetMergedState(); diff --git a/src/duckdb/src/function/aggregate/distributive/minmax.cpp b/src/duckdb/src/function/aggregate/distributive/minmax.cpp index 9a9c2d7b7..98298bbbb 100644 --- a/src/duckdb/src/function/aggregate/distributive/minmax.cpp +++ b/src/duckdb/src/function/aggregate/distributive/minmax.cpp @@ -331,6 +331,29 @@ static AggregateFunction GetMinMaxOperator(const LogicalType &type) { } } +const AggregateFunction &GetCollatedMinMaxFunction(ClientContext &context, const Identifier &name, + const vector &types) { + const auto function_name = name == "min" ? "arg_min" : "arg_max"; + QueryErrorContext error_context; + auto func = Catalog::GetEntry( + context, QualifiedName(Identifier(), Identifier(), Identifier(function_name)), OnEntryNotFound::RETURN_NULL, + error_context); + if (!func) { + throw NotImplementedException( + "Failure while binding function \"%s\" using collations - arg_min/arg_max do not exist in the " + "catalog - load the core_functions module to fix this issue", + name); + } + + FunctionBinder function_binder(context); + ErrorData error; + auto best_function = function_binder.BindFunction(func->name, func->functions, types, error); + if (!best_function.IsValid()) { + throw BinderException("Fail to find corresponding function for collation min/max: %s", error.Message()); + } + return func->functions.GetFunctionByOffset(best_function.GetIndex()); +} + template unique_ptr BindMinMax(BindAggregateFunctionInput &input) { auto &context = input.GetClientContext(); @@ -338,39 +361,19 @@ unique_ptr BindMinMax(BindAggregateFunctionInput &input) { auto &arguments = input.GetArguments(); auto input_type = arguments[0]->GetReturnType(); - // The generic non-VARCHAR collation path is not ready yet (see internal #8704). BIT uses an explicit - // binary-comparable key so min/max follows the same logical order as comparisons and ORDER BY. + // The generic non-VARCHAR collation path is not ready yet (see internal #8704). BIT and VARIANT use explicit + // binary-comparable keys so min/max follows the same logical order as comparisons and ORDER BY. const auto varchar_collation = input_type.id() == LogicalTypeId::VARCHAR && (!StringType::GetCollation(input_type).empty() || !Settings::Get(context).empty()); - const auto collation = input_type.id() == LogicalTypeId::BIT || varchar_collation; + const auto collation = + input_type.id() == LogicalTypeId::BIT || input_type.id() == LogicalTypeId::VARIANT || varchar_collation; auto collated_arg = collation ? arguments[0]->Copy() : nullptr; if (collation && ExpressionBinder::PushCollation(context, collated_arg, collated_arg->GetReturnType())) { // If aggr function is min/max and uses collations, replace bound_function with arg_min/arg_max // to make sure the result's correctness. - string function_name = function.GetName() == "min" ? "arg_min" : "arg_max"; - QueryErrorContext error_context; - auto func = Catalog::GetEntry( - context, QualifiedName(Identifier(), Identifier(), Identifier(function_name)), OnEntryNotFound::RETURN_NULL, - error_context); - if (!func) { - throw NotImplementedException( - "Failure while binding function \"%s\" using collations - arg_min/arg_max do not exist in the " - "catalog - load the core_functions module to fix this issue", - function.GetName()); - } - - auto &func_entry = *func; - - FunctionBinder function_binder(context); vector types {arguments[0]->GetReturnType(), collated_arg->GetReturnType()}; - ErrorData error; - auto best_function = function_binder.BindFunction(func_entry.name, func_entry.functions, types, error); - if (!best_function.IsValid()) { - throw BinderException(string("Fail to find corresponding function for collation min/max: ") + - error.Message()); - } - function.ReplaceImplementation(func_entry.functions.GetFunctionByOffset(best_function.GetIndex())); + function.ReplaceImplementation(GetCollatedMinMaxFunction(context, function.GetName(), types)); // Bind function like arg_min/arg_max. arguments.push_back(std::move(collated_arg)); @@ -532,13 +535,32 @@ void SpecializeMinMaxNFunction(PhysicalType arg_type, BoundAggregateFunction &fu template unique_ptr MinMaxNBind(BindAggregateFunctionInput &input) { + auto &context = input.GetClientContext(); auto &function = input.GetBoundFunction(); auto &arguments = input.GetArguments(); - for (auto &arg : arguments) { + for (const auto &arg : arguments) { if (arg->GetReturnType().id() == LogicalTypeId::UNKNOWN) { throw ParameterNotResolvedException(); } } + if (arguments[0]->GetReturnType().id() == LogicalTypeId::VARIANT) { + auto collated_arg = arguments[0]->Copy(); + if (ExpressionBinder::PushCollation(context, collated_arg, collated_arg->GetReturnType())) { + vector types {arguments[0]->GetReturnType(), collated_arg->GetReturnType(), + arguments[1]->GetReturnType()}; + auto &collated_function = GetCollatedMinMaxFunction(context, function.GetName(), types); + + vector> collated_arguments; + collated_arguments.reserve(3); + collated_arguments.push_back(std::move(arguments[0])); + collated_arguments.push_back(std::move(collated_arg)); + collated_arguments.push_back(std::move(arguments[1])); + auto expr = collated_function.Bind(context, std::move(collated_arguments)); + arguments = std::move(expr->GetChildrenMutable()); + function = std::move(expr->FunctionMutable()); + return std::move(expr->BindInfoMutable()); + } + } const auto val_type = arguments[0]->GetReturnType().InternalType(); diff --git a/src/duckdb/src/function/cast/variant/to_json.cpp b/src/duckdb/src/function/cast/variant/to_json.cpp index 5e16fb9c7..2c827843c 100644 --- a/src/duckdb/src/function/cast/variant/to_json.cpp +++ b/src/duckdb/src/function/cast/variant/to_json.cpp @@ -142,6 +142,11 @@ struct JSONConverter { } else { throw InternalException("Unhandled decimal type"); } + if (width == scale) { + // Decimal::ToString omits the zero before the decimal point when all digits are fractional. + // JSON numbers require an integer component. + val_str.insert(val_str[0] == '-' ? 1 : 0, 1, '0'); + } return yyjson_mut_rawncpy(doc, val_str.c_str(), val_str.size()); } diff --git a/src/duckdb/src/function/scalar/create_sort_key.cpp b/src/duckdb/src/function/scalar/create_sort_key.cpp index 1d1e5823f..2a3012549 100644 --- a/src/duckdb/src/function/scalar/create_sort_key.cpp +++ b/src/duckdb/src/function/scalar/create_sort_key.cpp @@ -1423,6 +1423,14 @@ void DecodeSortKeyRecursive(DecodeSortKeyData decode_data[], DecodeSortKeyVector } // namespace +bool CreateSortKeyHelpers::IsNullSortKey(const_data_ptr_t data, OrderByNullType null_order) { + auto null_byte = SortKeyVectorData::NULL_FIRST_BYTE; + if (null_order == OrderByNullType::NULLS_LAST) { + null_byte = SortKeyVectorData::NULL_LAST_BYTE; + } + return data[0] == null_byte; +} + idx_t CreateSortKeyHelpers::DecodeSortKey(string_t sort_key, Vector &result, idx_t result_idx, OrderModifiers modifiers) { DecodeSortKeyVectorData sort_key_data(result.GetType(), modifiers); diff --git a/src/duckdb/src/function/scalar/operator/arithmetic.cpp b/src/duckdb/src/function/scalar/operator/arithmetic.cpp index 1b017e0c5..24f94bdb6 100644 --- a/src/duckdb/src/function/scalar/operator/arithmetic.cpp +++ b/src/duckdb/src/function/scalar/operator/arithmetic.cpp @@ -1328,6 +1328,14 @@ timestamp_t InterpolateOperator::Operation(const timestamp_t &lo, const double d template <> hugeint_t InterpolateOperator::Operation(const hugeint_t &lo, const double d, const hugeint_t &hi) { + hugeint_t delta_hugeint = hi; + if (Hugeint::TrySubtractInPlace(delta_hugeint, lo)) { + const auto delta = Hugeint::Cast(delta_hugeint); + return lo + Hugeint::Convert(delta * d); + } + + // if delta overflows, we fall back to original subtraction to avoid UB + // only happens when lo and hi are so apart that their difference can't be stored in a hugeint return Hugeint::Convert(Operation(Hugeint::Cast(lo), d, Hugeint::Cast(hi))); } diff --git a/src/duckdb/src/function/scalar/string/like.cpp b/src/duckdb/src/function/scalar/string/like.cpp index 84857f434..ae30a84ec 100644 --- a/src/duckdb/src/function/scalar/string/like.cpp +++ b/src/duckdb/src/function/scalar/string/like.cpp @@ -397,7 +397,8 @@ struct LikeEscapeOperator { template static inline bool Operation(TA str, TB pattern, TC escape) { char escape_char = GetEscapeChar(escape); - return LikeOperatorFunction(str.GetData(), str.GetSize(), pattern.GetData(), pattern.GetSize(), escape_char); + return escape.GetSize() == 0 ? LikeOperatorFunction(str, pattern) + : LikeOperatorFunction(str, pattern, escape_char); } }; @@ -415,7 +416,8 @@ struct LikeOperator { } }; -bool ILikeOperatorFunction(string_t &str, string_t &pattern, char escape = '\0') { +template +bool ILikeOperatorFunctionInternal(string_t &str, string_t &pattern, char escape) { auto str_data = str.GetData(); auto str_size = str.GetSize(); auto pat_data = pattern.GetData(); @@ -431,18 +433,26 @@ bool ILikeOperatorFunction(string_t &str, string_t &pattern, char escape = '\0') LowerCase(pat_data, pat_size, pat_ldata.get()); string_t str_lcase(str_ldata.get(), UnsafeNumericCast(str_llength)); string_t pat_lcase(pat_ldata.get(), UnsafeNumericCast(pat_llength)); - // '\0' is the "no escape" sentinel: use the non-escape matcher so embedded NUL bytes are matched literally - if (escape == '\0') { + if (!HAS_ESCAPE) { return LikeOperatorFunction(str_lcase, pat_lcase); } return LikeOperatorFunction(str_lcase, pat_lcase, escape); } +bool ILikeOperatorFunction(string_t &str, string_t &pattern) { + return ILikeOperatorFunctionInternal(str, pattern, '\0'); +} + +bool ILikeOperatorFunction(string_t &str, string_t &pattern, char escape) { + return ILikeOperatorFunctionInternal(str, pattern, escape); +} + struct ILikeEscapeOperator { template static inline bool Operation(TA str, TB pattern, TC escape) { char escape_char = GetEscapeChar(escape); - return ILikeOperatorFunction(str, pattern, escape_char); + return escape.GetSize() == 0 ? ILikeOperatorFunction(str, pattern) + : ILikeOperatorFunction(str, pattern, escape_char); } }; @@ -523,6 +533,7 @@ void ILikeEscapeFunction(DataChunk &args, ExpressionState &state, Vector &result auto pattern = *ConstantVector::GetData(pattern_vec); auto escape = *ConstantVector::GetData(escape_vec); char escape_char = GetEscapeChar(escape); + bool has_escape = escape.GetSize() != 0; // lowercase the pattern exactly once, up front idx_t pat_llength = LowerLength(pattern.GetData(), pattern.GetSize()); @@ -533,8 +544,7 @@ void ILikeEscapeFunction(DataChunk &args, ExpressionState &state, Vector &result // the matcher cannot honor escape semantics, so only use it when the escape char never appears in the // (lowercased) pattern, in which case escape is irrelevant and the pattern is a plain LIKE pattern unique_ptr matcher; - bool escape_active = - escape_char != '\0' && memchr(pat_lcase.GetData(), escape_char, pat_lcase.GetSize()) != nullptr; + bool escape_active = has_escape && memchr(pat_lcase.GetData(), escape_char, pat_lcase.GetSize()) != nullptr; if (!escape_active) { matcher = LikeMatcher::CreateLikeMatcher(string(pat_lcase.GetData(), pat_lcase.GetSize())); } @@ -550,10 +560,9 @@ void ILikeEscapeFunction(DataChunk &args, ExpressionState &state, Vector &result } LowerCase(str.GetData(), str.GetSize(), scratch.get()); string_t str_lcase(scratch.get(), UnsafeNumericCast(str_llength)); - // '\0' escape means no escape: use the non-escape matcher so embedded NUL bytes are matched literally bool match = matcher ? matcher->Match(str_lcase) - : (escape_char == '\0' ? LikeOperatorFunction(str_lcase, pat_lcase) - : LikeOperatorFunction(str_lcase, pat_lcase, escape_char)); + : (has_escape ? LikeOperatorFunction(str_lcase, pat_lcase, escape_char) + : LikeOperatorFunction(str_lcase, pat_lcase)); return INVERT ? !match : match; }); return; diff --git a/src/duckdb/src/function/table/arrow.cpp b/src/duckdb/src/function/table/arrow.cpp index ca3068e76..bc09f3565 100644 --- a/src/duckdb/src/function/table/arrow.cpp +++ b/src/duckdb/src/function/table/arrow.cpp @@ -49,7 +49,7 @@ void ArrowTableFunction::PopulateArrowTableSchema(ClientContext &context, ArrowT unique_ptr ArrowTableFunction::ArrowScanBindDumb(ClientContext &context, TableFunctionBindInput &input, vector &return_types, - vector &names) { + vector &names) { auto bind_data = ArrowScanBind(context, input, return_types, names); auto &arrow_bind_data = bind_data->Cast(); arrow_bind_data.projection_pushdown_enabled = false; @@ -57,7 +57,8 @@ unique_ptr ArrowTableFunction::ArrowScanBindDumb(ClientContext &co } unique_ptr ArrowTableFunction::ArrowScanBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { if (input.inputs[0].IsNull() || input.inputs[1].IsNull() || input.inputs[2].IsNull()) { throw BinderException("arrow_scan: pointers cannot be null"); } @@ -80,7 +81,7 @@ unique_ptr ArrowTableFunction::ArrowScanBind(ClientContext &contex auto &data = *res; stream_factory_get_schema(reinterpret_cast(stream_factory_ptr), data.schema_root.arrow_schema); PopulateArrowTableSchema(context, res->arrow_table, data.schema_root.arrow_schema); - names = res->arrow_table.GetNames(); + names = StringsToIdentifiers(res->arrow_table.GetNames()); return_types = res->arrow_table.GetTypes(); res->all_types = return_types; if (return_types.empty()) { diff --git a/src/duckdb/src/function/table/arrow_conversion.cpp b/src/duckdb/src/function/table/arrow_conversion.cpp index e49deaf73..a229c8664 100644 --- a/src/duckdb/src/function/table/arrow_conversion.cpp +++ b/src/duckdb/src/function/table/arrow_conversion.cpp @@ -156,24 +156,31 @@ static ArrowListOffsetData ConvertArrowListViewOffsetsTemplated(Vector &vector, // for that reason we need to keep track of the lowest offset, so we can skip all the data that comes before it // when we scan the child data - auto lowest_offset = size ? offsets[0] : 0; + bool has_non_empty_entry = false; + BUFFER_TYPE lowest_offset = 0; + BUFFER_TYPE highest_offset = 0; auto list_data = FlatVector::GetDataMutable(vector); for (idx_t i = 0; i < size; i++) { auto &le = list_data[i]; le.offset = offsets[i]; le.length = sizes[i]; - list_size += le.length; - if (sizes[i] != 0) { - lowest_offset = MinValue(lowest_offset, offsets[i]); + if (le.length != 0) { + auto end_offset = offsets[i] + sizes[i]; + if (!has_non_empty_entry) { + lowest_offset = offsets[i]; + highest_offset = end_offset; + has_non_empty_entry = true; + } else { + lowest_offset = MinValue(lowest_offset, offsets[i]); + highest_offset = MaxValue(highest_offset, end_offset); + } } } - start_offset = lowest_offset; - if (start_offset) { - // We start scanning the child data at the 'start_offset' so we need to fix up the created list entries - for (idx_t i = 0; i < size; i++) { - auto &le = list_data[i]; - le.offset = le.offset <= start_offset ? 0 : le.offset - start_offset; - } + start_offset = has_non_empty_entry ? lowest_offset : 0; + list_size = has_non_empty_entry ? highest_offset - lowest_offset : 0; + for (idx_t i = 0; i < size; i++) { + auto &le = list_data[i]; + le.offset = le.length == 0 ? 0 : le.offset - start_offset; } return result; } diff --git a/src/duckdb/src/function/table/checkpoint.cpp b/src/duckdb/src/function/table/checkpoint.cpp index 8ba250d3d..3dd2474ed 100644 --- a/src/duckdb/src/function/table/checkpoint.cpp +++ b/src/duckdb/src/function/table/checkpoint.cpp @@ -25,7 +25,7 @@ struct CheckpointBindData : public FunctionData { }; static unique_ptr CheckpointBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.emplace_back(LogicalType::BOOLEAN); names.emplace_back("Success"); diff --git a/src/duckdb/src/function/table/glob.cpp b/src/duckdb/src/function/table/glob.cpp index 6aaeac9a4..b9ca555ab 100644 --- a/src/duckdb/src/function/table/glob.cpp +++ b/src/duckdb/src/function/table/glob.cpp @@ -12,7 +12,7 @@ struct GlobFunctionBindData : public TableFunctionData { }; static unique_ptr GlobFunctionBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); auto multi_file_reader = MultiFileReader::Create(input.table_function); result->file_list = multi_file_reader->CreateFileList(context, input.inputs[0], FileGlobOptions::ALLOW_EMPTY); diff --git a/src/duckdb/src/function/table/range.cpp b/src/duckdb/src/function/table/range.cpp index 4950d4d83..d777c40a2 100644 --- a/src/duckdb/src/function/table/range.cpp +++ b/src/duckdb/src/function/table/range.cpp @@ -56,7 +56,7 @@ struct RangeFunctionBindData : public TableFunctionData { template static unique_ptr RangeFunctionBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.emplace_back(LogicalType::BIGINT); if (GENERATE_SERIES) { names.emplace_back("generate_series"); @@ -222,7 +222,7 @@ struct RangeDateTimeBindData : public TableFunctionData { template static unique_ptr RangeDateTimeBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.push_back(LogicalType::TIMESTAMP); if (GENERATE_SERIES) { names.emplace_back("generate_series"); @@ -396,6 +396,7 @@ void RangeTableFunction::RegisterFunction(BuiltinFunctions &set) { range_function.cardinality = RangeCardinality; range_function.is_repeatable = RangeIsRepeatable; range_function.parallelism = TableFunctionParallelism::FORCE_SINGLE_THREADED; + range_function.return_type = TableFunctionReturnType::SET_RETURNING_FUNCTION; // single argument range: (end) - implicit start = 0 and increment = 1 range.AddFunction(range_function); @@ -411,6 +412,7 @@ void RangeTableFunction::RegisterFunction(BuiltinFunctions &set) { range_in_out.cardinality = RangeDateTimeCardinality; range_in_out.is_repeatable = RangeIsRepeatable; range_in_out.parallelism = TableFunctionParallelism::FORCE_SINGLE_THREADED; + range_in_out.return_type = TableFunctionReturnType::SET_RETURNING_FUNCTION; range.AddFunction(range_in_out); set.AddFunction(range); // generate_series: similar to range, but inclusive instead of exclusive bounds on the RHS diff --git a/src/duckdb/src/function/table/repeat.cpp b/src/duckdb/src/function/table/repeat.cpp index 7643ffedf..d6afd5342 100644 --- a/src/duckdb/src/function/table/repeat.cpp +++ b/src/duckdb/src/function/table/repeat.cpp @@ -18,11 +18,11 @@ struct RepeatOperatorData : public GlobalTableFunctionState { }; static unique_ptr RepeatBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { // the repeat function returns the type of the first argument auto &inputs = input.inputs; return_types.push_back(inputs[0].type()); - names.push_back(inputs[0].ToString()); + names.emplace_back(inputs[0].ToString()); if (inputs[1].IsNull()) { throw BinderException("Repeat second parameter cannot be NULL"); } diff --git a/src/duckdb/src/function/table/repeat_row.cpp b/src/duckdb/src/function/table/repeat_row.cpp index 82ae22675..9dd93a11e 100644 --- a/src/duckdb/src/function/table/repeat_row.cpp +++ b/src/duckdb/src/function/table/repeat_row.cpp @@ -19,11 +19,11 @@ struct RepeatRowOperatorData : public GlobalTableFunctionState { }; static unique_ptr RepeatRowBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto &inputs = input.inputs; for (idx_t input_idx = 0; input_idx < inputs.size(); input_idx++) { return_types.push_back(inputs[input_idx].type()); - names.push_back("column" + std::to_string(input_idx)); + names.emplace_back("column" + std::to_string(input_idx)); } auto entry = input.named_parameters.find("num_rows"); if (entry == input.named_parameters.end()) { diff --git a/src/duckdb/src/function/table/sniff_csv.cpp b/src/duckdb/src/function/table/sniff_csv.cpp index 184e4bf60..6d9ab72a8 100644 --- a/src/duckdb/src/function/table/sniff_csv.cpp +++ b/src/duckdb/src/function/table/sniff_csv.cpp @@ -37,7 +37,7 @@ static unique_ptr CSVSniffInitGlobal(ClientContext &co } static unique_ptr CSVSniffBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); if (input.inputs[0].IsNull()) { throw BinderException("sniff_csv cannot take NULL as a file path parameter"); diff --git a/src/duckdb/src/function/table/summary.cpp b/src/duckdb/src/function/table/summary.cpp index 4c8e3a223..8c93dfca8 100644 --- a/src/duckdb/src/function/table/summary.cpp +++ b/src/duckdb/src/function/table/summary.cpp @@ -8,13 +8,13 @@ namespace duckdb { static unique_ptr SummaryFunctionBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("summary"); for (idx_t i = 0; i < input.input_table_types.size(); i++) { return_types.push_back(input.input_table_types[i]); - names.emplace_back(input.input_table_names[i]); + names.push_back(input.input_table_names[i]); } return make_uniq(); diff --git a/src/duckdb/src/function/table/system/create_external_resource.cpp b/src/duckdb/src/function/table/system/create_external_resource.cpp index af2d97035..f848a3a5d 100644 --- a/src/duckdb/src/function/table/system/create_external_resource.cpp +++ b/src/duckdb/src/function/table/system/create_external_resource.cpp @@ -108,7 +108,8 @@ struct CreateExternalResourceState : public GlobalTableFunctionState { }; static unique_ptr CreateExternalResourceBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { auto result = make_uniq(); if (input.inputs[0].IsNull()) { throw InvalidInputException("create_external_resource: the type name must not be NULL"); @@ -389,7 +390,8 @@ struct DestroyExternalResourceState : public GlobalTableFunctionState { }; static unique_ptr DestroyExternalResourceBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { auto result = make_uniq(); if (input.inputs[0].IsNull() || StringValue::Get(input.inputs[0]).empty()) { throw InvalidInputException("destroy_external_resource: the deleter function must not be NULL or empty"); diff --git a/src/duckdb/src/function/table/system/duckdb_approx_database_count.cpp b/src/duckdb/src/function/table/system/duckdb_approx_database_count.cpp index e62a6a03e..ff7da6c40 100644 --- a/src/duckdb/src/function/table/system/duckdb_approx_database_count.cpp +++ b/src/duckdb/src/function/table/system/duckdb_approx_database_count.cpp @@ -12,7 +12,7 @@ struct DuckDBApproxDatabaseCountData : public GlobalTableFunctionState { static unique_ptr DuckDBApproxDatabaseCountBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, - vector &names) { + vector &names) { names.emplace_back("approx_count"); return_types.emplace_back(LogicalType::UBIGINT); return nullptr; diff --git a/src/duckdb/src/function/table/system/duckdb_columns.cpp b/src/duckdb/src/function/table/system/duckdb_columns.cpp index 9d71f3932..8e5206b5e 100644 --- a/src/duckdb/src/function/table/system/duckdb_columns.cpp +++ b/src/duckdb/src/function/table/system/duckdb_columns.cpp @@ -24,7 +24,7 @@ struct DuckDBColumnsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBColumnsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_connection_count.cpp b/src/duckdb/src/function/table/system/duckdb_connection_count.cpp index 33f49a187..df30262f7 100644 --- a/src/duckdb/src/function/table/system/duckdb_connection_count.cpp +++ b/src/duckdb/src/function/table/system/duckdb_connection_count.cpp @@ -14,7 +14,8 @@ struct DuckDBConnectionCountData : public GlobalTableFunctionState { }; static unique_ptr DuckDBConnectionCountBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { names.emplace_back("count"); return_types.emplace_back(LogicalType::UBIGINT); return nullptr; diff --git a/src/duckdb/src/function/table/system/duckdb_constraints.cpp b/src/duckdb/src/function/table/system/duckdb_constraints.cpp index 10e61bd92..182d74af0 100644 --- a/src/duckdb/src/function/table/system/duckdb_constraints.cpp +++ b/src/duckdb/src/function/table/system/duckdb_constraints.cpp @@ -42,7 +42,7 @@ struct DuckDBConstraintsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBConstraintsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_coordinate_systems.cpp b/src/duckdb/src/function/table/system/duckdb_coordinate_systems.cpp index 46a52d4af..f189107ba 100644 --- a/src/duckdb/src/function/table/system/duckdb_coordinate_systems.cpp +++ b/src/duckdb/src/function/table/system/duckdb_coordinate_systems.cpp @@ -21,7 +21,8 @@ struct DuckDBCoordinateSystemsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBCoordinateSystemsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_databases.cpp b/src/duckdb/src/function/table/system/duckdb_databases.cpp index 4b2abbbb0..1ea9359c6 100644 --- a/src/duckdb/src/function/table/system/duckdb_databases.cpp +++ b/src/duckdb/src/function/table/system/duckdb_databases.cpp @@ -14,7 +14,7 @@ struct DuckDBDatabasesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBDatabasesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_dependencies.cpp b/src/duckdb/src/function/table/system/duckdb_dependencies.cpp index 40bc783c1..85beac0d8 100644 --- a/src/duckdb/src/function/table/system/duckdb_dependencies.cpp +++ b/src/duckdb/src/function/table/system/duckdb_dependencies.cpp @@ -27,7 +27,7 @@ struct DuckDBDependenciesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBDependenciesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("classid"); return_types.emplace_back(LogicalType::BIGINT); diff --git a/src/duckdb/src/function/table/system/duckdb_eviction_queues.cpp b/src/duckdb/src/function/table/system/duckdb_eviction_queues.cpp index 0e1c29288..085a2a711 100644 --- a/src/duckdb/src/function/table/system/duckdb_eviction_queues.cpp +++ b/src/duckdb/src/function/table/system/duckdb_eviction_queues.cpp @@ -13,7 +13,7 @@ struct DuckDBEvictionQueuesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBEvictionQueuesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("queue_index"); return_types.emplace_back(LogicalType::BIGINT); diff --git a/src/duckdb/src/function/table/system/duckdb_extensions.cpp b/src/duckdb/src/function/table/system/duckdb_extensions.cpp index 83529d8b0..d72386763 100644 --- a/src/duckdb/src/function/table/system/duckdb_extensions.cpp +++ b/src/duckdb/src/function/table/system/duckdb_extensions.cpp @@ -35,7 +35,7 @@ struct DuckDBExtensionsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBExtensionsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("extension_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_external_file_cache.cpp b/src/duckdb/src/function/table/system/duckdb_external_file_cache.cpp index 3a87e393c..8a86c225b 100644 --- a/src/duckdb/src/function/table/system/duckdb_external_file_cache.cpp +++ b/src/duckdb/src/function/table/system/duckdb_external_file_cache.cpp @@ -12,7 +12,8 @@ struct DuckDBExternalFileCacheData : public GlobalTableFunctionState { }; static unique_ptr DuckDBExternalFileCacheBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { names.emplace_back("path"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_functions.cpp b/src/duckdb/src/function/table/system/duckdb_functions.cpp index 8f19feefa..61c8a0d24 100644 --- a/src/duckdb/src/function/table/system/duckdb_functions.cpp +++ b/src/duckdb/src/function/table/system/duckdb_functions.cpp @@ -31,7 +31,7 @@ struct DuckDBFunctionsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBFunctionsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_indexes.cpp b/src/duckdb/src/function/table/system/duckdb_indexes.cpp index 0d1c9f54b..b68505254 100644 --- a/src/duckdb/src/function/table/system/duckdb_indexes.cpp +++ b/src/duckdb/src/function/table/system/duckdb_indexes.cpp @@ -17,7 +17,7 @@ struct DuckDBIndexesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBIndexesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_keywords.cpp b/src/duckdb/src/function/table/system/duckdb_keywords.cpp index 3b0d48b29..56ea13ca8 100644 --- a/src/duckdb/src/function/table/system/duckdb_keywords.cpp +++ b/src/duckdb/src/function/table/system/duckdb_keywords.cpp @@ -15,7 +15,7 @@ struct DuckDBKeywordsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBKeywordsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("keyword_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_log.cpp b/src/duckdb/src/function/table/system/duckdb_log.cpp index 96c35853f..dfa8ca71f 100644 --- a/src/duckdb/src/function/table/system/duckdb_log.cpp +++ b/src/duckdb/src/function/table/system/duckdb_log.cpp @@ -23,7 +23,7 @@ struct DuckDBLogData : public GlobalTableFunctionState { }; static unique_ptr DuckDBLogBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("context_id"); return_types.emplace_back(LogicalType::UBIGINT); diff --git a/src/duckdb/src/function/table/system/duckdb_log_contexts.cpp b/src/duckdb/src/function/table/system/duckdb_log_contexts.cpp index d48b8debb..4c389d6d6 100644 --- a/src/duckdb/src/function/table/system/duckdb_log_contexts.cpp +++ b/src/duckdb/src/function/table/system/duckdb_log_contexts.cpp @@ -25,7 +25,7 @@ struct DuckDBLogContextData : public GlobalTableFunctionState { }; static unique_ptr DuckDBLogContextBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("context_id"); return_types.emplace_back(LogicalType::UBIGINT); diff --git a/src/duckdb/src/function/table/system/duckdb_memory.cpp b/src/duckdb/src/function/table/system/duckdb_memory.cpp index f2cce56cd..96b536127 100644 --- a/src/duckdb/src/function/table/system/duckdb_memory.cpp +++ b/src/duckdb/src/function/table/system/duckdb_memory.cpp @@ -12,7 +12,7 @@ struct DuckDBMemoryData : public GlobalTableFunctionState { }; static unique_ptr DuckDBMemoryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("tag"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_metrics.cpp b/src/duckdb/src/function/table/system/duckdb_metrics.cpp index 2dd25be1f..a9b09dd2e 100644 --- a/src/duckdb/src/function/table/system/duckdb_metrics.cpp +++ b/src/duckdb/src/function/table/system/duckdb_metrics.cpp @@ -13,7 +13,7 @@ struct DuckDBMetricsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBMetricsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("metric_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_optimizers.cpp b/src/duckdb/src/function/table/system/duckdb_optimizers.cpp index 3dd0f899c..719e10514 100644 --- a/src/duckdb/src/function/table/system/duckdb_optimizers.cpp +++ b/src/duckdb/src/function/table/system/duckdb_optimizers.cpp @@ -16,7 +16,7 @@ struct DuckDBOptimizersData : public GlobalTableFunctionState { }; static unique_ptr DuckDBOptimizersBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_prepared_statements.cpp b/src/duckdb/src/function/table/system/duckdb_prepared_statements.cpp index d80359033..0ee0f69f0 100644 --- a/src/duckdb/src/function/table/system/duckdb_prepared_statements.cpp +++ b/src/duckdb/src/function/table/system/duckdb_prepared_statements.cpp @@ -18,7 +18,8 @@ struct DuckDBPreparedStatementsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBPreparedStatementsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_schemas.cpp b/src/duckdb/src/function/table/system/duckdb_schemas.cpp index c3f492325..9dd7fb568 100644 --- a/src/duckdb/src/function/table/system/duckdb_schemas.cpp +++ b/src/duckdb/src/function/table/system/duckdb_schemas.cpp @@ -17,7 +17,7 @@ struct DuckDBSchemasData : public GlobalTableFunctionState { }; static unique_ptr DuckDBSchemasBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("oid"); return_types.emplace_back(LogicalType::BIGINT); diff --git a/src/duckdb/src/function/table/system/duckdb_secret_types.cpp b/src/duckdb/src/function/table/system/duckdb_secret_types.cpp index fe60d8365..69efc42dd 100644 --- a/src/duckdb/src/function/table/system/duckdb_secret_types.cpp +++ b/src/duckdb/src/function/table/system/duckdb_secret_types.cpp @@ -16,7 +16,7 @@ struct DuckDBSecretTypesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBSecretTypesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("type"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_secrets.cpp b/src/duckdb/src/function/table/system/duckdb_secrets.cpp index b698e9f8d..2361ac633 100644 --- a/src/duckdb/src/function/table/system/duckdb_secrets.cpp +++ b/src/duckdb/src/function/table/system/duckdb_secrets.cpp @@ -28,7 +28,7 @@ struct DuckDBSecretsBindData : public FunctionData { }; static unique_ptr DuckDBSecretsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); auto entry = input.named_parameters.find("redact"); diff --git a/src/duckdb/src/function/table/system/duckdb_sequences.cpp b/src/duckdb/src/function/table/system/duckdb_sequences.cpp index 441576a32..089dfd73d 100644 --- a/src/duckdb/src/function/table/system/duckdb_sequences.cpp +++ b/src/duckdb/src/function/table/system/duckdb_sequences.cpp @@ -19,7 +19,7 @@ struct DuckDBSequencesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBSequencesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_settings.cpp b/src/duckdb/src/function/table/system/duckdb_settings.cpp index 9184ae860..dea93d024 100644 --- a/src/duckdb/src/function/table/system/duckdb_settings.cpp +++ b/src/duckdb/src/function/table/system/duckdb_settings.cpp @@ -27,7 +27,7 @@ struct DuckDBSettingsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBSettingsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_tables.cpp b/src/duckdb/src/function/table/system/duckdb_tables.cpp index 965ceafb6..257b403aa 100644 --- a/src/duckdb/src/function/table/system/duckdb_tables.cpp +++ b/src/duckdb/src/function/table/system/duckdb_tables.cpp @@ -22,7 +22,7 @@ struct DuckDBTablesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBTablesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_temporary_files.cpp b/src/duckdb/src/function/table/system/duckdb_temporary_files.cpp index 690cf4da0..6f7f1742c 100644 --- a/src/duckdb/src/function/table/system/duckdb_temporary_files.cpp +++ b/src/duckdb/src/function/table/system/duckdb_temporary_files.cpp @@ -12,7 +12,7 @@ struct DuckDBTemporaryFilesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBTemporaryFilesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("path"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_triggers.cpp b/src/duckdb/src/function/table/system/duckdb_triggers.cpp index 7075e3f80..5e177aa96 100644 --- a/src/duckdb/src/function/table/system/duckdb_triggers.cpp +++ b/src/duckdb/src/function/table/system/duckdb_triggers.cpp @@ -19,7 +19,7 @@ struct DuckDBTriggersData : public GlobalTableFunctionState { }; static unique_ptr DuckDBTriggersBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_types.cpp b/src/duckdb/src/function/table/system/duckdb_types.cpp index 4f8370fa4..76bd9d706 100644 --- a/src/duckdb/src/function/table/system/duckdb_types.cpp +++ b/src/duckdb/src/function/table/system/duckdb_types.cpp @@ -21,7 +21,7 @@ struct DuckDBTypesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBTypesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_variables.cpp b/src/duckdb/src/function/table/system/duckdb_variables.cpp index d31194313..95d1c87aa 100644 --- a/src/duckdb/src/function/table/system/duckdb_variables.cpp +++ b/src/duckdb/src/function/table/system/duckdb_variables.cpp @@ -25,7 +25,7 @@ struct DuckDBVariablesData : public GlobalTableFunctionState { }; static unique_ptr DuckDBVariablesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_views.cpp b/src/duckdb/src/function/table/system/duckdb_views.cpp index 8cf4c5a77..04444568e 100644 --- a/src/duckdb/src/function/table/system/duckdb_views.cpp +++ b/src/duckdb/src/function/table/system/duckdb_views.cpp @@ -19,7 +19,7 @@ struct DuckDBViewsData : public GlobalTableFunctionState { }; static unique_ptr DuckDBViewsBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/duckdb_which_secret.cpp b/src/duckdb/src/function/table/system/duckdb_which_secret.cpp index 55ee22467..fc326fa41 100644 --- a/src/duckdb/src/function/table/system/duckdb_which_secret.cpp +++ b/src/duckdb/src/function/table/system/duckdb_which_secret.cpp @@ -25,7 +25,7 @@ struct DuckDBWhichSecretBindData : public TableFunctionData { }; static unique_ptr DuckDBWhichSecretBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/enable_profiling.cpp b/src/duckdb/src/function/table/system/enable_profiling.cpp index 26115ffb6..b1fd0984d 100644 --- a/src/duckdb/src/function/table/system/enable_profiling.cpp +++ b/src/duckdb/src/function/table/system/enable_profiling.cpp @@ -62,7 +62,7 @@ static void EnableProfiling(ClientContext &context, TableFunctionInput &data, Da } static unique_ptr BindEnableProfiling(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { if (input.inputs.size() > 1) { throw InvalidInputException("EnableProfiling: expected 0 or 1 parameter"); } @@ -123,7 +123,7 @@ static void DisableProfiling(ClientContext &context, TableFunctionInput &data, D } static unique_ptr BindDisableProfiling(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.emplace_back(LogicalType::BOOLEAN); names.emplace_back("Success"); diff --git a/src/duckdb/src/function/table/system/external_resource_types.cpp b/src/duckdb/src/function/table/system/external_resource_types.cpp index a7b41724b..9ae9eb692 100644 --- a/src/duckdb/src/function/table/system/external_resource_types.cpp +++ b/src/duckdb/src/function/table/system/external_resource_types.cpp @@ -22,7 +22,7 @@ struct RegisterExternalResourceTypeState : public GlobalTableFunctionState { static unique_ptr RegisterExternalResourceTypeBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, - vector &names) { + vector &names) { auto result = make_uniq(); auto &type = result->type; @@ -91,7 +91,8 @@ struct ExternalResourceTypesData : public GlobalTableFunctionState { }; static unique_ptr ExternalResourceTypesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { names.emplace_back("name"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("kind"); diff --git a/src/duckdb/src/function/table/system/external_resources.cpp b/src/duckdb/src/function/table/system/external_resources.cpp index f40ef6651..3926864db 100644 --- a/src/duckdb/src/function/table/system/external_resources.cpp +++ b/src/duckdb/src/function/table/system/external_resources.cpp @@ -46,7 +46,7 @@ struct ExternalResourcesGlobalState : public GlobalTableFunctionState { }; static unique_ptr ExternalResourcesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); for (auto &np : input.named_parameters) { if (StringUtil::Lower(np.first.GetIdentifierName()) == "discover" && !np.second.IsNull()) { @@ -233,7 +233,8 @@ struct RegisterExternalResourceState : public GlobalTableFunctionState { }; static unique_ptr RegisterExternalResourceBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { auto result = make_uniq(); auto &resource = result->resource; if (input.inputs[0].IsNull() || input.inputs[1].IsNull()) { @@ -312,7 +313,7 @@ struct DeregisterExternalResourceState : public GlobalTableFunctionState { static unique_ptr DeregisterExternalResourceBind(ClientContext &context, TableFunctionBindInput &input, vector &return_types, - vector &names) { + vector &names) { auto result = make_uniq(); if (input.inputs[0].IsNull() || StringValue::Get(input.inputs[0]).empty()) { throw InvalidInputException("deregister_external_resource: the name must not be NULL or empty"); diff --git a/src/duckdb/src/function/table/system/logging_utils.cpp b/src/duckdb/src/function/table/system/logging_utils.cpp index e90bab231..296dc179f 100644 --- a/src/duckdb/src/function/table/system/logging_utils.cpp +++ b/src/duckdb/src/function/table/system/logging_utils.cpp @@ -45,7 +45,7 @@ static void EnableLogging(ClientContext &context, TableFunctionInput &data, Data } static unique_ptr BindEnableLogging(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { if (input.inputs.size() > 1) { throw InvalidInputException("EnableLogging: expected 0 or 1 parameter"); } @@ -140,7 +140,7 @@ static void TruncateLogs(ClientContext &context, TableFunctionInput &data, DataC } static unique_ptr BindDisableLogging(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.emplace_back(LogicalType::BOOLEAN); names.emplace_back("Success"); @@ -148,7 +148,7 @@ static unique_ptr BindDisableLogging(ClientContext &context, Table } static unique_ptr BindTruncateLogs(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { return_types.emplace_back(LogicalType::BOOLEAN); names.emplace_back("Success"); diff --git a/src/duckdb/src/function/table/system/pragma_collations.cpp b/src/duckdb/src/function/table/system/pragma_collations.cpp index de16983cc..29c02abfb 100644 --- a/src/duckdb/src/function/table/system/pragma_collations.cpp +++ b/src/duckdb/src/function/table/system/pragma_collations.cpp @@ -16,7 +16,7 @@ struct PragmaCollateData : public GlobalTableFunctionState { }; static unique_ptr PragmaCollateBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("collname"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/pragma_database_size.cpp b/src/duckdb/src/function/table/system/pragma_database_size.cpp index 0bd4a3f69..fb784f52c 100644 --- a/src/duckdb/src/function/table/system/pragma_database_size.cpp +++ b/src/duckdb/src/function/table/system/pragma_database_size.cpp @@ -22,7 +22,7 @@ struct PragmaDatabaseSizeData : public GlobalTableFunctionState { }; static unique_ptr PragmaDatabaseSizeBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("database_name"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/pragma_metadata_info.cpp b/src/duckdb/src/function/table/system/pragma_metadata_info.cpp index 2daa75c69..5deb58c3f 100644 --- a/src/duckdb/src/function/table/system/pragma_metadata_info.cpp +++ b/src/duckdb/src/function/table/system/pragma_metadata_info.cpp @@ -21,7 +21,7 @@ struct PragmaMetadataOperatorData : public GlobalTableFunctionState { }; static unique_ptr PragmaMetadataInfoBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("block_id"); return_types.emplace_back(LogicalType::BIGINT); diff --git a/src/duckdb/src/function/table/system/pragma_storage_info.cpp b/src/duckdb/src/function/table/system/pragma_storage_info.cpp index e99860645..9525cc4d3 100644 --- a/src/duckdb/src/function/table/system/pragma_storage_info.cpp +++ b/src/duckdb/src/function/table/system/pragma_storage_info.cpp @@ -46,7 +46,7 @@ struct PragmaStorageLocalState : public LocalTableFunctionState { }; static unique_ptr PragmaStorageInfoBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("row_group_id"); return_types.emplace_back(LogicalType::BIGINT); diff --git a/src/duckdb/src/function/table/system/pragma_table_info.cpp b/src/duckdb/src/function/table/system/pragma_table_info.cpp index 8ed6ca7f7..7d5b52085 100644 --- a/src/duckdb/src/function/table/system/pragma_table_info.cpp +++ b/src/duckdb/src/function/table/system/pragma_table_info.cpp @@ -49,7 +49,7 @@ static Value DefaultValue(const ColumnDefinition &def) { } struct PragmaTableInfoHelper { - static void GetSchema(vector &return_types, vector &names) { + static void GetSchema(vector &return_types, vector &names) { names.emplace_back("cid"); return_types.emplace_back(LogicalType::INTEGER); @@ -102,7 +102,7 @@ struct PragmaTableInfoHelper { }; struct PragmaShowHelper { - static void GetSchema(vector &return_types, vector &names) { + static void GetSchema(vector &return_types, vector &names) { names.emplace_back("column_name"); return_types.emplace_back(LogicalType::VARCHAR); @@ -160,7 +160,7 @@ struct PragmaShowHelper { template static unique_ptr PragmaTableInfoBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { if (IS_PRAGMA_TABLE_INFO) { PragmaTableInfoHelper::GetSchema(return_types, names); } else { diff --git a/src/duckdb/src/function/table/system/pragma_table_sample.cpp b/src/duckdb/src/function/table/system/pragma_table_sample.cpp index 1c21a5e3e..eb22fecc9 100644 --- a/src/duckdb/src/function/table/system/pragma_table_sample.cpp +++ b/src/duckdb/src/function/table/system/pragma_table_sample.cpp @@ -31,7 +31,7 @@ struct DuckDBTableSampleOperatorData : public GlobalTableFunctionState { }; static unique_ptr DuckDBTableSampleBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { // look up the table name in the catalog auto qname = QualifiedName::Parse(input.inputs[0].GetValue()); CatalogEntryRetriever retriever(context); @@ -49,7 +49,7 @@ static unique_ptr DuckDBTableSampleBind(ClientContext &context, Ta for (idx_t i = 0; i < types.size(); i++) { auto logical_index = LogicalIndex(i); auto &col = table_entry.GetColumn(logical_index); - names.emplace_back(col.GetName().GetIdentifierName()); + names.push_back(col.GetName()); } return make_uniq(entry); diff --git a/src/duckdb/src/function/table/system/pragma_user_agent.cpp b/src/duckdb/src/function/table/system/pragma_user_agent.cpp index 8b6003906..82b04eca2 100644 --- a/src/duckdb/src/function/table/system/pragma_user_agent.cpp +++ b/src/duckdb/src/function/table/system/pragma_user_agent.cpp @@ -12,7 +12,7 @@ struct PragmaUserAgentData : public GlobalTableFunctionState { }; static unique_ptr PragmaUserAgentBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("user_agent"); return_types.emplace_back(LogicalType::VARCHAR); diff --git a/src/duckdb/src/function/table/system/test_all_types.cpp b/src/duckdb/src/function/table/system/test_all_types.cpp index 56ae62f89..2af0bbae7 100644 --- a/src/duckdb/src/function/table/system/test_all_types.cpp +++ b/src/duckdb/src/function/table/system/test_all_types.cpp @@ -385,7 +385,7 @@ struct TestAllTypesBindData : public TableFunctionData { }; static unique_ptr TestAllTypesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); bool use_large_enum = false; bool use_large_bignum = false; @@ -406,7 +406,7 @@ static unique_ptr TestAllTypesBind(ClientContext &context, TableFu result->test_types = TestAllTypesFun::GetTestTypes(use_large_enum, use_large_bignum); for (auto &test_type : result->test_types) { return_types.push_back(test_type.type); - names.push_back(test_type.name); + names.emplace_back(test_type.name); } return std::move(result); } diff --git a/src/duckdb/src/function/table/system/test_vector_types.cpp b/src/duckdb/src/function/table/system/test_vector_types.cpp index 3e046b117..a91964061 100644 --- a/src/duckdb/src/function/table/system/test_vector_types.cpp +++ b/src/duckdb/src/function/table/system/test_vector_types.cpp @@ -263,7 +263,7 @@ struct TestVectorDictionary { }; static unique_ptr TestVectorTypesBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto result = make_uniq(); for (idx_t i = 0; i < input.inputs.size(); i++) { string name = "test_vector"; diff --git a/src/duckdb/src/function/table/unnest.cpp b/src/duckdb/src/function/table/unnest.cpp index 450adf39b..76cf40ef6 100644 --- a/src/duckdb/src/function/table/unnest.cpp +++ b/src/duckdb/src/function/table/unnest.cpp @@ -43,7 +43,7 @@ struct UnnestLocalState : public LocalTableFunctionState { }; static unique_ptr UnnestBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { if (input.input_table_types.size() != 1 || (input.input_table_types[0].id() != LogicalTypeId::LIST && input.input_table_types[0].id() != LogicalTypeId::ARRAY)) { throw BinderException("UNNEST requires a single list or array as input"); diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index f25eac315..67ecfec4f 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "0-alpha36812" +#define DUCKDB_PATCH_VERSION "0-alpha37080" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 0 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 2 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v2.0.0-alpha36812" +#define DUCKDB_VERSION "v2.0.0-alpha37080" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "193754bd44" +#define DUCKDB_SOURCE_ID "e85c4d27d7" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" @@ -30,7 +30,7 @@ struct PragmaVersionData : public GlobalTableFunctionState { }; static unique_ptr PragmaVersionBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("library_version"); return_types.emplace_back(LogicalType::VARCHAR); names.emplace_back("source_id"); @@ -114,7 +114,7 @@ struct PragmaPlatformData : public GlobalTableFunctionState { }; static unique_ptr PragmaPlatformBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { names.emplace_back("platform"); return_types.emplace_back(LogicalType::VARCHAR); return nullptr; diff --git a/src/duckdb/src/include/duckdb/catalog/catalog.hpp b/src/duckdb/src/include/duckdb/catalog/catalog.hpp index 39810ef39..c430e2bec 100644 --- a/src/duckdb/src/include/duckdb/catalog/catalog.hpp +++ b/src/duckdb/src/include/duckdb/catalog/catalog.hpp @@ -410,14 +410,14 @@ class Catalog { } //! Returns the default schema of the catalog - virtual string GetDefaultSchema() const; + virtual Identifier GetDefaultSchema() const; //! The default table is used for `SELECT * FROM ;` //! FIXME: these should be virtual methods DUCKDB_API bool HasDefaultTable() const; DUCKDB_API void SetDefaultTable(const Identifier &schema, const Identifier &name); - DUCKDB_API string GetDefaultTable() const; - DUCKDB_API string GetDefaultTableSchema() const; + DUCKDB_API Identifier GetDefaultTable() const; + DUCKDB_API Identifier GetDefaultTableSchema() const; //! Returns the dependency manager of this catalog - if the catalog has any virtual optional_ptr GetDependencyManager(); @@ -487,8 +487,8 @@ class Catalog { AttachedDatabase &db; //! (optionally) a default table to query for `SELECT * FROM ;` - string default_table; - string default_table_schema; + Identifier default_table; + Identifier default_table_schema; public: //! Lookup an entry using TryLookupEntry, throws if entry not found and if_not_found == THROW_EXCEPTION diff --git a/src/duckdb/src/include/duckdb/common/enum_util.hpp b/src/duckdb/src/include/duckdb/common/enum_util.hpp index a30435634..9279ce093 100644 --- a/src/duckdb/src/include/duckdb/common/enum_util.hpp +++ b/src/duckdb/src/include/duckdb/common/enum_util.hpp @@ -184,6 +184,8 @@ enum class DefaultOrderByNullType : uint8_t; enum class DeferredRuntimeFilterType : uint8_t; +enum class DeleteIdState : uint8_t; + enum class DependencyEntryType : uint8_t; enum class DeprecatedIndexType : uint8_t; @@ -837,6 +839,9 @@ const char* EnumUtil::ToChars(DefaultOrderByNullType val template<> const char* EnumUtil::ToChars(DeferredRuntimeFilterType value); +template<> +const char* EnumUtil::ToChars(DeleteIdState value); + template<> const char* EnumUtil::ToChars(DependencyEntryType value); @@ -1702,6 +1707,9 @@ DefaultOrderByNullType EnumUtil::FromString(const char * template<> DeferredRuntimeFilterType EnumUtil::FromString(const char *value); +template<> +DeleteIdState EnumUtil::FromString(const char *value); + template<> DependencyEntryType EnumUtil::FromString(const char *value); diff --git a/src/duckdb/src/include/duckdb/common/enums/row_id_handling.hpp b/src/duckdb/src/include/duckdb/common/enums/row_id_handling.hpp index 1d89509b0..dd9f7fbe9 100644 --- a/src/duckdb/src/include/duckdb/common/enums/row_id_handling.hpp +++ b/src/duckdb/src/include/duckdb/common/enums/row_id_handling.hpp @@ -9,7 +9,7 @@ #pragma once #include "duckdb/common/constants.hpp" -#include "duckdb/common/windows_undefs.hpp" // test5 +#include "duckdb/common/windows_undefs.hpp" namespace duckdb { diff --git a/src/duckdb/src/include/duckdb/common/http_util.hpp b/src/duckdb/src/include/duckdb/common/http_util.hpp index 0ea4f0368..8f7693397 100644 --- a/src/duckdb/src/include/duckdb/common/http_util.hpp +++ b/src/duckdb/src/include/duckdb/common/http_util.hpp @@ -289,6 +289,17 @@ class HTTPUtil { HTTPUtil(const HTTPUtil &other) = delete; HTTPUtil &operator=(const HTTPUtil &) = delete; + template + TARGET &Cast() { + DynamicCastCheck(this); + return reinterpret_cast(*this); + } + template + const TARGET &Cast() const { + DynamicCastCheck(this); + return reinterpret_cast(*this); + } + public: static HTTPUtil &Get(DatabaseInstance &db); diff --git a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp index da80a7636..30eb0a9e1 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp @@ -101,7 +101,7 @@ class MultiFileFunction : public TableFunction { static unique_ptr MultiFileBindInternal(ClientContext &context, unique_ptr multi_file_reader_p, shared_ptr multi_file_list_p, - vector &return_types, vector &names, + vector &return_types, vector &names, MultiFileOptions file_options_p, unique_ptr options_p, unique_ptr interface_p) { @@ -120,7 +120,7 @@ class MultiFileFunction : public TableFunction { result->names.emplace_back("empty"); result->columns = MultiFileColumnDefinition::ColumnsFromNamesAndTypes(result->names, result->types); return_types = result->types; - names = IdentifiersToStrings(result->names); + names = result->names; return std::move(result); } @@ -143,7 +143,7 @@ class MultiFileFunction : public TableFunction { if (return_types.empty()) { // no expected types - just copy the types return_types = result->types; - names = IdentifiersToStrings(result->names); + names = result->names; } else { // We're deserializing from a previously successful bind call // verify that the amount of columns still matches @@ -183,14 +183,14 @@ class MultiFileFunction : public TableFunction { } // expected types - overwrite the types we want to read instead result->types = return_types; - result->table_columns = names; + result->table_columns = IdentifiersToStrings(names); } result->columns = MultiFileColumnDefinition::ColumnsFromNamesAndTypes(result->names, result->types); return std::move(result); } static unique_ptr MultiFileBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto interface = OP::CreateInterface(context); auto multi_file_reader = MultiFileReader::Create(input.table_function); @@ -249,8 +249,12 @@ class MultiFileFunction : public TableFunction { } interface->FinalizeCopyBind(context, *options, expected_names, expected_types); - return MultiFileBindInternal(context, std::move(multi_file_reader), std::move(file_list), expected_types, - expected_names, std::move(file_options), std::move(options), std::move(interface)); + // the COPY bind still operates on plain strings - convert around the table function bind + auto names = StringsToIdentifiers(expected_names); + auto result = MultiFileBindInternal(context, std::move(multi_file_reader), std::move(file_list), expected_types, + names, std::move(file_options), std::move(options), std::move(interface)); + expected_names = IdentifiersToStrings(names); + return result; } static unique_ptr MultiFileFilterPushdown(ClientContext &context, const MultiFileBindData &data, diff --git a/src/duckdb/src/include/duckdb/function/create_sort_key.hpp b/src/duckdb/src/include/duckdb/function/create_sort_key.hpp index e8d08aab3..eeffb0b42 100644 --- a/src/duckdb/src/include/duckdb/function/create_sort_key.hpp +++ b/src/duckdb/src/include/duckdb/function/create_sort_key.hpp @@ -49,6 +49,7 @@ struct CreateSortKeyHelpers { static void CreateSortKey(DataChunk &input, const vector &modifiers, Vector &result); static void CreateSortKey(const Vector &input, OrderModifiers modifiers, Vector &result); static void CreateSortKey(const Vector &input, idx_t count, OrderModifiers modifiers, Vector &result); + static bool IsNullSortKey(const_data_ptr_t data, OrderByNullType null_order); static idx_t DecodeSortKey(string_t sort_key, Vector &result, idx_t result_idx, OrderModifiers modifiers); static void DecodeSortKey(string_t sort_key, DataChunk &result, idx_t result_idx, const vector &modifiers); diff --git a/src/duckdb/src/include/duckdb/function/table/arrow.hpp b/src/duckdb/src/include/duckdb/function/table/arrow.hpp index 75ce9b37a..bd0b78398 100644 --- a/src/duckdb/src/include/duckdb/function/table/arrow.hpp +++ b/src/duckdb/src/include/duckdb/function/table/arrow.hpp @@ -207,9 +207,9 @@ struct ArrowTableFunction { public: //! Binds an arrow table static unique_ptr ArrowScanBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names); + vector &return_types, vector &names); static unique_ptr ArrowScanBindDumb(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names); + vector &return_types, vector &names); //! Actual conversion from Arrow to DuckDB static void ArrowToDuckDB(ArrowScanLocalState &scan_state, const arrow_column_map_t &arrow_convert_data, DataChunk &output, bool arrow_scan_is_projected = true, diff --git a/src/duckdb/src/include/duckdb/function/table_function.hpp b/src/duckdb/src/include/duckdb/function/table_function.hpp index 8cd6e451a..5bd2e0925 100644 --- a/src/duckdb/src/include/duckdb/function/table_function.hpp +++ b/src/duckdb/src/include/duckdb/function/table_function.hpp @@ -316,12 +316,12 @@ struct BindInfo { }; typedef unique_ptr (*table_function_bind_t)(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names); + vector &return_types, vector &names); typedef unique_ptr (*table_function_bind_replace_t)(ClientContext &context, TableFunctionBindInput &input); typedef unique_ptr (*table_function_bind_operator_t)(ClientContext &context, TableFunctionBindInput &input, TableIndex bind_index, - vector &return_names); + vector &return_names); typedef unique_ptr (*table_function_init_global_t)(ClientContext &context, TableFunctionInitInput &input); typedef unique_ptr (*table_function_init_local_t)(ExecutionContext &context, diff --git a/src/duckdb/src/include/duckdb/main/capi/capi_internal_table.hpp b/src/duckdb/src/include/duckdb/main/capi/capi_internal_table.hpp index b320556f1..6384f2ba6 100644 --- a/src/duckdb/src/include/duckdb/main/capi/capi_internal_table.hpp +++ b/src/duckdb/src/include/duckdb/main/capi/capi_internal_table.hpp @@ -57,7 +57,7 @@ struct CTableBindData : public TableFunctionData { struct CTableInternalBindInfo { CTableInternalBindInfo(ClientContext &context, const vector ¶meters, const named_parameter_map_t &named_parameters, vector &return_types, - vector &names, CTableBindData &bind_data, CTableFunctionInfo &function_info) + vector &names, CTableBindData &bind_data, CTableFunctionInfo &function_info) : context(context), parameters(parameters), named_parameters(named_parameters), return_types(return_types), names(names), bind_data(bind_data), function_info(function_info), success(true) { } @@ -68,7 +68,7 @@ struct CTableInternalBindInfo { named_parameter_map_t named_parameters; vector &return_types; - vector &names; + vector &names; CTableBindData &bind_data; CTableFunctionInfo &function_info; bool success; diff --git a/src/duckdb/src/include/duckdb/main/extension_entries.hpp b/src/duckdb/src/include/duckdb/main/extension_entries.hpp index 7cf9449f3..18e1b5208 100644 --- a/src/duckdb/src/include/duckdb/main/extension_entries.hpp +++ b/src/duckdb/src/include/duckdb/main/extension_entries.hpp @@ -1281,6 +1281,7 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = { {"enable_geoparquet_conversion", "parquet"}, {"enable_global_s3_configuration", "httpfs"}, {"enable_server_cert_verification", "httpfs"}, + {"extra_http_headers", "httpfs"}, {"force_download", "httpfs"}, {"force_download_threshold", "httpfs"}, {"geometry_always_xy", "spatial"}, diff --git a/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp b/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp index 96b7a5cca..a4f7898e0 100644 --- a/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +++ b/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp @@ -9,7 +9,7 @@ #include "duckdb/common/reference_map.hpp" #include "duckdb/optimizer/join_order/join_relation_set.hpp" -#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" namespace duckdb { diff --git a/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp b/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp index 1953ce49d..1aa886b9d 100644 --- a/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp +++ b/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp @@ -14,7 +14,7 @@ #include "duckdb/optimizer/join_order/cardinality_estimator.hpp" #include "duckdb/optimizer/join_order/join_relation_set.hpp" #include "duckdb/optimizer/join_order/join_order_operator.hpp" -#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" #include "duckdb/parser/expression_map.hpp" #include "duckdb/planner/column_binding_map.hpp" #include "duckdb/planner/logical_operator.hpp" @@ -60,14 +60,14 @@ class RelationManager { //! Extract the set of relations referred to inside an expression bool ExtractBindings(const Expression &expression, unordered_set &bindings); - void AddRelation(LogicalOperator &op, optional_ptr parent, const RelationStats &stats); + bool TryNormalizeBinding(ColumnBinding binding, ColumnBinding &normalized) const; + bool HasCompleteStats() const; + bool AddRelation(LogicalOperator &op, optional_ptr parent, const RelationStats &stats); //! Add an unnest relation which can come from a logical unnest or a logical get which has an unnest function - void AddRelationWithChildren(JoinOrderOptimizer &optimizer, LogicalOperator &op, LogicalOperator &input_op, + bool AddRelationWithChildren(JoinOrderOptimizer &optimizer, LogicalOperator &op, LogicalOperator &input_op, optional_ptr parent, RelationStats &child_stats, optional_ptr limit_op, vector> &datasource_filters); - void AddAggregateOrWindowRelation(LogicalOperator &op, optional_ptr parent, - const RelationStats &stats, LogicalOperatorType op_type); vector> GetRelations(); const vector GetRelationStats(); @@ -81,12 +81,16 @@ class RelationManager { JoinRelationSetManager &set_manager); void GetColumnBindingsFromExpression(const Expression &expression, column_binding_set_t &column_bindings); void GetColumnBindingsFromOperator(LogicalOperator &op, column_binding_set_t &column_bindings); + void RegisterRelationBindings(LogicalOperator &op, RelationIndex relation_id, const RelationStats &stats); + optional AlignStatsWithRelationRoot(LogicalOperator &op, const RelationStats &stats); private: ClientContext &context; //! Set of all relations considered in the join optimizer vector> relations; reference_map_t operator_relations; + column_binding_map_t normalized_bindings; + bool stats_complete = true; }; } // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp b/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp deleted file mode 100644 index f32af4dff..000000000 --- a/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +++ /dev/null @@ -1,86 +0,0 @@ -//===----------------------------------------------------------------------===// -// DuckDB -// -// duckdb/optimizer/join_order/statistics_extractor.hpp -// -// -//===----------------------------------------------------------------------===// -#pragma once - -#include "duckdb/planner/filter/expression_filter.hpp" -#include "duckdb/planner/logical_operator.hpp" - -namespace duckdb { - -class CardinalityEstimator; - -enum class DistinctCountSource : uint8_t { CARDINALITY, MIN_MAX, HLL, EXACT }; - -struct DistinctCount { - DistinctCount(idx_t distinct_count, DistinctCountSource source); - - idx_t distinct_count; - DistinctCountSource source; -}; - -struct ExpressionBinding { -public: - bool FoundExpression() const; - bool FoundColumnRef() const; - -public: - optional_ptr expression; - ColumnBinding child_binding; - bool expression_is_constant = false; -}; - -struct RelationStats { -public: - RelationStats(); - -public: - //! column_id -> estimated distinct count for column - vector column_distinct_count; - idx_t cardinality; - double filter_strength = 1; - bool stats_initialized = false; - - //! for debug, column names and tables - vector column_names; - Identifier table_name; -}; - -class RelationStatisticsHelper { -public: - static constexpr double DEFAULT_SELECTIVITY = 0.2; - -public: - static idx_t InspectTableFilter(idx_t cardinality, const TableFilter &filter, BaseStatistics &base_stats); - //! Extract Statistics from a LogicalGet. - static RelationStats ExtractGetStats(LogicalGet &get, ClientContext &context); - static RelationStats ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context); - //! Create the statistics for a projection using the statistics of the operator that sits underneath the - //! projection. Then also create statistics for any extra columns the projection creates. - static RelationStats ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context); - static RelationStats ExtractExpressionGetStats(LogicalExpressionGet &expression_get, ClientContext &context); - //! All relation extractors for blocking relations - static RelationStats ExtractProjectionStats(LogicalProjection &proj, RelationStats &child_stats); - static RelationStats ExtractAggregationStats(LogicalAggregate &aggr, RelationStats &child_stats); - static RelationStats ExtractWindowStats(LogicalWindow &window, RelationStats &child_stats); - static RelationStats ExtractEmptyResultStats(LogicalEmptyResult &empty); - //! Called after reordering a query plan with potentially 2+ relations. - static RelationStats CombineStatsOfReorderableOperator(vector &bindings, - vector relation_stats); - //! Called after reordering a query plan with potentially 2+ relations. - static RelationStats CombineStatsOfNonReorderableOperator(LogicalOperator &op, - const vector &child_stats); - static void CopyRelationStats(RelationStats &to, const RelationStats &from); - -private: - static unique_ptr GetColumnStatistics(LogicalGet &get, ClientContext &context, - const ColumnIndex &column_id); - static DistinctCount GetDistinctCount(LogicalGet &get, ClientContext &context, const ColumnIndex &column_id, - idx_t base_table_cardinality); -}; - -} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics.hpp b/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics.hpp new file mode 100644 index 000000000..334f3cde2 --- /dev/null +++ b/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics.hpp @@ -0,0 +1,57 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/optimizer/relation_statistics/relation_statistics.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/optional_idx.hpp" +#include "duckdb/common/optional_ptr.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/planner/column_binding.hpp" + +namespace duckdb { + +enum class DistinctCountSource : uint8_t { CARDINALITY, MIN_MAX, HLL, EXACT }; + +struct DistinctCount { +public: + DistinctCount(idx_t distinct_count, DistinctCountSource source); + +public: + idx_t distinct_count; + DistinctCountSource source; +}; + +struct RelationColumnStats { +public: + RelationColumnStats(ColumnBinding binding, DistinctCount distinct_count, Identifier name); + +public: + ColumnBinding binding; + DistinctCount distinct_count; + Identifier name; +}; + +struct RelationStats { +public: + RelationStats(); + +public: + optional_idx FindColumn(ColumnBinding binding) const; + optional_ptr GetColumnStats(ColumnBinding binding) const; + bool MatchesBindings(const vector &bindings) const; + void Verify(const vector &bindings) const; + +public: + vector columns; + idx_t cardinality; + double filter_strength = 1; + bool stats_initialized = false; + Identifier table_name; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics_extractor.hpp b/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics_extractor.hpp new file mode 100644 index 000000000..955f11d6f --- /dev/null +++ b/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics_extractor.hpp @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/optimizer/relation_statistics/relation_statistics_extractor.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/optional.hpp" +#include "duckdb/common/reference_map.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics.hpp" + +#include + +namespace duckdb { + +class ClientContext; +class LogicalCTERef; +class LogicalOperator; + +using relation_stats_cte_callback_t = std::function(TableIndex)>; + +class RelationStatsExtractor { +public: + explicit RelationStatsExtractor(ClientContext &context); + RelationStatsExtractor(ClientContext &context, relation_stats_cte_callback_t cte_callback); + +public: + optional_ptr Extract(LogicalOperator &op); + idx_t ExtractedOperatorCount() const; + +private: + optional ExtractInternal(LogicalOperator &op); + optional ExtractCTERef(LogicalCTERef &cte_ref); + +private: + ClientContext &context; + relation_stats_cte_callback_t cte_callback; + reference_map_t cache; + reference_set_t active_operators; + reference_set_t failed_operators; + idx_t extracted_operator_count = 0; +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp b/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp new file mode 100644 index 000000000..b251de571 --- /dev/null +++ b/src/duckdb/src/include/duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/optional.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics.hpp" +#include "duckdb/planner/filter/expression_filter.hpp" +#include "duckdb/planner/logical_operator.hpp" + +namespace duckdb { + +class ClientContext; +class LogicalAggregate; +class LogicalColumnDataGet; +class LogicalDelimGet; +class LogicalDistinct; +class LogicalDummyScan; +class LogicalEmptyResult; +class LogicalExpressionGet; +class LogicalGet; +class LogicalProjection; +class LogicalWindow; + +class RelationStatisticsHelper { +public: + static constexpr double DEFAULT_SELECTIVITY = 0.2; + +public: + static idx_t InspectTableFilter(idx_t cardinality, const TableFilter &filter, BaseStatistics &base_stats); + static RelationStats ExtractGetStats(LogicalGet &get, ClientContext &context); + static RelationStats ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context); + static RelationStats ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context); + static RelationStats ExtractExpressionGetStats(LogicalExpressionGet &expression_get, ClientContext &context); + static RelationStats ExtractColumnDataGetStats(LogicalColumnDataGet &column_data_get, ClientContext &context); + static RelationStats ExtractExplainStats(LogicalOperator &op); + static optional ExtractOperatorStats(LogicalOperator &op, ClientContext &context, + const vector> &child_stats); + static optional ExtractProjectionStats(LogicalProjection &projection, + const RelationStats &child_stats); + static optional ExtractAggregationStats(LogicalAggregate &aggregate, + const RelationStats &child_stats); + static optional ExtractWindowStats(LogicalWindow &window, const RelationStats &child_stats); + static optional ExtractDistinctStats(LogicalDistinct &distinct, const RelationStats &child_stats); + static RelationStats ExtractEmptyResultStats(LogicalEmptyResult &empty); + static optional ProjectOutputStats(const RelationStats &stats, LogicalOperator &op); + static optional RebindOutputStats(const RelationStats &stats, LogicalOperator &op); + static idx_t EstimateDistinctCardinality(const vector &distinct_counts, idx_t input_cardinality); + +private: + static unique_ptr GetColumnStatistics(LogicalGet &get, ClientContext &context, + const ColumnIndex &column_id); + static DistinctCount GetDistinctCount(LogicalGet &get, ClientContext &context, const ColumnIndex &column_id, + idx_t base_table_cardinality); +}; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp b/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp index 3c8507bc8..364e90738 100644 --- a/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp +++ b/src/duckdb/src/include/duckdb/parser/peg/inlined_grammar.hpp @@ -163,8 +163,9 @@ const char INLINED_PEG_GRAMMAR[] = { "MapType <- 'MAP' Parens(List(Type))?\n" "TupleType <- 'TUPLE' Parens(List(Type))\n" "ColIdType <- ColId Type\n" - "ArrayBounds <- SquareBracketsArray / ArrayKeyword\n" + "ArrayBounds <- ArrayKeywordWithBounds / SquareBracketsArray / ArrayKeyword\n" "ArrayKeyword <- 'ARRAY'\n" + "ArrayKeywordWithBounds <- 'ARRAY' SquareBracketsArray\n" "SquareBracketsArray <- '[' Expression? ']'\n" "TimeType <- TimeOrTimestamp TypeModifiers? TimeZone?\n" "TimeOrTimestamp <- TimeTypeId / TimestampTypeId\n" @@ -1609,7 +1610,7 @@ const char INLINED_PEG_GRAMMAR[] = { "RegularJoinClause <- Asof? JoinType? 'JOIN' TableRef JoinQualifier\n" "JoinByClause <- 'JOIN' 'BY' Parens('TYPE' ColLabel) TableRef JoinQualifier\n" "Asof <- 'ASOF'\n" - "JoinWithoutOnClause <- JoinPrefix 'JOIN' TableRef\n" + "JoinWithoutOnClause <- JoinPrefix 'JOIN' InnerTableRef\n" "JoinQualifier <- OnClause / UsingClause\n" "OnClause <- 'ON' Expression\n" "UsingClause <- 'USING' Parens(List(ColumnName))\n" diff --git a/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp b/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp index cea71e8a9..5336159ad 100644 --- a/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp +++ b/src/duckdb/src/include/duckdb/parser/peg/transformer/peg_transformer.hpp @@ -1050,6 +1050,11 @@ class PEGTransformerFactory { TransformStackFrame &frame); static unique_ptr FinalizeArrayKeywordTrampoline(PEGTransformer &transformer, TransformStack &stack, TransformStackFrame &frame); + static void InitializeArrayKeywordWithBoundsTrampoline(PEGTransformer &transformer, TransformStack &stack, + TransformStackFrame &frame); + static unique_ptr FinalizeArrayKeywordWithBoundsTrampoline(PEGTransformer &transformer, + TransformStack &stack, + TransformStackFrame &frame); static void InitializeSquareBracketsArrayTrampoline(PEGTransformer &transformer, TransformStack &stack, TransformStackFrame &frame); static unique_ptr FinalizeSquareBracketsArrayTrampoline(PEGTransformer &transformer, @@ -5279,6 +5284,9 @@ class PEGTransformerFactory { static unique_ptr TransformArrayKeywordInternal(PEGTransformer &transformer, ParseResult &parse_result); static int64_t TransformArrayKeyword(PEGTransformer &transformer); + static unique_ptr TransformArrayKeywordWithBoundsInternal(PEGTransformer &transformer, + ParseResult &parse_result); + static int64_t TransformArrayKeywordWithBounds(PEGTransformer &transformer, const int64_t &square_brackets_array); static unique_ptr TransformSquareBracketsArrayInternal(PEGTransformer &transformer, ParseResult &parse_result); static int64_t TransformSquareBracketsArray(PEGTransformer &transformer, @@ -7965,7 +7973,7 @@ class PEGTransformerFactory { static unique_ptr TransformJoinWithoutOnClauseInternal(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr TransformJoinWithoutOnClause(PEGTransformer &transformer, const JoinPrefix &join_prefix, - unique_ptr table_ref); + unique_ptr inner_table_ref); static unique_ptr TransformJoinQualifierInternal(PEGTransformer &transformer, ParseResult &parse_result); static unique_ptr TransformOnClauseInternal(PEGTransformer &transformer, diff --git a/src/duckdb/src/include/duckdb/planner/bind_context.hpp b/src/duckdb/src/include/duckdb/planner/bind_context.hpp index f27c22be6..77d898e36 100644 --- a/src/duckdb/src/include/duckdb/planner/bind_context.hpp +++ b/src/duckdb/src/include/duckdb/planner/bind_context.hpp @@ -119,6 +119,10 @@ class BindContext { void AddGenericBinding(TableIndex index, const Identifier &alias, const vector &names, const vector &types); + //! Registers an alternative name for a column of the binding with the given index + //! The alias can be bound like a regular column, but is not emitted by * + void AddColumnAlias(TableIndex index, const Identifier &column_alias, column_t column_index); + //! Adds a base table with the given alias to the CTE BindContext. //! We need this to correctly bind recursive CTEs with multiple references. void AddCTEBinding(TableIndex index, BindingAlias alias, const vector &names, diff --git a/src/duckdb/src/include/duckdb/planner/filter/expression_filter.hpp b/src/duckdb/src/include/duckdb/planner/filter/expression_filter.hpp index 58b67c366..fd4353a5d 100644 --- a/src/duckdb/src/include/duckdb/planner/filter/expression_filter.hpp +++ b/src/duckdb/src/include/duckdb/planner/filter/expression_filter.hpp @@ -8,11 +8,12 @@ #pragma once -#include "duckdb/planner/table_filter.hpp" +#include "duckdb/common/array_ptr.hpp" #include "duckdb/planner/expression.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_conjunction_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/table_filter.hpp" namespace duckdb { class ExpressionExecutor; @@ -51,6 +52,9 @@ class ExpressionFilter : public TableFilter { static FilterPropagateResult CheckExpressionStatistics(const Expression &expr, const BaseStatistics &stats); static FilterPropagateResult CheckExpressionStatistics(optional_ptr context_p, const Expression &expr, const BaseStatistics &stats); + //! Derive statistics for an expression over one or more input columns + static unique_ptr TryGetExpressionStatistics(ClientContext &context, const Expression &expr, + array_ptr input_stats); //! Check if an expression tree contains an internal function with the given name static bool ContainsInternalFunction(const Expression &expr, const string &func_name); //! Check if an expression tree is entirely optional filter semantics diff --git a/src/duckdb/src/include/duckdb/planner/table_binding.hpp b/src/duckdb/src/include/duckdb/planner/table_binding.hpp index f1614b580..1d7cf1f2e 100644 --- a/src/duckdb/src/include/duckdb/planner/table_binding.hpp +++ b/src/duckdb/src/include/duckdb/planner/table_binding.hpp @@ -42,6 +42,11 @@ struct Binding { bool TryGetBindingIndex(const Identifier &column_name, column_t &column_index); column_t GetBindingIndex(const Identifier &column_name); bool HasMatchingBinding(const Identifier &column_name); + //! Register an alternative name for an existing column - the alias can be bound, but is hidden from * + void AddColumnAlias(const Identifier &column_alias, column_t column_index); + //! Returns the name under which a column is registered in this binding (this can differ from the provided name + //! because of case insensitivity, or because the column is referenced through a column alias) + const Identifier &GetRegisteredColumnName(const Identifier &column_name); virtual ErrorData ColumnNotFoundError(const Identifier &column_name) const; virtual BindResult Bind(ColumnRefExpression &colref, idx_t depth); virtual optional_ptr GetStandardEntry(); @@ -77,6 +82,8 @@ struct Binding { protected: void Initialize(); + //! Set the alias of the column reference to the name under which the column is registered in this binding + void SetBoundColumnAlias(ColumnRefExpression &colref); protected: //! The type of Binding diff --git a/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp b/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp index b7249743c..4549fdee6 100644 --- a/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp +++ b/src/duckdb/src/include/duckdb/storage/table/chunk_info.hpp @@ -10,6 +10,7 @@ #include "duckdb/execution/index/index_pointer.hpp" #include "duckdb/common/enums/scan_options.hpp" +#include "duckdb/common/types/validity_mask.hpp" namespace duckdb { class RowGroup; @@ -42,6 +43,13 @@ enum class VersionCompressionResult : uint8_t { SETTLED }; +//! The storage state of the delete side of a ChunkVectorInfo. Exhaustive and mutually exclusive: +//! CONSTANT - all rows share constant_delete_id (NOT_DELETED_ID for none-deleted, or a single delete id) +//! MASKED - partially deleted, all deleted rows share one committed id (mask_delete_id): which rows are +//! deleted is stored in deleted_mask, deleted_data freed +//! ARRAY - per-row delete ids materialized in deleted_data +enum class DeleteIdState : uint8_t { CONSTANT, MASKED, ARRAY }; + class ChunkVectorInfo { public: explicit ChunkVectorInfo(FixedSizeAllocator &allocator, idx_t start, transaction_t insert_id = 0); @@ -106,6 +114,11 @@ class ChunkVectorInfo { IndexPointer GetInitializedDeletedPointer(); //! Frees the per-row delete ids (if any) void FreeDeleteData(); + //! ARRAY -> MASKED: record alive rows as invalid bits, free the per-row delete array. mask_id is the + //! shared committed id of the deleted rows (0 when they are already visible to all transactions) + void CompressDeleteToMask(transaction_t mask_id); + //! MASKED -> ARRAY: re-materialize the per-row delete array from the bitmask + void DecompressDeleteMask(); private: FixedSizeAllocator &allocator; @@ -118,6 +131,16 @@ class ChunkVectorInfo { IndexPointer deleted_data; //! The constant delete id (if there is only one, e.g. because the entire vector was deleted in one transaction) transaction_t constant_delete_id; + //! Bitmask used in the MASKED state: valid bit == the row is deleted, invalid bit == the row is alive. + //! Matches the on-disk VECTOR_INFO orientation. Only meaningful when delete_state == DeleteIdState::MASKED. + ValidityMask deleted_mask; + //! The single committed id shared by every deleted row in the MASKED state. 0 means the deletes are + //! visible to all transactions (the value used when read from disk); a non-zero committed id means the + //! mask was folded from one committed transaction whose delete is not yet visible to every snapshot. + //! Only meaningful when delete_state == DeleteIdState::MASKED. + transaction_t mask_delete_id = 0; + //! The current delete-side storage state - the single source of truth for the delete side + DeleteIdState delete_state = DeleteIdState::CONSTANT; //! Whether a compression pass could achieve anything for this vector: armed by any id modification, //! disarmed when a pass compresses the vector fully or finds it settled (live rows block the collapse //! until a further delete re-arms it). CompressVersionIds returns the cached SETTLED without diff --git a/src/duckdb/src/main/capi/copy_function-c.cpp b/src/duckdb/src/main/capi/copy_function-c.cpp index 6a6505bdc..914d90c25 100644 --- a/src/duckdb/src/main/capi/copy_function-c.cpp +++ b/src/duckdb/src/main/capi/copy_function-c.cpp @@ -694,12 +694,14 @@ unique_ptr CCopyFromBind(ClientContext &context, CopyFromFunctionB parameters.push_back(Value(info.info.file_path)); // Now bind, using the normal table function bind mechanism - CTableInternalBindInfo bind_info(context, parameters, named_parameters, expected_types, expected_names, *result, - tf_info); + // the COPY bind still operates on plain strings - convert around the table function bind + auto names = StringsToIdentifiers(expected_names); + CTableInternalBindInfo bind_info(context, parameters, named_parameters, expected_types, names, *result, tf_info); tf_info.bind(reinterpret_cast(&bind_info)); if (!bind_info.success) { throw BinderException(bind_info.error); } + expected_names = IdentifiersToStrings(names); return std::move(result); } diff --git a/src/duckdb/src/main/capi/table_function-c.cpp b/src/duckdb/src/main/capi/table_function-c.cpp index 06ff0970b..7fd048c33 100644 --- a/src/duckdb/src/main/capi/table_function-c.cpp +++ b/src/duckdb/src/main/capi/table_function-c.cpp @@ -106,7 +106,7 @@ duckdb_function_info ToCTableFunctionInfo(duckdb::CTableInternalFunctionInfo &in //===--------------------------------------------------------------------===// unique_ptr CTableFunctionBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { auto &info = input.info->Cast(); D_ASSERT(info.bind && info.function && info.init); diff --git a/src/duckdb/src/optimizer/build_probe_side_optimizer.cpp b/src/duckdb/src/optimizer/build_probe_side_optimizer.cpp index 5151b635f..8a3f1b6da 100644 --- a/src/duckdb/src/optimizer/build_probe_side_optimizer.cpp +++ b/src/duckdb/src/optimizer/build_probe_side_optimizer.cpp @@ -15,6 +15,7 @@ #include "duckdb/optimizer/column_binding_replacer.hpp" #include "duckdb/optimizer/join_filter_pushdown_optimizer.hpp" #include "duckdb/optimizer/optimizer.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/operator/logical_cross_product.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/main/settings.hpp" @@ -393,10 +394,66 @@ RecursiveProbeSidePreference BuildProbeSideOptimizer::GetRecursiveProbeSidePrefe return RecursiveProbeSidePreference::NONE; } +static optional_ptr FindCorrelatedDomainAttachment(LogicalCTE &cte) { + if (cte.correlated_columns.empty()) { + return nullptr; + } + + auto correlated_column_count = cte.correlated_columns.size(); + auto anchor_bindings = cte.children[0]->GetColumnBindings(); + if (anchor_bindings.size() != cte.column_count || anchor_bindings.size() < correlated_column_count) { + return nullptr; + } + + vector correlated_bindings( + anchor_bindings.end() - NumericCast::difference_type>(correlated_column_count), + anchor_bindings.end()); + auto current = cte.children[0].get(); + while (current->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = current->Cast(); + for (auto &binding : correlated_bindings) { + if (binding.table_index != projection.table_index || + binding.column_index.GetIndexUnsafe() >= projection.expressions.size()) { + return nullptr; + } + auto &expression = projection.GetExpression(binding); + if (expression.GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + return nullptr; + } + auto &column_ref = expression.Cast(); + if (column_ref.Depth() != 0) { + return nullptr; + } + binding = column_ref.Binding(); + } + if (projection.children.size() != 1) { + return nullptr; + } + current = projection.children[0].get(); + } + + if (current->type != LogicalOperatorType::LOGICAL_CROSS_PRODUCT) { + return nullptr; + } + auto domain_bindings = current->children[1]->GetColumnBindings(); + if (domain_bindings != correlated_bindings) { + return nullptr; + } + return current; +} + void BuildProbeSideOptimizer::VisitOperator(LogicalOperator &op) { if (op.type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) { - VisitOperator(*op.children[0]); - active_recursive_cte_indexes.push_back(op.Cast().table_index); + auto &cte = op.Cast(); + auto domain_attachment = FindCorrelatedDomainAttachment(cte); + if (domain_attachment) { + // FlattenDependentJoins appends the correlation domain to the recursive anchor. Optimize both inputs while + // preserving the orientation of this generated attachment. + VisitOperatorChildren(*domain_attachment); + } else { + VisitOperator(*op.children[0]); + } + active_recursive_cte_indexes.push_back(cte.table_index); VisitOperator(*op.children[1]); active_recursive_cte_indexes.pop_back(); return; diff --git a/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp b/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp index 051a80d7f..4d48fd3d2 100644 --- a/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +++ b/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp @@ -6,7 +6,7 @@ #include "duckdb/function/table/table_scan.hpp" #include "duckdb/optimizer/join_order/join_node.hpp" #include "duckdb/optimizer/join_order/query_graph_manager.hpp" -#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" #include "duckdb/planner/operator/logical_comparison_join.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" #include "duckdb/storage/data_table.hpp" @@ -952,14 +952,12 @@ void CardinalityEstimator::UpdateTotalDomains(optional_ptr set, D_ASSERT(set->count == 1); auto relation_id = set->relations[0]; //! Initialize the distinct count for all columns used in joins with the current relation. - // D_ASSERT(stats.column_distinct_count.size() >= 1); - - for (idx_t i = 0; i < stats.column_distinct_count.size(); i++) { + for (idx_t i = 0; i < stats.columns.size(); i++) { //! for every column used in a filter in the relation, get the distinct count via HLL, or assume it to be //! the cardinality // Update the relation_to_tdom set with the estimated distinct count (or tdom) calculated above auto key = ColumnBinding(TableIndex(relation_id.index), ProjectionIndex(i)); - auto distinct_count = stats.column_distinct_count.at(i); + auto distinct_count = stats.columns[i].distinct_count; for (auto &relation_to_tdom : state->relation_set_stats) { const auto &i_set = relation_to_tdom.equivalent_relations; if (i_set.find(key) == i_set.end()) { @@ -978,8 +976,8 @@ void CardinalityEstimator::AddRelationNamesToRelationStats(vector for (auto &binding : total_domain.equivalent_relations) { D_ASSERT(binding.table_index.index < stats.size()); string column_name; - if (binding.column_index < stats[binding.table_index.index].column_names.size()) { - column_name = stats[binding.table_index.index].column_names[binding.column_index].GetIdentifierName(); + if (binding.column_index < stats[binding.table_index.index].columns.size()) { + column_name = stats[binding.table_index.index].columns[binding.column_index].name.GetIdentifierName(); } else { column_name = "[unknown]"; } diff --git a/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp b/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp index b07b63a76..98a8f2b33 100644 --- a/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +++ b/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp @@ -12,6 +12,38 @@ namespace duckdb { +static optional CombineReorderableStats(const vector &bindings, + const vector &relation_stats) { + RelationStats result; + result.cardinality = 0; + result.stats_initialized = true; + for (auto &stats : relation_stats) { + if (!stats.stats_initialized) { + return {}; + } + result.cardinality = MaxValue(result.cardinality, stats.cardinality); + if (!result.table_name.empty()) { + result.table_name = Identifier(result.table_name + " joined with "); + } + result.table_name = Identifier(result.table_name + stats.table_name); + } + for (auto &binding : bindings) { + optional_ptr source; + for (auto &stats : relation_stats) { + source = stats.GetColumnStats(binding); + if (source) { + break; + } + } + if (!source) { + return {}; + } + result.columns.emplace_back(binding, source->distinct_count, source->name); + } + result.Verify(bindings); + return result; +} + JoinOrderOptimizer::JoinOrderOptimizer(ClientContext &context) : context(context), query_graph_manager(context), depth(1) { } @@ -76,11 +108,15 @@ unique_ptr JoinOrderOptimizer::Optimize(unique_ptrEstimateCardinality(context); - auto bindings = new_logical_plan->GetColumnBindings(); - auto new_stats = RelationStatisticsHelper::CombineStatsOfReorderableOperator(bindings, relation_stats); - new_stats.cardinality = cardinality; - RelationStatisticsHelper::CopyRelationStats(*stats, new_stats); + auto new_stats = query_graph_manager.relation_manager.HasCompleteStats() + ? CombineReorderableStats(new_logical_plan->GetColumnBindings(), relation_stats) + : optional(); + if (new_stats) { + new_stats->cardinality = new_logical_plan->EstimateCardinality(context); + *stats = std::move(*new_stats); + } else { + *stats = RelationStats(); + } } else { // starts recursively setting cardinality new_logical_plan->EstimateCardinality(context); diff --git a/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp b/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp index ffd5eb757..a38a9616a 100644 --- a/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +++ b/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp @@ -212,11 +212,12 @@ void QueryGraphManager::GetColumnBinding(const Expression &root_expr, ColumnBind root_expr, [&](const BoundColumnRefExpression &colref) { D_ASSERT(colref.Depth() == 0); D_ASSERT(colref.Binding().table_index.IsValid()); - // map the base table index to the relation index used by the JoinOrderOptimizer - D_ASSERT(relation_manager.relation_mapping.find(colref.Binding().table_index) != - relation_manager.relation_mapping.end()); - binding = ColumnBinding(TableIndex(relation_manager.relation_mapping[colref.Binding().table_index].index), - colref.Binding().column_index); + // Map the logical output binding to the relation-local statistics column. + auto normalized = relation_manager.TryNormalizeBinding(colref.Binding(), binding); + D_ASSERT(normalized); + if (!normalized) { + return; + } }); } @@ -228,9 +229,11 @@ void QueryGraphManager::GetEquivalenceBinding(const Expression &expression, Colu if (!colref.Binding().table_index.IsValid()) { return; } - auto entry = relation_manager.relation_mapping.find(colref.Binding().table_index); - D_ASSERT(entry != relation_manager.relation_mapping.end()); - binding = ColumnBinding(TableIndex(entry->second.index), colref.Binding().column_index); + auto normalized = relation_manager.TryNormalizeBinding(colref.Binding(), binding); + D_ASSERT(normalized); + if (!normalized) { + return; + } return; } case ExpressionClass::BOUND_FUNCTION: { diff --git a/src/duckdb/src/optimizer/join_order/relation_manager.cpp b/src/duckdb/src/optimizer/join_order/relation_manager.cpp index 2e0c38654..dac31f037 100644 --- a/src/duckdb/src/optimizer/join_order/relation_manager.cpp +++ b/src/duckdb/src/optimizer/join_order/relation_manager.cpp @@ -5,7 +5,7 @@ #include "duckdb/common/string_util.hpp" #include "duckdb/optimizer/join_order/join_order_optimizer.hpp" #include "duckdb/optimizer/join_order/join_relation_set.hpp" -#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp" +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" #include "duckdb/parser/expression_map.hpp" #include "duckdb/planner/expression/list.hpp" #include "duckdb/planner/expression_iterator.hpp" @@ -26,6 +26,24 @@ SingleJoinRelation::SingleJoinRelation(LogicalOperator &op, optional_ptr RelationManager::GetRelationStats() { vector ret; for (idx_t i = 0; i < relations.size(); i++) { @@ -42,78 +60,101 @@ idx_t RelationManager::NumRelations() { return relations.size(); } -void RelationManager::AddAggregateOrWindowRelation(LogicalOperator &op, optional_ptr parent, - const RelationStats &stats, LogicalOperatorType op_type) { - auto relation = make_uniq(op, parent, stats); - RelationIndex relation_id(relations.size()); - - auto op_bindings = op.GetColumnBindings(); - for (auto &binding : op_bindings) { - if (relation_mapping.find(binding.table_index) == relation_mapping.end()) { - relation_mapping[binding.table_index] = relation_id; - } - } - operator_relations[op] = relation_id; - relations.push_back(std::move(relation)); - op.estimated_cardinality = stats.cardinality; - op.has_estimated_cardinality = true; -} - -void RelationManager::AddRelation(LogicalOperator &op, optional_ptr parent, +bool RelationManager::AddRelation(LogicalOperator &op, optional_ptr parent, const RelationStats &stats) { // if parent is null, then this is a root relation // if parent is not null, it should have multiple children D_ASSERT(!parent || parent->children.size() >= 2); - auto relation = make_uniq(op, parent, stats); + auto aligned_stats = AlignStatsWithRelationRoot(op, stats); + if (!aligned_stats) { + stats_complete = false; + return false; + } + auto relation = make_uniq(op, parent, std::move(*aligned_stats)); RelationIndex relation_id(relations.size()); + unordered_set relation_bindings; auto table_indexes = op.GetTableIndex(); - bool is_mark = op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN && + auto is_mark = op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN && op.Cast().join_type == JoinType::MARK; - bool get_all_child_bindings = op.type == LogicalOperatorType::LOGICAL_UNNEST; - if (op.type == LogicalOperatorType::LOGICAL_GET) { - get_all_child_bindings = !op.children.empty(); - } if (table_indexes.empty() || is_mark) { - // relation represents a non-reorderable relation, most likely a join relation - // Get the tables referenced in the non-reorderable relation and add them to the relation mapping - // This should all table references, even if there are nested non-reorderable joins. - unordered_set table_references; - LogicalJoin::GetTableReferences(op, table_references); - D_ASSERT(!table_references.empty()); - for (auto &reference : table_references) { - D_ASSERT(relation_mapping.find(reference) == relation_mapping.end()); - relation_mapping[reference] = relation_id; - } - } else if (get_all_child_bindings) { - // logical get has a logical_get index, but if a function is present other bindings can refer to - // columns that are not unnested, and from the child of the logical get. - auto bindings = op.GetColumnBindings(); - for (auto &binding : bindings) { - if (relation_mapping.find(binding.table_index) == relation_mapping.end()) { - relation_mapping[binding.table_index] = relation_id; - } - } + LogicalJoin::GetTableReferences(op, relation_bindings); } else { - // Map all table indexes produced by this operator to this relation. - // Most operators have exactly one table index, but some (e.g. LogicalAggregate) - // return multiple. All should map to the same atomic relation in the join order. - D_ASSERT(!table_indexes.empty()); - for (auto &table_index : table_indexes) { - D_ASSERT(relation_mapping.find(table_index) == relation_mapping.end()); - relation_mapping[table_index] = relation_id; + relation_bindings.insert(table_indexes.begin(), table_indexes.end()); + } + for (auto &column : relation->stats.columns) { + relation_bindings.insert(column.binding.table_index); + } + for (auto &binding : relation_bindings) { + auto entry = relation_mapping.find(binding); + if (entry == relation_mapping.end()) { + relation_mapping.emplace(binding, relation_id); + } else { + D_ASSERT(entry->second == relation_id); } } + RegisterRelationBindings(op, relation_id, relation->stats); operator_relations[op] = relation_id; - relations.push_back(std::move(relation)); - op.estimated_cardinality = stats.cardinality; + op.estimated_cardinality = relation->stats.cardinality; op.has_estimated_cardinality = true; + relations.push_back(std::move(relation)); + return true; +} + +optional RelationManager::AlignStatsWithRelationRoot(LogicalOperator &op, const RelationStats &stats) { + auto projected = RelationStatisticsHelper::ProjectOutputStats(stats, op); + if (projected && (op.children.size() != 1 || CanReuseAlignedStats(op.type))) { + return projected; + } + if (op.children.size() != 1) { + return {}; + } + auto child_stats = AlignStatsWithRelationRoot(*op.children[0], stats); + if (!child_stats) { + return {}; + } + vector> children {*child_stats}; + auto derived = RelationStatisticsHelper::ExtractOperatorStats(op, context, children); + if (derived) { + return derived; + } + + return {}; +} + +bool RelationManager::HasCompleteStats() const { + return stats_complete; +} + +void RelationManager::RegisterRelationBindings(LogicalOperator &op, RelationIndex relation_id, + const RelationStats &stats) { + auto bindings = op.GetColumnBindings(); + stats.Verify(bindings); + for (idx_t column_idx = 0; column_idx < bindings.size(); column_idx++) { + auto normalized = ColumnBinding(TableIndex(relation_id.index), ProjectionIndex(column_idx)); + auto entry = normalized_bindings.find(bindings[column_idx]); + if (entry == normalized_bindings.end()) { + normalized_bindings.emplace(bindings[column_idx], normalized); + } else { + D_ASSERT(entry->second == normalized); + } + } +} + +bool RelationManager::TryNormalizeBinding(ColumnBinding binding, ColumnBinding &normalized) const { + auto entry = normalized_bindings.find(binding); + if (entry == normalized_bindings.end()) { + return false; + } + normalized = entry->second; + return true; } static bool OperatorNeedsRelation(LogicalOperatorType op_type) { switch (op_type) { case LogicalOperatorType::LOGICAL_PROJECTION: case LogicalOperatorType::LOGICAL_EXPRESSION_GET: + case LogicalOperatorType::LOGICAL_CHUNK_GET: case LogicalOperatorType::LOGICAL_GET: case LogicalOperatorType::LOGICAL_UNNEST: case LogicalOperatorType::LOGICAL_DELIM_GET: @@ -257,19 +298,92 @@ static void ModifyStatsIfLimit(optional_ptr limit_op, RelationS } } -void RelationManager::AddRelationWithChildren(JoinOrderOptimizer &optimizer, LogicalOperator &op, +static optional CombineNonReorderableStats(LogicalOperator &op, const vector &child_stats, + ClientContext &context) { + if (child_stats.size() != op.children.size()) { + return {}; + } + vector> child_references; + for (idx_t child_idx = 0; child_idx < child_stats.size(); child_idx++) { + if (!child_stats[child_idx].stats_initialized || + !child_stats[child_idx].MatchesBindings(op.children[child_idx]->GetColumnBindings())) { + return {}; + } + child_references.push_back(child_stats[child_idx]); + } + auto derived = RelationStatisticsHelper::ExtractOperatorStats(op, context, child_references); + if (derived) { + return derived; + } + + RelationStats result; + result.cardinality = 0; + result.stats_initialized = true; + for (auto &stats : child_stats) { + result.cardinality = MaxValue(result.cardinality, stats.cardinality); + } + if (op.type == LogicalOperatorType::LOGICAL_UNION) { + auto &setop = op.Cast(); + if (setop.setop_all) { + result.cardinality = 0; + for (auto &stats : child_stats) { + result.cardinality += stats.cardinality; + } + } + } else if (op.type == LogicalOperatorType::LOGICAL_INTERSECT && child_stats.size() == 2) { + result.cardinality = MinValue(child_stats[0].cardinality, child_stats[1].cardinality); + } else if (op.type == LogicalOperatorType::LOGICAL_EXCEPT && child_stats.size() == 2) { + result.cardinality = child_stats[0].cardinality; + } else if (op.type == LogicalOperatorType::LOGICAL_ASOF_JOIN && child_stats.size() == 2) { + auto &join = op.Cast(); + result.cardinality = join.join_type == JoinType::RIGHT || join.join_type == JoinType::OUTER + ? child_stats[1].cardinality + : child_stats[0].cardinality; + } + + auto bindings = op.GetColumnBindings(); + auto ordinal_set_output = op.type == LogicalOperatorType::LOGICAL_UNION || + op.type == LogicalOperatorType::LOGICAL_INTERSECT || + op.type == LogicalOperatorType::LOGICAL_EXCEPT; + for (idx_t column_idx = 0; column_idx < bindings.size(); column_idx++) { + optional_ptr source; + for (auto &stats : child_stats) { + source = stats.GetColumnStats(bindings[column_idx]); + if (source) { + break; + } + } + if (!source && ordinal_set_output && !child_stats.empty() && column_idx < child_stats[0].columns.size()) { + source = child_stats[0].columns[column_idx]; + } + if (!source) { + return {}; + } + result.columns.emplace_back(bindings[column_idx], source->distinct_count, source->name); + } + result.Verify(bindings); + return result; +} + +bool RelationManager::AddRelationWithChildren(JoinOrderOptimizer &optimizer, LogicalOperator &op, LogicalOperator &input_op, optional_ptr parent, RelationStats &child_stats, optional_ptr limit_op, vector> &datasource_filters) { D_ASSERT(!op.children.empty()); auto child_optimizer = optimizer.CreateChildOptimizer(); op.children[0] = child_optimizer.Optimize(std::move(op.children[0]), &child_stats); + vector> children_stats {child_stats}; + auto operator_stats = RelationStatisticsHelper::ExtractOperatorStats(op, context, children_stats); + if (!operator_stats) { + stats_complete = false; + return false; + } if (!datasource_filters.empty()) { - child_stats.cardinality = LossyNumericCast(static_cast(child_stats.cardinality) * - RelationStatisticsHelper::DEFAULT_SELECTIVITY); + operator_stats->cardinality = LossyNumericCast(static_cast(operator_stats->cardinality) * + RelationStatisticsHelper::DEFAULT_SELECTIVITY); } - ModifyStatsIfLimit(limit_op.get(), child_stats); - AddRelation(input_op, parent, child_stats); + ModifyStatsIfLimit(limit_op.get(), *operator_stats); + return AddRelation(input_op, parent, *operator_stats); } bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, LogicalOperator &input_op, @@ -322,14 +436,17 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica children_stats.push_back(stats); } - auto combined_stats = RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(*op, children_stats); - op->SetEstimatedCardinality(combined_stats.cardinality); + auto combined_stats = CombineNonReorderableStats(*op, children_stats, context); + if (!combined_stats) { + stats_complete = false; + return false; + } + op->SetEstimatedCardinality(combined_stats->cardinality); if (!datasource_filters.empty()) { - combined_stats.cardinality = (idx_t)MaxValue( - double(combined_stats.cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1); + combined_stats->cardinality = (idx_t)MaxValue( + double(combined_stats->cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1); } - AddRelation(input_op, parent, combined_stats); - return true; + return AddRelation(input_op, parent, *combined_stats); } switch (op->type) { @@ -340,15 +457,18 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats); auto &aggr = op->Cast(); auto operator_stats = RelationStatisticsHelper::ExtractAggregationStats(aggr, child_stats); + if (!operator_stats) { + stats_complete = false; + return false; + } // the extracted cardinality should be set for aggregate - aggr.SetEstimatedCardinality(operator_stats.cardinality); + aggr.SetEstimatedCardinality(operator_stats->cardinality); if (!datasource_filters.empty()) { - operator_stats.cardinality = LossyNumericCast(static_cast(operator_stats.cardinality) * - RelationStatisticsHelper::DEFAULT_SELECTIVITY); + operator_stats->cardinality = LossyNumericCast(static_cast(operator_stats->cardinality) * + RelationStatisticsHelper::DEFAULT_SELECTIVITY); } ModifyStatsIfLimit(limit_op.get(), child_stats); - AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type); - return true; + return AddRelation(input_op, parent, *operator_stats); } case LogicalOperatorType::LOGICAL_WINDOW: { // optimize children @@ -357,21 +477,23 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica op->children[0] = child_optimizer.Optimize(std::move(op->children[0]), &child_stats); auto &window = op->Cast(); auto operator_stats = RelationStatisticsHelper::ExtractWindowStats(window, child_stats); + if (!operator_stats) { + stats_complete = false; + return false; + } // the extracted cardinality should be set for window - window.SetEstimatedCardinality(operator_stats.cardinality); + window.SetEstimatedCardinality(operator_stats->cardinality); if (!datasource_filters.empty()) { - operator_stats.cardinality = LossyNumericCast(static_cast(operator_stats.cardinality) * - RelationStatisticsHelper::DEFAULT_SELECTIVITY); + operator_stats->cardinality = LossyNumericCast(static_cast(operator_stats->cardinality) * + RelationStatisticsHelper::DEFAULT_SELECTIVITY); } ModifyStatsIfLimit(limit_op.get(), child_stats); - AddAggregateOrWindowRelation(input_op, parent, operator_stats, op->type); - return true; + return AddRelation(input_op, parent, *operator_stats); } case LogicalOperatorType::LOGICAL_UNNEST: { // optimize children of unnest RelationStats child_stats; - AddRelationWithChildren(optimizer, *op, input_op, parent, child_stats, limit_op, datasource_filters); - return true; + return AddRelationWithChildren(optimizer, *op, input_op, parent, child_stats, limit_op, datasource_filters); } case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: { auto &join = op->Cast(); @@ -393,7 +515,9 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica // optimize the child and copy the stats auto child_optimizer = optimizer.CreateChildOptimizer(); op->children[1] = child_optimizer.Optimize(std::move(op->children[1]), &child_stats); - AddRelation(*op->children[1], op, child_stats); + if (!AddRelation(*op->children[1], op, child_stats)) { + return false; + } auto right_child_bindings = op->children[1]->GetColumnBindings(); for (auto &bindings : right_child_bindings) { relation_mapping[bindings.table_index] = RelationIndex(relations.size() - 1); @@ -411,16 +535,18 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica case LogicalOperatorType::LOGICAL_DUMMY_SCAN: { auto &dummy_scan = op->Cast(); auto stats = RelationStatisticsHelper::ExtractDummyScanStats(dummy_scan, context); - AddRelation(input_op, parent, stats); - return true; + return AddRelation(input_op, parent, stats); } case LogicalOperatorType::LOGICAL_EXPRESSION_GET: { // base table scan, add to set of relations. // create empty stats for dummy scan or logical expression get auto &expression_get = op->Cast(); auto stats = RelationStatisticsHelper::ExtractExpressionGetStats(expression_get, context); - AddRelation(input_op, parent, stats); - return true; + return AddRelation(input_op, parent, stats); + } + case LogicalOperatorType::LOGICAL_CHUNK_GET: { + auto stats = RelationStatisticsHelper::ExtractColumnDataGetStats(op->Cast(), context); + return AddRelation(input_op, parent, stats); } case LogicalOperatorType::LOGICAL_GET: { // TODO: Get stats from a logical GET @@ -430,8 +556,7 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica // be used in joins if (!op->children.empty()) { RelationStats child_stats; - AddRelationWithChildren(optimizer, *op, input_op, parent, child_stats, limit_op, datasource_filters); - return true; + return AddRelationWithChildren(optimizer, *op, input_op, parent, child_stats, limit_op, datasource_filters); } auto stats = RelationStatisticsHelper::ExtractGetStats(get, context); // if there is another logical filter that could not be pushed down into the @@ -442,8 +567,7 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica (idx_t)MaxValue(double(stats.cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY, (double)1); } ModifyStatsIfLimit(limit_op.get(), stats); - AddRelation(input_op, parent, stats); - return true; + return AddRelation(input_op, parent, stats); } case LogicalOperatorType::LOGICAL_PROJECTION: { RelationStats child_stats; @@ -453,10 +577,13 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica auto &proj = op->Cast(); // Projection can create columns so we need to add them here auto proj_stats = RelationStatisticsHelper::ExtractProjectionStats(proj, child_stats); - proj.SetEstimatedCardinality(proj_stats.cardinality); - ModifyStatsIfLimit(limit_op.get(), proj_stats); - AddRelation(input_op, parent, proj_stats); - return true; + if (!proj_stats) { + stats_complete = false; + return false; + } + proj.SetEstimatedCardinality(proj_stats->cardinality); + ModifyStatsIfLimit(limit_op.get(), *proj_stats); + return AddRelation(input_op, parent, *proj_stats); } case LogicalOperatorType::LOGICAL_EMPTY_RESULT: { // optimize the child and copy the stats @@ -464,8 +591,7 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica // Projection can create columns so we need to add them here auto stats = RelationStatisticsHelper::ExtractEmptyResultStats(empty_result); empty_result.SetEstimatedCardinality(stats.cardinality); - AddRelation(input_op, parent, stats); - return true; + return AddRelation(input_op, parent, stats); } case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: { @@ -487,15 +613,34 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica // create the stats for the CTE auto child_2_card = rhs_stats.stats_initialized ? rhs_stats.cardinality : 0; + idx_t cte_cardinality; if (op->type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) { // we cannot really estimate the cardinality of a recursive CTE // because we don't know how many times it will be executed // we just assume it will be executed 1000 times - op->SetEstimatedCardinality(child_1_card + child_2_card * 1000); + cte_cardinality = child_1_card + child_2_card * 1000; } else if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { // for a materialized CTE, we just take the cardinality of the right children - op->SetEstimatedCardinality(child_2_card); + cte_cardinality = child_2_card; + } else { + throw InternalException("Unexpected CTE type in join order optimizer"); + } + op->SetEstimatedCardinality(cte_cardinality); + auto output_stats = RelationStatisticsHelper::ProjectOutputStats(rhs_stats, input_op); + if (!output_stats) { + output_stats = RelationStatisticsHelper::RebindOutputStats(rhs_stats, input_op); + } + if (output_stats && op->type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) { + output_stats->cardinality = cte_cardinality; + // Statistics from one recursive term do not describe the complete fixpoint. Keep the output layout, but + // represent each fixpoint domain using the standard cardinality-based fallback. + for (auto &column : output_stats->columns) { + column.distinct_count = DistinctCount(cte_cardinality, DistinctCountSource::CARDINALITY); + } + } + if (!output_stats || !AddRelation(input_op, parent, *output_stats)) { + stats_complete = false; } return false; @@ -503,8 +648,15 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica case LogicalOperatorType::LOGICAL_CTE_REF: { auto &cte_ref = op->Cast(); auto cte_stats = optimizer.GetMaterializedCTEStats(cte_ref.cte_index); + auto rebound_stats = RelationStatisticsHelper::RebindOutputStats(cte_stats, cte_ref); + if (!rebound_stats) { + stats_complete = false; + return false; + } cte_ref.SetEstimatedCardinality(cte_stats.cardinality); - AddRelation(input_op, parent, cte_stats); + if (!AddRelation(input_op, parent, *rebound_stats)) { + return false; + } auto is_recursive = optimizer.recursive_cte_indexes.find(cte_ref.cte_index); if (is_recursive != optimizer.recursive_cte_indexes.end()) { @@ -533,11 +685,15 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica dummy_aggr->groups.push_back(delim_col->Copy()); } auto lhs_delim_stats = RelationStatisticsHelper::ExtractAggregationStats(*dummy_aggr, lhs_stats); + if (!lhs_delim_stats) { + stats_complete = false; + return false; + } // optimize the other child, which will now have access to the stats RelationStats rhs_stats; auto rhs_optimizer = optimizer.CreateChildOptimizer(); - rhs_optimizer.AddDelimScanStats(lhs_delim_stats); + rhs_optimizer.AddDelimScanStats(*lhs_delim_stats); op->children[1] = rhs_optimizer.Optimize(std::move(op->children[1]), rhs_stats); RelationStats dj_stats; @@ -568,7 +724,9 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica 1); } - AddAggregateOrWindowRelation(input_op, parent, dj_stats, op->type); + if (!AddRelation(input_op, parent, dj_stats)) { + return false; + } return false; } @@ -577,9 +735,13 @@ bool RelationManager::ExtractJoinRelations(JoinOrderOptimizer &optimizer, Logica // but ran into terrible join orders (see internal issue #596), so we removed it again // We now have proper statistics for DelimGets, and get an even better query plan for #596 auto delim_scan_stats = optimizer.GetDelimScanStats(); + auto rebound_stats = RelationStatisticsHelper::RebindOutputStats(delim_scan_stats, *op); + if (!rebound_stats) { + stats_complete = false; + return false; + } op->SetEstimatedCardinality(delim_scan_stats.cardinality); - AddAggregateOrWindowRelation(input_op, parent, delim_scan_stats, op->type); - return true; + return AddRelation(input_op, parent, *rebound_stats); } default: return false; @@ -594,13 +756,11 @@ void RelationManager::GetColumnBindingsFromExpression(const Expression &expressi auto &colref = expression.Cast(); D_ASSERT(colref.Depth() == 0); D_ASSERT(colref.Binding().table_index.IsValid()); - // only add column bindings that map to relations. - if (relation_mapping.find(colref.Binding().table_index) == relation_mapping.end()) { + ColumnBinding normalized; + if (!TryNormalizeBinding(colref.Binding(), normalized)) { return; } - // map the base table index to the relation index used by the JoinOrderOptimizer - column_bindings.insert(ColumnBinding(TableIndex(relation_mapping[colref.Binding().table_index].index), - colref.Binding().column_index)); + column_bindings.insert(normalized); } // TODO: handle inequality filters with functions. @@ -610,11 +770,11 @@ void RelationManager::GetColumnBindingsFromExpression(const Expression &expressi void RelationManager::GetColumnBindingsFromOperator(LogicalOperator &op, column_binding_set_t &column_bindings) { for (auto &binding : op.GetColumnBindings()) { - auto entry = relation_mapping.find(binding.table_index); - if (entry == relation_mapping.end()) { + ColumnBinding normalized; + if (!TryNormalizeBinding(binding, normalized)) { continue; } - column_bindings.insert(ColumnBinding(TableIndex(entry->second.index), binding.column_index)); + column_bindings.insert(normalized); } } @@ -986,10 +1146,8 @@ void RelationManager::PrintRelationStats() { for (idx_t i = 0; i < relations.size(); i++) { auto &relation = relations.at(i); auto &stats = relation->stats; - D_ASSERT(stats.column_names.size() == stats.column_distinct_count.size()); - for (idx_t i = 0; i < stats.column_names.size(); i++) { - to_print = stats.column_names.at(i) + " has estimated distinct count " + - to_string(stats.column_distinct_count.at(i).distinct_count); + for (auto &column : stats.columns) { + to_print = column.name + " has estimated distinct count " + to_string(column.distinct_count.distinct_count); Printer::Print(to_print); } to_print = stats.table_name + " has estimated cardinality " + to_string(stats.cardinality); diff --git a/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp b/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp deleted file mode 100644 index 4beda1e45..000000000 --- a/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +++ /dev/null @@ -1,639 +0,0 @@ -#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp" -#include "duckdb/planner/expression/list.hpp" -#include "duckdb/planner/operator/list.hpp" -#include "duckdb/planner/filter/expression_filter.hpp" -#include "duckdb/planner/expression/bound_comparison_expression.hpp" -#include "duckdb/planner/expression/bound_conjunction_expression.hpp" -#include "duckdb/planner/expression_iterator.hpp" -#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" -#include "duckdb/common/operator/add.hpp" -#include "duckdb/common/operator/subtract.hpp" -#include "duckdb/common/types/hugeint.hpp" -#include "duckdb/function/table/table_scan.hpp" -#include "duckdb/planner/operator/logical_get.hpp" -#include "duckdb/storage/data_table.hpp" -#include "duckdb/storage/statistics/numeric_stats.hpp" - -#include - -namespace duckdb { - -bool ExpressionBinding::FoundExpression() const { - return expression; -} - -bool ExpressionBinding::FoundColumnRef() const { - if (!FoundExpression()) { - return false; - } - return expression->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF; -} - -RelationStats::RelationStats() : cardinality(1), filter_strength(1), stats_initialized(false) { -} - -DistinctCount::DistinctCount(idx_t distinct_count, DistinctCountSource source) - : distinct_count(distinct_count), source(source) { -} - -static idx_t CapMinMaxDistinctCount(uint64_t distinct_count, idx_t base_table_cardinality) { - if (base_table_cardinality == 0) { - return 0; - } - if (distinct_count == 0) { - return 0; - } - auto capped_distinct_count = MinValue(distinct_count, base_table_cardinality); - return capped_distinct_count == NumericLimits::Maximum() ? 0 : capped_distinct_count; -} - -static idx_t GetMinMaxSpanDistinctCount(uint64_t span, idx_t base_table_cardinality) { - uint64_t distinct_count; - if (!TryAddOperator::Operation(span, 1, distinct_count)) { - return 0; - } - return CapMinMaxDistinctCount(distinct_count, base_table_cardinality); -} - -template -static idx_t GetSignedMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { - auto min_value = NumericStats::Min(base_stats).GetValueUnsafe(); - auto max_value = NumericStats::Max(base_stats).GetValueUnsafe(); - if (max_value < min_value) { - return 0; - } - hugeint_t span; - if (!TrySubtractOperator::Operation(hugeint_t(static_cast(max_value)), - hugeint_t(static_cast(min_value)), span)) { - return 0; - } - uint64_t unsigned_span; - if (!Hugeint::TryCast(span, unsigned_span)) { - return 0; - } - return GetMinMaxSpanDistinctCount(unsigned_span, base_table_cardinality); -} - -template -static idx_t GetUnsignedMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { - auto min_value = NumericStats::Min(base_stats).GetValueUnsafe(); - auto max_value = NumericStats::Max(base_stats).GetValueUnsafe(); - T span; - if (!TrySubtractOperator::Operation(max_value, min_value, span)) { - return 0; - } - return GetMinMaxSpanDistinctCount(static_cast(span), base_table_cardinality); -} - -static idx_t GetBooleanMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { - auto min_value = NumericStats::Min(base_stats).GetValueUnsafe(); - auto max_value = NumericStats::Max(base_stats).GetValueUnsafe(); - idx_t distinct_count = min_value == max_value ? idx_t(1) : idx_t(2); - return CapMinMaxDistinctCount(distinct_count, base_table_cardinality); -} - -static idx_t GetMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { - if (base_table_cardinality == 0 || base_stats.GetStatsType() != StatisticsType::NUMERIC_STATS || - !NumericStats::HasMinMax(base_stats)) { - return 0; - } - - switch (base_stats.GetType().InternalType()) { - case PhysicalType::BOOL: - return GetBooleanMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::INT8: - return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::INT16: - return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::INT32: - return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::INT64: - return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::UINT8: - return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::UINT16: - return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::UINT32: - return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); - case PhysicalType::UINT64: - return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); - default: - return 0; - } -} - -static DistinctCount GetDistinctCountFromStats(BaseStatistics &base_stats, idx_t base_table_cardinality) { - auto distinct_count = base_stats.GetDistinctCount(); - if (distinct_count > 0) { - return DistinctCount(distinct_count, DistinctCountSource::HLL); - } - distinct_count = GetMinMaxDistinctCount(base_stats, base_table_cardinality); - if (distinct_count > 0) { - return DistinctCount(distinct_count, DistinctCountSource::MIN_MAX); - } - return DistinctCount(0, DistinctCountSource::CARDINALITY); -} - -static ExpressionBinding GetChildColumnBinding(Expression &expr) { - auto ret = ExpressionBinding(); - switch (expr.GetExpressionClass()) { - case ExpressionClass::BOUND_FUNCTION: { - // TODO: Other expression classes that can have 0 children? - auto &func = expr.Cast(); - // no children some sort of gen_random_uuid() or equivalent. - if (func.GetChildren().empty()) { - ret.expression = expr; - ret.expression_is_constant = true; - return ret; - } - break; - } - case ExpressionClass::BOUND_COLUMN_REF: { - ret.expression = expr; - auto &new_col_ref = expr.Cast(); - ret.child_binding = ColumnBinding(new_col_ref.Binding().table_index, new_col_ref.Binding().column_index); - return ret; - } - case ExpressionClass::BOUND_LAMBDA_REF: - case ExpressionClass::BOUND_CONSTANT: - case ExpressionClass::BOUND_DEFAULT: - case ExpressionClass::BOUND_PARAMETER: - case ExpressionClass::BOUND_REF: - ret.expression = expr; - ret.expression_is_constant = true; - return ret; - default: - break; - } - ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr &child) { - if (ret.FoundColumnRef()) { - //! Already found a column ref expression - return; - } - auto recursive_result = GetChildColumnBinding(*child); - if (recursive_result.FoundExpression()) { - ret = recursive_result; - return; - } - }); - // we didn't find a Bound Column Ref - return ret; -} - -unique_ptr RelationStatisticsHelper::GetColumnStatistics(LogicalGet &get, ClientContext &context, - const ColumnIndex &column_id) { - if (!get.bind_data || (!get.function.statistics && !get.function.statistics_extended)) { - return nullptr; - } - if (get.function.statistics_extended) { - TableFunctionGetStatisticsInput input(get.bind_data.get(), column_id); - return get.function.statistics_extended(context, input); - } - D_ASSERT(get.function.statistics); - return get.function.statistics(context, get.bind_data.get(), column_id.GetPrimaryIndex()); -} - -DistinctCount RelationStatisticsHelper::GetDistinctCount(LogicalGet &get, ClientContext &context, - const ColumnIndex &column_id, idx_t base_table_cardinality) { - auto column_statistics = GetColumnStatistics(get, context, column_id); - if (!column_statistics) { - return DistinctCount(0, DistinctCountSource::CARDINALITY); - } - return GetDistinctCountFromStats(*column_statistics, base_table_cardinality); -} - -RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientContext &context) { - auto return_stats = RelationStats(); - - auto base_table_cardinality = get.EstimateCardinality(context); - auto cardinality_after_filters = base_table_cardinality; - - auto catalog_table = get.GetTable(); - auto name = string("some table"); - if (catalog_table) { - name = catalog_table->name.GetIdentifierName(); - return_stats.table_name = Identifier(name); - } - - // first push back basic distinct counts for each column (if we have them). - auto &column_ids = get.GetColumnIds(); - for (idx_t i = 0; i < column_ids.size(); i++) { - auto column_id = column_ids[i].GetPrimaryIndex(); - auto distinct_count = GetDistinctCount(get, context, column_ids[i], base_table_cardinality); - if (distinct_count.distinct_count > 0) { - return_stats.column_distinct_count.emplace_back(distinct_count.distinct_count, distinct_count.source); - return_stats.column_names.push_back(Identifier(name + "." + get.names.at(column_id))); - } else { - // treat the cardinality as the distinct count. - // the cardinality estimator will update these distinct counts based - // on the extra columns that are joined on. - return_stats.column_distinct_count.emplace_back(cardinality_after_filters, - DistinctCountSource::CARDINALITY); - auto column_name = string("column"); - if (column_id < get.names.size()) { - column_name = get.names.at(column_id).GetIdentifierName(); - } - return_stats.column_names.push_back(Identifier(get.GetName() + "." + column_name)); - } - } - - if (get.table_filters.HasFilters()) { - bool has_non_optional_filters = false; - for (auto &entry : get.table_filters) { - auto &column_index = get.GetColumnIndex(entry.GetIndex()); - auto column_statistics = GetColumnStatistics(get, context, column_index); - - if (column_statistics) { - idx_t cardinality_with_filter = - InspectTableFilter(base_table_cardinality, entry.Filter(), *column_statistics); - cardinality_after_filters = MinValue(cardinality_after_filters, cardinality_with_filter); - } - - if (!ExpressionFilter::IsOptionalFilter(entry.Filter())) { - has_non_optional_filters = true; - } - } - // if the above code didn't find an equality filter (i.e country_code = "[us]") - // and there are other table filters (i.e cost > 50), use default selectivity. - bool has_equality_filter = (cardinality_after_filters != base_table_cardinality); - if (!has_equality_filter && has_non_optional_filters) { - cardinality_after_filters = MaxValue( - LossyNumericCast(double(base_table_cardinality) * RelationStatisticsHelper::DEFAULT_SELECTIVITY), - 1U); - } - if (base_table_cardinality == 0) { - cardinality_after_filters = 0; - } - } - - return_stats.cardinality = cardinality_after_filters; - // update the estimated cardinality of the get as well. - // This is not updated during plan reconstruction. - get.estimated_cardinality = cardinality_after_filters; - get.has_estimated_cardinality = true; - D_ASSERT(base_table_cardinality >= cardinality_after_filters); - return_stats.stats_initialized = true; - return return_stats; -} - -RelationStats RelationStatisticsHelper::ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context) { - RelationStats stats; - stats.table_name = Identifier(delim_get.GetName()); - idx_t card = delim_get.EstimateCardinality(context); - stats.cardinality = card; - stats.stats_initialized = true; - for (auto &binding : delim_get.GetColumnBindings()) { - stats.column_distinct_count.emplace_back(1, DistinctCountSource::CARDINALITY); - stats.column_names.push_back(Identifier("column" + to_string(binding.column_index))); - } - return stats; -} - -RelationStats RelationStatisticsHelper::ExtractProjectionStats(LogicalProjection &proj, RelationStats &child_stats) { - auto proj_stats = RelationStats(); - proj_stats.cardinality = child_stats.cardinality; - proj_stats.table_name = Identifier(proj.GetName()); - for (auto &expr : proj.expressions) { - proj_stats.column_names.emplace_back(expr->GetName()); - auto res = GetChildColumnBinding(*expr); - D_ASSERT(res.FoundExpression()); - if (res.expression_is_constant) { - proj_stats.column_distinct_count.emplace_back(1, DistinctCountSource::EXACT); - } else { - auto column_index = res.child_binding.column_index; - if (column_index >= child_stats.column_distinct_count.size() && expr->ToString() == "count_star()") { - // only one value for a count star - proj_stats.column_distinct_count.emplace_back(1, DistinctCountSource::EXACT); - } else { - // TODO: add this back in - // D_ASSERT(column_index < stats.column_distinct_count.size()); - if (column_index < child_stats.column_distinct_count.size()) { - proj_stats.column_distinct_count.push_back(child_stats.column_distinct_count.at(column_index)); - } else { - proj_stats.column_distinct_count.emplace_back(proj_stats.cardinality, - DistinctCountSource::CARDINALITY); - } - } - } - } - proj_stats.stats_initialized = true; - return proj_stats; -} - -RelationStats RelationStatisticsHelper::ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context) { - auto stats = RelationStats(); - idx_t card = dummy_scan.EstimateCardinality(context); - stats.cardinality = card; - for (idx_t i = 0; i < dummy_scan.GetColumnBindings().size(); i++) { - stats.column_distinct_count.emplace_back(card, DistinctCountSource::CARDINALITY); - stats.column_names.push_back("dummy_scan_column"); - } - stats.stats_initialized = true; - stats.table_name = "dummy scan"; - return stats; -} - -void RelationStatisticsHelper::CopyRelationStats(RelationStats &to, const RelationStats &from) { - to.column_distinct_count = from.column_distinct_count; - to.column_names = from.column_names; - to.cardinality = from.cardinality; - to.table_name = from.table_name; - to.stats_initialized = from.stats_initialized; -} - -RelationStats RelationStatisticsHelper::CombineStatsOfReorderableOperator(vector &bindings, - vector relation_stats) { - RelationStats stats; - idx_t max_card = 0; - for (auto &child_stats : relation_stats) { - for (idx_t i = 0; i < child_stats.column_distinct_count.size(); i++) { - stats.column_distinct_count.push_back(child_stats.column_distinct_count.at(i)); - stats.column_names.push_back(child_stats.column_names.at(i)); - } - stats.table_name = Identifier(stats.table_name + "joined with " + child_stats.table_name); - max_card = MaxValue(max_card, child_stats.cardinality); - } - stats.stats_initialized = true; - stats.cardinality = max_card; - return stats; -} - -RelationStats RelationStatisticsHelper::CombineStatsOfNonReorderableOperator(LogicalOperator &op, - const vector &child_stats) { - RelationStats ret; - ret.cardinality = 0; - - // default predicted cardinality is the max of all child cardinalities - vector child_cardinalities; - for (auto &stats : child_stats) { - idx_t child_cardinality = stats.stats_initialized ? stats.cardinality : 0; - ret.cardinality = MaxValue(ret.cardinality, child_cardinality); - child_cardinalities.push_back(child_cardinality); - } - switch (op.type) { - case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: { - D_ASSERT(child_stats.size() == 2); - auto &join = op.Cast(); - switch (join.join_type) { - case JoinType::RIGHT_ANTI: - case JoinType::RIGHT_SEMI: - ret.cardinality = child_cardinalities[1]; - break; - case JoinType::ANTI: - case JoinType::SEMI: - case JoinType::SINGLE: - case JoinType::MARK: - ret.cardinality = child_cardinalities[0]; - break; - default: - break; - } - break; - } - case LogicalOperatorType::LOGICAL_UNION: { - auto &setop = op.Cast(); - if (setop.setop_all) { - // setop returns all records - ret.cardinality = 0; - for (auto &child_cardinality : child_cardinalities) { - ret.cardinality += child_cardinality; - } - } - break; - } - case LogicalOperatorType::LOGICAL_INTERSECT: { - D_ASSERT(child_stats.size() == 2); - ret.cardinality = MinValue(child_cardinalities[0], child_cardinalities[1]); - break; - } - case LogicalOperatorType::LOGICAL_EXCEPT: { - D_ASSERT(child_stats.size() == 2); - ret.cardinality = child_cardinalities[0]; - break; - } - case LogicalOperatorType::LOGICAL_ASOF_JOIN: { - D_ASSERT(child_stats.size() == 2); - auto &join = op.Cast(); - switch (join.join_type) { - case JoinType::RIGHT: - case JoinType::OUTER: - ret.cardinality = child_cardinalities[1]; - break; - default: - // ASOF Joins are usually a table lookup that produce single matches for the LHS - ret.cardinality = child_cardinalities[0]; - break; - } - break; - } - default: - break; - } - - ret.stats_initialized = true; - ret.filter_strength = 1; - ret.table_name = Identifier(string()); - for (auto &stats : child_stats) { - if (!ret.table_name.empty()) { - ret.table_name = Identifier(ret.table_name + " joined with "); - } - ret.table_name = Identifier(ret.table_name + stats.table_name); - // MARK joins are nonreorderable. They won't return initialized stats - // continue in this case. - if (!stats.stats_initialized) { - continue; - } - for (auto &distinct_count : stats.column_distinct_count) { - ret.column_distinct_count.push_back(distinct_count); - } - for (auto &column_name : stats.column_names) { - ret.column_names.push_back(column_name); - } - } - return ret; -} - -RelationStats RelationStatisticsHelper::ExtractExpressionGetStats(LogicalExpressionGet &expression_get, - ClientContext &context) { - auto stats = RelationStats(); - idx_t card = expression_get.EstimateCardinality(context); - stats.cardinality = card; - for (idx_t i = 0; i < expression_get.GetColumnBindings().size(); i++) { - stats.column_distinct_count.emplace_back(card, DistinctCountSource::CARDINALITY); - stats.column_names.push_back("expression_get_column"); - } - stats.stats_initialized = true; - stats.table_name = "expression_get"; - return stats; -} - -RelationStats RelationStatisticsHelper::ExtractWindowStats(LogicalWindow &window, RelationStats &child_stats) { - RelationStats stats; - stats.cardinality = child_stats.cardinality; - stats.column_distinct_count = child_stats.column_distinct_count; - stats.column_names = child_stats.column_names; - stats.stats_initialized = true; - auto num_child_columns = window.GetColumnBindings().size(); - - for (idx_t column_index = child_stats.column_distinct_count.size(); column_index < num_child_columns; - column_index++) { - stats.column_distinct_count.emplace_back(child_stats.cardinality, DistinctCountSource::CARDINALITY); - stats.column_names.push_back("window"); - } - return stats; -} - -RelationStats RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate &aggr, RelationStats &child_stats) { - RelationStats stats; - // TODO: look at child distinct count to better estimate cardinality. - stats.cardinality = child_stats.cardinality; - stats.column_distinct_count = child_stats.column_distinct_count; - vector distinct_counts; - for (auto &g_set : aggr.grouping_sets) { - vector set_distinct_counts; - for (auto &ind : g_set) { - auto &group = aggr.GetGroupExpression(ind); - if (group.GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { - continue; - } - auto &bound_col = group.Cast(); - auto col_index = bound_col.Binding().column_index; - if (col_index >= child_stats.column_distinct_count.size()) { - // it is possible the column index of the grouping_set is not in the child stats. - // this can happen when delim joins are present, since delim scans are not currently - // reorderable. Meaning they don't add a relation or column_ids that could potentially - // be grouped by. Hopefully this can be fixed with duckdb-internal#606 - continue; - } - double distinct_count = static_cast(child_stats.column_distinct_count[col_index].distinct_count); - set_distinct_counts.push_back(distinct_count == 0 ? 1 : distinct_count); - } - // We use the grouping set with the most group key columns for cardinality estimation - if (set_distinct_counts.size() > distinct_counts.size()) { - distinct_counts = std::move(set_distinct_counts); - } - } - - double new_card; - if (distinct_counts.empty()) { - // We have no good statistics on distinct count. - // most likely we are running on parquet files. Therefore we divide by 2. - new_card = static_cast(child_stats.cardinality) / 2.0; - } else { - // Multiply distinct counts - double product = 1; - for (const auto &distinct_count : distinct_counts) { - product *= distinct_count; - } - - // Assume slight correlation for each grouping column - const auto correction = pow(0.95, static_cast(distinct_counts.size() - 1)); - product *= correction; - - // Estimate using the "Occupancy Problem", - // where "product" is number of bins, and "child_stats.cardinality" is number of balls - const auto mult = 1.0 - exp(-static_cast(child_stats.cardinality) / product); - if (mult == 0) { // Can become 0 with very large estimates due to double imprecision - new_card = static_cast(child_stats.cardinality); - } else { - new_card = product * mult; - } - new_card = MinValue(new_card, static_cast(child_stats.cardinality)); - } - - // an ungrouped aggregate has 1 row - stats.cardinality = aggr.groups.empty() ? 1 : LossyNumericCast(new_card); - stats.column_names = child_stats.column_names; - stats.stats_initialized = true; - const auto aggr_column_bindings = aggr.GetColumnBindings(); - auto num_child_columns = aggr_column_bindings.size(); - - for (idx_t column_index = 0; column_index < num_child_columns; column_index++) { - const auto &binding = aggr_column_bindings[column_index]; - if (binding.table_index == aggr.group_index && column_index < distinct_counts.size()) { - // Group column that we have the HLL of - stats.column_distinct_count.emplace_back(LossyNumericCast(distinct_counts[column_index]), - DistinctCountSource::HLL); - } else { - // Non-group column, or we don't have the HLL - stats.column_distinct_count.emplace_back(child_stats.cardinality, DistinctCountSource::CARDINALITY); - } - stats.column_names.push_back("aggregate"); - } - return stats; -} - -RelationStats RelationStatisticsHelper::ExtractEmptyResultStats(LogicalEmptyResult &empty) { - RelationStats stats; - for (idx_t i = 0; i < empty.GetColumnBindings().size(); i++) { - stats.column_distinct_count.emplace_back(0, DistinctCountSource::CARDINALITY); - stats.column_names.push_back("empty_result_column"); - } - stats.stats_initialized = true; - return stats; -} - -idx_t RelationStatisticsHelper::InspectTableFilter(idx_t cardinality, const TableFilter &filter, - BaseStatistics &base_stats) { - auto cardinality_after_filters = cardinality; - auto &expr_filter = ExpressionFilter::GetExpressionFilter(filter, "RelationStatisticsHelper::InspectTableFilter"); - auto &expr = *expr_filter.expr; - if (expr.GetExpressionType() == ExpressionType::CONJUNCTION_AND) { - auto &conj = expr.Cast(); - for (auto &child : conj.GetChildren()) { - ExpressionFilter child_filter(child->Copy()); - cardinality_after_filters = - MinValue(cardinality_after_filters, InspectTableFilter(cardinality, child_filter, base_stats)); - } - return cardinality_after_filters; - } - if (!BoundComparisonExpression::IsComparison(expr)) { - return cardinality_after_filters; - } - auto &comparison = expr.Cast(); - if (comparison.GetExpressionType() != ExpressionType::COMPARE_EQUAL) { - return cardinality_after_filters; - } - auto column_count = GetDistinctCountFromStats(base_stats, cardinality).distinct_count; - // column_count = 0 when there is no HLL and no usable min/max proxy. - if (column_count > 0) { - // we want the ceil of cardinality/column_count. We also want to avoid compiler errors - cardinality_after_filters = (cardinality + column_count - 1) / column_count; - } - return cardinality_after_filters; -} - -// TODO: Currently only simple AND filters are pushed into table scans. -// When OR filters are pushed this function can be added -// idx_t RelationStatisticsHelper::InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter -// &filter, -// BaseStatistics &base_stats) { -// auto has_equality_filter = false; -// auto cardinality_after_filters = cardinality; -// for (auto &child_filter : filter.child_filters) { -// if (child_filter->filter_type != TableFilterType::CONSTANT_COMPARISON) { -// continue; -// } -// auto &comparison_filter = child_filter->Cast(); -// if (comparison_filter.comparison_type == ExpressionType::COMPARE_EQUAL) { -// auto column_count = base_stats.GetDistinctCount(); -// auto increment = MaxValue(((cardinality + column_count - 1) / column_count), 1); -// if (has_equality_filter) { -// cardinality_after_filters += increment; -// } else { -// cardinality_after_filters = increment; -// } -// has_equality_filter = true; -// } -// if (child_filter->filter_type == TableFilterType::CONJUNCTION_AND) { -// auto &and_filter = child_filter->Cast(); -// cardinality_after_filters = RelationStatisticsHelper::InspectConjunctionAND( -// cardinality_after_filters, column_index, and_filter, base_stats); -// continue; -// } -// } -// D_ASSERT(cardinality_after_filters > 0); -// return cardinality_after_filters; -//} - -} // namespace duckdb diff --git a/src/duckdb/src/optimizer/relation_statistics/relation_statistics.cpp b/src/duckdb/src/optimizer/relation_statistics/relation_statistics.cpp new file mode 100644 index 000000000..7426e56e5 --- /dev/null +++ b/src/duckdb/src/optimizer/relation_statistics/relation_statistics.cpp @@ -0,0 +1,49 @@ +#include "duckdb/optimizer/relation_statistics/relation_statistics.hpp" + +namespace duckdb { + +DistinctCount::DistinctCount(idx_t distinct_count, DistinctCountSource source) + : distinct_count(distinct_count), source(source) { +} + +RelationColumnStats::RelationColumnStats(ColumnBinding binding, DistinctCount distinct_count, Identifier name) + : binding(binding), distinct_count(distinct_count), name(std::move(name)) { +} + +RelationStats::RelationStats() : cardinality(1), filter_strength(1), stats_initialized(false) { +} + +optional_idx RelationStats::FindColumn(ColumnBinding binding) const { + for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) { + if (columns[column_idx].binding == binding) { + return column_idx; + } + } + return {}; +} + +optional_ptr RelationStats::GetColumnStats(ColumnBinding binding) const { + auto column_idx = FindColumn(binding); + if (!column_idx.IsValid()) { + return nullptr; + } + return columns[column_idx.GetIndex()]; +} + +bool RelationStats::MatchesBindings(const vector &bindings) const { + if (columns.size() != bindings.size()) { + return false; + } + for (idx_t column_idx = 0; column_idx < columns.size(); column_idx++) { + if (columns[column_idx].binding != bindings[column_idx]) { + return false; + } + } + return true; +} + +void RelationStats::Verify(const vector &bindings) const { + D_ASSERT(!stats_initialized || MatchesBindings(bindings)); +} + +} // namespace duckdb diff --git a/src/duckdb/src/optimizer/relation_statistics/relation_statistics_extractor.cpp b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_extractor.cpp new file mode 100644 index 000000000..48411609e --- /dev/null +++ b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_extractor.cpp @@ -0,0 +1,75 @@ +#include "duckdb/optimizer/relation_statistics/relation_statistics_extractor.hpp" + +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" +#include "duckdb/planner/operator/logical_cteref.hpp" + +namespace duckdb { + +RelationStatsExtractor::RelationStatsExtractor(ClientContext &context) : context(context) { +} + +RelationStatsExtractor::RelationStatsExtractor(ClientContext &context, relation_stats_cte_callback_t cte_callback) + : context(context), cte_callback(std::move(cte_callback)) { +} + +optional_ptr RelationStatsExtractor::Extract(LogicalOperator &op) { + auto cache_entry = cache.find(op); + if (cache_entry != cache.end()) { + return cache_entry->second; + } + if (failed_operators.find(op) != failed_operators.end() || !active_operators.insert(op).second) { + return nullptr; + } + + extracted_operator_count++; + auto result = ExtractInternal(op); + active_operators.erase(op); + if (!result || !result->stats_initialized || !result->MatchesBindings(op.GetColumnBindings())) { + failed_operators.insert(op); + return nullptr; + } + result->Verify(op.GetColumnBindings()); + auto inserted = cache.emplace(op, std::move(*result)); + return inserted.first->second; +} + +idx_t RelationStatsExtractor::ExtractedOperatorCount() const { + return extracted_operator_count; +} + +optional RelationStatsExtractor::ExtractInternal(LogicalOperator &op) { + if (op.type == LogicalOperatorType::LOGICAL_CTE_REF) { + return ExtractCTERef(op.Cast()); + } + if (op.type == LogicalOperatorType::LOGICAL_EXPLAIN) { + return RelationStatisticsHelper::ExtractExplainStats(op); + } + + vector> child_stats; + child_stats.reserve(op.children.size()); + for (auto &child : op.children) { + auto stats = Extract(*child); + if (!stats) { + return {}; + } + child_stats.push_back(*stats); + } + return RelationStatisticsHelper::ExtractOperatorStats(op, context, child_stats); +} + +optional RelationStatsExtractor::ExtractCTERef(LogicalCTERef &cte_ref) { + if (cte_ref.is_recurring || !cte_callback) { + return {}; + } + auto definition = cte_callback(cte_ref.cte_index); + if (!definition) { + return {}; + } + auto definition_stats = Extract(*definition); + if (!definition_stats) { + return {}; + } + return RelationStatisticsHelper::RebindOutputStats(*definition_stats, cte_ref); +} + +} // namespace duckdb diff --git a/src/duckdb/src/optimizer/relation_statistics/relation_statistics_helper.cpp b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_helper.cpp new file mode 100644 index 000000000..9886312c1 --- /dev/null +++ b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_helper.cpp @@ -0,0 +1,276 @@ +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" + +#include "duckdb/planner/expression/list.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/operator/list.hpp" + +#include + +namespace duckdb { + +struct ExpressionBinding { + bool FoundExpression() const { + return expression; + } + + bool FoundColumnRef() const { + return FoundExpression() && expression->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF; + } + + optional_ptr expression; + ColumnBinding child_binding; + bool expression_is_constant = false; +}; + +static ExpressionBinding GetChildColumnBinding(Expression &expr) { + ExpressionBinding result; + switch (expr.GetExpressionClass()) { + case ExpressionClass::BOUND_FUNCTION: { + auto &function = expr.Cast(); + if (function.GetChildren().empty()) { + result.expression = expr; + result.expression_is_constant = true; + return result; + } + break; + } + case ExpressionClass::BOUND_COLUMN_REF: { + result.expression = expr; + result.child_binding = expr.Cast().Binding(); + return result; + } + case ExpressionClass::BOUND_LAMBDA_REF: + case ExpressionClass::BOUND_CONSTANT: + case ExpressionClass::BOUND_DEFAULT: + case ExpressionClass::BOUND_PARAMETER: + case ExpressionClass::BOUND_REF: + result.expression = expr; + result.expression_is_constant = true; + return result; + default: + break; + } + ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr &child) { + if (result.FoundColumnRef()) { + return; + } + auto child_result = GetChildColumnBinding(*child); + if (child_result.FoundExpression()) { + result = child_result; + } + }); + return result; +} + +optional RelationStatisticsHelper::ExtractProjectionStats(LogicalProjection &projection, + const RelationStats &child_stats) { + RelationStats result; + result.cardinality = child_stats.cardinality; + result.table_name = Identifier(projection.GetName()); + result.stats_initialized = true; + auto bindings = projection.GetColumnBindings(); + D_ASSERT(bindings.size() == projection.expressions.size()); + for (idx_t expression_idx = 0; expression_idx < projection.expressions.size(); expression_idx++) { + auto &expression = *projection.expressions[expression_idx]; + auto expression_binding = GetChildColumnBinding(expression); + DistinctCount distinct_count(result.cardinality, DistinctCountSource::CARDINALITY); + if (expression_binding.expression_is_constant) { + distinct_count = DistinctCount(MinValue(result.cardinality, 1), DistinctCountSource::EXACT); + } else if (expression_binding.FoundColumnRef()) { + auto child_column = child_stats.GetColumnStats(expression_binding.child_binding); + if (!child_column) { + return {}; + } + distinct_count = child_column->distinct_count; + } + result.columns.emplace_back(bindings[expression_idx], distinct_count, Identifier(expression.GetName())); + } + result.Verify(bindings); + return result; +} + +idx_t RelationStatisticsHelper::EstimateDistinctCardinality(const vector &distinct_counts, + idx_t input_cardinality) { + if (distinct_counts.empty()) { + return input_cardinality / 2; + } + double product = 1; + for (auto &distinct_count : distinct_counts) { + product *= static_cast(MaxValue(distinct_count.distinct_count, 1)); + } + product *= pow(0.95, static_cast(distinct_counts.size() - 1)); + const auto multiplier = 1.0 - exp(-static_cast(input_cardinality) / product); + const auto estimate = multiplier == 0 ? static_cast(input_cardinality) : product * multiplier; + auto result = LossyNumericCast(MinValue(estimate, static_cast(input_cardinality))); + return input_cardinality > 0 ? MaxValue(result, 1) : 0; +} + +optional RelationStatisticsHelper::ExtractAggregationStats(LogicalAggregate &aggregate, + const RelationStats &child_stats) { + vector cardinality_counts; + for (auto &grouping_set : aggregate.grouping_sets) { + vector set_counts; + for (auto group_idx : grouping_set) { + auto &group = aggregate.GetGroupExpression(group_idx); + if (group.GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + continue; + } + auto column = child_stats.GetColumnStats(group.Cast().Binding()); + if (!column) { + return {}; + } + auto count = column->distinct_count; + count.distinct_count = MaxValue(count.distinct_count, 1); + set_counts.push_back(count); + } + if (set_counts.size() > cardinality_counts.size()) { + cardinality_counts = std::move(set_counts); + } + } + + RelationStats result; + result.cardinality = + aggregate.groups.empty() ? 1 : EstimateDistinctCardinality(cardinality_counts, child_stats.cardinality); + result.table_name = Identifier(aggregate.GetName()); + result.stats_initialized = true; + auto bindings = aggregate.GetColumnBindings(); + for (idx_t group_idx = 0; group_idx < aggregate.groups.size(); group_idx++) { + auto &group = *aggregate.groups[group_idx]; + DistinctCount distinct_count(result.cardinality, DistinctCountSource::CARDINALITY); + if (group.GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { + auto child_column = child_stats.GetColumnStats(group.Cast().Binding()); + if (!child_column) { + return {}; + } + distinct_count = child_column->distinct_count; + } + result.columns.emplace_back(bindings[result.columns.size()], distinct_count, Identifier(group.GetName())); + } + for (auto &expression : aggregate.expressions) { + result.columns.emplace_back(bindings[result.columns.size()], + DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier(expression->GetName())); + } + for (idx_t grouping_idx = 0; grouping_idx < aggregate.grouping_functions.size(); grouping_idx++) { + auto grouping_count = MinValue(result.cardinality, MaxValue(aggregate.grouping_sets.size(), 1)); + result.columns.emplace_back(bindings[result.columns.size()], + DistinctCount(grouping_count, DistinctCountSource::CARDINALITY), + Identifier("grouping")); + } + result.Verify(bindings); + return result; +} + +optional RelationStatisticsHelper::ExtractWindowStats(LogicalWindow &window, + const RelationStats &child_stats) { + RelationStats result; + result.cardinality = child_stats.cardinality; + result.table_name = Identifier(window.GetName()); + result.stats_initialized = true; + for (auto &binding : window.GetColumnBindings()) { + auto child_column = child_stats.GetColumnStats(binding); + if (child_column) { + result.columns.emplace_back(binding, child_column->distinct_count, child_column->name); + } else if (binding.table_index == window.window_index) { + result.columns.emplace_back(binding, DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier("window")); + } else { + return {}; + } + } + result.Verify(window.GetColumnBindings()); + return result; +} + +static optional GetDistinctTargetCount(Expression &target, const RelationStats &child_stats) { + switch (target.GetExpressionClass()) { + case ExpressionClass::BOUND_COLUMN_REF: { + auto column = child_stats.GetColumnStats(target.Cast().Binding()); + return column ? optional(column->distinct_count) : optional(); + } + case ExpressionClass::BOUND_REF: { + auto index = target.Cast().Index(); + return index < child_stats.columns.size() ? optional(child_stats.columns[index].distinct_count) + : optional(); + } + case ExpressionClass::BOUND_CONSTANT: + return DistinctCount(MinValue(child_stats.cardinality, 1), DistinctCountSource::EXACT); + default: + return {}; + } +} + +optional RelationStatisticsHelper::ExtractDistinctStats(LogicalDistinct &distinct, + const RelationStats &child_stats) { + auto result = ProjectOutputStats(child_stats, distinct); + if (!result) { + return {}; + } + vector distinct_counts; + if (distinct.distinct_targets.empty()) { + for (auto &column : child_stats.columns) { + distinct_counts.push_back(column.distinct_count); + } + } else { + for (auto &target : distinct.distinct_targets) { + auto count = GetDistinctTargetCount(*target, child_stats); + if (!count) { + return result; + } + distinct_counts.push_back(*count); + } + } + result->cardinality = EstimateDistinctCardinality(distinct_counts, child_stats.cardinality); + for (auto &column : result->columns) { + column.distinct_count.distinct_count = MinValue(column.distinct_count.distinct_count, result->cardinality); + } + return result; +} + +RelationStats RelationStatisticsHelper::ExtractEmptyResultStats(LogicalEmptyResult &empty) { + RelationStats result; + result.cardinality = 0; + result.table_name = Identifier(empty.GetName()); + result.stats_initialized = true; + for (auto &binding : empty.GetColumnBindings()) { + result.columns.emplace_back(binding, DistinctCount(0, DistinctCountSource::CARDINALITY), + Identifier("empty_result_column")); + } + result.Verify(empty.GetColumnBindings()); + return result; +} + +optional RelationStatisticsHelper::ProjectOutputStats(const RelationStats &stats, LogicalOperator &op) { + if (!stats.stats_initialized) { + return {}; + } + RelationStats result; + result.cardinality = stats.cardinality; + result.filter_strength = stats.filter_strength; + result.stats_initialized = true; + result.table_name = stats.table_name; + for (auto &binding : op.GetColumnBindings()) { + auto column = stats.GetColumnStats(binding); + if (!column) { + return {}; + } + result.columns.push_back(*column); + } + result.Verify(op.GetColumnBindings()); + return result; +} + +optional RelationStatisticsHelper::RebindOutputStats(const RelationStats &stats, LogicalOperator &op) { + auto bindings = op.GetColumnBindings(); + if (!stats.stats_initialized || bindings.size() != stats.columns.size()) { + return {}; + } + auto result = stats; + for (idx_t column_idx = 0; column_idx < bindings.size(); column_idx++) { + result.columns[column_idx].binding = bindings[column_idx]; + } + result.Verify(bindings); + return result; +} + +} // namespace duckdb diff --git a/src/duckdb/src/optimizer/relation_statistics/relation_statistics_operator.cpp b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_operator.cpp new file mode 100644 index 000000000..f1ba72352 --- /dev/null +++ b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_operator.cpp @@ -0,0 +1,213 @@ +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" + +#include "duckdb/common/operator/multiply.hpp" +#include "duckdb/planner/operator/list.hpp" + +namespace duckdb { + +static optional +ProjectChildStats(LogicalOperator &op, const vector> &children, idx_t cardinality) { + RelationStats result; + result.cardinality = cardinality; + result.stats_initialized = true; + result.table_name = Identifier(op.GetName()); + for (auto &binding : op.GetColumnBindings()) { + optional_ptr source; + for (auto &child : children) { + source = child.get().GetColumnStats(binding); + if (source) { + break; + } + } + if (!source) { + return {}; + } + result.columns.emplace_back(binding, source->distinct_count, source->name); + } + result.Verify(op.GetColumnBindings()); + return result; +} + +static idx_t JoinCardinality(LogicalComparisonJoin &join, const RelationStats &left, const RelationStats &right) { + switch (join.join_type) { + case JoinType::RIGHT_ANTI: + case JoinType::RIGHT_SEMI: + return right.cardinality; + case JoinType::ANTI: + case JoinType::SEMI: + case JoinType::SINGLE: + case JoinType::MARK: + return left.cardinality; + default: + return MaxValue(left.cardinality, right.cardinality); + } +} + +static optional ExtractGetWithChildStats(LogicalGet &get, ClientContext &context, + const RelationStats &child_stats) { + auto result = RelationStatisticsHelper::ExtractGetStats(get, context); + result.cardinality = child_stats.cardinality; + for (auto &binding : get.GetColumnBindings()) { + if (binding.table_index == get.table_index) { + continue; + } + auto child_column = child_stats.GetColumnStats(binding); + if (!child_column) { + return {}; + } + result.columns.emplace_back(binding, child_column->distinct_count, child_column->name); + } + result.Verify(get.GetColumnBindings()); + return result; +} + +static optional ExtractUnnestStats(LogicalOperator &op, const RelationStats &child_stats) { + auto &unnest = op.Cast(); + RelationStats result; + result.cardinality = child_stats.cardinality; + result.stats_initialized = true; + result.table_name = Identifier(op.GetName()); + for (auto &binding : op.GetColumnBindings()) { + auto child_column = child_stats.GetColumnStats(binding); + if (child_column) { + result.columns.emplace_back(binding, child_column->distinct_count, child_column->name); + } else if (binding.table_index == unnest.unnest_index) { + result.columns.emplace_back(binding, DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier("unnest")); + } else { + return {}; + } + } + result.Verify(op.GetColumnBindings()); + return result; +} + +static optional ExtractComparisonJoinStats(LogicalComparisonJoin &join, + const vector> &child_stats) { + if (child_stats.size() != 2) { + return {}; + } + auto cardinality = JoinCardinality(join, child_stats[0], child_stats[1]); + auto result = ProjectChildStats(join, child_stats, cardinality); + if (result || join.join_type != JoinType::MARK) { + return result; + } + + RelationStats mark_result; + mark_result.cardinality = cardinality; + mark_result.stats_initialized = true; + mark_result.table_name = Identifier(join.GetName()); + for (auto &binding : join.GetColumnBindings()) { + auto column = child_stats[0].get().GetColumnStats(binding); + if (column) { + mark_result.columns.emplace_back(binding, column->distinct_count, column->name); + } else if (binding.table_index == join.mark_index) { + mark_result.columns.emplace_back( + binding, DistinctCount(MinValue(cardinality, 3), DistinctCountSource::CARDINALITY), + Identifier("mark")); + } else { + return {}; + } + } + mark_result.Verify(join.GetColumnBindings()); + return mark_result; +} + +static optional ExtractCrossProductStats(LogicalOperator &op, + const vector> &child_stats) { + if (child_stats.size() != 2) { + return {}; + } + idx_t cardinality; + if (!TryMultiplyOperator::Operation(child_stats[0].get().cardinality, child_stats[1].get().cardinality, + cardinality)) { + cardinality = NumericLimits::Maximum(); + } + return ProjectChildStats(op, child_stats, cardinality); +} + +RelationStats RelationStatisticsHelper::ExtractExplainStats(LogicalOperator &op) { + RelationStats result; + result.cardinality = 3; + result.stats_initialized = true; + result.table_name = Identifier(op.GetName()); + for (auto &binding : op.GetColumnBindings()) { + result.columns.emplace_back(binding, DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier("explain")); + } + result.Verify(op.GetColumnBindings()); + return result; +} + +optional +RelationStatisticsHelper::ExtractOperatorStats(LogicalOperator &op, ClientContext &context, + const vector> &child_stats) { + if (child_stats.size() != op.children.size()) { + return {}; + } + for (idx_t child_idx = 0; child_idx < child_stats.size(); child_idx++) { + auto &stats = child_stats[child_idx].get(); + if (!stats.stats_initialized || !stats.MatchesBindings(op.children[child_idx]->GetColumnBindings())) { + return {}; + } + } + switch (op.type) { + case LogicalOperatorType::LOGICAL_GET: + if (child_stats.empty()) { + return ExtractGetStats(op.Cast(), context); + } + return child_stats.size() == 1 ? ExtractGetWithChildStats(op.Cast(), context, child_stats[0].get()) + : optional(); + case LogicalOperatorType::LOGICAL_DELIM_GET: + return ExtractDelimGetStats(op.Cast(), context); + case LogicalOperatorType::LOGICAL_DUMMY_SCAN: + return ExtractDummyScanStats(op.Cast(), context); + case LogicalOperatorType::LOGICAL_EXPRESSION_GET: + return ExtractExpressionGetStats(op.Cast(), context); + case LogicalOperatorType::LOGICAL_CHUNK_GET: + return ExtractColumnDataGetStats(op.Cast(), context); + case LogicalOperatorType::LOGICAL_PROJECTION: + return ExtractProjectionStats(op.Cast(), child_stats[0].get()); + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + return ExtractAggregationStats(op.Cast(), child_stats[0].get()); + case LogicalOperatorType::LOGICAL_WINDOW: + return ExtractWindowStats(op.Cast(), child_stats[0].get()); + case LogicalOperatorType::LOGICAL_DISTINCT: + return ExtractDistinctStats(op.Cast(), child_stats[0].get()); + case LogicalOperatorType::LOGICAL_FILTER: { + if (child_stats.size() != 1) { + return {}; + } + auto cardinality = child_stats[0].get().cardinality; + if (cardinality > 0) { + cardinality = MaxValue(LossyNumericCast(double(cardinality) * DEFAULT_SELECTIVITY), 1); + } + return ProjectChildStats(op, child_stats, cardinality); + } + case LogicalOperatorType::LOGICAL_UNNEST: + return ExtractUnnestStats(op, child_stats[0].get()); + case LogicalOperatorType::LOGICAL_LIMIT: { + if (child_stats.size() != 1) { + return {}; + } + auto cardinality = child_stats[0].get().cardinality; + auto &limit = op.Cast(); + if (limit.limit_val.Type() == LimitNodeType::CONSTANT_VALUE) { + cardinality = MinValue(cardinality, limit.limit_val.GetConstantValue()); + } + return ProjectChildStats(op, child_stats, cardinality); + } + case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: + return ExtractComparisonJoinStats(op.Cast(), child_stats); + case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: + return ExtractCrossProductStats(op, child_stats); + case LogicalOperatorType::LOGICAL_EMPTY_RESULT: + return ExtractEmptyResultStats(op.Cast()); + case LogicalOperatorType::LOGICAL_EXPLAIN: + return ExtractExplainStats(op); + default: + return {}; + } +} + +} // namespace duckdb diff --git a/src/duckdb/src/optimizer/relation_statistics/relation_statistics_scan.cpp b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_scan.cpp new file mode 100644 index 000000000..bb9b0079b --- /dev/null +++ b/src/duckdb/src/optimizer/relation_statistics/relation_statistics_scan.cpp @@ -0,0 +1,264 @@ +#include "duckdb/optimizer/relation_statistics/relation_statistics_helper.hpp" + +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/common/operator/add.hpp" +#include "duckdb/common/operator/subtract.hpp" +#include "duckdb/common/types/hugeint.hpp" +#include "duckdb/function/table/table_scan.hpp" +#include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_conjunction_expression.hpp" +#include "duckdb/planner/operator/logical_delim_get.hpp" +#include "duckdb/planner/operator/logical_column_data_get.hpp" +#include "duckdb/planner/operator/logical_dummy_scan.hpp" +#include "duckdb/planner/operator/logical_expression_get.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/storage/statistics/numeric_stats.hpp" + +namespace duckdb { + +static idx_t CapMinMaxDistinctCount(uint64_t distinct_count, idx_t base_table_cardinality) { + if (base_table_cardinality == 0 || distinct_count == 0) { + return 0; + } + auto capped_distinct_count = MinValue(distinct_count, base_table_cardinality); + return capped_distinct_count == NumericLimits::Maximum() ? 0 : capped_distinct_count; +} + +static idx_t GetMinMaxSpanDistinctCount(uint64_t span, idx_t base_table_cardinality) { + uint64_t distinct_count; + if (!TryAddOperator::Operation(span, 1, distinct_count)) { + return 0; + } + return CapMinMaxDistinctCount(distinct_count, base_table_cardinality); +} + +template +static idx_t GetSignedMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { + auto min_value = NumericStats::Min(base_stats).GetValueUnsafe(); + auto max_value = NumericStats::Max(base_stats).GetValueUnsafe(); + if (max_value < min_value) { + return 0; + } + hugeint_t span; + if (!TrySubtractOperator::Operation(hugeint_t(static_cast(max_value)), + hugeint_t(static_cast(min_value)), span)) { + return 0; + } + uint64_t unsigned_span; + if (!Hugeint::TryCast(span, unsigned_span)) { + return 0; + } + return GetMinMaxSpanDistinctCount(unsigned_span, base_table_cardinality); +} + +template +static idx_t GetUnsignedMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { + auto min_value = NumericStats::Min(base_stats).GetValueUnsafe(); + auto max_value = NumericStats::Max(base_stats).GetValueUnsafe(); + T span; + if (!TrySubtractOperator::Operation(max_value, min_value, span)) { + return 0; + } + return GetMinMaxSpanDistinctCount(static_cast(span), base_table_cardinality); +} + +static idx_t GetBooleanMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { + auto min_value = NumericStats::Min(base_stats).GetValueUnsafe(); + auto max_value = NumericStats::Max(base_stats).GetValueUnsafe(); + return CapMinMaxDistinctCount(min_value == max_value ? idx_t(1) : idx_t(2), base_table_cardinality); +} + +static idx_t GetMinMaxDistinctCount(const BaseStatistics &base_stats, idx_t base_table_cardinality) { + if (base_table_cardinality == 0 || base_stats.GetStatsType() != StatisticsType::NUMERIC_STATS || + !NumericStats::HasMinMax(base_stats)) { + return 0; + } + + switch (base_stats.GetType().InternalType()) { + case PhysicalType::BOOL: + return GetBooleanMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::INT8: + return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::INT16: + return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::INT32: + return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::INT64: + return GetSignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::UINT8: + return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::UINT16: + return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::UINT32: + return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); + case PhysicalType::UINT64: + return GetUnsignedMinMaxDistinctCount(base_stats, base_table_cardinality); + default: + return 0; + } +} + +static DistinctCount GetDistinctCountFromStats(BaseStatistics &base_stats, idx_t base_table_cardinality) { + auto distinct_count = base_stats.GetDistinctCount(); + if (distinct_count > 0) { + return DistinctCount(distinct_count, DistinctCountSource::HLL); + } + distinct_count = GetMinMaxDistinctCount(base_stats, base_table_cardinality); + if (distinct_count > 0) { + return DistinctCount(distinct_count, DistinctCountSource::MIN_MAX); + } + return DistinctCount(0, DistinctCountSource::CARDINALITY); +} + +unique_ptr RelationStatisticsHelper::GetColumnStatistics(LogicalGet &get, ClientContext &context, + const ColumnIndex &column_id) { + if (!get.bind_data || (!get.function.statistics && !get.function.statistics_extended)) { + return nullptr; + } + if (get.function.statistics_extended) { + TableFunctionGetStatisticsInput input(get.bind_data.get(), column_id); + return get.function.statistics_extended(context, input); + } + return get.function.statistics(context, get.bind_data.get(), column_id.GetPrimaryIndex()); +} + +DistinctCount RelationStatisticsHelper::GetDistinctCount(LogicalGet &get, ClientContext &context, + const ColumnIndex &column_id, idx_t base_table_cardinality) { + auto column_statistics = GetColumnStatistics(get, context, column_id); + if (!column_statistics) { + return DistinctCount(0, DistinctCountSource::CARDINALITY); + } + return GetDistinctCountFromStats(*column_statistics, base_table_cardinality); +} + +RelationStats RelationStatisticsHelper::ExtractGetStats(LogicalGet &get, ClientContext &context) { + RelationStats result; + auto base_table_cardinality = get.EstimateCardinality(context); + auto cardinality_after_filters = base_table_cardinality; + result.table_name = get.GetTable() ? get.GetTable()->name : Identifier(get.GetName()); + + if (get.table_filters.HasFilters()) { + bool has_non_optional_filters = false; + for (auto &entry : get.table_filters) { + auto &column_index = get.GetColumnIndex(entry.GetIndex()); + auto column_statistics = GetColumnStatistics(get, context, column_index); + if (column_statistics) { + cardinality_after_filters = + MinValue(cardinality_after_filters, + InspectTableFilter(base_table_cardinality, entry.Filter(), *column_statistics)); + } + if (!ExpressionFilter::IsOptionalFilter(entry.Filter())) { + has_non_optional_filters = true; + } + } + if (cardinality_after_filters == base_table_cardinality && has_non_optional_filters) { + cardinality_after_filters = + MaxValue(LossyNumericCast(double(base_table_cardinality) * DEFAULT_SELECTIVITY), 1); + } + if (base_table_cardinality == 0) { + cardinality_after_filters = 0; + } + } + + result.cardinality = cardinality_after_filters; + for (auto &binding : get.GetColumnBindings()) { + if (binding.table_index != get.table_index) { + continue; + } + ColumnIndex fallback_column(get.GetAnyColumn()); + auto column_id = get.GetColumnIds().empty() ? fallback_column : get.GetColumnIndex(binding); + auto distinct_count = GetDistinctCount(get, context, column_id, base_table_cardinality); + if (distinct_count.distinct_count == 0) { + distinct_count = DistinctCount(cardinality_after_filters, DistinctCountSource::CARDINALITY); + } + result.columns.emplace_back(binding, distinct_count, + Identifier(get.GetName() + "." + get.GetColumnName(column_id))); + } + result.stats_initialized = true; + D_ASSERT(base_table_cardinality >= cardinality_after_filters); + return result; +} + +RelationStats RelationStatisticsHelper::ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context) { + RelationStats result; + result.table_name = Identifier(delim_get.GetName()); + result.cardinality = delim_get.EstimateCardinality(context); + result.stats_initialized = true; + for (auto &binding : delim_get.GetColumnBindings()) { + result.columns.emplace_back(binding, DistinctCount(1, DistinctCountSource::CARDINALITY), + Identifier("column" + to_string(binding.column_index))); + } + result.Verify(delim_get.GetColumnBindings()); + return result; +} + +RelationStats RelationStatisticsHelper::ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context) { + RelationStats result; + result.cardinality = dummy_scan.EstimateCardinality(context); + result.table_name = "dummy scan"; + result.stats_initialized = true; + for (auto &binding : dummy_scan.GetColumnBindings()) { + result.columns.emplace_back(binding, DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier("dummy_scan_column")); + } + result.Verify(dummy_scan.GetColumnBindings()); + return result; +} + +RelationStats RelationStatisticsHelper::ExtractExpressionGetStats(LogicalExpressionGet &expression_get, + ClientContext &context) { + RelationStats result; + result.cardinality = expression_get.EstimateCardinality(context); + result.table_name = "expression_get"; + result.stats_initialized = true; + for (auto &binding : expression_get.GetColumnBindings()) { + result.columns.emplace_back(binding, DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier("expression_get_column")); + } + result.Verify(expression_get.GetColumnBindings()); + return result; +} + +RelationStats RelationStatisticsHelper::ExtractColumnDataGetStats(LogicalColumnDataGet &column_data_get, + ClientContext &) { + RelationStats result; + result.cardinality = column_data_get.collection->Count(); + result.table_name = Identifier(column_data_get.GetName()); + result.stats_initialized = true; + for (auto &binding : column_data_get.GetColumnBindings()) { + result.columns.emplace_back(binding, DistinctCount(result.cardinality, DistinctCountSource::CARDINALITY), + Identifier("column_data")); + } + result.Verify(column_data_get.GetColumnBindings()); + return result; +} + +idx_t RelationStatisticsHelper::InspectTableFilter(idx_t cardinality, const TableFilter &filter, + BaseStatistics &base_stats) { + auto cardinality_after_filters = cardinality; + auto &expr_filter = ExpressionFilter::GetExpressionFilter(filter, "RelationStatisticsHelper::InspectTableFilter"); + auto &expr = *expr_filter.expr; + if (expr.GetExpressionType() == ExpressionType::CONJUNCTION_AND) { + auto &conjunction = expr.Cast(); + for (auto &child : conjunction.GetChildren()) { + ExpressionFilter child_filter(child->Copy()); + cardinality_after_filters = + MinValue(cardinality_after_filters, InspectTableFilter(cardinality, child_filter, base_stats)); + } + return cardinality_after_filters; + } + if (!BoundComparisonExpression::IsComparison(expr)) { + return cardinality_after_filters; + } + auto &comparison = expr.Cast(); + if (comparison.GetExpressionType() != ExpressionType::COMPARE_EQUAL) { + return cardinality_after_filters; + } + auto column_count = GetDistinctCountFromStats(base_stats, cardinality).distinct_count; + if (column_count > 0) { + cardinality_after_filters = (cardinality + column_count - 1) / column_count; + } + return cardinality_after_filters; +} + +} // namespace duckdb diff --git a/src/duckdb/src/optimizer/remove_unused_columns.cpp b/src/duckdb/src/optimizer/remove_unused_columns.cpp index 78e06f3d6..f02a2c604 100644 --- a/src/duckdb/src/optimizer/remove_unused_columns.cpp +++ b/src/duckdb/src/optimizer/remove_unused_columns.cpp @@ -1102,6 +1102,33 @@ void RemoveUnusedColumns::RemoveColumnsFromLogicalGet(LogicalGet &get, unique_pt get.table_filters = std::move(remapped_filters); } + if (get.function.filter_prune) { + // Now set the projection cols by matching the "selection vector" that excludes filter columns + // with the "selection vector" that includes filter columns + idx_t col_idx = 0; + get.projection_ids.clear(); + vector filtered_original_ids; + //! Find matching indices between the proj_sel and the col_sel + for (auto to_keep : proj_sel) { + for (; col_idx < col_sel.size(); col_idx++) { + if (to_keep == col_sel[col_idx]) { + filtered_original_ids.push_back(to_keep); + break; + } + } + } + col_idx = 0; + for (auto col : filtered_original_ids) { + for (; col_idx < original_ids.size(); col_idx++) { + if (original_ids[col_idx] == col) { + get.projection_ids.push_back(ProjectionIndex(col_idx)); + } else if (original_ids[col_idx] > col) { + break; + } + } + } + } + if (has_pushdown_extract && !filter_expressions.empty()) { // if we have performed pushdown extract and we have filter expressions we might have adjusted the filter // expressions remove the table filters and push a filter, then try to re-push the filters with the new set of @@ -1114,35 +1141,15 @@ void RemoveUnusedColumns::RemoveColumnsFromLogicalGet(LogicalGet &get, unique_pt // try to push filters back into the table scan FilterPushdown pushdown(optimizer); op_ref = pushdown.Rewrite(std::move(filter)); - return; - } - - if (!get.function.filter_prune) { - return; - } - // Now set the projection cols by matching the "selection vector" that excludes filter columns - // with the "selection vector" that includes filter columns - idx_t col_idx = 0; - get.projection_ids.clear(); - vector filtered_original_ids; - //! Find matching indices between the proj_sel and the col_sel - for (auto to_keep : proj_sel) { - for (; col_idx < col_sel.size(); col_idx++) { - if (to_keep == col_sel[col_idx]) { - filtered_original_ids.push_back(to_keep); - break; - } - } - } - col_idx = 0; - for (auto col : filtered_original_ids) { - for (; col_idx < original_ids.size(); col_idx++) { - if (original_ids[col_idx] == col) { - get.projection_ids.push_back(ProjectionIndex(col_idx)); - } else if (original_ids[col_idx] > col) { - break; - } + // if the filters could not be pushed back into the scan, the columns referenced by the + // remaining filter have to be included in the output of the scan. Since the projection can + // only prune columns when it projects out fewer columns than are scanned (see + // TableFunctionInitInput::CanRemoveFilterColumns), we fall back to outputting all columns + // in scan order + if (op_ref->type == LogicalOperatorType::LOGICAL_FILTER) { + get.projection_ids.clear(); } + return; } } diff --git a/src/duckdb/src/optimizer/rule/regex_optimizations.cpp b/src/duckdb/src/optimizer/rule/regex_optimizations.cpp index 3b6d02ca4..509c3ede2 100644 --- a/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +++ b/src/duckdb/src/optimizer/rule/regex_optimizations.cpp @@ -154,6 +154,12 @@ unique_ptr RegexOptimizationRule::Apply(LogicalOperator &op, vector< auto &root = bindings[0].get().Cast(); auto &constant_expr = bindings[2].get().Cast(); D_ASSERT(root.GetChildrenMutable().size() == 2 || root.GetChildrenMutable().size() == 3); + for (idx_t i = 0; i < 2; i++) { + const auto &type = root.GetChildren()[i]->GetReturnType(); + if (type.id() == LogicalTypeId::VARCHAR && !StringType::GetCollation(type).empty()) { + return nullptr; + } + } auto regexp_bind_data = root.BindInfo().get()->Cast(); auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr); diff --git a/src/duckdb/src/optimizer/window_self_join.cpp b/src/duckdb/src/optimizer/window_self_join.cpp index 41aeb9144..5cae25f20 100644 --- a/src/duckdb/src/optimizer/window_self_join.cpp +++ b/src/duckdb/src/optimizer/window_self_join.cpp @@ -13,6 +13,25 @@ namespace duckdb { +class VolatileExpressionCounter : public LogicalOperatorVisitor { +public: + static idx_t HasVolatiles(LogicalOperator &op) { + VolatileExpressionCounter counter; + counter.VisitOperator(op); + return counter.volatiles; + } + + VolatileExpressionCounter() { + } + + void VisitExpression(unique_ptr *expression) override { + volatiles += (*expression)->IsVolatile(); + LogicalOperatorVisitor::VisitExpression(expression); + } + + idx_t volatiles = 0; +}; + static bool IsOrderableDistinctAggregate(const BoundWindowExpression &w_expr) { // If the aggregate is order-sensitive and distinct, // then the ORDER BYs need to be functional dependencies of the arguments. @@ -180,6 +199,11 @@ unique_ptr WindowSelfJoinOptimizer::OptimizeInternal(unique_ptr // Check recursively window.children[0] = OptimizeInternal(std::move(window.children[0]), replacer); + // We cannot perform self-join when there are volatile functions below us. + if (VolatileExpressionCounter::HasVolatiles(window)) { + return op; + } + if (!CanOptimize(*window.children[0])) { return op; } diff --git a/src/duckdb/src/parser/peg/tokenizer/base_tokenizer.cpp b/src/duckdb/src/parser/peg/tokenizer/base_tokenizer.cpp index ff7d7735d..69ecdb184 100644 --- a/src/duckdb/src/parser/peg/tokenizer/base_tokenizer.cpp +++ b/src/duckdb/src/parser/peg/tokenizer/base_tokenizer.cpp @@ -236,8 +236,10 @@ void BaseTokenizer::TokenizeInput() { bool BaseTokenizer::TokenizeInputInternal() { auto state = TokenizeState::STANDARD; idx_t last_pos = 0; + bool escape_string = false; string dollar_quote_marker; idx_t dollar_marker_start = 0; + idx_t multi_line_comment_depth = 0; for (idx_t i = 0; i < sql.size(); i++) { auto c = sql[i]; switch (state) { @@ -245,6 +247,7 @@ bool BaseTokenizer::TokenizeInputInternal() { if (c == '\'') { state = TokenizeState::STRING_LITERAL; last_pos = i; + escape_string = false; break; } if (c == '"') { @@ -305,6 +308,7 @@ bool BaseTokenizer::TokenizeInputInternal() { if (c == '/' && i + 1 < sql.size() && sql[i + 1] == '*') { i++; state = TokenizeState::MULTI_LINE_COMMENT; + multi_line_comment_depth = 1; break; } if (StringUtil::CharacterIsSpace(c)) { @@ -337,6 +341,9 @@ bool BaseTokenizer::TokenizeInputInternal() { if (i + 1 < sql.size() && sql[i + 1] == '\'') { state = TokenizeState::STRING_LITERAL; last_pos = i; + if (c == 'E' || c == 'e') { + escape_string = true; + } i++; break; } @@ -436,6 +443,10 @@ bool BaseTokenizer::TokenizeInputInternal() { } break; case TokenizeState::STRING_LITERAL: + if (escape_string && c == '\\' && i + 1 < sql.size()) { + i++; + break; + } if (c == '\'') { if (i + 1 < sql.size() && sql[i + 1] == '\'') { // escaped - skip escape @@ -443,6 +454,7 @@ bool BaseTokenizer::TokenizeInputInternal() { } else { PushToken(last_pos, i + 1, TokenType::STRING_LITERAL); last_pos = i + 1; + escape_string = false; state = TokenizeState::STANDARD; } } @@ -467,11 +479,17 @@ bool BaseTokenizer::TokenizeInputInternal() { } break; case TokenizeState::MULTI_LINE_COMMENT: - if (c == '*' && i + 1 < sql.size() && sql[i + 1] == '/') { + if (c == '/' && i + 1 < sql.size() && sql[i + 1] == '*') { i++; - PushToken(last_pos, i + 1, TokenType::COMMENT); - last_pos = i + 1; - state = TokenizeState::STANDARD; + multi_line_comment_depth++; + } else if (c == '*' && i + 1 < sql.size() && sql[i + 1] == '/') { + i++; + multi_line_comment_depth--; + if (multi_line_comment_depth == 0) { + PushToken(last_pos, i + 1, TokenType::COMMENT); + last_pos = i + 1; + state = TokenizeState::STANDARD; + } } break; case TokenizeState::DOLLAR_QUOTED_STRING: { diff --git a/src/duckdb/src/parser/peg/transformer/transform_common.cpp b/src/duckdb/src/parser/peg/transformer/transform_common.cpp index 66358b885..64e386271 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_common.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_common.cpp @@ -77,6 +77,11 @@ int64_t PEGTransformerFactory::TransformArrayKeyword(PEGTransformer &transformer return -1; } +int64_t PEGTransformerFactory::TransformArrayKeywordWithBounds(PEGTransformer &transformer, + const int64_t &square_brackets_array) { + return square_brackets_array; +} + int64_t PEGTransformerFactory::TransformSquareBracketsArray(PEGTransformer &transformer, optional> expression) { if (!expression) { diff --git a/src/duckdb/src/parser/peg/transformer/transform_generated.cpp b/src/duckdb/src/parser/peg/transformer/transform_generated.cpp index c2b364d99..8dd5c317a 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_generated.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_generated.cpp @@ -1335,6 +1335,14 @@ unique_ptr PEGTransformerFactory::TransformArrayKeywordInt return make_uniq>(result); } +unique_ptr +PEGTransformerFactory::TransformArrayKeywordWithBoundsInternal(PEGTransformer &transformer, ParseResult &parse_result) { + auto &list_pr = parse_result.Cast(); + auto square_brackets_array = transformer.Transform(list_pr.GetChild(1)); + auto result = TransformArrayKeywordWithBounds(transformer, square_brackets_array); + return make_uniq>(result); +} + unique_ptr PEGTransformerFactory::TransformSquareBracketsArrayInternal(PEGTransformer &transformer, ParseResult &parse_result) { auto &list_pr = parse_result.Cast(); @@ -9310,8 +9318,8 @@ unique_ptr PEGTransformerFactory::TransformJoinWithoutOnClauseInternal(PEGTransformer &transformer, ParseResult &parse_result) { auto &list_pr = parse_result.Cast(); auto join_prefix = transformer.Transform(list_pr.GetChild(0)); - auto table_ref = transformer.Transform>(list_pr.GetChild(2)); - auto result = TransformJoinWithoutOnClause(transformer, join_prefix, std::move(table_ref)); + auto inner_table_ref = transformer.Transform>(list_pr.GetChild(2)); + auto result = TransformJoinWithoutOnClause(transformer, join_prefix, std::move(inner_table_ref)); return make_uniq>>(std::move(result)); } @@ -10730,6 +10738,7 @@ void PEGTransformerFactory::RegisterGenerated() { {"ColIdType", &PEGTransformerFactory::TransformColIdTypeInternal}, {"ArrayBounds", &PEGTransformerFactory::TransformArrayBoundsInternal}, {"ArrayKeyword", &PEGTransformerFactory::TransformArrayKeywordInternal}, + {"ArrayKeywordWithBounds", &PEGTransformerFactory::TransformArrayKeywordWithBoundsInternal}, {"SquareBracketsArray", &PEGTransformerFactory::TransformSquareBracketsArrayInternal}, {"TimeType", &PEGTransformerFactory::TransformTimeTypeInternal}, {"TimeOrTimestamp", &PEGTransformerFactory::TransformTimeOrTimestampInternal}, diff --git a/src/duckdb/src/parser/peg/transformer/transform_generated_trampoline.cpp b/src/duckdb/src/parser/peg/transformer/transform_generated_trampoline.cpp index edd7e4128..e3ca3ed6a 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_generated_trampoline.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_generated_trampoline.cpp @@ -389,6 +389,9 @@ static const TransformFrameOps ARRAY_BOUNDS_OPS = {"ArrayBounds", static const TransformFrameOps ARRAY_KEYWORD_OPS = {"ArrayKeyword", &PEGTransformerFactory::InitializeArrayKeywordTrampoline, &PEGTransformerFactory::FinalizeArrayKeywordTrampoline}; +static const TransformFrameOps ARRAY_KEYWORD_WITH_BOUNDS_OPS = { + "ArrayKeywordWithBounds", &PEGTransformerFactory::InitializeArrayKeywordWithBoundsTrampoline, + &PEGTransformerFactory::FinalizeArrayKeywordWithBoundsTrampoline}; static const TransformFrameOps SQUARE_BRACKETS_ARRAY_OPS = { "SquareBracketsArray", &PEGTransformerFactory::InitializeSquareBracketsArrayTrampoline, &PEGTransformerFactory::FinalizeSquareBracketsArrayTrampoline}; @@ -2993,6 +2996,7 @@ const case_insensitive_map_t &PEGTransformerFactory:: {"ColIdType", &COL_ID_TYPE_OPS}, {"ArrayBounds", &ARRAY_BOUNDS_OPS}, {"ArrayKeyword", &ARRAY_KEYWORD_OPS}, + {"ArrayKeywordWithBounds", &ARRAY_KEYWORD_WITH_BOUNDS_OPS}, {"SquareBracketsArray", &SQUARE_BRACKETS_ARRAY_OPS}, {"TimeType", &TIME_TYPE_OPS}, {"TimeOrTimestamp", &TIME_OR_TIMESTAMP_OPS}, @@ -6534,6 +6538,22 @@ unique_ptr PEGTransformerFactory::FinalizeArrayKeywordTram return make_uniq>(result); } +void PEGTransformerFactory::InitializeArrayKeywordWithBoundsTrampoline(PEGTransformer &transformer, + TransformStack &stack, + TransformStackFrame &frame) { + auto &list_pr = frame.parse_result.Cast(); + frame.ReserveChildSlots(1); + stack.PushFrame(list_pr.GetChild(1), SQUARE_BRACKETS_ARRAY_OPS, TransformFrameResultTarget(frame.frame_index, 0)); +} + +unique_ptr +PEGTransformerFactory::FinalizeArrayKeywordWithBoundsTrampoline(PEGTransformer &transformer, TransformStack &stack, + TransformStackFrame &frame) { + auto square_brackets_array = frame.TakeResult(0); + auto result = TransformArrayKeywordWithBounds(transformer, square_brackets_array); + return make_uniq>(result); +} + void PEGTransformerFactory::InitializeSquareBracketsArrayTrampoline(PEGTransformer &transformer, TransformStack &stack, TransformStackFrame &frame) { auto &list_pr = frame.parse_result.Cast(); @@ -22040,7 +22060,7 @@ void PEGTransformerFactory::InitializeJoinWithoutOnClauseTrampoline(PEGTransform TransformStackFrame &frame) { auto &list_pr = frame.parse_result.Cast(); frame.ReserveChildSlots(2); - stack.PushFrame(list_pr.GetChild(2), TABLE_REF_OPS, TransformFrameResultTarget(frame.frame_index, 1)); + stack.PushFrame(list_pr.GetChild(2), INNER_TABLE_REF_OPS, TransformFrameResultTarget(frame.frame_index, 1)); stack.PushFrame(list_pr.GetChild(0), JOIN_PREFIX_OPS, TransformFrameResultTarget(frame.frame_index, 0)); } @@ -22048,8 +22068,8 @@ unique_ptr PEGTransformerFactory::FinalizeJoinWithoutOnClauseTrampoline(PEGTransformer &transformer, TransformStack &stack, TransformStackFrame &frame) { auto join_prefix = frame.TakeResult(0); - auto table_ref = frame.TakeResult>(1); - auto result = TransformJoinWithoutOnClause(transformer, join_prefix, std::move(table_ref)); + auto inner_table_ref = frame.TakeResult>(1); + auto result = TransformJoinWithoutOnClause(transformer, join_prefix, std::move(inner_table_ref)); return make_uniq>>(std::move(result)); } diff --git a/src/duckdb/src/parser/peg/transformer/transform_select.cpp b/src/duckdb/src/parser/peg/transformer/transform_select.cpp index a6c00f08b..9ca06da20 100644 --- a/src/duckdb/src/parser/peg/transformer/transform_select.cpp +++ b/src/duckdb/src/parser/peg/transformer/transform_select.cpp @@ -441,51 +441,6 @@ QualifiedName PEGTransformerFactory::TransformSchemaReservedIdentifierOrStringLi return result; } -static bool IsConditionlessJoin(const JoinRef &join) { - if (join.condition || !join.using_columns.empty()) { - return false; - } - if (join.ref_type != JoinRefType::CROSS && join.ref_type != JoinRefType::POSITIONAL && - join.ref_type != JoinRefType::NATURAL) { - return false; - } - return true; -} - -static unique_ptr ReassociateJoins(unique_ptr root) { - // Left-rotate while the current node is a conditionless join and its right child is a join. - // This converts right-associative join trees (from PEG grammar) to left-associative. - while (root->type == TableReferenceType::JOIN) { - auto ¤t = root->Cast(); - if (!IsConditionlessJoin(current) || !current.right || current.right->type != TableReferenceType::JOIN) { - break; - } - // Left rotation: - // current(left=A, right=inner(left=B, right=C)) - // => inner(left=current(left=A, right=B), right=C) - auto inner = std::move(current.right); - auto &inner_join = inner->Cast(); - current.right = std::move(inner_join.left); - inner_join.left = std::move(root); - root = std::move(inner); - } - return root; -} - -//! Check whether the RHS TableRef of a JoinOrPivot parse result has its own JoinOrPivot* entries. -//! This distinguishes PEG right-recursion (has entries) from parenthesized joins (no entries). -//! Navigation: JoinOrPivot → Choice → JoinClause → Choice → JoinWithoutOnClause → child(2)=TableRef → child(1)=Optional -static bool RHSTableRefHasJoinOrPivot(ParseResult &join_or_pivot_pr) { - auto &jop_list = join_or_pivot_pr.Cast(); - auto &jop_choice = jop_list.Child(0); - auto &join_clause = jop_choice.GetResult().Cast(); - auto &jc_choice = join_clause.Child(0); - auto &join_impl = jc_choice.GetResult().Cast(); - // For JoinWithoutOnClause the TableRef is at index 2 - auto &table_ref = join_impl.Child(2); - return table_ref.Child(1).HasResult(); -} - unique_ptr PEGTransformerFactory::TransformTableRef(PEGTransformer &transformer, ParseResult &parse_result) { auto &list_pr = parse_result.Cast(); auto inner_table_ref = transformer.Transform>(list_pr.Child(0)); @@ -499,11 +454,7 @@ unique_ptr PEGTransformerFactory::TransformTableRef(PEGTransformer &tr if (transform_join_or_pivot->type == TableReferenceType::JOIN) { auto &join_ref = transform_join_or_pivot->Cast(); join_ref.left = std::move(inner_table_ref); - if (IsConditionlessJoin(join_ref) && RHSTableRefHasJoinOrPivot(join_or_pivot)) { - inner_table_ref = ReassociateJoins(std::move(transform_join_or_pivot)); - } else { - inner_table_ref = std::move(transform_join_or_pivot); - } + inner_table_ref = std::move(transform_join_or_pivot); } else if (transform_join_or_pivot->type == TableReferenceType::PIVOT) { auto &pivot_ref = transform_join_or_pivot->Cast(); pivot_ref.source = std::move(inner_table_ref); @@ -553,11 +504,7 @@ unique_ptr PEGTransformerFactory::FinalizeTableRefTrampoli if (transform_join_or_pivot->type == TableReferenceType::JOIN) { auto &join_ref = transform_join_or_pivot->Cast(); join_ref.left = std::move(inner_table_ref); - if (IsConditionlessJoin(join_ref) && RHSTableRefHasJoinOrPivot(repeat_children[i].get())) { - inner_table_ref = ReassociateJoins(std::move(transform_join_or_pivot)); - } else { - inner_table_ref = std::move(transform_join_or_pivot); - } + inner_table_ref = std::move(transform_join_or_pivot); } else if (transform_join_or_pivot->type == TableReferenceType::PIVOT) { auto &pivot_ref = transform_join_or_pivot->Cast(); pivot_ref.source = std::move(inner_table_ref); @@ -1739,11 +1686,11 @@ unique_ptr PEGTransformerFactory::TransformAtSpecifier(PEGTransformer unique_ptr PEGTransformerFactory::TransformJoinWithoutOnClause(PEGTransformer &transformer, const JoinPrefix &join_prefix, - unique_ptr table_ref) { + unique_ptr inner_table_ref) { auto result = make_uniq(); result->ref_type = join_prefix.ref_type; result->type = join_prefix.join_type; - result->right = std::move(table_ref); + result->right = std::move(inner_table_ref); return std::move(result); } diff --git a/src/duckdb/src/planner/bind_context.cpp b/src/duckdb/src/planner/bind_context.cpp index c6d9cd403..a8f43d334 100644 --- a/src/duckdb/src/planner/bind_context.cpp +++ b/src/duckdb/src/planner/bind_context.cpp @@ -243,11 +243,11 @@ unique_ptr BindContext::CreateColumnReference(const BindingAli if (bind_type == ColumnBindType::EXPAND_GENERATED_COLUMNS && ColumnIsGenerated(*binding, column_index)) { return ExpandGeneratedColumn(binding->Cast(), column_name); } - auto &column_names = binding->GetColumnNames(); - if (column_index < column_names.size() && column_names[column_index] != column_name) { + auto ®istered_name = binding->GetRegisteredColumnName(column_name); + if (registered_name != column_name) { // because of case insensitivity in the binder we rename the column to the original name // as it appears in the binding itself - result->SetAlias(column_names[column_index]); + result->SetAlias(registered_name); } return std::move(result); } @@ -285,11 +285,11 @@ unique_ptr BindContext::CreateColumnReference(const Identifier if (bind_type == ColumnBindType::EXPAND_GENERATED_COLUMNS && ColumnIsGenerated(*binding, column_index)) { return ExpandGeneratedColumn(binding->Cast(), column_name); } - auto &column_names = binding->GetColumnNames(); - if (column_index < column_names.size() && column_names[column_index] != column_name) { + auto ®istered_name = binding->GetRegisteredColumnName(column_name); + if (registered_name != column_name) { // because of case insensitivity in the binder we rename the column to the original name // as it appears in the binding itself - result->SetAlias(column_names[column_index]); + result->SetAlias(registered_name); } return std::move(result); } @@ -785,6 +785,16 @@ void BindContext::AddGenericBinding(TableIndex index, const Identifier &alias, c AddBinding(make_uniq(BindingType::BASE, BindingAlias(alias), types, names, index)); } +void BindContext::AddColumnAlias(TableIndex index, const Identifier &column_alias, column_t column_index) { + for (auto &binding : bindings_list) { + if (binding->GetIndex() == index) { + binding->AddColumnAlias(column_alias, column_index); + return; + } + } + throw InternalException("AddColumnAlias - no binding found with the given table index"); +} + void BindContext::AddCTEBinding(unique_ptr binding) { for (auto &cte_binding : cte_bindings) { if (cte_binding->GetBindingAlias() == binding->GetBindingAlias()) { diff --git a/src/duckdb/src/planner/binder/statement/bind_copy.cpp b/src/duckdb/src/planner/binder/statement/bind_copy.cpp index 977b6ed88..cc910424a 100644 --- a/src/duckdb/src/planner/binder/statement/bind_copy.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_copy.cpp @@ -183,6 +183,10 @@ struct CopyToParsedOptions { bool Partitioned() const { return !partition_cols.empty(); } + + bool PartitionedOrOrdered() const { + return Partitioned() || !order_columns.empty(); + } }; struct CopyToResolvedOptions { @@ -209,6 +213,10 @@ struct CopyToResolvedOptions { bool Partitioned() const { return !partition_cols.empty(); } + + bool PartitionedOrOrdered() const { + return Partitioned() || !order_columns.empty(); + } }; static bool ResolveUseTmpFile(ClientContext &context, const string &file_path, const CopyToParsedOptions &options) { @@ -222,7 +230,7 @@ static bool ResolveUseTmpFile(ClientContext &context, const string &file_path, c auto &fs = FileSystem::GetFileSystem(context); bool is_file_and_exists = fs.FileExists(file_path); bool is_stdout = file_path == "/dev/stdout"; - return is_file_and_exists && !options.PerThreadOutput() && !options.Partitioned() && !is_stdout; + return is_file_and_exists && !options.PerThreadOutput() && !options.PartitionedOrOrdered() && !is_stdout; } static CopyToResolvedOptions ResolveCopyToOptions(ClientContext &context, const string &file_path, @@ -260,9 +268,15 @@ static void ValidateCopyToOptionCombinations(const CopyToParsedOptions &options, if (options.UserSetUseTmpFile() && options.Partitioned()) { throw NotImplementedException("Can't combine USE_TMP_FILE and PARTITION_BY for COPY"); } + if (options.UserSetUseTmpFile() && !options.order_columns.empty()) { + throw NotImplementedException("Can't combine USE_TMP_FILE and ORDER_BY for COPY"); + } if (options.PerThreadOutput() && options.Partitioned()) { throw NotImplementedException("Can't combine PER_THREAD_OUTPUT and PARTITION_BY for COPY"); } + if (options.PerThreadOutput() && !options.order_columns.empty()) { + throw NotImplementedException("Can't combine PER_THREAD_OUTPUT and ORDER_BY for COPY"); + } if (options.Rotate() && (!function.prepare_batch || !function.flush_batch)) { throw NotImplementedException("Can't use file rotation (e.g., ROW_GROUPS_PER_FILE) with FORMAT %s", function.name); @@ -274,14 +288,14 @@ static void ValidateCopyToOptionCombinations(const CopyToParsedOptions &options, if (options.Partitioned()) { throw NotImplementedException("Can't combine WRITE_EMPTY_FILE false with PARTITION_BY"); } + if (!options.order_columns.empty()) { + throw NotImplementedException("Can't combine WRITE_EMPTY_FILE false with ORDER_BY"); + } } if (options.ReturnType() == CopyFunctionReturnType::WRITTEN_FILE_STATISTICS && !function.copy_to_get_written_statistics) { throw NotImplementedException("RETURN_STATS is not supported for the \"%s\" copy format", format); } - if (!options.order_columns.empty() && !options.Partitioned()) { - throw NotImplementedException("ORDER_BY is not supported without PARTITION_BY"); - } } static void ValidateCopyToOutputColumns(const CopyToResolvedOptions &options, idx_t column_count) { @@ -466,7 +480,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt, const CopyFunction &funct copy->batches_per_file = resolved_options.batches_per_file; copy->file_size_bytes = resolved_options.file_size_bytes; copy->rotate = resolved_options.Rotate(); - copy->partition_output = resolved_options.Partitioned(); + copy->partition_output = resolved_options.PartitionedOrOrdered(); copy->write_partition_columns = resolved_options.write_partition_columns; copy->partition_columns = std::move(resolved_options.partition_cols); copy->write_empty_file = resolved_options.write_empty_file; diff --git a/src/duckdb/src/planner/binder/statement/bind_create.cpp b/src/duckdb/src/planner/binder/statement/bind_create.cpp index 448214390..885ed70c0 100644 --- a/src/duckdb/src/planner/binder/statement/bind_create.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_create.cpp @@ -167,7 +167,7 @@ void Binder::SearchSchema(CreateInfo &info) { schema_path.push_back(default_entry.GetSchema()); } else if (schema_path.empty()) { // a catalog was given but no schema: use the catalog's default schema - schema_path.push_back(Identifier(search_path->GetDefaultSchema(context, catalog))); + schema_path.push_back(search_path->GetDefaultSchema(context, catalog)); } else if (IsInvalidCatalog(catalog)) { // a schema was given but no catalog: resolve the catalog that holds it catalog = Identifier(search_path->GetDefaultCatalog(schema_path[0])); diff --git a/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp b/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp index 9f7bd5244..94275f9cd 100644 --- a/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +++ b/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp @@ -109,7 +109,7 @@ vector Binder::GetSearchPath(Catalog &catalog, const Identif } auto default_schema = catalog.GetDefaultSchema(); if (schema_name.empty() && schema_name != default_schema) { - view_search_path.emplace_back(catalog_name, Identifier(default_schema)); + view_search_path.emplace_back(catalog_name, default_schema); } //! Signal that this catalog should be checked, regardless of the schema in the reference view_search_path.emplace_back(catalog_name, INVALID_SCHEMA, default_schema_precedence); diff --git a/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp b/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp index 2c27d617b..8907a4af0 100644 --- a/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +++ b/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp @@ -177,13 +177,28 @@ static string GetAlias(const TableFunctionRef &ref) { return string(); } -static void ApplyPostgresSetofAliasCompatibility(const TableFunction &table_function, const TableFunctionRef &ref, - vector &return_names) { +//! Postgres names the single column of a set-returning function after the alias of the function +//! (e.g. "SELECT t FROM generate_series(1, 2) t" returns the values, not a struct) +//! returns the original column name, which is kept available as a column alias +static Identifier ApplyPostgresSetofAliasCompatibility(const TableFunction &table_function, const TableFunctionRef &ref, + vector &return_names) { if (table_function.return_type != TableFunctionReturnType::SET_RETURNING_FUNCTION || ref.alias.empty() || !ref.column_name_alias.empty() || return_names.size() != 1) { - return; + return Identifier(); } + auto original_name = return_names[0]; return_names[0] = ref.alias; + return original_name; +} + +//! Keep the original column name of a set-returning function bindable, e.g. "SELECT t.generate_series FROM +//! generate_series(1, 2) t" - the original name is hidden from * since the column is emitted under the alias +static void AddPostgresSetofColumnAlias(BindContext &bind_context, TableIndex bind_index, + const Identifier &original_name) { + if (original_name.empty()) { + return; + } + bind_context.AddColumnAlias(bind_index, original_name, 0); } BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, const TableFunctionRef &ref, @@ -205,9 +220,7 @@ BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, TableFunctionBindInput bind_input(parameters, named_parameters, input_table_types, input_table_names, table_function.function_info.get(), this, table_function, ref, input_plan); if (table_function.bind_operator) { - vector operator_names; - auto new_plan = table_function.bind_operator(context, bind_input, bind_index, operator_names); - return_names = StringsToIdentifiers(operator_names); + auto new_plan = table_function.bind_operator(context, bind_input, bind_index, return_names); if (new_plan) { new_plan->ResolveOperatorTypes(); if (new_plan->types.size() != return_names.size()) { @@ -221,9 +234,10 @@ BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, table_function.name); } } - ApplyPostgresSetofAliasCompatibility(table_function, ref, return_names); + auto setof_column_name = ApplyPostgresSetofAliasCompatibility(table_function, ref, return_names); BoundStatement result; bind_context.AddGenericBinding(bind_index, Identifier(function_name), return_names, new_plan->types); + AddPostgresSetofColumnAlias(bind_context, bind_index, setof_column_name); result.names = return_names; result.types = new_plan->types; result.plan = std::move(new_plan); @@ -246,9 +260,7 @@ BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, throw BinderException("Failed to bind \"%s\": nullptr returned from bind_replace without bind function", table_function.name); } - vector bind_names; - bind_data = table_function.bind(context, bind_input, return_types, bind_names); - return_names = StringsToIdentifiers(bind_names); + bind_data = table_function.bind(context, bind_input, return_types, return_names); if (ref.with_ordinality == OrdinalityType::WITH_ORDINALITY) { // check if column name 'ordinality' already exists and if so, replace it iteratively until free name is // found @@ -284,7 +296,7 @@ BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, throw InternalException("Failed to bind \"%s\": Table function must return at least one column", table_function.name); } - ApplyPostgresSetofAliasCompatibility(table_function, ref, return_names); + auto setof_column_name = ApplyPostgresSetofAliasCompatibility(table_function, ref, return_names); // overwrite the names with any supplied aliases for (idx_t i = 0; i < column_name_alias.size() && i < return_names.size(); i++) { return_names[i] = column_name_alias[i]; @@ -317,6 +329,7 @@ BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, if (ref.with_ordinality == OrdinalityType::WITH_ORDINALITY && correlated_columns.empty()) { bind_context.AddTableFunction(bind_index, Identifier(function_name), return_names, return_types, get->GetMutableColumnIds(), get->GetTable().get(), std::move(virtual_columns)); + AddPostgresSetofColumnAlias(bind_context, bind_index, setof_column_name); auto window_index = GenerateTableIndex(); auto window = make_uniq(window_index); @@ -348,6 +361,7 @@ BoundStatement Binder::BindTableFunctionInternal(TableFunction &table_function, BoundStatement result; bind_context.AddTableFunction(bind_index, Identifier(function_name), return_names, return_types, get->GetMutableColumnIds(), get->GetTable().get(), std::move(virtual_columns)); + AddPostgresSetofColumnAlias(bind_context, bind_index, setof_column_name); result.names = std::move(return_names); result.types = std::move(return_types); result.plan = std::move(get); diff --git a/src/duckdb/src/planner/expression_binder/base_select_binder.cpp b/src/duckdb/src/planner/expression_binder/base_select_binder.cpp index bbab12bb4..576d57456 100644 --- a/src/duckdb/src/planner/expression_binder/base_select_binder.cpp +++ b/src/duckdb/src/planner/expression_binder/base_select_binder.cpp @@ -118,7 +118,7 @@ BindResult BaseSelectBinder::BindGroup(ParsedExpression &expr, idx_t depth, Proj if (node.groups.grouping_sets.size() <= 1) { // if there are no more than two grouping sets, you can return the uncollated first expression. - // "first" meaning the aggreagte function. + // "first" meaning the aggregate function. return BindResult(std::move(uncollated_first_expression)); } diff --git a/src/duckdb/src/planner/filter/expression_filter.cpp b/src/duckdb/src/planner/filter/expression_filter.cpp index 4b49d82d5..c49c333b6 100644 --- a/src/duckdb/src/planner/filter/expression_filter.cpp +++ b/src/duckdb/src/planner/filter/expression_filter.cpp @@ -201,12 +201,18 @@ static FilterPropagateResult CheckZonemapAgainstConstants(const BaseStatistics & } } -static optional_ptr TryGetFilterStats(optional_ptr context_p, - const Expression &expr, const BaseStatistics &stats, - vector> &owned_stats) { +static optional_ptr TryGetExpressionStats(optional_ptr context_p, + const Expression &expr, + array_ptr input_stats, + vector> &owned_stats) { switch (expr.GetExpressionClass()) { - case ExpressionClass::BOUND_REF: - return &stats; + case ExpressionClass::BOUND_REF: { + auto index = expr.Cast().Index(); + if (index >= input_stats.size()) { + return nullptr; + } + return &input_stats[index]; + } case ExpressionClass::BOUND_CONSTANT: { auto &constant = expr.Cast().GetValue(); owned_stats.push_back(BaseStatistics::FromConstant(constant).ToUnique()); @@ -217,7 +223,7 @@ static optional_ptr TryGetFilterStats(optional_ptr TryGetFilterStats(optional_ptrGetType().id() != LogicalTypeId::STRUCT) { return nullptr; } @@ -251,7 +257,7 @@ static optional_ptr TryGetFilterStats(optional_ptr child_stats; child_stats.reserve(func.GetChildren().size()); for (auto &child_expr : func.GetChildren()) { - auto child_stat = TryGetFilterStats(context_p, *child_expr, stats, owned_stats); + auto child_stat = TryGetExpressionStats(context_p, *child_expr, input_stats, owned_stats); if (!child_stat) { return nullptr; } @@ -273,7 +279,7 @@ static optional_ptr TryGetFilterStats(optional_ptr child_stats; child_stats.reserve(func.GetChildren().size()); for (auto &child_expr : func.GetChildren()) { - auto child_stat = TryGetFilterStats(context_p, *child_expr, stats, owned_stats); + auto child_stat = TryGetExpressionStats(context_p, *child_expr, input_stats, owned_stats); child_stats.push_back(child_stat ? child_stat->Copy() : BaseStatistics::CreateUnknown(child_expr->GetReturnType())); } @@ -291,6 +297,19 @@ static optional_ptr TryGetFilterStats(optional_ptr TryGetFilterStats(optional_ptr context_p, + const Expression &expr, const BaseStatistics &stats, + vector> &owned_stats) { + return TryGetExpressionStats(context_p, expr, array_ptr(stats), owned_stats); +} + +unique_ptr ExpressionFilter::TryGetExpressionStatistics(ClientContext &context, const Expression &expr, + array_ptr input_stats) { + vector> owned_stats; + auto result = TryGetExpressionStats(&context, expr, input_stats, owned_stats); + return result ? result->ToUnique() : nullptr; +} + static bool TryGetVariantComparisonStatsType(const LogicalType &typed_type, const LogicalType &constant_type, LogicalType &comparison_type) { if (typed_type == constant_type) { diff --git a/src/duckdb/src/planner/operator/logical_get.cpp b/src/duckdb/src/planner/operator/logical_get.cpp index 60d79b63c..0f7acdaab 100644 --- a/src/duckdb/src/planner/operator/logical_get.cpp +++ b/src/duckdb/src/planner/operator/logical_get.cpp @@ -380,7 +380,7 @@ unique_ptr LogicalGet::Deserialize(Deserializer &deserializer) empty_ref, nullptr); vector bind_return_types; - vector bind_names; + vector bind_names; if (!function.bind) { throw InternalException("Table function \"%s\" has neither bind nor (de)serialize", function.name); } diff --git a/src/duckdb/src/planner/table_binding.cpp b/src/duckdb/src/planner/table_binding.cpp index 385437c2d..b6e83cbee 100644 --- a/src/duckdb/src/planner/table_binding.cpp +++ b/src/duckdb/src/planner/table_binding.cpp @@ -92,6 +92,27 @@ bool Binding::HasMatchingBinding(const Identifier &column_name) { return TryGetBindingIndex(column_name, result); } +void Binding::AddColumnAlias(const Identifier &column_alias, column_t column_index) { + D_ASSERT(column_index < names.size()); + if (name_map.find(column_alias) != name_map.end()) { + // a column with this name already exists - the alias is not required + return; + } + name_map[column_alias] = column_index; +} + +const Identifier &Binding::GetRegisteredColumnName(const Identifier &column_name) { + auto entry = name_map.find(column_name); + return entry == name_map.end() ? column_name : entry->first; +} + +void Binding::SetBoundColumnAlias(ColumnRefExpression &colref) { + if (!colref.GetAlias().empty()) { + return; + } + colref.SetAlias(GetRegisteredColumnName(colref.GetColumnName())); +} + ErrorData Binding::ColumnNotFoundError(const Identifier &column_name) const { return ErrorData(ExceptionType::BINDER, StringUtil::Format("Values list \"%s\" does not have a column named \"%s\"", GetAlias(), column_name)); @@ -108,9 +129,7 @@ BindResult Binding::Bind(ColumnRefExpression &colref, idx_t depth) { binding.table_index = index; binding.column_index = ProjectionIndex(column_index); LogicalType sql_type = types[column_index]; - if (colref.GetAlias().empty()) { - colref.SetAlias(names[column_index]); - } + SetBoundColumnAlias(colref); return BindResult(make_uniq(Identifier(colref.GetName()), sql_type, binding, depth)); } @@ -288,9 +307,7 @@ BindResult TableBinding::Bind(ColumnRefExpression &colref, idx_t depth) { } else { // normal column: fetch type from base column col_type = types[column_index]; - if (colref.GetAlias().empty()) { - colref.SetAlias(names[column_index]); - } + SetBoundColumnAlias(colref); } ColumnBinding binding = GetColumnBinding(column_index); return BindResult(make_uniq(Identifier(colref.GetName()), col_type, binding, depth)); diff --git a/src/duckdb/src/storage/table/chunk_info.cpp b/src/duckdb/src/storage/table/chunk_info.cpp index f7abb0d66..f37259ae2 100644 --- a/src/duckdb/src/storage/table/chunk_info.cpp +++ b/src/duckdb/src/storage/table/chunk_info.cpp @@ -67,7 +67,8 @@ ChunkVectorInfo::~ChunkVectorInfo() { template idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transaction_t transaction_id, optional_ptr sel_vector, idx_t max_count) const { - if (HasConstantDeleteId()) { + switch (delete_state) { + case DeleteIdState::CONSTANT: { // all tuples have the same deleted id if (DELETE_OP::IsDeleted(start_time, transaction_id, ConstantDeleteId())) { // all tuples are deleted @@ -98,16 +99,60 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti } return count; } - if (HasConstantInsertionId()) { - if (!INSERT_OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { - return 0; + case DeleteIdState::MASKED: { + // every deleted row shares mask_delete_id and alive rows are NOT_DELETED_ID (never deleted), so the + // delete decision is a single constant for the whole vector + const bool masked_deleted = DELETE_OP::IsDeleted(start_time, transaction_id, mask_delete_id); + if (HasConstantInsertionId()) { + if (!INSERT_OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { + return 0; + } + if (!masked_deleted) { + // the delete is not visible to this transaction - every row is visible + return max_count; + } + // only the alive (mask-invalid) rows are visible + if (!sel_vector) { + return max_count - deleted_mask.CountValid(max_count); + } + // scan the mask a word at a time: skip fully-deleted words, take fully-alive words wholesale, + // only extract bits for mixed words + idx_t count = 0; + const idx_t entry_count = ValidityMask::EntryCount(max_count); + for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) { + auto entry = deleted_mask.GetValidityEntry(entry_idx); + if (ValidityMask::AllValid(entry)) { + // every row in this word is deleted - skip + continue; + } + const idx_t base = entry_idx * ValidityMask::BITS_PER_VALUE; + const idx_t entry_end = MinValue(base + ValidityMask::BITS_PER_VALUE, max_count); + if (ValidityMask::NoneValid(entry)) { + // every row in this word is alive - select them all + for (idx_t i = base; i < entry_end; i++) { + sel_vector->set_index(count++, i); + } + continue; + } + for (idx_t i = base; i < entry_end; i++) { + if (!ValidityMask::RowIsValid(entry, i - base)) { + sel_vector->set_index(count++, i); + } + } + } + return count; } - // have to check deleted flag + // per-row insert ids: the mask cannot collapse the insert check, but the delete decision is still + // the constant masked_deleted + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); idx_t count = 0; - auto segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = segment.GetPtr(); for (idx_t i = 0; i < max_count; i++) { - if (DELETE_OP::IsDeleted(start_time, transaction_id, deleted[i])) { + if (!INSERT_OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { + continue; + } + if (masked_deleted && deleted_mask.RowIsValid(i)) { + // the row is deleted and the delete is visible to this transaction continue; } if (sel_vector) { @@ -117,27 +162,51 @@ idx_t ChunkVectorInfo::TemplatedGetSelVector(transaction_t start_time, transacti } return count; } + case DeleteIdState::ARRAY: { + if (HasConstantInsertionId()) { + if (!INSERT_OP::UseInsertedVersion(start_time, transaction_id, ConstantInsertId())) { + return 0; + } + // have to check deleted flag + idx_t count = 0; + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); + for (idx_t i = 0; i < max_count; i++) { + if (DELETE_OP::IsDeleted(start_time, transaction_id, deleted[i])) { + continue; + } + if (sel_vector) { + sel_vector->set_index(count, i); + } + count++; + } + return count; + } - idx_t count = 0; - // have to check both flags - auto insert_segment = allocator.GetHandle(GetInsertedPointer()); - auto inserted = insert_segment.GetPtr(); + idx_t count = 0; + // have to check both flags + auto insert_segment = allocator.GetHandle(GetInsertedPointer()); + auto inserted = insert_segment.GetPtr(); - auto delete_segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = delete_segment.GetPtr(); - for (idx_t i = 0; i < max_count; i++) { - if (!INSERT_OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { - continue; - } - if (DELETE_OP::IsDeleted(start_time, transaction_id, deleted[i])) { - continue; - } - if (sel_vector) { - sel_vector->set_index(count, i); + auto delete_segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = delete_segment.GetPtr(); + for (idx_t i = 0; i < max_count; i++) { + if (!INSERT_OP::UseInsertedVersion(start_time, transaction_id, inserted[i])) { + continue; + } + if (DELETE_OP::IsDeleted(start_time, transaction_id, deleted[i])) { + continue; + } + if (sel_vector) { + sel_vector->set_index(count, i); + } + count++; } - count++; + return count; + } + default: + throw InternalException("Unknown DeleteIdState in TemplatedGetSelVector"); } - return count; } idx_t ChunkVectorInfo::GetSelVector(ScanOptions options, optional_ptr sel_vector, @@ -176,7 +245,6 @@ idx_t ChunkVectorInfo::GetSelVector(ScanOptions options, optional_ptr(); fetch_insert_id = inserted[row]; } - if (HasConstantDeleteId()) { + transaction_t fetch_deleted_id; + switch (delete_state) { + case DeleteIdState::CONSTANT: fetch_deleted_id = ConstantDeleteId(); - } else { + break; + case DeleteIdState::MASKED: + fetch_deleted_id = deleted_mask.RowIsValid(row) ? mask_delete_id : NOT_DELETED_ID; + break; + case DeleteIdState::ARRAY: { auto delete_segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = delete_segment.GetPtr(); - fetch_deleted_id = deleted[row]; + fetch_deleted_id = delete_segment.GetPtr()[row]; + break; + } + default: + throw InternalException("Unknown DeleteIdState in Fetch"); } return UseVersion(transaction, fetch_insert_id) && !UseVersion(transaction, fetch_deleted_id); @@ -203,8 +280,9 @@ IndexPointer ChunkVectorInfo::GetInsertedPointer() const { } IndexPointer ChunkVectorInfo::GetDeletedPointer() const { - if (HasConstantDeleteId()) { - throw InternalException("ChunkVectorInfo: deleted id requested but deletions were not initialized"); + if (delete_state != DeleteIdState::ARRAY) { + throw InternalException( + "ChunkVectorInfo: deleted id array requested but delete side is not in the ARRAY state"); } return deleted_data; } @@ -225,6 +303,10 @@ IndexPointer ChunkVectorInfo::GetInitializedInsertedPointer() { } IndexPointer ChunkVectorInfo::GetInitializedDeletedPointer() { + if (delete_state == DeleteIdState::MASKED) { + // re-materialize the per-row array so callers can write into it + DecompressDeleteMask(); + } if (HasConstantDeleteId()) { transaction_t constant_id = ConstantDeleteId(); @@ -235,16 +317,56 @@ IndexPointer ChunkVectorInfo::GetInitializedDeletedPointer() { for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { deleted[i] = constant_id; } + delete_state = DeleteIdState::ARRAY; } return deleted_data; } void ChunkVectorInfo::FreeDeleteData() { - if (HasConstantDeleteId()) { - return; + if (delete_state == DeleteIdState::ARRAY) { + allocator.Free(deleted_data); + deleted_data = IndexPointer(); } + deleted_mask.Reset(); + delete_state = DeleteIdState::CONSTANT; +} + +void ChunkVectorInfo::CompressDeleteToMask(transaction_t mask_id) { + D_ASSERT(delete_state == DeleteIdState::ARRAY); + // the mask can only carry a single committed id shared by every deleted row + D_ASSERT(mask_id < TRANSACTION_ID_START); + // start all-valid (== all deleted), then mark the alive rows invalid + deleted_mask.Initialize(STANDARD_VECTOR_SIZE); + { + auto segment = allocator.GetHandle(deleted_data); + auto deleted = segment.GetPtr(); + for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { + if (deleted[i] == NOT_DELETED_ID) { + deleted_mask.SetInvalid(i); + } + } + } // release the read handle before freeing the buffer allocator.Free(deleted_data); deleted_data = IndexPointer(); + mask_delete_id = mask_id; + delete_state = DeleteIdState::MASKED; +} + +void ChunkVectorInfo::DecompressDeleteMask() { + D_ASSERT(delete_state == DeleteIdState::MASKED); + // re-materialize the per-row array: deleted rows == mask_delete_id, alive rows == NOT_DELETED_ID + deleted_data = allocator.New(); + deleted_data.SetMetadata(1); + { + auto segment = allocator.GetHandle(deleted_data); + auto deleted = segment.GetPtr(); + for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { + deleted[i] = deleted_mask.RowIsValid(i) ? mask_delete_id : NOT_DELETED_ID; + } + } + deleted_mask.Reset(); + delete_state = DeleteIdState::ARRAY; + recheck_compression = true; } static bool DeletesEntireVector(const row_t rows[], idx_t count) { @@ -354,18 +476,13 @@ void ChunkVectorInfo::VerifyCachedCompressionState() const { // modification re-arms it. Both conditions below are independent of the lowest active start: // per-row insert ids all become visible (or are reverted) eventually, so they must already be compressed D_ASSERT(HasConstantInsertionId()); - if (!HasConstantDeleteId()) { - // per-row delete ids may only remain because a live row blocks the collapse - auto segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = segment.GetPtr(); - bool rows_alive = false; - for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { - if (deleted[i] == NOT_DELETED_ID) { - rows_alive = true; - break; - } - } - D_ASSERT(rows_alive); + if (delete_state != DeleteIdState::CONSTANT) { + // a settled, non-constant delete side can only be the terminal masked state + D_ASSERT(delete_state == DeleteIdState::MASKED); + // the mask must hold at least one deleted row and one alive row: a fully deleted vector + // collapses to a constant, and a vector with no deletes carries no delete-side info + idx_t deleted_rows = deleted_mask.CountValid(STANDARD_VECTOR_SIZE); + D_ASSERT(deleted_rows > 0 && deleted_rows < STANDARD_VECTOR_SIZE); } #endif } @@ -381,32 +498,58 @@ VersionCompressionResult ChunkVectorInfo::CompressVersionIds(transaction_t lowes : VersionCompressionResult::SETTLED; } bool pending = false; - if (!HasConstantDeleteId()) { - // check if all rows are deleted, with all deletes visible to all active and future transactions - // if so, the per-row delete ids are equivalent to a single constant delete id + if (delete_state == DeleteIdState::ARRAY) { + // scan the per-row delete ids to decide how far they can collapse bool rows_alive = false; bool deletes_pending = false; + bool deletes_uncommitted = false; + bool deletes_equal = true; transaction_t max_delete_id = 0; + transaction_t shared_delete_id = NOT_DELETED_ID; { auto segment = allocator.GetHandle(GetDeletedPointer()); auto deleted = segment.GetPtr(); for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { if (deleted[i] == NOT_DELETED_ID) { - // the row is not deleted - the ids cannot compress until it is + // the row is not deleted - the ids cannot fully collapse until it is rows_alive = true; - } else if (deleted[i] >= lowest_active_start) { - // deleted, but the delete is not yet visible to all transactions - the ids can - // compress once the lowest active start advances past the delete id + continue; + } + if (deleted[i] >= lowest_active_start) { + // deleted, but the delete is not yet visible to all transactions deletes_pending = true; + if (deleted[i] >= TRANSACTION_ID_START) { + // the delete is not even committed yet - the array must be kept + deletes_uncommitted = true; + } } else { max_delete_id = MaxValue(max_delete_id, deleted[i]); } + // track whether every deleted row shares a single id + if (shared_delete_id == NOT_DELETED_ID) { + shared_delete_id = deleted[i]; + } else if (deleted[i] != shared_delete_id) { + deletes_equal = false; + } } } if (!rows_alive && !deletes_pending) { + // entire vector deleted and visible to all - collapse to a constant FreeDeleteData(); constant_delete_id = max_delete_id; } else if (!rows_alive) { + // entire vector deleted but a delete is still pending - retry next pass + pending = true; + } else if (!deletes_pending) { + // partially deleted, every delete visible to all - compress to a mask (terminal) + CompressDeleteToMask(0); + } else if (deletes_equal && !deletes_uncommitted) { + // partially deleted, every delete committed by the same transaction but not yet visible to + // all - compress to a mask carrying that single committed id (terminal). Older snapshots still + // see the rows via the id comparison, and on reload every id is visible so it becomes 0. + CompressDeleteToMask(shared_delete_id); + } else { + // partially deleted with pending deletes from multiple or uncommitted transactions - retry pending = true; } } @@ -513,18 +656,25 @@ bool ChunkVectorInfo::HasDeletes(transaction_t transaction_id) const { if (transaction_id == MAX_TRANSACTION_ID) { return true; } - if (HasConstantDeleteId()) { + switch (delete_state) { + case DeleteIdState::CONSTANT: return ConstantDeleteId() <= transaction_id; - } - auto segment = allocator.GetHandle(deleted_data); - auto deleted = segment.GetPtr(); - - for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { - if (deleted[i] <= transaction_id) { - return true; + case DeleteIdState::MASKED: + // AnyDeleted() above guaranteed at least one deleted row; they all share mask_delete_id + return mask_delete_id <= transaction_id; + case DeleteIdState::ARRAY: { + auto segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = segment.GetPtr(); + for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { + if (deleted[i] <= transaction_id) { + return true; + } } + return false; + } + default: + throw InternalException("Unknown DeleteIdState in HasDeletes"); } - return false; } bool ChunkVectorInfo::HasUncommittedChanges() const { @@ -541,24 +691,40 @@ bool ChunkVectorInfo::HasUncommittedChanges() const { } } } - if (HasConstantDeleteId()) { + switch (delete_state) { + case DeleteIdState::CONSTANT: return ConstantDeleteId() != NOT_DELETED_ID && ConstantDeleteId() >= TRANSACTION_ID_START; - } - auto delete_segment = allocator.GetHandle(GetDeletedPointer()); - auto deleted = delete_segment.GetPtr(); - for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { - if (deleted[i] != NOT_DELETED_ID && deleted[i] >= TRANSACTION_ID_START) { - return true; + case DeleteIdState::MASKED: + // the mask is only ever folded from committed deletes, so mask_delete_id is always committed + D_ASSERT(mask_delete_id < TRANSACTION_ID_START); + return false; + case DeleteIdState::ARRAY: { + auto delete_segment = allocator.GetHandle(GetDeletedPointer()); + auto deleted = delete_segment.GetPtr(); + for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { + if (deleted[i] != NOT_DELETED_ID && deleted[i] >= TRANSACTION_ID_START) { + return true; + } } + return false; + } + default: + throw InternalException("Unknown DeleteIdState in HasUncommittedChanges"); } - return false; } bool ChunkVectorInfo::AnyDeleted() const { - if (HasConstantDeleteId()) { + switch (delete_state) { + case DeleteIdState::CONSTANT: return ConstantDeleteId() != NOT_DELETED_ID; + case DeleteIdState::MASKED: + // a masked vector always contains at least one deleted row + return true; + case DeleteIdState::ARRAY: + return true; + default: + throw InternalException("Unknown DeleteIdState in AnyDeleted"); } - return true; } bool ChunkVectorInfo::HasConstantInsertionId() const { @@ -566,7 +732,7 @@ bool ChunkVectorInfo::HasConstantInsertionId() const { } bool ChunkVectorInfo::HasConstantDeleteId() const { - return !deleted_data.HasMetadata(); + return delete_state == DeleteIdState::CONSTANT; } string ChunkVectorInfo::ToString(idx_t max_count) const { @@ -588,11 +754,25 @@ string ChunkVectorInfo::ToString(idx_t max_count) const { } result += "]"; } - if (HasConstantDeleteId()) { + switch (delete_state) { + case DeleteIdState::CONSTANT: if (ConstantDeleteId() != NOT_DELETED_ID) { result += ", Delete Id: " + to_string(constant_delete_id); } - } else { + break; + case DeleteIdState::MASKED: { + result += ", Delete Id: " + to_string(mask_delete_id); + result += ", Deleted (mask): ["; + for (idx_t idx = 0; idx < max_count; idx++) { + if (idx > 0) { + result += ", "; + } + result += deleted_mask.RowIsValid(idx) ? "1" : "0"; + } + result += "]"; + break; + } + case DeleteIdState::ARRAY: { result += ", Delete Ids: ["; auto segment = allocator.GetHandle(GetDeletedPointer()); auto deleted = segment.GetPtr(); @@ -604,6 +784,10 @@ string ChunkVectorInfo::ToString(idx_t max_count) const { result += to_string(deleted[idx]); } result += "]"; + break; + } + default: + throw InternalException("Unknown DeleteIdState in ToString"); } result += "]"; return result; @@ -665,26 +849,26 @@ unique_ptr ChunkVectorInfo::Read(FixedSizeAllocator &allocator, return result; } case ChunkInfoType::VECTOR_INFO: { - // a partially deleted vector - the deleted rows are stored as a boolean mask + // a partially deleted vector - the deleted rows are stored as a boolean mask, all committed and + // visible to every transaction. The on-disk orientation (valid == deleted) matches the in-memory + // MASKED state, so load it straight into deleted_mask without materializing a per-row array. auto start = reader.Read(); auto result = make_uniq(allocator, start); - ValidityMask mask; - mask.Read(reader, STANDARD_VECTOR_SIZE); - - bool rows_alive = false; - auto segment = allocator.GetHandle(result->GetInitializedDeletedPointer()); - auto deleted = segment.GetPtr(); - for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) { - if (mask.RowIsValid(i)) { - deleted[i] = 0; - } else { - rows_alive = true; - } - } - // the reconstructed ids are all visible to all transactions, so the vector can only be - // compressible if every row is deleted (in which case Write emits CONSTANT_INFO instead, - // so with the current format the ids are always settled here) - result->recheck_compression = !rows_alive; + result->deleted_mask.Read(reader, STANDARD_VECTOR_SIZE); + // Write only emits VECTOR_INFO for a partial delete: an all-deleted vector becomes + // CONSTANT_INFO and an undeleted one becomes EMPTY_INFO, so the mask must have at least one + // deleted (valid) and one alive (invalid) bit - never all-valid, never all-invalid. + if (result->deleted_mask.CheckAllValid(STANDARD_VECTOR_SIZE) || + result->deleted_mask.CheckAllInvalid(STANDARD_VECTOR_SIZE)) { + throw DataCorruptionException( + "Partial-delete vector info mask marks either all rows deleted or all rows alive, but a " + "VECTOR_INFO block must always encode a partial delete. The database file may be corrupted."); + } + result->delete_state = DeleteIdState::MASKED; + // on-disk deletes are all committed and visible to every transaction - the shared id is 0 + result->mask_delete_id = 0; + // every id is already visible to all transactions - nothing left to compress + result->recheck_compression = false; return result; } default: diff --git a/src/duckdb/ub_src_optimizer_join_order.cpp b/src/duckdb/ub_src_optimizer_join_order.cpp index 9dd609cc0..4c7f3234c 100644 --- a/src/duckdb/ub_src_optimizer_join_order.cpp +++ b/src/duckdb/ub_src_optimizer_join_order.cpp @@ -22,5 +22,3 @@ #include "src/optimizer/join_order/relation_manager.cpp" -#include "src/optimizer/join_order/relation_statistics_helper.cpp" - diff --git a/src/duckdb/ub_src_optimizer_relation_statistics.cpp b/src/duckdb/ub_src_optimizer_relation_statistics.cpp new file mode 100644 index 000000000..45770beb5 --- /dev/null +++ b/src/duckdb/ub_src_optimizer_relation_statistics.cpp @@ -0,0 +1,10 @@ +#include "src/optimizer/relation_statistics/relation_statistics.cpp" + +#include "src/optimizer/relation_statistics/relation_statistics_extractor.cpp" + +#include "src/optimizer/relation_statistics/relation_statistics_helper.cpp" + +#include "src/optimizer/relation_statistics/relation_statistics_operator.cpp" + +#include "src/optimizer/relation_statistics/relation_statistics_scan.cpp" +