Skip to content

Commit 7d0ad9d

Browse files
manuzhangclaude
andcommitted
refactor(inspect): align the metadata scan API with apache#1066
Make the scan surface a subset of the API proposed in apache#1066 so the two can converge: - MetadataTable::NewScan() is virtual and const, and ScanProjection() is const. - MetadataScanBuilder::Select() resolves names immediately with the current CaseSensitive() setting and records errors for Build(); Project() sets the output schema directly, and Build() rejects a null projection. - MetadataScan::ToArrowArrayStream() is virtual and const. - Projection validation rejects unknown columns, type changes, and projecting a nullable column as required, with the same messages as apache#1066. - Drop the time-travel options together with TimeTravelMetadataTable and SnapshotSelection, which apache#1066 also removes; nothing on main implements them. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent a6de0a9 commit 7d0ad9d

12 files changed

Lines changed: 163 additions & 323 deletions

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(std::shared_ptr<Table>
5151
}
5252

5353
Result<ArrowArrayStream> HistoryTable::ScanProjection(
54-
const Schema& /*projected_schema*/) {
54+
const Schema& /*projected_schema*/) const {
5555
return NotSupported("Scan is not supported for the history table");
5656
}
5757

‎src/iceberg/inspect/history_table.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class ICEBERG_EXPORT HistoryTable : public MetadataTable {
4343
const std::shared_ptr<Schema>& schema() const override;
4444

4545
protected:
46-
Result<ArrowArrayStream> ScanProjection(const Schema& projected_schema) override;
46+
Result<ArrowArrayStream> ScanProjection(const Schema& projected_schema) const override;
4747

4848
private:
4949
explicit HistoryTable(std::shared_ptr<Table> table);

‎src/iceberg/inspect/metadata_scan.cc‎

Lines changed: 18 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -28,76 +28,47 @@
2828

2929
namespace iceberg {
3030

31-
MetadataScan::MetadataScan(MetadataTable& table, std::shared_ptr<Schema> schema,
32-
SnapshotSelection snapshot_selection)
33-
: table_(&table),
34-
schema_(std::move(schema)),
35-
snapshot_selection_(std::move(snapshot_selection)) {}
31+
MetadataScan::MetadataScan(const MetadataTable& table, std::shared_ptr<Schema> schema)
32+
: table_(&table), schema_(std::move(schema)) {}
3633

3734
MetadataScan::~MetadataScan() = default;
3835

3936
const std::shared_ptr<Schema>& MetadataScan::schema() const { return schema_; }
4037

41-
const SnapshotSelection& MetadataScan::snapshot_selection() const {
42-
return snapshot_selection_;
43-
}
44-
4538
Result<ArrowArrayStream> MetadataScan::ToArrowArrayStream() const {
46-
return table_->ScanSnapshot(snapshot_selection_, *schema_);
39+
return table_->ScanProjection(*schema_);
4740
}
4841

49-
MetadataScanBuilder::MetadataScanBuilder(MetadataTable& table) : table_(&table) {}
42+
MetadataScanBuilder::MetadataScanBuilder(const MetadataTable& table)
43+
: table_(&table), projected_schema_(table.schema()) {}
5044

5145
MetadataScanBuilder::~MetadataScanBuilder() = default;
5246

5347
MetadataScanBuilder& MetadataScanBuilder::Select(
54-
const std::vector<std::string>& column_names) {
55-
selected_columns_ = column_names;
56-
return *this;
57-
}
58-
59-
MetadataScanBuilder& MetadataScanBuilder::CaseSensitive(bool case_sensitive) {
60-
case_sensitive_ = case_sensitive;
61-
return *this;
62-
}
63-
64-
MetadataScanBuilder& MetadataScanBuilder::UseSnapshot(int64_t snapshot_id) {
65-
ICEBERG_BUILDER_CHECK(
66-
std::holds_alternative<std::monostate>(snapshot_selection_.snapshot),
67-
"Cannot override snapshot, a snapshot or timestamp is already set");
68-
snapshot_selection_.snapshot = snapshot_id;
48+
const std::vector<std::string>& columns) {
49+
auto projected = table_->schema()->Select(columns, case_sensitive_);
50+
if (projected) {
51+
projected_schema_ = std::move(projected).value();
52+
} else {
53+
AddError(projected.error());
54+
}
6955
return *this;
7056
}
7157

72-
MetadataScanBuilder& MetadataScanBuilder::UseRef(const std::string& ref) {
73-
ICEBERG_BUILDER_CHECK(snapshot_selection_.ref_name.empty(),
74-
"Cannot override ref, already set ref={}",
75-
snapshot_selection_.ref_name);
76-
ICEBERG_BUILDER_CHECK(!ref.empty(), "Ref name cannot be empty");
77-
snapshot_selection_.ref_name = ref;
58+
MetadataScanBuilder& MetadataScanBuilder::Project(std::shared_ptr<Schema> schema) {
59+
projected_schema_ = std::move(schema);
7860
return *this;
7961
}
8062

81-
MetadataScanBuilder& MetadataScanBuilder::AsOfTime(int64_t timestamp_millis) {
82-
ICEBERG_BUILDER_CHECK(
83-
std::holds_alternative<std::monostate>(snapshot_selection_.snapshot),
84-
"Cannot override snapshot, a snapshot or timestamp is already set");
85-
snapshot_selection_.snapshot = TimePointMsFromUnixMs(timestamp_millis);
63+
MetadataScanBuilder& MetadataScanBuilder::CaseSensitive(bool value) {
64+
case_sensitive_ = value;
8665
return *this;
8766
}
8867

8968
Result<std::unique_ptr<MetadataScan>> MetadataScanBuilder::Build() {
9069
ICEBERG_RETURN_UNEXPECTED(CheckErrors());
91-
if (!snapshot_selection_.IsCurrentSnapshot() && !table_->supports_time_travel()) {
92-
return NotSupported("Metadata table does not support time travel");
93-
}
94-
95-
std::shared_ptr<Schema> schema = table_->schema();
96-
if (selected_columns_.has_value()) {
97-
ICEBERG_ASSIGN_OR_RAISE(schema, schema->Select(*selected_columns_, case_sensitive_));
98-
}
99-
return std::unique_ptr<MetadataScan>(
100-
new MetadataScan(*table_, std::move(schema), snapshot_selection_));
70+
ICEBERG_PRECHECK(projected_schema_ != nullptr, "Projected schema cannot be null");
71+
return std::unique_ptr<MetadataScan>(new MetadataScan(*table_, projected_schema_));
10172
}
10273

10374
} // namespace iceberg

‎src/iceberg/inspect/metadata_scan.h‎

Lines changed: 32 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -22,79 +22,49 @@
2222
/// \file iceberg/inspect/metadata_scan.h
2323
/// \brief Scan configuration for Iceberg metadata tables.
2424

25-
#include <cstdint>
2625
#include <memory>
27-
#include <optional>
2826
#include <string>
29-
#include <variant>
3027
#include <vector>
3128

3229
#include "iceberg/arrow_c_data.h"
3330
#include "iceberg/iceberg_export.h"
3431
#include "iceberg/result.h"
3532
#include "iceberg/type_fwd.h"
3633
#include "iceberg/util/error_collector.h"
37-
#include "iceberg/util/timepoint.h"
3834

3935
namespace iceberg {
4036

41-
/// \brief Snapshot selection parameters for a time-travel scan.
42-
struct SnapshotSelection {
43-
/// \brief Select the current snapshot, a snapshot ID, or an as-of timestamp.
44-
///
45-
/// std::monostate selects the current snapshot.
46-
std::variant<std::monostate, int64_t, TimePointMs> snapshot;
47-
48-
/// \brief Resolve the snapshot relative to this branch or tag.
49-
///
50-
/// An empty string uses the main branch.
51-
std::string ref_name;
52-
53-
/// \brief Return whether this selects the current snapshot on the main branch.
54-
bool IsCurrentSnapshot() const {
55-
return std::holds_alternative<std::monostate>(snapshot) && ref_name.empty();
56-
}
57-
};
58-
59-
/// \brief A configured scan of a metadata table.
37+
/// \brief A metadata table scan that reads its rows as Arrow.
6038
///
6139
/// Scans are created by MetadataScanBuilder::Build(). The metadata table that created
6240
/// the builder must outlive the scan.
6341
class ICEBERG_EXPORT MetadataScan {
6442
public:
65-
~MetadataScan();
43+
virtual ~MetadataScan();
6644

67-
/// \brief Return the schema of the rows this scan emits.
68-
///
69-
/// This is schema() of the metadata table projected to the selected columns.
45+
/// \brief Return the output schema: schema() of the metadata table projected to the
46+
/// selected columns.
7047
const std::shared_ptr<Schema>& schema() const;
7148

72-
/// \brief Return the snapshot this scan reads.
73-
const SnapshotSelection& snapshot_selection() const;
74-
75-
/// \brief Read the scan as an Arrow stream.
49+
/// \brief Read the scan as one Arrow stream with the output schema.
7650
///
7751
/// Each call returns a new stream. The caller owns the returned stream and must
7852
/// invoke its `release` callback when the stream is no longer needed.
79-
///
80-
/// \return An Arrow stream containing the selected columns, or an error.
81-
Result<ArrowArrayStream> ToArrowArrayStream() const;
53+
virtual Result<ArrowArrayStream> ToArrowArrayStream() const;
8254

8355
private:
8456
friend class MetadataScanBuilder;
8557

86-
MetadataScan(MetadataTable& table, std::shared_ptr<Schema> schema,
87-
SnapshotSelection snapshot_selection);
58+
MetadataScan(const MetadataTable& table, std::shared_ptr<Schema> schema);
8859

89-
MetadataTable* table_;
60+
const MetadataTable* table_;
9061
std::shared_ptr<Schema> schema_;
91-
SnapshotSelection snapshot_selection_;
9262
};
9363

94-
/// \brief Builder for metadata table scans.
64+
/// \brief Configure a scan over a metadata table.
9565
///
96-
/// Builders are created by MetadataTable::NewScan(). Options are validated when the
97-
/// scan is built, and the metadata table must outlive the builder.
66+
/// Builders are created by MetadataTable::NewScan(), and the metadata table must
67+
/// outlive the builder.
9868
///
9969
/// \code
10070
/// ICEBERG_ASSIGN_OR_RAISE(auto builder, snapshots_table->NewScan());
@@ -105,66 +75,44 @@ class ICEBERG_EXPORT MetadataScanBuilder : public ErrorCollector {
10575
public:
10676
~MetadataScanBuilder() override;
10777

108-
/// \brief Request this scan to return the given columns.
78+
/// \brief Select columns by name; nested columns use dot-separated paths.
10979
///
110-
/// Nested names are dot-concatenated. Columns are returned in schema order, and
111-
/// names that match no column are ignored. An empty list selects no columns, and a
112-
/// list containing `*` selects all columns. If this method is not called, all
113-
/// columns are selected. Selecting a map's value returns the whole map; selecting
114-
/// only its key is an error.
80+
/// Columns are returned in schema order, and names that match no column are
81+
/// ignored. An empty list selects no columns; `*` selects all columns. Names are
82+
/// resolved immediately, using the current CaseSensitive() setting. Selecting a
83+
/// map's value returns the whole map; selecting only its key is an error.
11584
///
116-
/// \param column_names Column names from the metadata table's schema.
117-
MetadataScanBuilder& Select(const std::vector<std::string>& column_names);
85+
/// \param columns Column names from the metadata table's schema.
86+
MetadataScanBuilder& Select(const std::vector<std::string>& columns);
11887

119-
/// \brief Control whether Select() matches column names case-sensitively.
88+
/// \brief Set the output schema to a projection of the metadata table schema.
12089
///
121-
/// Default is true.
90+
/// Each field must select a whole column of the metadata table by field id, with
91+
/// the same type and without tightening its nullability. Fields may be reordered.
12292
///
123-
/// \param case_sensitive Whether column names are matched case-sensitively.
124-
MetadataScanBuilder& CaseSensitive(bool case_sensitive);
93+
/// \param schema The output schema.
94+
MetadataScanBuilder& Project(std::shared_ptr<Schema> schema);
12595

126-
/// \brief Request this scan to use the given snapshot by ID.
127-
///
128-
/// Only metadata tables that support time travel accept a snapshot; Build()
129-
/// returns NotSupported otherwise. The table resolves the snapshot when the scan
130-
/// is read.
96+
/// \brief Use case-sensitive column names in Select(); enabled by default.
13197
///
132-
/// \param snapshot_id A snapshot ID.
133-
MetadataScanBuilder& UseSnapshot(int64_t snapshot_id);
98+
/// \param value Whether column names are matched case-sensitively.
99+
MetadataScanBuilder& CaseSensitive(bool value);
134100

135-
/// \brief Request this scan to resolve the snapshot on the given branch or tag.
101+
/// \brief Validate the settings and build a scan with its own copy of them.
136102
///
137-
/// Only metadata tables that support time travel accept a reference; Build()
138-
/// returns NotSupported otherwise. The table resolves the reference when the scan
139-
/// is read.
140-
///
141-
/// \param ref A branch or tag name.
142-
MetadataScanBuilder& UseRef(const std::string& ref);
143-
144-
/// \brief Request this scan to use the most recent snapshot as of the given time.
145-
///
146-
/// The snapshot is resolved on the branch set by UseRef(), or main if no branch
147-
/// is set. Only metadata tables that support time travel accept a timestamp;
148-
/// Build() returns NotSupported otherwise.
149-
///
150-
/// \param timestamp_millis A timestamp in milliseconds since the Unix epoch.
151-
MetadataScanBuilder& AsOfTime(int64_t timestamp_millis);
152-
153-
/// \brief Validate the options and build the scan.
103+
/// Later builder changes do not change the returned scan.
154104
///
155-
/// \return The scan, or an error if an option is invalid, the selected columns
156-
/// cannot be projected, or the table does not support the requested time travel.
105+
/// \return The scan, or the validation errors.
157106
Result<std::unique_ptr<MetadataScan>> Build();
158107

159108
private:
160109
friend class MetadataTable;
161110

162-
explicit MetadataScanBuilder(MetadataTable& table);
111+
explicit MetadataScanBuilder(const MetadataTable& table);
163112

164-
MetadataTable* table_;
165-
std::optional<std::vector<std::string>> selected_columns_;
113+
const MetadataTable* table_;
114+
std::shared_ptr<Schema> projected_schema_;
166115
bool case_sensitive_ = true;
167-
SnapshotSelection snapshot_selection_;
168116
};
169117

170118
} // namespace iceberg

‎src/iceberg/inspect/metadata_table.cc‎

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -31,34 +31,12 @@ MetadataTable::MetadataTable(std::shared_ptr<Table> source_table)
3131

3232
MetadataTable::~MetadataTable() = default;
3333

34-
bool MetadataTable::supports_time_travel() const noexcept { return false; }
35-
3634
const std::shared_ptr<Table>& MetadataTable::source_table() const {
3735
return source_table_;
3836
}
3937

40-
Result<std::unique_ptr<MetadataScanBuilder>> MetadataTable::NewScan() {
38+
Result<std::unique_ptr<MetadataScanBuilder>> MetadataTable::NewScan() const {
4139
return std::unique_ptr<MetadataScanBuilder>(new MetadataScanBuilder(*this));
4240
}
4341

44-
Result<ArrowArrayStream> MetadataTable::ScanSnapshot(
45-
const SnapshotSelection& snapshot_selection, const Schema& projected_schema) {
46-
if (!snapshot_selection.IsCurrentSnapshot()) {
47-
return NotSupported("Metadata table does not support time travel");
48-
}
49-
return ScanProjection(projected_schema);
50-
}
51-
52-
TimeTravelMetadataTable::TimeTravelMetadataTable(std::shared_ptr<Table> source_table)
53-
: MetadataTable(std::move(source_table)) {}
54-
55-
TimeTravelMetadataTable::~TimeTravelMetadataTable() = default;
56-
57-
bool TimeTravelMetadataTable::supports_time_travel() const noexcept { return true; }
58-
59-
Result<ArrowArrayStream> TimeTravelMetadataTable::ScanProjection(
60-
const Schema& projected_schema) {
61-
return ScanSnapshot(SnapshotSelection{}, projected_schema);
62-
}
63-
6442
} // namespace iceberg

0 commit comments

Comments
 (0)