Skip to content

Commit a0fb6c8

Browse files
committed
feat(inspect): add task-based scans for snapshots and history
- Add Table::Inspect() and metadata scan builders for column selection, projection, and case-sensitive name resolution. - Separate task planning from reading with reusable static tasks backed by captured table metadata. - Preserve scan configuration and metadata across builder changes and table refreshes. - Produce Arrow batches lazily through MetadataScanTaskReader, with reader-controlled batch sizes and independent stream lifetimes. - Keep canonical schemas and row conversion in each metadata table implementation, sharing batch construction and projection validation. - Add common Arrow table comparison utilities and tests for schemas, full reads, reordered projections, and stream behavior
1 parent 5045427 commit a0fb6c8

26 files changed

Lines changed: 1355 additions & 616 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ set(ICEBERG_SOURCES
5959
incremental_changelog_scan.cc
6060
inspect/history_table.cc
6161
inspect/metadata_table.cc
62+
inspect/metadata_scan.cc
63+
inspect/metadata_scan_task.cc
64+
inspect/metadata_scan_task_reader.cc
6265
inspect/snapshots_table.cc
6366
inheritable_metadata.cc
6467
json_serde.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@
8080
#include "iceberg/incremental_changelog_scan.h"
8181
#include "iceberg/inheritable_metadata.h"
8282
#include "iceberg/inspect/history_table.h"
83+
#include "iceberg/inspect/metadata_scan.h"
84+
#include "iceberg/inspect/metadata_scan_task.h"
85+
#include "iceberg/inspect/metadata_scan_task_reader.h"
8386
#include "iceberg/inspect/metadata_table.h"
8487
#include "iceberg/inspect/snapshots_table.h"
8588
#include "iceberg/labels.h"

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 98 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,34 @@
2020
#include "iceberg/inspect/history_table.h"
2121

2222
#include <memory>
23+
#include <unordered_map>
24+
#include <unordered_set>
2325
#include <utility>
24-
#include <vector>
2526

27+
#include "iceberg/arrow_row_builder_internal.h"
28+
#include "iceberg/constants.h"
29+
#include "iceberg/inspect/metadata_scan_task.h"
30+
#include "iceberg/inspect/metadata_scan_task_reader.h"
31+
#include "iceberg/inspect/metadata_scan_util_internal.h"
2632
#include "iceberg/schema.h"
2733
#include "iceberg/schema_field.h"
34+
#include "iceberg/snapshot.h"
2835
#include "iceberg/table.h"
36+
#include "iceberg/table_metadata.h"
2937
#include "iceberg/type.h"
3038
#include "iceberg/util/macros.h"
3139

3240
namespace iceberg {
3341

34-
HistoryTable::HistoryTable(std::shared_ptr<Table> table)
42+
// HistoryTable implementation
43+
HistoryTable::HistoryTable(std::shared_ptr<const Table> table)
3544
: MetadataTable(std::move(table)) {}
3645

3746
HistoryTable::~HistoryTable() = default;
3847

39-
const std::shared_ptr<Schema>& HistoryTable::schema() const {
48+
const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
49+
50+
const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
4051
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
4152
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
4253
SchemaField::MakeRequired(2, "snapshot_id", int64()),
@@ -45,13 +56,94 @@ const std::shared_ptr<Schema>& HistoryTable::schema() const {
4556
return schema;
4657
}
4758

48-
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(std::shared_ptr<Table> table) {
59+
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(
60+
std::shared_ptr<const Table> table) {
4961
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
5062
return std::unique_ptr<HistoryTable>(new HistoryTable(std::move(table)));
5163
}
5264

53-
Result<ArrowArrayStream> HistoryTable::Scan() {
54-
return NotSupported("Scan is not supported for the history table");
65+
Result<ArrowArrayStream> HistoryTable::ReadRows(
66+
std::shared_ptr<const TableMetadata> metadata,
67+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) {
68+
std::unordered_map<int64_t, const Snapshot*> snapshots_by_id;
69+
for (const auto& snapshot : metadata->snapshots) {
70+
if (snapshot) {
71+
snapshots_by_id.emplace(snapshot->snapshot_id, snapshot.get());
72+
}
73+
}
74+
75+
// Walk the current snapshot's ancestry; the insert check stops on cycles.
76+
std::unordered_set<int64_t> ancestors;
77+
for (auto current = metadata->current_snapshot_id;
78+
current != kInvalidSnapshotId && ancestors.insert(current).second;) {
79+
auto it = snapshots_by_id.find(current);
80+
if (it == snapshots_by_id.end() || !it->second->parent_snapshot_id) {
81+
break;
82+
}
83+
current = *it->second->parent_snapshot_id;
84+
}
85+
86+
const size_t num_rows = metadata->snapshot_log.size();
87+
return internal::MakeMetadataBatchStream(
88+
CanonicalSchema(), projected_schema, num_rows, batch_size,
89+
[metadata = std::move(metadata), snapshots_by_id = std::move(snapshots_by_id),
90+
ancestors = std::move(ancestors)](ArrowArray* column, int32_t field_id,
91+
size_t row) -> Status {
92+
const auto& entry = metadata->snapshot_log[row];
93+
switch (field_id) {
94+
case 1: // made_current_at
95+
return AppendInt(column, UnixMsFromTimePointMs(entry.timestamp_ms) * 1000);
96+
case 2: // snapshot_id
97+
return AppendInt(column, entry.snapshot_id);
98+
case 3: { // parent_id; null when the snapshot has been expired
99+
auto it = snapshots_by_id.find(entry.snapshot_id);
100+
return it != snapshots_by_id.end() && it->second->parent_snapshot_id
101+
? AppendInt(column, *it->second->parent_snapshot_id)
102+
: AppendNull(column);
103+
}
104+
case 4: // is_current_ancestor
105+
return AppendBoolean(column, ancestors.contains(entry.snapshot_id));
106+
default:
107+
return InvalidArgument("Unknown history metadata column id: {}", field_id);
108+
}
109+
});
110+
}
111+
112+
// HistoryScan implementation
113+
HistoryScan::HistoryScan(internal::MetadataScanContext context,
114+
std::shared_ptr<StaticMetadataTask> task)
115+
: MetadataScan(std::move(context)), task_(std::move(task)) {}
116+
117+
HistoryScan::~HistoryScan() = default;
118+
119+
Result<std::unique_ptr<HistoryScan>> HistoryScan::Make(
120+
internal::MetadataScanContext context) {
121+
ICEBERG_PRECHECK(context.kind == MetadataTableKind::kHistory,
122+
"Unexpected metadata table kind for history scan");
123+
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
124+
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
125+
ICEBERG_PRECHECK(context.projected_schema != nullptr,
126+
"Projected schema cannot be null");
127+
ICEBERG_ASSIGN_OR_RAISE(auto task,
128+
StaticMetadataTask::Make(context.kind, context.table_metadata));
129+
return std::unique_ptr<HistoryScan>(
130+
new HistoryScan(std::move(context), std::move(task)));
131+
}
132+
133+
Result<MetadataScanTaskStreamPtr> HistoryScan::PlanTasksStream() const {
134+
return MetadataScanTaskStream::FromVector({task_});
135+
}
136+
137+
Result<std::vector<std::shared_ptr<MetadataScanTask>>> HistoryScan::PlanTasks() const {
138+
return std::vector<std::shared_ptr<MetadataScanTask>>{task_};
139+
}
140+
141+
Result<ArrowArrayStream> HistoryScan::ToArrowArrayStream() const {
142+
ICEBERG_ASSIGN_OR_RAISE(
143+
auto reader,
144+
MetadataScanTaskReader::Make(
145+
{.projected_schema = context().projected_schema, .io = context().io}));
146+
return reader->Open(*task_);
55147
}
56148

57149
} // namespace iceberg

‎src/iceberg/inspect/history_table.h‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,12 @@
2222
/// \file iceberg/inspect/history_table.h
2323
/// \brief Define the history metadata table.
2424

25+
#include <cstdint>
2526
#include <memory>
2627

28+
#include "iceberg/arrow_c_data.h"
2729
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_scan.h"
2831
#include "iceberg/inspect/metadata_table.h"
2932
#include "iceberg/result.h"
3033
#include "iceberg/type_fwd.h"
@@ -34,18 +37,47 @@ namespace iceberg {
3437
/// \brief History metadata table.
3538
class ICEBERG_EXPORT HistoryTable : public MetadataTable {
3639
public:
37-
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<Table> table);
40+
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<const Table> table);
41+
42+
/// \brief Return the canonical schema of the history metadata table.
43+
static const std::shared_ptr<Schema>& CanonicalSchema();
3844

3945
~HistoryTable() override;
4046

41-
Kind kind() const noexcept override { return Kind::kHistory; }
47+
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }
4248

4349
const std::shared_ptr<Schema>& schema() const override;
4450

45-
Result<ArrowArrayStream> Scan() override;
51+
private:
52+
friend class MetadataScanTaskReader;
53+
54+
explicit HistoryTable(std::shared_ptr<const Table> table);
55+
56+
/// \brief Read the snapshot log in metadata as Arrow batches of projected_schema.
57+
static Result<ArrowArrayStream> ReadRows(
58+
std::shared_ptr<const TableMetadata> metadata,
59+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size);
60+
};
61+
62+
/// \brief Scan the history metadata captured from a table.
63+
class ICEBERG_EXPORT HistoryScan final : public MetadataScan {
64+
public:
65+
/// \brief Create a scan from captured metadata table settings.
66+
static Result<std::unique_ptr<HistoryScan>> Make(internal::MetadataScanContext context);
67+
68+
~HistoryScan() override;
69+
70+
Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override;
71+
72+
Result<std::vector<std::shared_ptr<MetadataScanTask>>> PlanTasks() const override;
73+
74+
Result<ArrowArrayStream> ToArrowArrayStream() const override;
4675

4776
private:
48-
explicit HistoryTable(std::shared_ptr<Table> table);
77+
HistoryScan(internal::MetadataScanContext context,
78+
std::shared_ptr<StaticMetadataTask> task);
79+
80+
const std::shared_ptr<StaticMetadataTask> task_;
4981
};
5082

5183
} // namespace iceberg
Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
#include "iceberg/inspect/metadata_scan.h"
2+
3+
#include <optional>
4+
#include <utility>
5+
6+
#include "iceberg/arrow_c_data_util_internal.h"
7+
#include "iceberg/inspect/history_table.h"
8+
#include "iceberg/inspect/metadata_scan_task_reader.h"
9+
#include "iceberg/inspect/metadata_table.h"
10+
#include "iceberg/inspect/snapshots_table.h"
11+
#include "iceberg/schema.h"
12+
#include "iceberg/schema_internal.h"
13+
#include "iceberg/table.h"
14+
#include "iceberg/util/macros.h"
15+
#include "iceberg/util/stream.h"
16+
17+
namespace iceberg {
18+
namespace {
19+
20+
class TaskArrowStreamSource {
21+
public:
22+
TaskArrowStreamSource(std::vector<std::shared_ptr<MetadataScanTask>> tasks,
23+
std::unique_ptr<MetadataScanTaskReader> reader,
24+
std::shared_ptr<Schema> schema)
25+
: tasks_(std::move(tasks)),
26+
reader_(std::move(reader)),
27+
schema_(std::move(schema)) {}
28+
29+
~TaskArrowStreamSource() { std::ignore = Close(); }
30+
31+
Status Close() {
32+
if (current_.release != nullptr) {
33+
current_.release(&current_);
34+
}
35+
return {};
36+
}
37+
38+
Result<ArrowSchema> Schema() {
39+
ArrowSchema schema{};
40+
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*schema_, &schema));
41+
return schema;
42+
}
43+
44+
Result<std::optional<ArrowArray>> Next() {
45+
if (error_) {
46+
return iceberg::unexpected(*error_);
47+
}
48+
auto result = NextBatch();
49+
if (!result) {
50+
error_ = result.error();
51+
}
52+
return result;
53+
}
54+
55+
Result<std::optional<ArrowArray>> NextBatch() {
56+
while (true) {
57+
if (current_.release == nullptr) {
58+
if (next_task_ == tasks_.size()) {
59+
return std::nullopt;
60+
}
61+
ICEBERG_ASSIGN_OR_RAISE(current_, reader_->Open(*tasks_[next_task_++]));
62+
}
63+
64+
ArrowArray array{};
65+
const int error = current_.get_next(&current_, &array);
66+
if (error != 0) {
67+
std::string message = current_.get_last_error != nullptr &&
68+
current_.get_last_error(&current_) != nullptr
69+
? current_.get_last_error(&current_)
70+
: "Metadata task stream failed";
71+
current_.release(&current_);
72+
return IOError("{}", message);
73+
}
74+
if (array.release != nullptr) {
75+
return array;
76+
}
77+
current_.release(&current_);
78+
}
79+
}
80+
81+
const std::vector<std::shared_ptr<MetadataScanTask>> tasks_;
82+
const std::unique_ptr<MetadataScanTaskReader> reader_;
83+
const std::shared_ptr<iceberg::Schema> schema_;
84+
size_t next_task_ = 0;
85+
ArrowArrayStream current_{};
86+
std::optional<Error> error_;
87+
};
88+
89+
} // namespace
90+
91+
MetadataScanBuilder::MetadataScanBuilder(std::shared_ptr<const Table> source,
92+
MetadataTableKind kind,
93+
std::shared_ptr<Schema> schema)
94+
: context_{.kind = kind,
95+
.table_metadata = source->metadata(),
96+
.metadata_file_location = std::string(source->metadata_file_location()),
97+
.schema = schema,
98+
.projected_schema = std::move(schema),
99+
.io = source->io()} {}
100+
101+
Result<std::unique_ptr<MetadataScanBuilder>> MetadataScanBuilder::Make(
102+
std::shared_ptr<const Table> source, MetadataTableKind kind,
103+
std::shared_ptr<Schema> schema) {
104+
ICEBERG_PRECHECK(source != nullptr, "Source table cannot be null");
105+
ICEBERG_PRECHECK(schema != nullptr, "Metadata table schema cannot be null");
106+
return std::unique_ptr<MetadataScanBuilder>(
107+
new MetadataScanBuilder(std::move(source), kind, std::move(schema)));
108+
}
109+
110+
MetadataScanBuilder& MetadataScanBuilder::Select(
111+
const std::vector<std::string>& columns) {
112+
auto projected = context_.schema->Select(columns, context_.case_sensitive);
113+
if (projected) {
114+
context_.projected_schema = std::move(projected).value();
115+
} else {
116+
AddError(projected.error());
117+
}
118+
return *this;
119+
}
120+
121+
MetadataScanBuilder& MetadataScanBuilder::Project(std::shared_ptr<Schema> schema) {
122+
context_.projected_schema = std::move(schema);
123+
return *this;
124+
}
125+
126+
MetadataScanBuilder& MetadataScanBuilder::CaseSensitive(bool value) {
127+
context_.case_sensitive = value;
128+
return *this;
129+
}
130+
131+
Result<std::unique_ptr<MetadataScan>> MetadataScanBuilder::Build() {
132+
ICEBERG_RETURN_UNEXPECTED(CheckErrors());
133+
ICEBERG_PRECHECK(context_.projected_schema != nullptr,
134+
"Projected schema cannot be null");
135+
switch (context_.kind) {
136+
case MetadataTableKind::kSnapshots:
137+
return SnapshotsScan::Make(context_);
138+
case MetadataTableKind::kHistory:
139+
return HistoryScan::Make(context_);
140+
default:
141+
return NotImplemented("Metadata table kind is not implemented");
142+
}
143+
}
144+
145+
MetadataScan::MetadataScan(internal::MetadataScanContext context)
146+
: context_(std::move(context)) {}
147+
148+
MetadataScan::~MetadataScan() = default;
149+
150+
Result<std::vector<std::shared_ptr<MetadataScanTask>>> MetadataScan::PlanTasks() const {
151+
ICEBERG_ASSIGN_OR_RAISE(auto stream, PlanTasksStream());
152+
return stream->ToVector();
153+
}
154+
155+
Result<ArrowArrayStream> MetadataScan::ToArrowArrayStream() const {
156+
ICEBERG_ASSIGN_OR_RAISE(auto tasks, PlanTasks());
157+
ICEBERG_ASSIGN_OR_RAISE(auto reader, MetadataScanTaskReader::Make(
158+
{.projected_schema = context_.projected_schema,
159+
.io = context_.io}));
160+
return MakeArrowArrayStream(std::make_unique<TaskArrowStreamSource>(
161+
std::move(tasks), std::move(reader), context_.projected_schema));
162+
}
163+
164+
} // namespace iceberg

0 commit comments

Comments
 (0)