Skip to content

Commit 346d3b7

Browse files
committed
feat(inspect): add task-based scans for snapshots and history
- Add Table::Inspect() and metadata scan builders for column selection, projection, and case-sensitive name resolution. - Separate task planning from reading with reusable static tasks backed by captured table metadata. - Preserve scan configuration and metadata across builder changes and table refreshes. - Produce Arrow batches lazily through MetadataScanTaskReader, with reader-controlled batch sizes and independent stream lifetimes. - Keep canonical schemas and row conversion in each metadata table implementation, sharing batch construction and projection validation. - Add common Arrow table comparison utilities and tests for schemas, full reads, reordered projections, and stream behavior
1 parent 5045427 commit 346d3b7

26 files changed

Lines changed: 1381 additions & 616 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ set(ICEBERG_SOURCES
5959
incremental_changelog_scan.cc
6060
inspect/history_table.cc
6161
inspect/metadata_table.cc
62+
inspect/metadata_scan.cc
63+
inspect/metadata_scan_task.cc
64+
inspect/metadata_scan_task_reader.cc
6265
inspect/snapshots_table.cc
6366
inheritable_metadata.cc
6467
json_serde.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@
8080
#include "iceberg/incremental_changelog_scan.h"
8181
#include "iceberg/inheritable_metadata.h"
8282
#include "iceberg/inspect/history_table.h"
83+
#include "iceberg/inspect/metadata_scan.h"
84+
#include "iceberg/inspect/metadata_scan_task.h"
85+
#include "iceberg/inspect/metadata_scan_task_reader.h"
8386
#include "iceberg/inspect/metadata_table.h"
8487
#include "iceberg/inspect/snapshots_table.h"
8588
#include "iceberg/labels.h"

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 98 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,34 @@
2020
#include "iceberg/inspect/history_table.h"
2121

2222
#include <memory>
23+
#include <unordered_map>
24+
#include <unordered_set>
2325
#include <utility>
24-
#include <vector>
2526

27+
#include "iceberg/arrow_row_builder_internal.h"
28+
#include "iceberg/constants.h"
29+
#include "iceberg/inspect/metadata_scan_task.h"
30+
#include "iceberg/inspect/metadata_scan_task_reader.h"
31+
#include "iceberg/inspect/metadata_scan_util_internal.h"
2632
#include "iceberg/schema.h"
2733
#include "iceberg/schema_field.h"
34+
#include "iceberg/snapshot.h"
2835
#include "iceberg/table.h"
36+
#include "iceberg/table_metadata.h"
2937
#include "iceberg/type.h"
3038
#include "iceberg/util/macros.h"
3139

3240
namespace iceberg {
3341

34-
HistoryTable::HistoryTable(std::shared_ptr<Table> table)
42+
// HistoryTable implementation
43+
HistoryTable::HistoryTable(std::shared_ptr<const Table> table)
3544
: MetadataTable(std::move(table)) {}
3645

3746
HistoryTable::~HistoryTable() = default;
3847

39-
const std::shared_ptr<Schema>& HistoryTable::schema() const {
48+
const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
49+
50+
const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
4051
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
4152
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
4253
SchemaField::MakeRequired(2, "snapshot_id", int64()),
@@ -45,13 +56,94 @@ const std::shared_ptr<Schema>& HistoryTable::schema() const {
4556
return schema;
4657
}
4758

48-
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(std::shared_ptr<Table> table) {
59+
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(
60+
std::shared_ptr<const Table> table) {
4961
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
5062
return std::unique_ptr<HistoryTable>(new HistoryTable(std::move(table)));
5163
}
5264

53-
Result<ArrowArrayStream> HistoryTable::Scan() {
54-
return NotSupported("Scan is not supported for the history table");
65+
Result<ArrowArrayStream> HistoryTable::ReadRows(
66+
std::shared_ptr<const TableMetadata> metadata,
67+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) {
68+
std::unordered_map<int64_t, const Snapshot*> snapshots_by_id;
69+
for (const auto& snapshot : metadata->snapshots) {
70+
if (snapshot) {
71+
snapshots_by_id.emplace(snapshot->snapshot_id, snapshot.get());
72+
}
73+
}
74+
75+
// Walk the current snapshot's ancestry; the insert check stops on cycles.
76+
std::unordered_set<int64_t> ancestors;
77+
for (auto current = metadata->current_snapshot_id;
78+
current != kInvalidSnapshotId && ancestors.insert(current).second;) {
79+
auto it = snapshots_by_id.find(current);
80+
if (it == snapshots_by_id.end() || !it->second->parent_snapshot_id) {
81+
break;
82+
}
83+
current = *it->second->parent_snapshot_id;
84+
}
85+
86+
const size_t num_rows = metadata->snapshot_log.size();
87+
return internal::MakeMetadataBatchStream(
88+
CanonicalSchema(), projected_schema, num_rows, batch_size,
89+
[metadata = std::move(metadata), snapshots_by_id = std::move(snapshots_by_id),
90+
ancestors = std::move(ancestors)](ArrowArray* column, int32_t field_id,
91+
size_t row) -> Status {
92+
const auto& entry = metadata->snapshot_log[row];
93+
switch (field_id) {
94+
case 1: // made_current_at
95+
return AppendInt(column, UnixMsFromTimePointMs(entry.timestamp_ms) * 1000);
96+
case 2: // snapshot_id
97+
return AppendInt(column, entry.snapshot_id);
98+
case 3: { // parent_id; null when the snapshot has been expired
99+
auto it = snapshots_by_id.find(entry.snapshot_id);
100+
return it != snapshots_by_id.end() && it->second->parent_snapshot_id
101+
? AppendInt(column, *it->second->parent_snapshot_id)
102+
: AppendNull(column);
103+
}
104+
case 4: // is_current_ancestor
105+
return AppendBoolean(column, ancestors.contains(entry.snapshot_id));
106+
default:
107+
return InvalidArgument("Unknown history metadata column id: {}", field_id);
108+
}
109+
});
110+
}
111+
112+
// HistoryScan implementation
113+
HistoryScan::HistoryScan(internal::MetadataScanContext context,
114+
std::shared_ptr<StaticMetadataTask> task)
115+
: MetadataScan(std::move(context)), task_(std::move(task)) {}
116+
117+
HistoryScan::~HistoryScan() = default;
118+
119+
Result<std::unique_ptr<HistoryScan>> HistoryScan::Make(
120+
internal::MetadataScanContext context) {
121+
ICEBERG_PRECHECK(context.kind == MetadataTableKind::kHistory,
122+
"Unexpected metadata table kind for history scan");
123+
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
124+
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
125+
ICEBERG_PRECHECK(context.projected_schema != nullptr,
126+
"Projected schema cannot be null");
127+
ICEBERG_ASSIGN_OR_RAISE(auto task,
128+
StaticMetadataTask::Make(context.kind, context.table_metadata));
129+
return std::unique_ptr<HistoryScan>(
130+
new HistoryScan(std::move(context), std::move(task)));
131+
}
132+
133+
Result<MetadataScanTaskStreamPtr> HistoryScan::PlanTasksStream() const {
134+
return MetadataScanTaskStream::FromVector({task_});
135+
}
136+
137+
Result<std::vector<std::shared_ptr<MetadataScanTask>>> HistoryScan::PlanTasks() const {
138+
return std::vector<std::shared_ptr<MetadataScanTask>>{task_};
139+
}
140+
141+
Result<ArrowArrayStream> HistoryScan::ToArrowArrayStream() const {
142+
ICEBERG_ASSIGN_OR_RAISE(
143+
auto reader,
144+
MetadataScanTaskReader::Make(
145+
{.projected_schema = context().projected_schema, .io = context().io}));
146+
return reader->Open(*task_);
55147
}
56148

57149
} // namespace iceberg

‎src/iceberg/inspect/history_table.h‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,12 @@
2222
/// \file iceberg/inspect/history_table.h
2323
/// \brief Define the history metadata table.
2424

25+
#include <cstdint>
2526
#include <memory>
2627

28+
#include "iceberg/arrow_c_data.h"
2729
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_scan.h"
2831
#include "iceberg/inspect/metadata_table.h"
2932
#include "iceberg/result.h"
3033
#include "iceberg/type_fwd.h"
@@ -34,18 +37,47 @@ namespace iceberg {
3437
/// \brief History metadata table.
3538
class ICEBERG_EXPORT HistoryTable : public MetadataTable {
3639
public:
37-
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<Table> table);
40+
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<const Table> table);
41+
42+
/// \brief Return the canonical schema of the history metadata table.
43+
static const std::shared_ptr<Schema>& CanonicalSchema();
3844

3945
~HistoryTable() override;
4046

41-
Kind kind() const noexcept override { return Kind::kHistory; }
47+
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }
4248

4349
const std::shared_ptr<Schema>& schema() const override;
4450

45-
Result<ArrowArrayStream> Scan() override;
51+
private:
52+
friend class MetadataScanTaskReader;
53+
54+
explicit HistoryTable(std::shared_ptr<const Table> table);
55+
56+
/// \brief Read the snapshot log in metadata as Arrow batches of projected_schema.
57+
static Result<ArrowArrayStream> ReadRows(
58+
std::shared_ptr<const TableMetadata> metadata,
59+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size);
60+
};
61+
62+
/// \brief Scan the history metadata captured from a table.
63+
class ICEBERG_EXPORT HistoryScan final : public MetadataScan {
64+
public:
65+
/// \brief Create a scan from captured metadata table settings.
66+
static Result<std::unique_ptr<HistoryScan>> Make(internal::MetadataScanContext context);
67+
68+
~HistoryScan() override;
69+
70+
Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override;
71+
72+
Result<std::vector<std::shared_ptr<MetadataScanTask>>> PlanTasks() const override;
73+
74+
Result<ArrowArrayStream> ToArrowArrayStream() const override;
4675

4776
private:
48-
explicit HistoryTable(std::shared_ptr<Table> table);
77+
HistoryScan(internal::MetadataScanContext context,
78+
std::shared_ptr<StaticMetadataTask> task);
79+
80+
const std::shared_ptr<StaticMetadataTask> task_;
4981
};
5082

5183
} // namespace iceberg

0 commit comments

Comments
 (0)