Skip to content

Commit 8b60bf1

Browse files
committed
feat(inspect): add task-based scans for snapshots and history
- Add Table::Inspect() and task-based metadata scans while retaining column selection, projection, case sensitivity, and time-travel builder configuration from the merged inspection API. - Separate task planning from reading with reusable static tasks backed by captured table metadata. - Preserve scan configuration and metadata across builder changes and table refreshes. - Produce Arrow batches lazily through MetadataScanTaskReader, with reader-controlled batch sizes and independent stream lifetimes. - Keep canonical schemas and row conversion in each metadata table implementation, sharing batch construction and projection validation. - Consolidate projected-column append helpers with lazy batch construction. - Preserve all merged metadata inspection test scenarios and cover planned projections, task reuse, stream lifetimes, batching, and vector<bool>.
1 parent d0ca49f commit 8b60bf1

25 files changed

Lines changed: 1308 additions & 463 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ set(ICEBERG_SOURCES
6161
inspect/history_table.cc
6262
inspect/metadata_scan.cc
6363
inspect/metadata_table.cc
64+
inspect/metadata_scan_task.cc
65+
inspect/metadata_scan_task_reader.cc
6466
inspect/snapshots_table.cc
6567
inheritable_metadata.cc
6668
json_serde.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@
8282
#include "iceberg/inheritable_metadata.h"
8383
#include "iceberg/inspect/history_table.h"
8484
#include "iceberg/inspect/metadata_scan.h"
85+
#include "iceberg/inspect/metadata_scan_task.h"
86+
#include "iceberg/inspect/metadata_scan_task_reader.h"
8587
#include "iceberg/inspect/metadata_table.h"
8688
#include "iceberg/inspect/snapshots_table.h"
8789
#include "iceberg/labels.h"

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 117 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,67 @@
1919

2020
#include "iceberg/inspect/history_table.h"
2121

22+
#include <array>
2223
#include <memory>
24+
#include <unordered_map>
25+
#include <unordered_set>
2326
#include <utility>
24-
#include <vector>
2527

28+
#include "iceberg/arrow_row_builder_internal.h"
29+
#include "iceberg/constants.h"
30+
#include "iceberg/inspect/metadata_scan_task.h"
31+
#include "iceberg/inspect/metadata_scan_util_internal.h"
2632
#include "iceberg/schema.h"
2733
#include "iceberg/schema_field.h"
34+
#include "iceberg/snapshot.h"
2835
#include "iceberg/table.h"
36+
#include "iceberg/table_metadata.h"
2937
#include "iceberg/type.h"
3038
#include "iceberg/util/macros.h"
3139

3240
namespace iceberg {
41+
namespace {
3342

34-
HistoryTable::HistoryTable(std::shared_ptr<Table> table)
43+
struct HistoryRow {
44+
const SnapshotLogEntry& entry;
45+
const Snapshot* snapshot;
46+
bool is_current_ancestor;
47+
};
48+
49+
constexpr std::array<internal::MetadataTableColumn<HistoryRow>, 4> kHistoryColumns = {
50+
{{.field_id = 1,
51+
.append =
52+
[](ArrowArray* column, const HistoryRow& row) {
53+
return AppendInt(column,
54+
UnixMsFromTimePointMs(row.entry.timestamp_ms) * 1000);
55+
}},
56+
{.field_id = 2,
57+
.append =
58+
[](ArrowArray* column, const HistoryRow& row) {
59+
return AppendInt(column, row.entry.snapshot_id);
60+
}},
61+
{.field_id = 3,
62+
.append =
63+
[](ArrowArray* column, const HistoryRow& row) {
64+
return row.snapshot != nullptr && row.snapshot->parent_snapshot_id
65+
? AppendInt(column, *row.snapshot->parent_snapshot_id)
66+
: AppendNull(column);
67+
}},
68+
{.field_id = 4, .append = [](ArrowArray* column, const HistoryRow& row) {
69+
return AppendBoolean(column, row.is_current_ancestor);
70+
}}}};
71+
72+
} // namespace
73+
74+
// HistoryTable implementation
75+
HistoryTable::HistoryTable(std::shared_ptr<const Table> table)
3576
: MetadataTable(std::move(table)) {}
3677

3778
HistoryTable::~HistoryTable() = default;
3879

39-
const std::shared_ptr<Schema>& HistoryTable::schema() const {
80+
const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
81+
82+
const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
4083
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
4184
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
4285
SchemaField::MakeRequired(2, "snapshot_id", int64()),
@@ -45,14 +88,81 @@ const std::shared_ptr<Schema>& HistoryTable::schema() const {
4588
return schema;
4689
}
4790

48-
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(std::shared_ptr<Table> table) {
91+
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(
92+
std::shared_ptr<const Table> table) {
4993
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
5094
return std::unique_ptr<HistoryTable>(new HistoryTable(std::move(table)));
5195
}
5296

53-
Result<ArrowArrayStream> HistoryTable::ScanProjection(
54-
const Schema& /*projected_schema*/) const {
55-
return NotSupported("Scan is not supported for the history table");
97+
Result<std::unique_ptr<MetadataScan>> HistoryTable::MakeScan(
98+
internal::MetadataScanContext context) const {
99+
return HistoryScan::Make(std::move(context));
100+
}
101+
102+
Result<ArrowArrayStream> HistoryTable::ReadRows(
103+
std::shared_ptr<const TableMetadata> metadata,
104+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) {
105+
std::unordered_map<int64_t, const Snapshot*> snapshots_by_id;
106+
for (const auto& snapshot : metadata->snapshots) {
107+
if (snapshot) {
108+
snapshots_by_id.emplace(snapshot->snapshot_id, snapshot.get());
109+
}
110+
}
111+
112+
// Walk the current snapshot's ancestry; the insert check stops on cycles.
113+
std::unordered_set<int64_t> ancestors;
114+
for (auto current = metadata->current_snapshot_id;
115+
current != kInvalidSnapshotId && ancestors.insert(current).second;) {
116+
auto it = snapshots_by_id.find(current);
117+
if (it == snapshots_by_id.end() || !it->second->parent_snapshot_id) {
118+
break;
119+
}
120+
current = *it->second->parent_snapshot_id;
121+
}
122+
123+
const size_t num_rows = metadata->snapshot_log.size();
124+
return internal::MakeMetadataBatchStream<HistoryRow>(
125+
CanonicalSchema(), projected_schema, kHistoryColumns, num_rows, batch_size,
126+
[metadata = std::move(metadata), snapshots_by_id = std::move(snapshots_by_id),
127+
ancestors = std::move(ancestors)](size_t index) -> HistoryRow {
128+
const auto& entry = metadata->snapshot_log[index];
129+
auto it = snapshots_by_id.find(entry.snapshot_id);
130+
return {.entry = entry,
131+
.snapshot = it != snapshots_by_id.end() ? it->second : nullptr,
132+
.is_current_ancestor = ancestors.contains(entry.snapshot_id)};
133+
});
134+
}
135+
136+
// HistoryScan implementation
137+
HistoryScan::HistoryScan(internal::MetadataScanContext context,
138+
std::shared_ptr<MetadataScanTask> task)
139+
: MetadataScan(std::move(context)), task_(std::move(task)) {}
140+
141+
HistoryScan::~HistoryScan() = default;
142+
143+
Result<std::unique_ptr<HistoryScan>> HistoryScan::Make(
144+
internal::MetadataScanContext context) {
145+
ICEBERG_PRECHECK(context.kind == MetadataTableKind::kHistory,
146+
"Unexpected metadata table kind for history scan");
147+
if (context.snapshot_id.has_value() || !context.ref_name.empty() ||
148+
context.as_of_time_in_millis.has_value()) {
149+
return NotSupported("Metadata table does not support time travel");
150+
}
151+
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
152+
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
153+
ICEBERG_PRECHECK(context.projected_schema != nullptr,
154+
"Projected schema cannot be null");
155+
ICEBERG_RETURN_UNEXPECTED(internal::SelectAppendFns<HistoryRow>(
156+
*HistoryTable::CanonicalSchema(), *context.projected_schema, kHistoryColumns));
157+
ICEBERG_ASSIGN_OR_RAISE(auto task,
158+
StaticMetadataTask::Make(context.kind, context.table_metadata,
159+
context.projected_schema));
160+
return std::unique_ptr<HistoryScan>(
161+
new HistoryScan(std::move(context), std::move(task)));
162+
}
163+
164+
Result<MetadataScanTaskStreamPtr> HistoryScan::PlanTasksStream() const {
165+
return MetadataScanTaskStream::FromVector({task_});
56166
}
57167

58168
} // namespace iceberg

‎src/iceberg/inspect/history_table.h‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,12 @@
2222
/// \file iceberg/inspect/history_table.h
2323
/// \brief Define the history metadata table.
2424

25+
#include <cstdint>
2526
#include <memory>
2627

28+
#include "iceberg/arrow_c_data.h"
2729
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_scan.h"
2831
#include "iceberg/inspect/metadata_table.h"
2932
#include "iceberg/result.h"
3033
#include "iceberg/type_fwd.h"
@@ -34,19 +37,46 @@ namespace iceberg {
3437
/// \brief History metadata table.
3538
class ICEBERG_EXPORT HistoryTable : public MetadataTable {
3639
public:
37-
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<Table> table);
40+
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<const Table> table);
41+
42+
/// \brief Return the canonical schema of the history metadata table.
43+
static const std::shared_ptr<Schema>& CanonicalSchema();
3844

3945
~HistoryTable() override;
4046

41-
Kind kind() const noexcept override { return Kind::kHistory; }
47+
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }
4248

4349
const std::shared_ptr<Schema>& schema() const override;
4450

4551
protected:
46-
Result<ArrowArrayStream> ScanProjection(const Schema& projected_schema) const override;
52+
Result<std::unique_ptr<MetadataScan>> MakeScan(
53+
internal::MetadataScanContext context) const override;
4754

4855
private:
49-
explicit HistoryTable(std::shared_ptr<Table> table);
56+
friend class MetadataScanTaskReader;
57+
58+
static Result<ArrowArrayStream> ReadRows(
59+
std::shared_ptr<const TableMetadata> metadata,
60+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size);
61+
62+
explicit HistoryTable(std::shared_ptr<const Table> table);
63+
};
64+
65+
/// \brief Scan the history metadata captured from a table.
66+
class ICEBERG_EXPORT HistoryScan final : public MetadataScan {
67+
public:
68+
/// \brief Create a scan from captured metadata table settings.
69+
static Result<std::unique_ptr<HistoryScan>> Make(internal::MetadataScanContext context);
70+
71+
~HistoryScan() override;
72+
73+
Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override;
74+
75+
private:
76+
HistoryScan(internal::MetadataScanContext context,
77+
std::shared_ptr<MetadataScanTask> task);
78+
79+
const std::shared_ptr<MetadataScanTask> task_;
5080
};
5181

5282
} // namespace iceberg

0 commit comments

Comments
 (0)