Skip to content

Commit af6e0e0

Browse files
committed
feat(inspect): add task-based scans for snapshots and history
- Add Table::Inspect() and task-based metadata scans while retaining column selection, projection, case sensitivity, and time-travel builder configuration from the merged inspection API. - Separate task planning from reading with reusable static tasks backed by captured table metadata. - Preserve scan configuration and metadata across builder changes and table refreshes. - Produce Arrow batches lazily through MetadataScanTaskReader, with reader-controlled batch sizes and independent stream lifetimes. - Keep canonical schemas and row conversion in each metadata table implementation, sharing batch construction and projection validation. - Consolidate projected-column append helpers with lazy batch construction. - Preserve all merged metadata inspection test scenarios and cover planned projections, task reuse, stream lifetimes, batching, and vector<bool>.
1 parent d0ca49f commit af6e0e0

25 files changed

Lines changed: 1323 additions & 368 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ set(ICEBERG_SOURCES
6161
inspect/history_table.cc
6262
inspect/metadata_scan.cc
6363
inspect/metadata_table.cc
64+
inspect/metadata_scan_task.cc
65+
inspect/metadata_scan_task_reader.cc
6466
inspect/snapshots_table.cc
6567
inheritable_metadata.cc
6668
json_serde.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@
8282
#include "iceberg/inheritable_metadata.h"
8383
#include "iceberg/inspect/history_table.h"
8484
#include "iceberg/inspect/metadata_scan.h"
85+
#include "iceberg/inspect/metadata_scan_task.h"
86+
#include "iceberg/inspect/metadata_scan_task_reader.h"
8587
#include "iceberg/inspect/metadata_table.h"
8688
#include "iceberg/inspect/snapshots_table.h"
8789
#include "iceberg/labels.h"

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 111 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,64 @@
2020
#include "iceberg/inspect/history_table.h"
2121

2222
#include <memory>
23+
#include <unordered_map>
24+
#include <unordered_set>
2325
#include <utility>
24-
#include <vector>
2526

27+
#include "iceberg/arrow_row_builder_internal.h"
28+
#include "iceberg/constants.h"
29+
#include "iceberg/inspect/metadata_scan_task.h"
30+
#include "iceberg/inspect/metadata_scan_util_internal.h"
2631
#include "iceberg/schema.h"
2732
#include "iceberg/schema_field.h"
33+
#include "iceberg/snapshot.h"
2834
#include "iceberg/table.h"
35+
#include "iceberg/table_metadata.h"
2936
#include "iceberg/type.h"
3037
#include "iceberg/util/macros.h"
3138

3239
namespace iceberg {
40+
namespace {
3341

34-
HistoryTable::HistoryTable(std::shared_ptr<Table> table)
42+
struct HistoryRow {
43+
const SnapshotLogEntry& entry;
44+
const Snapshot* snapshot;
45+
bool is_current_ancestor;
46+
};
47+
48+
constexpr internal::MetadataTableColumn<HistoryRow> kHistoryColumns[] = {
49+
{.field_id = 1,
50+
.append =
51+
[](ArrowArray* column, const HistoryRow& row) {
52+
return AppendInt(column, UnixMsFromTimePointMs(row.entry.timestamp_ms) * 1000);
53+
}},
54+
{.field_id = 2,
55+
.append =
56+
[](ArrowArray* column, const HistoryRow& row) {
57+
return AppendInt(column, row.entry.snapshot_id);
58+
}},
59+
{.field_id = 3,
60+
.append =
61+
[](ArrowArray* column, const HistoryRow& row) {
62+
return row.snapshot != nullptr && row.snapshot->parent_snapshot_id
63+
? AppendInt(column, *row.snapshot->parent_snapshot_id)
64+
: AppendNull(column);
65+
}},
66+
{.field_id = 4, .append = [](ArrowArray* column, const HistoryRow& row) {
67+
return AppendBoolean(column, row.is_current_ancestor);
68+
}}};
69+
70+
} // namespace
71+
72+
// HistoryTable implementation
73+
HistoryTable::HistoryTable(std::shared_ptr<const Table> table)
3574
: MetadataTable(std::move(table)) {}
3675

3776
HistoryTable::~HistoryTable() = default;
3877

39-
const std::shared_ptr<Schema>& HistoryTable::schema() const {
78+
const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
79+
80+
const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
4081
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
4182
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
4283
SchemaField::MakeRequired(2, "snapshot_id", int64()),
@@ -45,14 +86,77 @@ const std::shared_ptr<Schema>& HistoryTable::schema() const {
4586
return schema;
4687
}
4788

48-
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(std::shared_ptr<Table> table) {
89+
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(
90+
std::shared_ptr<const Table> table) {
4991
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
5092
return std::unique_ptr<HistoryTable>(new HistoryTable(std::move(table)));
5193
}
5294

53-
Result<ArrowArrayStream> HistoryTable::ScanProjection(
54-
const Schema& /*projected_schema*/) const {
55-
return NotSupported("Scan is not supported for the history table");
95+
Result<std::unique_ptr<MetadataScan>> HistoryTable::MakeScan(
96+
internal::MetadataScanContext context) const {
97+
return HistoryScan::Make(std::move(context));
98+
}
99+
100+
Result<ArrowArrayStream> HistoryTable::ReadRows(
101+
std::shared_ptr<const TableMetadata> metadata,
102+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) {
103+
std::unordered_map<int64_t, const Snapshot*> snapshots_by_id;
104+
for (const auto& snapshot : metadata->snapshots) {
105+
if (snapshot) {
106+
snapshots_by_id.emplace(snapshot->snapshot_id, snapshot.get());
107+
}
108+
}
109+
110+
// Walk the current snapshot's ancestry; the insert check stops on cycles.
111+
std::unordered_set<int64_t> ancestors;
112+
for (auto current = metadata->current_snapshot_id;
113+
current != kInvalidSnapshotId && ancestors.insert(current).second;) {
114+
auto it = snapshots_by_id.find(current);
115+
if (it == snapshots_by_id.end() || !it->second->parent_snapshot_id) {
116+
break;
117+
}
118+
current = *it->second->parent_snapshot_id;
119+
}
120+
121+
const size_t num_rows = metadata->snapshot_log.size();
122+
return internal::MakeMetadataBatchStream<HistoryRow>(
123+
CanonicalSchema(), projected_schema, kHistoryColumns, num_rows, batch_size,
124+
[metadata = std::move(metadata), snapshots_by_id = std::move(snapshots_by_id),
125+
ancestors = std::move(ancestors)](size_t index) -> HistoryRow {
126+
const auto& entry = metadata->snapshot_log[index];
127+
auto it = snapshots_by_id.find(entry.snapshot_id);
128+
return {.entry = entry,
129+
.snapshot = it != snapshots_by_id.end() ? it->second : nullptr,
130+
.is_current_ancestor = ancestors.contains(entry.snapshot_id)};
131+
});
132+
}
133+
134+
// HistoryScan implementation
135+
HistoryScan::HistoryScan(internal::MetadataScanContext context,
136+
std::shared_ptr<MetadataScanTask> task)
137+
: MetadataScan(std::move(context)), task_(std::move(task)) {}
138+
139+
HistoryScan::~HistoryScan() = default;
140+
141+
Result<std::unique_ptr<HistoryScan>> HistoryScan::Make(
142+
internal::MetadataScanContext context) {
143+
ICEBERG_PRECHECK(context.kind == MetadataTableKind::kHistory,
144+
"Unexpected metadata table kind for history scan");
145+
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
146+
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
147+
ICEBERG_PRECHECK(context.projected_schema != nullptr,
148+
"Projected schema cannot be null");
149+
ICEBERG_RETURN_UNEXPECTED(internal::SelectAppendFns<HistoryRow>(
150+
*HistoryTable::CanonicalSchema(), *context.projected_schema, kHistoryColumns));
151+
ICEBERG_ASSIGN_OR_RAISE(auto task,
152+
StaticMetadataTask::Make(context.kind, context.table_metadata,
153+
context.projected_schema));
154+
return std::unique_ptr<HistoryScan>(
155+
new HistoryScan(std::move(context), std::move(task)));
156+
}
157+
158+
Result<MetadataScanTaskStreamPtr> HistoryScan::PlanTasksStream() const {
159+
return MetadataScanTaskStream::FromVector({task_});
56160
}
57161

58162
} // namespace iceberg

‎src/iceberg/inspect/history_table.h‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,12 @@
2222
/// \file iceberg/inspect/history_table.h
2323
/// \brief Define the history metadata table.
2424

25+
#include <cstdint>
2526
#include <memory>
2627

28+
#include "iceberg/arrow_c_data.h"
2729
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_scan.h"
2831
#include "iceberg/inspect/metadata_table.h"
2932
#include "iceberg/result.h"
3033
#include "iceberg/type_fwd.h"
@@ -34,19 +37,46 @@ namespace iceberg {
3437
/// \brief History metadata table.
3538
class ICEBERG_EXPORT HistoryTable : public MetadataTable {
3639
public:
37-
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<Table> table);
40+
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<const Table> table);
41+
42+
/// \brief Return the canonical schema of the history metadata table.
43+
static const std::shared_ptr<Schema>& CanonicalSchema();
3844

3945
~HistoryTable() override;
4046

41-
Kind kind() const noexcept override { return Kind::kHistory; }
47+
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }
4248

4349
const std::shared_ptr<Schema>& schema() const override;
4450

4551
protected:
46-
Result<ArrowArrayStream> ScanProjection(const Schema& projected_schema) const override;
52+
Result<std::unique_ptr<MetadataScan>> MakeScan(
53+
internal::MetadataScanContext context) const override;
4754

4855
private:
49-
explicit HistoryTable(std::shared_ptr<Table> table);
56+
friend class MetadataScanTaskReader;
57+
58+
static Result<ArrowArrayStream> ReadRows(
59+
std::shared_ptr<const TableMetadata> metadata,
60+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size);
61+
62+
explicit HistoryTable(std::shared_ptr<const Table> table);
63+
};
64+
65+
/// \brief Scan the history metadata captured from a table.
66+
class ICEBERG_EXPORT HistoryScan final : public MetadataScan {
67+
public:
68+
/// \brief Create a scan from captured metadata table settings.
69+
static Result<std::unique_ptr<HistoryScan>> Make(internal::MetadataScanContext context);
70+
71+
~HistoryScan() override;
72+
73+
Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override;
74+
75+
private:
76+
HistoryScan(internal::MetadataScanContext context,
77+
std::shared_ptr<MetadataScanTask> task);
78+
79+
const std::shared_ptr<MetadataScanTask> task_;
5080
};
5181

5282
} // namespace iceberg

0 commit comments

Comments
 (0)