Skip to content

Commit 49b84a0

Browse files
authored
feat(inspect): add task-based scans for snapshots and history (#1066)
- Add Table::Inspect() and task-based metadata scans while retaining column selection, projection, case sensitivity, and time-travel builder configuration from the merged inspection API. - Separate task planning from reading with reusable static tasks backed by captured table metadata. - Preserve scan configuration and metadata across builder changes and table refreshes. - Produce Arrow batches lazily through MetadataScanTaskReader, with reader-controlled batch sizes and independent stream lifetimes. - Keep canonical schemas and row conversion in each metadata table implementation, sharing batch construction and projection validation. - Consolidate projected-column append helpers with lazy batch construction. - Preserve all merged metadata inspection test scenarios and cover planned projections, task reuse, stream lifetimes, batching, and vector<bool>.
1 parent d5970d9 commit 49b84a0

25 files changed

Lines changed: 1342 additions & 487 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ set(ICEBERG_SOURCES
6161
inspect/history_table.cc
6262
inspect/metadata_scan.cc
6363
inspect/metadata_table.cc
64+
inspect/metadata_scan_task.cc
65+
inspect/metadata_scan_task_reader.cc
6466
inspect/snapshots_table.cc
6567
inheritable_metadata.cc
6668
json_serde.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@
8282
#include "iceberg/inheritable_metadata.h"
8383
#include "iceberg/inspect/history_table.h"
8484
#include "iceberg/inspect/metadata_scan.h"
85+
#include "iceberg/inspect/metadata_scan_task.h"
86+
#include "iceberg/inspect/metadata_scan_task_reader.h"
8587
#include "iceberg/inspect/metadata_table.h"
8688
#include "iceberg/inspect/snapshots_table.h"
8789
#include "iceberg/labels.h"

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 120 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,67 @@
1919

2020
#include "iceberg/inspect/history_table.h"
2121

22+
#include <array>
2223
#include <memory>
24+
#include <unordered_map>
25+
#include <unordered_set>
2326
#include <utility>
24-
#include <vector>
2527

28+
#include "iceberg/arrow_row_builder_internal.h"
29+
#include "iceberg/constants.h"
30+
#include "iceberg/inspect/metadata_scan_task.h"
31+
#include "iceberg/inspect/metadata_scan_util_internal.h"
2632
#include "iceberg/schema.h"
2733
#include "iceberg/schema_field.h"
34+
#include "iceberg/snapshot.h"
2835
#include "iceberg/table.h"
36+
#include "iceberg/table_metadata.h"
2937
#include "iceberg/type.h"
3038
#include "iceberg/util/macros.h"
3139

3240
namespace iceberg {
41+
namespace {
3342

34-
HistoryTable::HistoryTable(std::shared_ptr<Table> table)
43+
struct HistoryRow {
44+
const SnapshotLogEntry& entry;
45+
const Snapshot* snapshot;
46+
bool is_current_ancestor;
47+
};
48+
49+
constexpr std::array<internal::MetadataTableColumn<HistoryRow>, 4> kHistoryColumns = {
50+
{{.field_id = 1,
51+
.append =
52+
[](ArrowArray* column, const HistoryRow& row) {
53+
return AppendInt(column,
54+
UnixMsFromTimePointMs(row.entry.timestamp_ms) * 1000);
55+
}},
56+
{.field_id = 2,
57+
.append =
58+
[](ArrowArray* column, const HistoryRow& row) {
59+
return AppendInt(column, row.entry.snapshot_id);
60+
}},
61+
{.field_id = 3,
62+
.append =
63+
[](ArrowArray* column, const HistoryRow& row) {
64+
return row.snapshot != nullptr && row.snapshot->parent_snapshot_id
65+
? AppendInt(column, *row.snapshot->parent_snapshot_id)
66+
: AppendNull(column);
67+
}},
68+
{.field_id = 4, .append = [](ArrowArray* column, const HistoryRow& row) {
69+
return AppendBoolean(column, row.is_current_ancestor);
70+
}}}};
71+
72+
} // namespace
73+
74+
// HistoryTable implementation
75+
HistoryTable::HistoryTable(std::shared_ptr<const Table> table)
3576
: MetadataTable(std::move(table)) {}
3677

3778
HistoryTable::~HistoryTable() = default;
3879

39-
const std::shared_ptr<Schema>& HistoryTable::schema() const {
80+
const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
81+
82+
const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
4083
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
4184
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
4285
SchemaField::MakeRequired(2, "snapshot_id", int64()),
@@ -45,14 +88,84 @@ const std::shared_ptr<Schema>& HistoryTable::schema() const {
4588
return schema;
4689
}
4790

48-
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(std::shared_ptr<Table> table) {
91+
Result<std::unique_ptr<HistoryTable>> HistoryTable::Make(
92+
std::shared_ptr<const Table> table) {
4993
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
5094
return std::unique_ptr<HistoryTable>(new HistoryTable(std::move(table)));
5195
}
5296

53-
Result<ArrowArrayStream> HistoryTable::ScanProjection(
54-
const Schema& /*projected_schema*/) const {
55-
return NotSupported("Scan is not supported for the history table");
97+
Result<std::unique_ptr<MetadataScan>> HistoryTable::MakeScan(
98+
internal::MetadataScanContext context) const {
99+
return HistoryScan::Make(std::move(context));
100+
}
101+
102+
Result<ArrowArrayStream> HistoryTable::ReadRows(
103+
std::shared_ptr<const TableMetadata> metadata,
104+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) {
105+
std::unordered_map<int64_t, const Snapshot*> snapshots_by_id;
106+
for (const auto& snapshot : metadata->snapshots) {
107+
if (snapshot) {
108+
snapshots_by_id.emplace(snapshot->snapshot_id, snapshot.get());
109+
}
110+
}
111+
112+
// Walk the current snapshot's ancestry; the insert check stops on cycles.
113+
std::unordered_set<int64_t> ancestors;
114+
for (auto current = metadata->current_snapshot_id; current != kInvalidSnapshotId;) {
115+
auto it = snapshots_by_id.find(current);
116+
// Stop at a missing snapshot without adding its ID to the ancestors.
117+
if (it == snapshots_by_id.end() || !ancestors.insert(current).second) {
118+
break;
119+
}
120+
if (!it->second->parent_snapshot_id) {
121+
break;
122+
}
123+
current = *it->second->parent_snapshot_id;
124+
}
125+
126+
const size_t num_rows = metadata->snapshot_log.size();
127+
return internal::MakeMetadataBatchStream<HistoryRow>(
128+
CanonicalSchema(), projected_schema, kHistoryColumns, num_rows, batch_size,
129+
[metadata = std::move(metadata), snapshots_by_id = std::move(snapshots_by_id),
130+
ancestors = std::move(ancestors)](size_t index) -> HistoryRow {
131+
const auto& entry = metadata->snapshot_log[index];
132+
auto it = snapshots_by_id.find(entry.snapshot_id);
133+
return {.entry = entry,
134+
.snapshot = it != snapshots_by_id.end() ? it->second : nullptr,
135+
.is_current_ancestor = ancestors.contains(entry.snapshot_id)};
136+
});
137+
}
138+
139+
// HistoryScan implementation
140+
HistoryScan::HistoryScan(internal::MetadataScanContext context,
141+
std::shared_ptr<MetadataScanTask> task)
142+
: MetadataScan(std::move(context)), task_(std::move(task)) {}
143+
144+
HistoryScan::~HistoryScan() = default;
145+
146+
Result<std::unique_ptr<HistoryScan>> HistoryScan::Make(
147+
internal::MetadataScanContext context) {
148+
ICEBERG_PRECHECK(context.kind == MetadataTableKind::kHistory,
149+
"Unexpected metadata table kind for history scan");
150+
if (context.snapshot_id.has_value() || !context.ref_name.empty() ||
151+
context.as_of_time_in_millis.has_value()) {
152+
return NotSupported("Metadata table does not support time travel");
153+
}
154+
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
155+
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
156+
ICEBERG_PRECHECK(context.projected_schema != nullptr,
157+
"Projected schema cannot be null");
158+
ICEBERG_RETURN_UNEXPECTED(internal::SelectAppendFns<HistoryRow>(
159+
*HistoryTable::CanonicalSchema(), *context.projected_schema, kHistoryColumns));
160+
ICEBERG_ASSIGN_OR_RAISE(auto task,
161+
StaticMetadataTask::Make(context.kind, context.table_metadata,
162+
context.projected_schema));
163+
return std::unique_ptr<HistoryScan>(
164+
new HistoryScan(std::move(context), std::move(task)));
165+
}
166+
167+
Result<MetadataScanTaskStreamPtr> HistoryScan::PlanTasksStream() const {
168+
return MetadataScanTaskStream::FromVector({task_});
56169
}
57170

58171
} // namespace iceberg

‎src/iceberg/inspect/history_table.h‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,12 @@
2222
/// \file iceberg/inspect/history_table.h
2323
/// \brief Define the history metadata table.
2424

25+
#include <cstdint>
2526
#include <memory>
2627

28+
#include "iceberg/arrow_c_data.h"
2729
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_scan.h"
2831
#include "iceberg/inspect/metadata_table.h"
2932
#include "iceberg/result.h"
3033
#include "iceberg/type_fwd.h"
@@ -34,19 +37,46 @@ namespace iceberg {
3437
/// \brief History metadata table.
3538
class ICEBERG_EXPORT HistoryTable : public MetadataTable {
3639
public:
37-
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<Table> table);
40+
static Result<std::unique_ptr<HistoryTable>> Make(std::shared_ptr<const Table> table);
41+
42+
/// \brief Return the canonical schema of the history metadata table.
43+
static const std::shared_ptr<Schema>& CanonicalSchema();
3844

3945
~HistoryTable() override;
4046

41-
Kind kind() const noexcept override { return Kind::kHistory; }
47+
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }
4248

4349
const std::shared_ptr<Schema>& schema() const override;
4450

4551
protected:
46-
Result<ArrowArrayStream> ScanProjection(const Schema& projected_schema) const override;
52+
Result<std::unique_ptr<MetadataScan>> MakeScan(
53+
internal::MetadataScanContext context) const override;
4754

4855
private:
49-
explicit HistoryTable(std::shared_ptr<Table> table);
56+
friend class MetadataScanTaskReader;
57+
58+
static Result<ArrowArrayStream> ReadRows(
59+
std::shared_ptr<const TableMetadata> metadata,
60+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size);
61+
62+
explicit HistoryTable(std::shared_ptr<const Table> table);
63+
};
64+
65+
/// \brief Scan the history metadata captured from a table.
66+
class ICEBERG_EXPORT HistoryScan final : public MetadataScan {
67+
public:
68+
/// \brief Create a scan from captured metadata table settings.
69+
static Result<std::unique_ptr<HistoryScan>> Make(internal::MetadataScanContext context);
70+
71+
~HistoryScan() override;
72+
73+
Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override;
74+
75+
private:
76+
HistoryScan(internal::MetadataScanContext context,
77+
std::shared_ptr<MetadataScanTask> task);
78+
79+
const std::shared_ptr<MetadataScanTask> task_;
5080
};
5181

5282
} // namespace iceberg

0 commit comments

Comments
 (0)