Skip to content

Commit eb929a7

Browse files
authored
feat(inspect): add refs metadata table (#1007)
1 parent 36e0d1a commit eb929a7

15 files changed

Lines changed: 559 additions & 10 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ set(ICEBERG_SOURCES
6363
inspect/metadata_table.cc
6464
inspect/metadata_scan_task.cc
6565
inspect/metadata_scan_task_reader.cc
66+
inspect/refs_table.cc
6667
inspect/snapshots_table.cc
6768
inheritable_metadata.cc
6869
json_serde.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
#include "iceberg/inspect/metadata_scan_task.h"
8686
#include "iceberg/inspect/metadata_scan_task_reader.h"
8787
#include "iceberg/inspect/metadata_table.h"
88+
#include "iceberg/inspect/refs_table.h"
8889
#include "iceberg/inspect/snapshots_table.h"
8990
#include "iceberg/labels.h"
9091
#include "iceberg/location_provider.h"

‎src/iceberg/inspect/metadata_scan_task.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ Result<std::shared_ptr<StaticMetadataTask>> StaticMetadataTask::Make(
3434
ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null");
3535
ICEBERG_PRECHECK(projected_schema != nullptr, "Projected schema cannot be null");
3636
if (table_kind != MetadataTableKind::kSnapshots &&
37-
table_kind != MetadataTableKind::kHistory) {
37+
table_kind != MetadataTableKind::kHistory &&
38+
table_kind != MetadataTableKind::kRefs) {
3839
return NotImplemented("Static metadata table kind is not implemented");
3940
}
4041
return std::shared_ptr<StaticMetadataTask>(new StaticMetadataTask(

‎src/iceberg/inspect/metadata_scan_task.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class ICEBERG_EXPORT MetadataScanTask {
3535
public:
3636
/// \brief Task category, determines how the reader opens the task.
3737
enum class TaskKind {
38-
/// StaticMetadataTask reads in-memory table metadata (snapshots, history).
38+
/// StaticMetadataTask reads in-memory table metadata (snapshots, history, refs).
3939
kStatic,
4040
};
4141

@@ -48,7 +48,7 @@ class ICEBERG_EXPORT MetadataScanTask {
4848
virtual MetadataTableKind table_kind() const = 0;
4949
};
5050

51-
/// \brief Task for static metadata tables (snapshots, history).
51+
/// \brief Task for static metadata tables (snapshots, history, refs).
5252
///
5353
/// Holds the metadata captured for a table scan; readers produce Arrow batches
5454
/// on demand. The metadata and its nested objects must not be modified while
@@ -57,7 +57,7 @@ class ICEBERG_EXPORT StaticMetadataTask : public MetadataScanTask {
5757
public:
5858
/// \brief Create a static metadata task.
5959
///
60-
/// \param table_kind The metadata table kind (kSnapshots or kHistory)
60+
/// \param table_kind The metadata table kind (kSnapshots, kHistory, or kRefs)
6161
/// \param metadata The table metadata to scan
6262
/// \param projected_schema The planned output schema
6363
/// \return A task, or an error for null metadata or an unsupported table kind.

‎src/iceberg/inspect/metadata_scan_task_reader.cc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "iceberg/inspect/history_table.h"
2626
#include "iceberg/inspect/metadata_scan_task.h"
2727
#include "iceberg/inspect/metadata_table.h"
28+
#include "iceberg/inspect/refs_table.h"
2829
#include "iceberg/inspect/snapshots_table.h"
2930
#include "iceberg/util/checked_cast.h"
3031
#include "iceberg/util/macros.h"
@@ -53,6 +54,9 @@ Result<ArrowArrayStream> MetadataScanTaskReader::Open(
5354
return SnapshotsTable::ReadRows(static_task.metadata(),
5455
static_task.projected_schema(),
5556
options_.batch_size);
57+
case MetadataTableKind::kRefs:
58+
return RefsTable::ReadRows(static_task.metadata(),
59+
static_task.projected_schema(), options_.batch_size);
5660
case MetadataTableKind::kHistory:
5761
return HistoryTable::ReadRows(static_task.metadata(),
5862
static_task.projected_schema(),

‎src/iceberg/inspect/metadata_table.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
#include "iceberg/inspect/history_table.h"
2626
#include "iceberg/inspect/metadata_scan.h"
27+
#include "iceberg/inspect/refs_table.h"
2728
#include "iceberg/inspect/snapshots_table.h"
2829

2930
namespace iceberg {
@@ -38,6 +39,8 @@ Result<std::unique_ptr<MetadataTable>> MetadataTable::Make(
3839
switch (kind) {
3940
case MetadataTableKind::kSnapshots:
4041
return SnapshotsTable::Make(std::move(table));
42+
case MetadataTableKind::kRefs:
43+
return RefsTable::Make(std::move(table));
4144
case MetadataTableKind::kHistory:
4245
return HistoryTable::Make(std::move(table));
4346
default:

‎src/iceberg/inspect/refs_table.cc‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/inspect/refs_table.h"
21+
22+
#include <algorithm>
23+
#include <array>
24+
#include <cstdint>
25+
#include <memory>
26+
#include <optional>
27+
#include <string>
28+
#include <utility>
29+
#include <variant>
30+
#include <vector>
31+
32+
#include "iceberg/arrow_row_builder_internal.h"
33+
#include "iceberg/inspect/metadata_scan_task.h"
34+
#include "iceberg/inspect/metadata_scan_util_internal.h"
35+
#include "iceberg/schema.h"
36+
#include "iceberg/schema_field.h"
37+
#include "iceberg/snapshot.h"
38+
#include "iceberg/table.h"
39+
#include "iceberg/table_metadata.h"
40+
#include "iceberg/type.h"
41+
#include "iceberg/util/macros.h"
42+
43+
namespace iceberg {
44+
namespace {
45+
46+
struct RefRow {
47+
std::string name;
48+
std::string type;
49+
int64_t snapshot_id;
50+
std::optional<int64_t> max_ref_age_ms;
51+
std::optional<int32_t> min_snapshots_to_keep;
52+
std::optional<int64_t> max_snapshot_age_ms;
53+
};
54+
55+
Status AppendOptional(ArrowArray* array, const auto& value) {
56+
if (!value.has_value()) {
57+
return AppendNull(array);
58+
}
59+
return AppendInt(array, static_cast<int64_t>(*value));
60+
}
61+
62+
constexpr std::array<internal::MetadataTableColumn<RefRow>, 6> kRefColumns = {
63+
{{.field_id = 1,
64+
.append = [](ArrowArray* column,
65+
const RefRow& row) { return AppendString(column, row.name); }},
66+
{.field_id = 2,
67+
.append = [](ArrowArray* column,
68+
const RefRow& row) { return AppendString(column, row.type); }},
69+
{.field_id = 3,
70+
.append = [](ArrowArray* column,
71+
const RefRow& row) { return AppendInt(column, row.snapshot_id); }},
72+
{.field_id = 4,
73+
.append =
74+
[](ArrowArray* column, const RefRow& row) {
75+
return AppendOptional(column, row.max_ref_age_ms);
76+
}},
77+
{.field_id = 5,
78+
.append =
79+
[](ArrowArray* column, const RefRow& row) {
80+
return AppendOptional(column, row.min_snapshots_to_keep);
81+
}},
82+
{.field_id = 6, .append = [](ArrowArray* column, const RefRow& row) {
83+
return AppendOptional(column, row.max_snapshot_age_ms);
84+
}}}};
85+
86+
} // namespace
87+
88+
// RefsTable implementation
89+
RefsTable::RefsTable(std::shared_ptr<const Table> table)
90+
: MetadataTable(std::move(table)) {}
91+
92+
RefsTable::~RefsTable() = default;
93+
94+
const std::shared_ptr<Schema>& RefsTable::schema() const { return CanonicalSchema(); }
95+
96+
const std::shared_ptr<Schema>& RefsTable::CanonicalSchema() {
97+
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
98+
SchemaField::MakeRequired(1, "name", string()),
99+
SchemaField::MakeRequired(2, "type", string()),
100+
SchemaField::MakeRequired(3, "snapshot_id", int64()),
101+
SchemaField::MakeOptional(4, "max_reference_age_in_ms", int64()),
102+
SchemaField::MakeOptional(5, "min_snapshots_to_keep", int32()),
103+
SchemaField::MakeOptional(6, "max_snapshot_age_in_ms", int64()),
104+
});
105+
return schema;
106+
}
107+
108+
Result<std::unique_ptr<RefsTable>> RefsTable::Make(std::shared_ptr<const Table> table) {
109+
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
110+
return std::unique_ptr<RefsTable>(new RefsTable(std::move(table)));
111+
}
112+
113+
Result<std::unique_ptr<MetadataScan>> RefsTable::MakeScan(
114+
internal::MetadataScanContext context) const {
115+
return RefsScan::Make(std::move(context));
116+
}
117+
118+
Result<ArrowArrayStream> RefsTable::ReadRows(
119+
std::shared_ptr<const TableMetadata> metadata,
120+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) {
121+
std::vector<RefRow> rows;
122+
const auto& refs = metadata->refs;
123+
rows.reserve(refs.size());
124+
for (const auto& [name, ref] : refs) {
125+
if (ref == nullptr) {
126+
return Invalid("Snapshot reference '{}' is null", name);
127+
}
128+
const auto* branch = std::get_if<SnapshotRef::Branch>(&ref->retention);
129+
RefRow row{.name = name,
130+
.type = branch != nullptr ? "BRANCH" : "TAG",
131+
.snapshot_id = ref->snapshot_id,
132+
.max_ref_age_ms =
133+
branch != nullptr
134+
? branch->max_ref_age_ms
135+
: std::get<SnapshotRef::Tag>(ref->retention).max_ref_age_ms};
136+
if (branch != nullptr) {
137+
row.min_snapshots_to_keep = branch->min_snapshots_to_keep;
138+
row.max_snapshot_age_ms = branch->max_snapshot_age_ms;
139+
}
140+
rows.push_back(std::move(row));
141+
}
142+
std::ranges::sort(rows, {}, &RefRow::name);
143+
const size_t num_rows = rows.size();
144+
return internal::MakeMetadataBatchStream<RefRow>(
145+
CanonicalSchema(), projected_schema, kRefColumns, num_rows, batch_size,
146+
[rows = std::move(rows)](size_t index) -> const RefRow& { return rows[index]; });
147+
}
148+
149+
// RefsScan implementation
150+
RefsScan::RefsScan(internal::MetadataScanContext context,
151+
std::shared_ptr<MetadataScanTask> task)
152+
: MetadataScan(std::move(context)), task_(std::move(task)) {}
153+
154+
RefsScan::~RefsScan() = default;
155+
156+
Result<std::unique_ptr<RefsScan>> RefsScan::Make(internal::MetadataScanContext context) {
157+
ICEBERG_PRECHECK(context.kind == MetadataTableKind::kRefs,
158+
"Unexpected metadata table kind for refs scan");
159+
if (context.snapshot_id.has_value() || !context.ref_name.empty() ||
160+
context.as_of_time_in_millis.has_value()) {
161+
return NotSupported("Metadata table does not support time travel");
162+
}
163+
ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null");
164+
ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null");
165+
ICEBERG_PRECHECK(context.projected_schema != nullptr,
166+
"Projected schema cannot be null");
167+
ICEBERG_RETURN_UNEXPECTED(internal::SelectAppendFns<RefRow>(
168+
*RefsTable::CanonicalSchema(), *context.projected_schema, kRefColumns));
169+
ICEBERG_ASSIGN_OR_RAISE(auto task,
170+
StaticMetadataTask::Make(context.kind, context.table_metadata,
171+
context.projected_schema));
172+
return std::unique_ptr<RefsScan>(new RefsScan(std::move(context), std::move(task)));
173+
}
174+
175+
Result<MetadataScanTaskStreamPtr> RefsScan::PlanTasksStream() const {
176+
return MetadataScanTaskStream::FromVector({task_});
177+
}
178+
179+
} // namespace iceberg

‎src/iceberg/inspect/refs_table.h‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/inspect/refs_table.h
23+
/// \brief Define the refs metadata table.
24+
25+
#include <cstdint>
26+
#include <memory>
27+
28+
#include "iceberg/arrow_c_data.h"
29+
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_scan.h"
31+
#include "iceberg/inspect/metadata_table.h"
32+
#include "iceberg/result.h"
33+
#include "iceberg/type_fwd.h"
34+
35+
namespace iceberg {
36+
37+
/// \brief Refs metadata table.
38+
class ICEBERG_EXPORT RefsTable : public MetadataTable {
39+
public:
40+
static Result<std::unique_ptr<RefsTable>> Make(std::shared_ptr<const Table> table);
41+
42+
/// \brief Return the canonical schema of the refs metadata table.
43+
static const std::shared_ptr<Schema>& CanonicalSchema();
44+
45+
~RefsTable() override;
46+
47+
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kRefs; }
48+
49+
const std::shared_ptr<Schema>& schema() const override;
50+
51+
protected:
52+
Result<std::unique_ptr<MetadataScan>> MakeScan(
53+
internal::MetadataScanContext context) const override;
54+
55+
private:
56+
friend class MetadataScanTaskReader;
57+
58+
static Result<ArrowArrayStream> ReadRows(
59+
std::shared_ptr<const TableMetadata> metadata,
60+
const std::shared_ptr<Schema>& projected_schema, int64_t batch_size);
61+
62+
explicit RefsTable(std::shared_ptr<const Table> table);
63+
};
64+
65+
/// \brief Scan the refs metadata captured from a table.
66+
class ICEBERG_EXPORT RefsScan final : public MetadataScan {
67+
public:
68+
/// \brief Create a scan from captured metadata table settings.
69+
static Result<std::unique_ptr<RefsScan>> Make(internal::MetadataScanContext context);
70+
71+
~RefsScan() override;
72+
73+
Result<MetadataScanTaskStreamPtr> PlanTasksStream() const override;
74+
75+
private:
76+
RefsScan(internal::MetadataScanContext context, std::shared_ptr<MetadataScanTask> task);
77+
78+
const std::shared_ptr<MetadataScanTask> task_;
79+
};
80+
81+
} // namespace iceberg

‎src/iceberg/test/CMakeLists.txt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -206,9 +206,10 @@ if(ICEBERG_BUILD_BUNDLE)
206206
add_iceberg_test(metadata_table_test
207207
USE_BUNDLE
208208
SOURCES
209-
history_table_test.cc
210-
metadata_table_test.cc
211-
snapshots_table_test.cc)
209+
inspect/history_table_test.cc
210+
inspect/metadata_table_test.cc
211+
inspect/refs_table_test.cc
212+
inspect/snapshots_table_test.cc)
212213

213214
add_iceberg_test(eval_expr_test
214215
USE_BUNDLE

src/iceberg/test/history_table_test.cc renamed to src/iceberg/test/inspect/history_table_test.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232
#include "iceberg/snapshot.h"
3333
#include "iceberg/table_metadata.h"
3434
#include "iceberg/test/arrow_test_util.h"
35+
#include "iceberg/test/inspect/metadata_table_test_base.h"
3536
#include "iceberg/test/matchers.h"
36-
#include "iceberg/test/metadata_table_test_base.h"
3737
#include "iceberg/type.h"
3838
#include "iceberg/util/timepoint.h"
3939

0 commit comments

Comments
 (0)