Skip to content

Commit e5a8a09

Browse files
manuzhangclaude
andcommitted
feat(inspect): scan metadata tables through data tasks
Metadata tables are now read through a scan. MetadataTable::NewScan() returns a builder that selects columns and, for tables that support time travel, a snapshot. The built scan plans DataTasks whose rows are StructLike rather than Arrow arrays. ReadDataTasks() converts the rows of any data task to an ArrowArrayStream, and ReadMetadataTable() reads a whole metadata table scan that way. MetadataTable::Scan() and kBatchSize are removed, so the metadata table interface no longer uses Arrow types. SnapshotsTable moves to the new interface, and HistoryTable gains a scan that returns one row per snapshot log entry. TimeTravelMetadataTable and SnapshotSelection are removed; a table that supports time travel overrides supports_time_travel() instead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0bdf7ce commit e5a8a09

23 files changed

Lines changed: 1620 additions & 183 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ set(ICEBERG_SOURCES
2727
catalog/session_catalog.cc
2828
catalog/session_context.cc
2929
data_table_scan.cc
30+
data_task_reader.cc
3031
delete_file_index.cc
3132
deletes/dv_util.cc
3233
deletes/dv_writer.cc
@@ -59,7 +60,10 @@ set(ICEBERG_SOURCES
5960
incremental_changelog_scan.cc
6061
inspect/history_table.cc
6162
inspect/metadata_table.cc
63+
inspect/metadata_table_reader.cc
64+
inspect/metadata_table_scan.cc
6265
inspect/snapshots_table.cc
66+
inspect/static_data_task_internal.cc
6367
inheritable_metadata.cc
6468
json_serde.cc
6569
location_provider.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#include "iceberg/data/position_delete_writer.h"
4444
#include "iceberg/data/writer.h"
4545
#include "iceberg/data_table_scan.h"
46+
#include "iceberg/data_task_reader.h"
4647
#include "iceberg/delete_file_index.h"
4748
#include "iceberg/deletes/dv_writer.h"
4849
#include "iceberg/deletes/position_delete_index.h"
@@ -81,6 +82,8 @@
8182
#include "iceberg/inheritable_metadata.h"
8283
#include "iceberg/inspect/history_table.h"
8384
#include "iceberg/inspect/metadata_table.h"
85+
#include "iceberg/inspect/metadata_table_reader.h"
86+
#include "iceberg/inspect/metadata_table_scan.h"
8487
#include "iceberg/inspect/snapshots_table.h"
8588
#include "iceberg/labels.h"
8689
#include "iceberg/location_provider.h"

‎src/iceberg/data_task_reader.cc‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/data_task_reader.h"
21+
22+
#include <cstddef>
23+
#include <cstdint>
24+
#include <memory>
25+
#include <optional>
26+
#include <span>
27+
#include <string_view>
28+
#include <tuple>
29+
#include <type_traits>
30+
#include <utility>
31+
#include <variant>
32+
#include <vector>
33+
34+
#include <nanoarrow/nanoarrow.h>
35+
36+
#include "iceberg/arrow_c_data_util_internal.h"
37+
#include "iceberg/arrow_row_builder_internal.h"
38+
#include "iceberg/nanoarrow_status_internal.h"
39+
#include "iceberg/row/struct_like.h"
40+
#include "iceberg/schema.h"
41+
#include "iceberg/schema_internal.h"
42+
#include "iceberg/table_scan.h"
43+
#include "iceberg/util/decimal.h"
44+
#include "iceberg/util/macros.h"
45+
46+
namespace iceberg {
47+
namespace {
48+
49+
Status AppendScalar(ArrowArray* array, const Scalar& value);
50+
51+
Status AppendStruct(ArrowArray* array, const StructLike& value) {
52+
if (std::cmp_not_equal(value.num_fields(), array->n_children)) {
53+
return InvalidArgument("Cannot append a struct with {} fields to {} Arrow children",
54+
value.num_fields(), array->n_children);
55+
}
56+
for (int64_t i = 0; i < array->n_children; ++i) {
57+
ICEBERG_ASSIGN_OR_RAISE(auto field, value.GetField(static_cast<size_t>(i)));
58+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[i], field));
59+
}
60+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
61+
return {};
62+
}
63+
64+
Status AppendList(ArrowArray* array, const ArrayLike& value) {
65+
if (array->n_children != 1) {
66+
return InvalidArrowData("List array must have exactly 1 child");
67+
}
68+
for (size_t i = 0; i < value.size(); ++i) {
69+
ICEBERG_ASSIGN_OR_RAISE(auto element, value.GetElement(i));
70+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[0], element));
71+
}
72+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
73+
return {};
74+
}
75+
76+
Status AppendMap(ArrowArray* array, const MapLike& value) {
77+
// A map array is a list of struct<key, value>.
78+
if (array->n_children != 1 || array->children[0]->n_children != 2) {
79+
return InvalidArrowData("Map array must have one entries child with 2 children");
80+
}
81+
ArrowArray* entries = array->children[0];
82+
for (size_t i = 0; i < value.size(); ++i) {
83+
ICEBERG_ASSIGN_OR_RAISE(auto key, value.GetKey(i));
84+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[0], key));
85+
ICEBERG_ASSIGN_OR_RAISE(auto item, value.GetValue(i));
86+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[1], item));
87+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(entries));
88+
}
89+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
90+
return {};
91+
}
92+
93+
// The Arrow array built from the schema decides the physical encoding, so the scalar
94+
// alternative alone selects the append call. For example, int64_t serves long, time,
95+
// and timestamp columns, and std::string_view serves string, binary, fixed, and uuid.
96+
Status AppendScalar(ArrowArray* array, const Scalar& value) {
97+
return std::visit(
98+
[array]<typename T>(const T& v) -> Status {
99+
if constexpr (std::is_same_v<T, std::monostate>) {
100+
return AppendNull(array);
101+
} else if constexpr (std::is_same_v<T, bool>) {
102+
return AppendBoolean(array, v);
103+
} else if constexpr (std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t>) {
104+
return AppendInt(array, v);
105+
} else if constexpr (std::is_same_v<T, float> || std::is_same_v<T, double>) {
106+
return AppendDouble(array, v);
107+
} else if constexpr (std::is_same_v<T, std::string_view>) {
108+
return AppendBytes(array,
109+
std::span<const uint8_t>(
110+
reinterpret_cast<const uint8_t*>(v.data()), // NOLINT
111+
v.size()));
112+
} else if constexpr (std::is_same_v<T, Decimal>) {
113+
return AppendDecimal(array, v);
114+
} else if constexpr (std::is_same_v<T, std::shared_ptr<StructLike>>) {
115+
return v == nullptr ? AppendNull(array) : AppendStruct(array, *v);
116+
} else if constexpr (std::is_same_v<T, std::shared_ptr<ArrayLike>>) {
117+
return v == nullptr ? AppendNull(array) : AppendList(array, *v);
118+
} else {
119+
static_assert(std::is_same_v<T, std::shared_ptr<MapLike>>);
120+
return v == nullptr ? AppendNull(array) : AppendMap(array, *v);
121+
}
122+
},
123+
value);
124+
}
125+
126+
Status AppendRow(ArrowRowBuilder& builder, const StructLike& row) {
127+
if (std::cmp_not_equal(row.num_fields(), builder.num_columns())) {
128+
return InvalidArgument("Data task row has {} fields but the schema has {} columns",
129+
row.num_fields(), builder.num_columns());
130+
}
131+
for (int64_t i = 0; i < builder.num_columns(); ++i) {
132+
ICEBERG_ASSIGN_OR_RAISE(auto field, row.GetField(static_cast<size_t>(i)));
133+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(builder.column(i), field));
134+
}
135+
return builder.FinishRow();
136+
}
137+
138+
class DataTaskArrowSource {
139+
public:
140+
static Result<std::unique_ptr<DataTaskArrowSource>> Make(
141+
const iceberg::Schema& schema, std::vector<std::shared_ptr<DataTask>> tasks,
142+
int64_t batch_size) {
143+
ArrowSchema arrow_schema{};
144+
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(schema, &arrow_schema));
145+
return std::unique_ptr<DataTaskArrowSource>(
146+
new DataTaskArrowSource(std::move(tasks), arrow_schema, batch_size));
147+
}
148+
149+
~DataTaskArrowSource() { std::ignore = Close(); }
150+
151+
Status Close() {
152+
rows_.reset();
153+
tasks_.clear();
154+
if (arrow_schema_.release != nullptr) {
155+
ArrowSchemaRelease(&arrow_schema_);
156+
}
157+
return {};
158+
}
159+
160+
Result<std::optional<ArrowArray>> Next() {
161+
if (arrow_schema_.release == nullptr) [[unlikely]] {
162+
return InvalidArgument("Cannot read from a closed data task stream");
163+
}
164+
165+
// Fetch a row before allocating a builder, so an exhausted stream allocates none.
166+
ICEBERG_ASSIGN_OR_RAISE(auto row, NextRow());
167+
if (row == nullptr) {
168+
return std::nullopt;
169+
}
170+
171+
ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(&arrow_schema_));
172+
while (true) {
173+
ICEBERG_RETURN_UNEXPECTED(AppendRow(builder, *row));
174+
if (builder.num_rows() == batch_size_) {
175+
break;
176+
}
177+
ICEBERG_ASSIGN_OR_RAISE(row, NextRow());
178+
if (row == nullptr) {
179+
break;
180+
}
181+
}
182+
183+
ICEBERG_ASSIGN_OR_RAISE(auto array, std::move(builder).Finish());
184+
return array;
185+
}
186+
187+
Result<ArrowSchema> Schema() {
188+
if (arrow_schema_.release == nullptr) [[unlikely]] {
189+
return InvalidArgument("Cannot read schema from a closed data task stream");
190+
}
191+
ArrowSchema schema_copy{};
192+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(
193+
ArrowSchemaDeepCopy(&arrow_schema_, &schema_copy));
194+
return schema_copy;
195+
}
196+
197+
private:
198+
DataTaskArrowSource(std::vector<std::shared_ptr<DataTask>> tasks,
199+
ArrowSchema arrow_schema, int64_t batch_size)
200+
: tasks_(std::move(tasks)), arrow_schema_(arrow_schema), batch_size_(batch_size) {}
201+
202+
// Return the next row across all tasks, or nullptr once every task is exhausted.
203+
Result<std::shared_ptr<StructLike>> NextRow() {
204+
while (true) {
205+
if (rows_ == nullptr) {
206+
if (next_task_ == tasks_.size()) {
207+
return nullptr;
208+
}
209+
ICEBERG_ASSIGN_OR_RAISE(rows_, tasks_[next_task_++]->Rows());
210+
}
211+
ICEBERG_ASSIGN_OR_RAISE(auto row, rows_->Next());
212+
if (!row.has_value()) {
213+
rows_.reset();
214+
continue;
215+
}
216+
if (*row == nullptr) [[unlikely]] {
217+
return InvalidArgument("Data task returned a null row");
218+
}
219+
return std::move(*row);
220+
}
221+
}
222+
223+
std::vector<std::shared_ptr<DataTask>> tasks_;
224+
size_t next_task_ = 0;
225+
std::unique_ptr<StructLikeStream> rows_;
226+
ArrowSchema arrow_schema_{};
227+
int64_t batch_size_;
228+
};
229+
230+
} // namespace
231+
232+
Result<ArrowArrayStream> ReadDataTasks(std::shared_ptr<Schema> schema,
233+
std::vector<std::shared_ptr<DataTask>> tasks,
234+
int64_t batch_size) {
235+
ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null");
236+
ICEBERG_PRECHECK(batch_size > 0, "Batch size must be positive, got {}", batch_size);
237+
for (const auto& task : tasks) {
238+
ICEBERG_PRECHECK(task != nullptr, "Data task cannot be null");
239+
ICEBERG_PRECHECK(task->schema() != nullptr, "Data task schema cannot be null");
240+
ICEBERG_PRECHECK(task->schema() == schema || *task->schema() == *schema,
241+
"Data task schema does not match the requested schema");
242+
}
243+
244+
ICEBERG_ASSIGN_OR_RAISE(
245+
auto source, DataTaskArrowSource::Make(*schema, std::move(tasks), batch_size));
246+
return MakeArrowArrayStream(std::move(source));
247+
}
248+
249+
} // namespace iceberg

‎src/iceberg/data_task_reader.h‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/data_task_reader.h
23+
/// \brief Read DataTask rows as Arrow C Data batches.
24+
25+
#include <cstdint>
26+
#include <memory>
27+
#include <vector>
28+
29+
#include "iceberg/arrow_c_data.h"
30+
#include "iceberg/iceberg_export.h"
31+
#include "iceberg/result.h"
32+
#include "iceberg/type_fwd.h"
33+
34+
namespace iceberg {
35+
36+
/// \brief Default maximum number of rows in each batch read from data tasks.
37+
inline constexpr int64_t kDefaultDataTaskBatchSize = 1024;
38+
39+
/// \brief Read the rows of data tasks as one Arrow stream.
40+
///
41+
/// Tasks are read in order and a batch may hold rows from more than one task, so a
42+
/// caller that wants to read tasks in parallel passes one task per call.
43+
///
44+
/// \param schema Schema of the returned stream. Every task must return rows in it.
45+
/// \param tasks Tasks to read. The stream keeps them alive until it is released.
46+
/// \param batch_size Maximum number of rows in each batch. Must be positive.
47+
/// \return A stream that the caller owns and must release by invoking its `release`
48+
/// callback, or an error.
49+
ICEBERG_EXPORT Result<ArrowArrayStream> ReadDataTasks(
50+
std::shared_ptr<Schema> schema, std::vector<std::shared_ptr<DataTask>> tasks,
51+
int64_t batch_size = kDefaultDataTaskBatchSize);
52+
53+
} // namespace iceberg

0 commit comments

Comments
 (0)