Skip to content

Commit eebab02

Browse files
manuzhangclaude
andcommitted
feat(inspect): scan metadata tables through data tasks
Metadata tables now follow Java's scan model. MetadataTable::NewScan() returns a builder that selects columns and, for tables that support time travel, a snapshot. The built scan plans DataTasks whose rows are StructLike rather than Arrow arrays, and ReadDataTasks() converts the rows of any data task to an ArrowArrayStream. Scan() still returns Arrow batches by running these steps with defaults. SnapshotsTable moves to the new interface, and HistoryTable gains a scan that returns one row per snapshot log entry. TimeTravelMetadataTable and SnapshotSelection are removed; a table that supports time travel overrides supports_time_travel() instead. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0bdf7ce commit eebab02

20 files changed

Lines changed: 1538 additions & 166 deletions

‎src/iceberg/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ set(ICEBERG_SOURCES
2727
catalog/session_catalog.cc
2828
catalog/session_context.cc
2929
data_table_scan.cc
30+
data_task_reader.cc
3031
delete_file_index.cc
3132
deletes/dv_util.cc
3233
deletes/dv_writer.cc
@@ -59,7 +60,9 @@ set(ICEBERG_SOURCES
5960
incremental_changelog_scan.cc
6061
inspect/history_table.cc
6162
inspect/metadata_table.cc
63+
inspect/metadata_table_scan.cc
6264
inspect/snapshots_table.cc
65+
inspect/static_data_task_internal.cc
6366
inheritable_metadata.cc
6467
json_serde.cc
6568
location_provider.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#include "iceberg/data/position_delete_writer.h"
4444
#include "iceberg/data/writer.h"
4545
#include "iceberg/data_table_scan.h"
46+
#include "iceberg/data_task_reader.h"
4647
#include "iceberg/delete_file_index.h"
4748
#include "iceberg/deletes/dv_writer.h"
4849
#include "iceberg/deletes/position_delete_index.h"
@@ -81,6 +82,7 @@
8182
#include "iceberg/inheritable_metadata.h"
8283
#include "iceberg/inspect/history_table.h"
8384
#include "iceberg/inspect/metadata_table.h"
85+
#include "iceberg/inspect/metadata_table_scan.h"
8486
#include "iceberg/inspect/snapshots_table.h"
8587
#include "iceberg/labels.h"
8688
#include "iceberg/location_provider.h"

‎src/iceberg/data_task_reader.cc‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/data_task_reader.h"
21+
22+
#include <cstddef>
23+
#include <cstdint>
24+
#include <memory>
25+
#include <optional>
26+
#include <span>
27+
#include <string_view>
28+
#include <tuple>
29+
#include <type_traits>
30+
#include <utility>
31+
#include <variant>
32+
#include <vector>
33+
34+
#include <nanoarrow/nanoarrow.h>
35+
36+
#include "iceberg/arrow_c_data_util_internal.h"
37+
#include "iceberg/arrow_row_builder_internal.h"
38+
#include "iceberg/nanoarrow_status_internal.h"
39+
#include "iceberg/row/struct_like.h"
40+
#include "iceberg/schema.h"
41+
#include "iceberg/schema_internal.h"
42+
#include "iceberg/table_scan.h"
43+
#include "iceberg/util/decimal.h"
44+
#include "iceberg/util/macros.h"
45+
46+
namespace iceberg {
47+
namespace {
48+
49+
Status AppendScalar(ArrowArray* array, const Scalar& value);
50+
51+
Status AppendStruct(ArrowArray* array, const StructLike& value) {
52+
if (std::cmp_not_equal(value.num_fields(), array->n_children)) {
53+
return InvalidArgument("Cannot append a struct with {} fields to {} Arrow children",
54+
value.num_fields(), array->n_children);
55+
}
56+
for (int64_t i = 0; i < array->n_children; ++i) {
57+
ICEBERG_ASSIGN_OR_RAISE(auto field, value.GetField(static_cast<size_t>(i)));
58+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[i], field));
59+
}
60+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
61+
return {};
62+
}
63+
64+
Status AppendList(ArrowArray* array, const ArrayLike& value) {
65+
if (array->n_children != 1) {
66+
return InvalidArrowData("List array must have exactly 1 child");
67+
}
68+
for (size_t i = 0; i < value.size(); ++i) {
69+
ICEBERG_ASSIGN_OR_RAISE(auto element, value.GetElement(i));
70+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[0], element));
71+
}
72+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
73+
return {};
74+
}
75+
76+
Status AppendMap(ArrowArray* array, const MapLike& value) {
77+
// A map array is a list of struct<key, value>.
78+
if (array->n_children != 1 || array->children[0]->n_children != 2) {
79+
return InvalidArrowData("Map array must have one entries child with 2 children");
80+
}
81+
ArrowArray* entries = array->children[0];
82+
for (size_t i = 0; i < value.size(); ++i) {
83+
ICEBERG_ASSIGN_OR_RAISE(auto key, value.GetKey(i));
84+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[0], key));
85+
ICEBERG_ASSIGN_OR_RAISE(auto item, value.GetValue(i));
86+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[1], item));
87+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(entries));
88+
}
89+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array));
90+
return {};
91+
}
92+
93+
// The Arrow array built from the schema decides the physical encoding, so the scalar
94+
// alternative alone selects the append call. For example, int64_t serves long, time,
95+
// and timestamp columns, and std::string_view serves string, binary, fixed, and uuid.
96+
Status AppendScalar(ArrowArray* array, const Scalar& value) {
97+
return std::visit(
98+
[array]<typename T>(const T& v) -> Status {
99+
if constexpr (std::is_same_v<T, std::monostate>) {
100+
return AppendNull(array);
101+
} else if constexpr (std::is_same_v<T, bool>) {
102+
return AppendBoolean(array, v);
103+
} else if constexpr (std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t>) {
104+
return AppendInt(array, v);
105+
} else if constexpr (std::is_same_v<T, float> || std::is_same_v<T, double>) {
106+
return AppendDouble(array, v);
107+
} else if constexpr (std::is_same_v<T, std::string_view>) {
108+
return AppendBytes(array,
109+
std::span<const uint8_t>(
110+
reinterpret_cast<const uint8_t*>(v.data()), // NOLINT
111+
v.size()));
112+
} else if constexpr (std::is_same_v<T, Decimal>) {
113+
return AppendDecimal(array, v);
114+
} else if constexpr (std::is_same_v<T, std::shared_ptr<StructLike>>) {
115+
return v == nullptr ? AppendNull(array) : AppendStruct(array, *v);
116+
} else if constexpr (std::is_same_v<T, std::shared_ptr<ArrayLike>>) {
117+
return v == nullptr ? AppendNull(array) : AppendList(array, *v);
118+
} else {
119+
static_assert(std::is_same_v<T, std::shared_ptr<MapLike>>);
120+
return v == nullptr ? AppendNull(array) : AppendMap(array, *v);
121+
}
122+
},
123+
value);
124+
}
125+
126+
Status AppendRow(ArrowRowBuilder& builder, const StructLike& row) {
127+
if (std::cmp_not_equal(row.num_fields(), builder.num_columns())) {
128+
return InvalidArgument("Data task row has {} fields but the schema has {} columns",
129+
row.num_fields(), builder.num_columns());
130+
}
131+
for (int64_t i = 0; i < builder.num_columns(); ++i) {
132+
ICEBERG_ASSIGN_OR_RAISE(auto field, row.GetField(static_cast<size_t>(i)));
133+
ICEBERG_RETURN_UNEXPECTED(AppendScalar(builder.column(i), field));
134+
}
135+
return builder.FinishRow();
136+
}
137+
138+
class DataTaskArrowSource {
139+
public:
140+
static Result<std::unique_ptr<DataTaskArrowSource>> Make(
141+
const iceberg::Schema& schema, std::vector<std::shared_ptr<DataTask>> tasks,
142+
int64_t batch_size) {
143+
ArrowSchema arrow_schema{};
144+
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(schema, &arrow_schema));
145+
return std::unique_ptr<DataTaskArrowSource>(
146+
new DataTaskArrowSource(std::move(tasks), arrow_schema, batch_size));
147+
}
148+
149+
~DataTaskArrowSource() { std::ignore = Close(); }
150+
151+
Status Close() {
152+
rows_.reset();
153+
tasks_.clear();
154+
if (arrow_schema_.release != nullptr) {
155+
ArrowSchemaRelease(&arrow_schema_);
156+
}
157+
return {};
158+
}
159+
160+
Result<std::optional<ArrowArray>> Next() {
161+
if (arrow_schema_.release == nullptr) [[unlikely]] {
162+
return InvalidArgument("Cannot read from a closed data task stream");
163+
}
164+
165+
// Fetch a row before allocating a builder, so an exhausted stream allocates none.
166+
ICEBERG_ASSIGN_OR_RAISE(auto row, NextRow());
167+
if (row == nullptr) {
168+
return std::nullopt;
169+
}
170+
171+
ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(&arrow_schema_));
172+
while (true) {
173+
ICEBERG_RETURN_UNEXPECTED(AppendRow(builder, *row));
174+
if (builder.num_rows() == batch_size_) {
175+
break;
176+
}
177+
ICEBERG_ASSIGN_OR_RAISE(row, NextRow());
178+
if (row == nullptr) {
179+
break;
180+
}
181+
}
182+
183+
ICEBERG_ASSIGN_OR_RAISE(auto array, std::move(builder).Finish());
184+
return array;
185+
}
186+
187+
Result<ArrowSchema> Schema() {
188+
if (arrow_schema_.release == nullptr) [[unlikely]] {
189+
return InvalidArgument("Cannot read schema from a closed data task stream");
190+
}
191+
ArrowSchema schema_copy{};
192+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(
193+
ArrowSchemaDeepCopy(&arrow_schema_, &schema_copy));
194+
return schema_copy;
195+
}
196+
197+
private:
198+
DataTaskArrowSource(std::vector<std::shared_ptr<DataTask>> tasks,
199+
ArrowSchema arrow_schema, int64_t batch_size)
200+
: tasks_(std::move(tasks)), arrow_schema_(arrow_schema), batch_size_(batch_size) {}
201+
202+
// Return the next row across all tasks, or nullptr once every task is exhausted.
203+
Result<std::shared_ptr<StructLike>> NextRow() {
204+
while (true) {
205+
if (rows_ == nullptr) {
206+
if (next_task_ == tasks_.size()) {
207+
return nullptr;
208+
}
209+
ICEBERG_ASSIGN_OR_RAISE(rows_, tasks_[next_task_++]->Rows());
210+
}
211+
ICEBERG_ASSIGN_OR_RAISE(auto row, rows_->Next());
212+
if (!row.has_value()) {
213+
rows_.reset();
214+
continue;
215+
}
216+
if (*row == nullptr) [[unlikely]] {
217+
return InvalidArgument("Data task returned a null row");
218+
}
219+
return std::move(*row);
220+
}
221+
}
222+
223+
std::vector<std::shared_ptr<DataTask>> tasks_;
224+
size_t next_task_ = 0;
225+
std::unique_ptr<StructLikeStream> rows_;
226+
ArrowSchema arrow_schema_{};
227+
int64_t batch_size_;
228+
};
229+
230+
} // namespace
231+
232+
Result<ArrowArrayStream> ReadDataTasks(std::shared_ptr<Schema> schema,
233+
std::vector<std::shared_ptr<DataTask>> tasks,
234+
int64_t batch_size) {
235+
ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null");
236+
ICEBERG_PRECHECK(batch_size > 0, "Batch size must be positive, got {}", batch_size);
237+
for (const auto& task : tasks) {
238+
ICEBERG_PRECHECK(task != nullptr, "Data task cannot be null");
239+
ICEBERG_PRECHECK(task->schema() != nullptr, "Data task schema cannot be null");
240+
ICEBERG_PRECHECK(task->schema() == schema || *task->schema() == *schema,
241+
"Data task schema does not match the requested schema");
242+
}
243+
244+
ICEBERG_ASSIGN_OR_RAISE(
245+
auto source, DataTaskArrowSource::Make(*schema, std::move(tasks), batch_size));
246+
return MakeArrowArrayStream(std::move(source));
247+
}
248+
249+
} // namespace iceberg

‎src/iceberg/data_task_reader.h‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/data_task_reader.h
23+
/// \brief Read DataTask rows as Arrow C Data batches.
24+
25+
#include <cstdint>
26+
#include <memory>
27+
#include <vector>
28+
29+
#include "iceberg/arrow_c_data.h"
30+
#include "iceberg/iceberg_export.h"
31+
#include "iceberg/result.h"
32+
#include "iceberg/type_fwd.h"
33+
34+
namespace iceberg {
35+
36+
/// \brief Default maximum number of rows in each batch read from data tasks.
37+
inline constexpr int64_t kDefaultDataTaskBatchSize = 1024;
38+
39+
/// \brief Read the rows of data tasks as one Arrow stream.
40+
///
41+
/// Tasks are read in order and a batch may hold rows from more than one task, so a
42+
/// caller that wants to read tasks in parallel passes one task per call.
43+
///
44+
/// \param schema Schema of the returned stream. Every task must return rows in it.
45+
/// \param tasks Tasks to read. The stream keeps them alive until it is released.
46+
/// \param batch_size Maximum number of rows in each batch. Must be positive.
47+
/// \return A stream that the caller owns and must release by invoking its `release`
48+
/// callback, or an error.
49+
ICEBERG_EXPORT Result<ArrowArrayStream> ReadDataTasks(
50+
std::shared_ptr<Schema> schema, std::vector<std::shared_ptr<DataTask>> tasks,
51+
int64_t batch_size = kDefaultDataTaskBatchSize);
52+
53+
} // namespace iceberg

0 commit comments

Comments
 (0)