Skip to content

Commit dff9ebd

Browse files
manuzhangcodex
andcommitted
feat(inspect): scan metadata tables through data tasks
Metadata tables are now read through a scan. MetadataTable::NewScan() returns a builder that selects columns and, for tables that support time travel, a snapshot. The built scan plans DataTasks whose rows are StructLike rather than Arrow arrays. ReadDataTasks() converts the rows of any data task to an ArrowArrayStream, and ReadMetadataTable() reads a whole metadata table scan that way. MetadataTable::Scan() and kBatchSize are removed, so the metadata table interface no longer uses Arrow types. SnapshotsTable moves to the new interface, and HistoryTable gains a scan that returns one row per snapshot log entry. TimeTravelMetadataTable and SnapshotSelection are removed; a table that supports time travel overrides supports_time_travel() instead. example/metadata_table_example.cc reads the snapshots table as Arrow batches and the history table as rows. Co-authored-by: Codex <codex@openai.com>
1 parent 0bdf7ce commit dff9ebd

25 files changed

Lines changed: 1945 additions & 183 deletions

‎example/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,9 @@ target_link_libraries(demo_example
4747
PRIVATE "$<IF:$<TARGET_EXISTS:iceberg::iceberg_bundle_shared>,iceberg::iceberg_bundle_shared,iceberg::iceberg_bundle_static>"
4848
"$<IF:$<TARGET_EXISTS:iceberg::iceberg_rest_shared>,iceberg::iceberg_rest_shared,iceberg::iceberg_rest_static>"
4949
)
50+
51+
add_executable(metadata_table_example metadata_table_example.cc)
52+
53+
target_link_libraries(metadata_table_example
54+
PRIVATE "$<IF:$<TARGET_EXISTS:iceberg::iceberg_bundle_shared>,iceberg::iceberg_bundle_shared,iceberg::iceberg_bundle_static>"
55+
)

‎example/metadata_table_example.cc‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
// Reads the snapshots and history metadata tables of an existing Iceberg table: the
21+
// snapshots table as Arrow batches, and the history table as rows without Arrow.
22+
23+
#include <cstdint>
24+
#include <iostream>
25+
#include <string>
26+
#include <variant>
27+
28+
#include "iceberg/arrow/arrow_io_util.h"
29+
#include "iceberg/catalog/memory/in_memory_catalog.h"
30+
#include "iceberg/inspect/history_table.h"
31+
#include "iceberg/inspect/metadata_table.h"
32+
#include "iceberg/inspect/metadata_table_reader.h"
33+
#include "iceberg/inspect/snapshots_table.h"
34+
#include "iceberg/result.h"
35+
#include "iceberg/row/struct_like.h"
36+
#include "iceberg/table.h"
37+
#include "iceberg/table_scan.h"
38+
#include "iceberg/util/macros.h"
39+
40+
namespace {
41+
42+
// ICEBERG_ASSIGN_OR_RAISE names Error without its namespace.
43+
using iceberg::Error;
44+
45+
// Read the snapshots table as an ArrowArrayStream. Any Arrow consumer can take the
46+
// stream; this one counts rows through the Arrow C stream interface.
47+
iceberg::Status PrintSnapshotCount(const iceberg::SnapshotsTable& snapshots) {
48+
ICEBERG_ASSIGN_OR_RAISE(
49+
auto scan, snapshots.NewScan()->Select({"snapshot_id", "operation"}).Build());
50+
ICEBERG_ASSIGN_OR_RAISE(ArrowArrayStream stream, iceberg::ReadMetadataTable(*scan));
51+
52+
int64_t num_rows = 0;
53+
while (true) {
54+
ArrowArray batch{};
55+
if (stream.get_next(&stream, &batch) != 0) {
56+
const char* message = stream.get_last_error(&stream);
57+
std::string error = message != nullptr ? message : "unknown error";
58+
stream.release(&stream);
59+
return iceberg::IOError("Failed to read the snapshots table: {}", error);
60+
}
61+
if (batch.release == nullptr) {
62+
break;
63+
}
64+
num_rows += batch.length;
65+
batch.release(&batch);
66+
}
67+
stream.release(&stream);
68+
69+
std::cout << "Snapshots: " << num_rows << std::endl;
70+
return {};
71+
}
72+
73+
// Read the history table row by row, without converting to Arrow.
74+
iceberg::Status PrintHistory(const iceberg::HistoryTable& history) {
75+
ICEBERG_ASSIGN_OR_RAISE(
76+
auto scan,
77+
history.NewScan()->Select({"snapshot_id", "is_current_ancestor"}).Build());
78+
ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan->PlanFiles());
79+
80+
std::cout << "History:" << std::endl;
81+
for (const auto& task : tasks) {
82+
ICEBERG_ASSIGN_OR_RAISE(auto rows, task->Rows());
83+
while (true) {
84+
ICEBERG_ASSIGN_OR_RAISE(auto row, rows->Next());
85+
if (!row.has_value()) {
86+
break;
87+
}
88+
// Fields follow the scan's schema, which keeps the table's column order.
89+
ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, (*row)->GetField(0));
90+
ICEBERG_ASSIGN_OR_RAISE(auto is_current_ancestor, (*row)->GetField(1));
91+
std::cout << " - " << std::get<int64_t>(snapshot_id)
92+
<< (std::get<bool>(is_current_ancestor) ? ""
93+
: " (not a current ancestor)")
94+
<< std::endl;
95+
}
96+
}
97+
return {};
98+
}
99+
100+
iceberg::Status Run(const std::string& warehouse_location, const std::string& table_name,
101+
const std::string& metadata_location) {
102+
ICEBERG_ASSIGN_OR_RAISE(auto catalog, iceberg::InMemoryCatalog::Make(
103+
"example", iceberg::arrow::MakeLocalFileIO(),
104+
warehouse_location, /*properties=*/{}));
105+
ICEBERG_ASSIGN_OR_RAISE(
106+
auto table, catalog->RegisterTable({.name = table_name}, metadata_location));
107+
108+
ICEBERG_ASSIGN_OR_RAISE(auto snapshots,
109+
iceberg::MetadataTable::Make<iceberg::SnapshotsTable>(table));
110+
ICEBERG_RETURN_UNEXPECTED(PrintSnapshotCount(*snapshots));
111+
112+
ICEBERG_ASSIGN_OR_RAISE(auto history,
113+
iceberg::MetadataTable::Make<iceberg::HistoryTable>(table));
114+
return PrintHistory(*history);
115+
}
116+
117+
} // namespace
118+
119+
int main(int argc, char** argv) {
120+
if (argc != 4) {
121+
std::cerr << "Usage: " << argv[0]
122+
<< " <warehouse_location> <table_name> <metadata_location>" << std::endl;
123+
return 0;
124+
}
125+
126+
auto status = Run(argv[1], argv[2], argv[3]);
127+
if (!status.has_value()) {
128+
std::cerr << status.error().message << std::endl;
129+
return 1;
130+
}
131+
return 0;
132+
}

‎src/iceberg/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ set(ICEBERG_SOURCES
2727
catalog/session_catalog.cc
2828
catalog/session_context.cc
2929
data_table_scan.cc
30+
data_task_reader.cc
3031
delete_file_index.cc
3132
deletes/dv_util.cc
3233
deletes/dv_writer.cc
@@ -59,7 +60,10 @@ set(ICEBERG_SOURCES
5960
incremental_changelog_scan.cc
6061
inspect/history_table.cc
6162
inspect/metadata_table.cc
63+
inspect/metadata_table_reader.cc
64+
inspect/metadata_table_scan.cc
6265
inspect/snapshots_table.cc
66+
inspect/static_data_task_internal.cc
6367
inheritable_metadata.cc
6468
json_serde.cc
6569
location_provider.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#include "iceberg/data/position_delete_writer.h"
4444
#include "iceberg/data/writer.h"
4545
#include "iceberg/data_table_scan.h"
46+
#include "iceberg/data_task_reader.h"
4647
#include "iceberg/delete_file_index.h"
4748
#include "iceberg/deletes/dv_writer.h"
4849
#include "iceberg/deletes/position_delete_index.h"
@@ -81,6 +82,8 @@
8182
#include "iceberg/inheritable_metadata.h"
8283
#include "iceberg/inspect/history_table.h"
8384
#include "iceberg/inspect/metadata_table.h"
85+
#include "iceberg/inspect/metadata_table_reader.h"
86+
#include "iceberg/inspect/metadata_table_scan.h"
8487
#include "iceberg/inspect/snapshots_table.h"
8588
#include "iceberg/labels.h"
8689
#include "iceberg/location_provider.h"

0 commit comments

Comments
 (0)