|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +// Reads the snapshots and history metadata tables of an existing Iceberg table: the |
| 21 | +// snapshots table as Arrow batches, and the history table as rows without Arrow. |
| 22 | + |
| 23 | +#include <cstdint> |
| 24 | +#include <iostream> |
| 25 | +#include <string> |
| 26 | +#include <variant> |
| 27 | + |
| 28 | +#include "iceberg/arrow/arrow_io_util.h" |
| 29 | +#include "iceberg/catalog/memory/in_memory_catalog.h" |
| 30 | +#include "iceberg/inspect/history_table.h" |
| 31 | +#include "iceberg/inspect/metadata_table.h" |
| 32 | +#include "iceberg/inspect/metadata_table_reader.h" |
| 33 | +#include "iceberg/inspect/snapshots_table.h" |
| 34 | +#include "iceberg/result.h" |
| 35 | +#include "iceberg/row/struct_like.h" |
| 36 | +#include "iceberg/table.h" |
| 37 | +#include "iceberg/table_scan.h" |
| 38 | +#include "iceberg/util/macros.h" |
| 39 | + |
| 40 | +namespace { |
| 41 | + |
| 42 | +// ICEBERG_ASSIGN_OR_RAISE names Error without its namespace. |
| 43 | +using iceberg::Error; |
| 44 | + |
| 45 | +// Read the snapshots table as an ArrowArrayStream. Any Arrow consumer can take the |
| 46 | +// stream; this one counts rows through the Arrow C stream interface. |
| 47 | +iceberg::Status PrintSnapshotCount(const iceberg::SnapshotsTable& snapshots) { |
| 48 | + ICEBERG_ASSIGN_OR_RAISE( |
| 49 | + auto scan, snapshots.NewScan()->Select({"snapshot_id", "operation"}).Build()); |
| 50 | + ICEBERG_ASSIGN_OR_RAISE(ArrowArrayStream stream, iceberg::ReadMetadataTable(*scan)); |
| 51 | + |
| 52 | + int64_t num_rows = 0; |
| 53 | + while (true) { |
| 54 | + ArrowArray batch{}; |
| 55 | + if (stream.get_next(&stream, &batch) != 0) { |
| 56 | + const char* message = stream.get_last_error(&stream); |
| 57 | + std::string error = message != nullptr ? message : "unknown error"; |
| 58 | + stream.release(&stream); |
| 59 | + return iceberg::IOError("Failed to read the snapshots table: {}", error); |
| 60 | + } |
| 61 | + if (batch.release == nullptr) { |
| 62 | + break; |
| 63 | + } |
| 64 | + num_rows += batch.length; |
| 65 | + batch.release(&batch); |
| 66 | + } |
| 67 | + stream.release(&stream); |
| 68 | + |
| 69 | + std::cout << "Snapshots: " << num_rows << std::endl; |
| 70 | + return {}; |
| 71 | +} |
| 72 | + |
| 73 | +// Read the history table row by row, without converting to Arrow. |
| 74 | +iceberg::Status PrintHistory(const iceberg::HistoryTable& history) { |
| 75 | + ICEBERG_ASSIGN_OR_RAISE( |
| 76 | + auto scan, |
| 77 | + history.NewScan()->Select({"snapshot_id", "is_current_ancestor"}).Build()); |
| 78 | + ICEBERG_ASSIGN_OR_RAISE(auto tasks, scan->PlanFiles()); |
| 79 | + |
| 80 | + std::cout << "History:" << std::endl; |
| 81 | + for (const auto& task : tasks) { |
| 82 | + ICEBERG_ASSIGN_OR_RAISE(auto rows, task->Rows()); |
| 83 | + while (true) { |
| 84 | + ICEBERG_ASSIGN_OR_RAISE(auto row, rows->Next()); |
| 85 | + if (!row.has_value()) { |
| 86 | + break; |
| 87 | + } |
| 88 | + // Fields follow the scan's schema, which keeps the table's column order. |
| 89 | + ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, (*row)->GetField(0)); |
| 90 | + ICEBERG_ASSIGN_OR_RAISE(auto is_current_ancestor, (*row)->GetField(1)); |
| 91 | + std::cout << " - " << std::get<int64_t>(snapshot_id) |
| 92 | + << (std::get<bool>(is_current_ancestor) ? "" |
| 93 | + : " (not a current ancestor)") |
| 94 | + << std::endl; |
| 95 | + } |
| 96 | + } |
| 97 | + return {}; |
| 98 | +} |
| 99 | + |
| 100 | +iceberg::Status Run(const std::string& warehouse_location, const std::string& table_name, |
| 101 | + const std::string& metadata_location) { |
| 102 | + ICEBERG_ASSIGN_OR_RAISE(auto catalog, iceberg::InMemoryCatalog::Make( |
| 103 | + "example", iceberg::arrow::MakeLocalFileIO(), |
| 104 | + warehouse_location, /*properties=*/{})); |
| 105 | + ICEBERG_ASSIGN_OR_RAISE( |
| 106 | + auto table, catalog->RegisterTable({.name = table_name}, metadata_location)); |
| 107 | + |
| 108 | + ICEBERG_ASSIGN_OR_RAISE(auto snapshots, |
| 109 | + iceberg::MetadataTable::Make<iceberg::SnapshotsTable>(table)); |
| 110 | + ICEBERG_RETURN_UNEXPECTED(PrintSnapshotCount(*snapshots)); |
| 111 | + |
| 112 | + ICEBERG_ASSIGN_OR_RAISE(auto history, |
| 113 | + iceberg::MetadataTable::Make<iceberg::HistoryTable>(table)); |
| 114 | + return PrintHistory(*history); |
| 115 | +} |
| 116 | + |
| 117 | +} // namespace |
| 118 | + |
| 119 | +int main(int argc, char** argv) { |
| 120 | + if (argc != 4) { |
| 121 | + std::cerr << "Usage: " << argv[0] |
| 122 | + << " <warehouse_location> <table_name> <metadata_location>" << std::endl; |
| 123 | + return 0; |
| 124 | + } |
| 125 | + |
| 126 | + auto status = Run(argv[1], argv[2], argv[3]); |
| 127 | + if (!status.has_value()) { |
| 128 | + std::cerr << status.error().message << std::endl; |
| 129 | + return 1; |
| 130 | + } |
| 131 | + return 0; |
| 132 | +} |
0 commit comments