|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +#include "iceberg/inspect/refs_table.h" |
| 21 | + |
| 22 | +#include <algorithm> |
| 23 | +#include <array> |
| 24 | +#include <cstdint> |
| 25 | +#include <memory> |
| 26 | +#include <optional> |
| 27 | +#include <string> |
| 28 | +#include <utility> |
| 29 | +#include <variant> |
| 30 | +#include <vector> |
| 31 | + |
| 32 | +#include "iceberg/arrow_row_builder_internal.h" |
| 33 | +#include "iceberg/inspect/metadata_scan_task.h" |
| 34 | +#include "iceberg/inspect/metadata_scan_util_internal.h" |
| 35 | +#include "iceberg/schema.h" |
| 36 | +#include "iceberg/schema_field.h" |
| 37 | +#include "iceberg/snapshot.h" |
| 38 | +#include "iceberg/table.h" |
| 39 | +#include "iceberg/table_metadata.h" |
| 40 | +#include "iceberg/type.h" |
| 41 | +#include "iceberg/util/macros.h" |
| 42 | + |
| 43 | +namespace iceberg { |
| 44 | +namespace { |
| 45 | + |
| 46 | +struct RefRow { |
| 47 | + std::string name; |
| 48 | + std::string type; |
| 49 | + int64_t snapshot_id; |
| 50 | + std::optional<int64_t> max_ref_age_ms; |
| 51 | + std::optional<int32_t> min_snapshots_to_keep; |
| 52 | + std::optional<int64_t> max_snapshot_age_ms; |
| 53 | +}; |
| 54 | + |
| 55 | +Status AppendOptional(ArrowArray* array, const auto& value) { |
| 56 | + if (!value.has_value()) { |
| 57 | + return AppendNull(array); |
| 58 | + } |
| 59 | + return AppendInt(array, static_cast<int64_t>(*value)); |
| 60 | +} |
| 61 | + |
| 62 | +constexpr std::array<internal::MetadataTableColumn<RefRow>, 6> kRefColumns = { |
| 63 | + {{.field_id = 1, |
| 64 | + .append = [](ArrowArray* column, |
| 65 | + const RefRow& row) { return AppendString(column, row.name); }}, |
| 66 | + {.field_id = 2, |
| 67 | + .append = [](ArrowArray* column, |
| 68 | + const RefRow& row) { return AppendString(column, row.type); }}, |
| 69 | + {.field_id = 3, |
| 70 | + .append = [](ArrowArray* column, |
| 71 | + const RefRow& row) { return AppendInt(column, row.snapshot_id); }}, |
| 72 | + {.field_id = 4, |
| 73 | + .append = |
| 74 | + [](ArrowArray* column, const RefRow& row) { |
| 75 | + return AppendOptional(column, row.max_ref_age_ms); |
| 76 | + }}, |
| 77 | + {.field_id = 5, |
| 78 | + .append = |
| 79 | + [](ArrowArray* column, const RefRow& row) { |
| 80 | + return AppendOptional(column, row.min_snapshots_to_keep); |
| 81 | + }}, |
| 82 | + {.field_id = 6, .append = [](ArrowArray* column, const RefRow& row) { |
| 83 | + return AppendOptional(column, row.max_snapshot_age_ms); |
| 84 | + }}}}; |
| 85 | + |
| 86 | +} // namespace |
| 87 | + |
| 88 | +// RefsTable implementation |
| 89 | +RefsTable::RefsTable(std::shared_ptr<const Table> table) |
| 90 | + : MetadataTable(std::move(table)) {} |
| 91 | + |
| 92 | +RefsTable::~RefsTable() = default; |
| 93 | + |
| 94 | +const std::shared_ptr<Schema>& RefsTable::schema() const { return CanonicalSchema(); } |
| 95 | + |
| 96 | +const std::shared_ptr<Schema>& RefsTable::CanonicalSchema() { |
| 97 | + static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{ |
| 98 | + SchemaField::MakeRequired(1, "name", string()), |
| 99 | + SchemaField::MakeRequired(2, "type", string()), |
| 100 | + SchemaField::MakeRequired(3, "snapshot_id", int64()), |
| 101 | + SchemaField::MakeOptional(4, "max_reference_age_in_ms", int64()), |
| 102 | + SchemaField::MakeOptional(5, "min_snapshots_to_keep", int32()), |
| 103 | + SchemaField::MakeOptional(6, "max_snapshot_age_in_ms", int64()), |
| 104 | + }); |
| 105 | + return schema; |
| 106 | +} |
| 107 | + |
| 108 | +Result<std::unique_ptr<RefsTable>> RefsTable::Make(std::shared_ptr<const Table> table) { |
| 109 | + ICEBERG_PRECHECK(table != nullptr, "Table cannot be null"); |
| 110 | + return std::unique_ptr<RefsTable>(new RefsTable(std::move(table))); |
| 111 | +} |
| 112 | + |
| 113 | +Result<std::unique_ptr<MetadataScan>> RefsTable::MakeScan( |
| 114 | + internal::MetadataScanContext context) const { |
| 115 | + return RefsScan::Make(std::move(context)); |
| 116 | +} |
| 117 | + |
| 118 | +Result<ArrowArrayStream> RefsTable::ReadRows( |
| 119 | + std::shared_ptr<const TableMetadata> metadata, |
| 120 | + const std::shared_ptr<Schema>& projected_schema, int64_t batch_size) { |
| 121 | + std::vector<RefRow> rows; |
| 122 | + const auto& refs = metadata->refs; |
| 123 | + rows.reserve(refs.size()); |
| 124 | + for (const auto& [name, ref] : refs) { |
| 125 | + if (ref == nullptr) { |
| 126 | + return Invalid("Snapshot reference '{}' is null", name); |
| 127 | + } |
| 128 | + const auto* branch = std::get_if<SnapshotRef::Branch>(&ref->retention); |
| 129 | + RefRow row{.name = name, |
| 130 | + .type = branch != nullptr ? "BRANCH" : "TAG", |
| 131 | + .snapshot_id = ref->snapshot_id, |
| 132 | + .max_ref_age_ms = |
| 133 | + branch != nullptr |
| 134 | + ? branch->max_ref_age_ms |
| 135 | + : std::get<SnapshotRef::Tag>(ref->retention).max_ref_age_ms}; |
| 136 | + if (branch != nullptr) { |
| 137 | + row.min_snapshots_to_keep = branch->min_snapshots_to_keep; |
| 138 | + row.max_snapshot_age_ms = branch->max_snapshot_age_ms; |
| 139 | + } |
| 140 | + rows.push_back(std::move(row)); |
| 141 | + } |
| 142 | + std::ranges::sort(rows, {}, &RefRow::name); |
| 143 | + const size_t num_rows = rows.size(); |
| 144 | + return internal::MakeMetadataBatchStream<RefRow>( |
| 145 | + CanonicalSchema(), projected_schema, kRefColumns, num_rows, batch_size, |
| 146 | + [rows = std::move(rows)](size_t index) -> const RefRow& { return rows[index]; }); |
| 147 | +} |
| 148 | + |
| 149 | +// RefsScan implementation |
| 150 | +RefsScan::RefsScan(internal::MetadataScanContext context, |
| 151 | + std::shared_ptr<MetadataScanTask> task) |
| 152 | + : MetadataScan(std::move(context)), task_(std::move(task)) {} |
| 153 | + |
| 154 | +RefsScan::~RefsScan() = default; |
| 155 | + |
| 156 | +Result<std::unique_ptr<RefsScan>> RefsScan::Make(internal::MetadataScanContext context) { |
| 157 | + ICEBERG_PRECHECK(context.kind == MetadataTableKind::kRefs, |
| 158 | + "Unexpected metadata table kind for refs scan"); |
| 159 | + if (context.snapshot_id.has_value() || !context.ref_name.empty() || |
| 160 | + context.as_of_time_in_millis.has_value()) { |
| 161 | + return NotSupported("Metadata table does not support time travel"); |
| 162 | + } |
| 163 | + ICEBERG_PRECHECK(context.table_metadata != nullptr, "Table metadata cannot be null"); |
| 164 | + ICEBERG_PRECHECK(context.schema != nullptr, "Metadata table schema cannot be null"); |
| 165 | + ICEBERG_PRECHECK(context.projected_schema != nullptr, |
| 166 | + "Projected schema cannot be null"); |
| 167 | + ICEBERG_RETURN_UNEXPECTED(internal::SelectAppendFns<RefRow>( |
| 168 | + *RefsTable::CanonicalSchema(), *context.projected_schema, kRefColumns)); |
| 169 | + ICEBERG_ASSIGN_OR_RAISE(auto task, |
| 170 | + StaticMetadataTask::Make(context.kind, context.table_metadata, |
| 171 | + context.projected_schema)); |
| 172 | + return std::unique_ptr<RefsScan>(new RefsScan(std::move(context), std::move(task))); |
| 173 | +} |
| 174 | + |
| 175 | +Result<MetadataScanTaskStreamPtr> RefsScan::PlanTasksStream() const { |
| 176 | + return MetadataScanTaskStream::FromVector({task_}); |
| 177 | +} |
| 178 | + |
| 179 | +} // namespace iceberg |
0 commit comments