|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +#include "iceberg/data_task_reader.h" |
| 21 | + |
| 22 | +#include <cstddef> |
| 23 | +#include <cstdint> |
| 24 | +#include <memory> |
| 25 | +#include <optional> |
| 26 | +#include <span> |
| 27 | +#include <string_view> |
| 28 | +#include <tuple> |
| 29 | +#include <type_traits> |
| 30 | +#include <utility> |
| 31 | +#include <variant> |
| 32 | +#include <vector> |
| 33 | + |
| 34 | +#include <nanoarrow/nanoarrow.h> |
| 35 | + |
| 36 | +#include "iceberg/arrow_c_data_util_internal.h" |
| 37 | +#include "iceberg/arrow_row_builder_internal.h" |
| 38 | +#include "iceberg/nanoarrow_status_internal.h" |
| 39 | +#include "iceberg/row/struct_like.h" |
| 40 | +#include "iceberg/schema.h" |
| 41 | +#include "iceberg/schema_internal.h" |
| 42 | +#include "iceberg/table_scan.h" |
| 43 | +#include "iceberg/util/decimal.h" |
| 44 | +#include "iceberg/util/macros.h" |
| 45 | + |
| 46 | +namespace iceberg { |
| 47 | +namespace { |
| 48 | + |
| 49 | +Status AppendScalar(ArrowArray* array, const Scalar& value); |
| 50 | + |
| 51 | +Status AppendStruct(ArrowArray* array, const StructLike& value) { |
| 52 | + if (std::cmp_not_equal(value.num_fields(), array->n_children)) { |
| 53 | + return InvalidArgument("Cannot append a struct with {} fields to {} Arrow children", |
| 54 | + value.num_fields(), array->n_children); |
| 55 | + } |
| 56 | + for (int64_t i = 0; i < array->n_children; ++i) { |
| 57 | + ICEBERG_ASSIGN_OR_RAISE(auto field, value.GetField(static_cast<size_t>(i))); |
| 58 | + ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[i], field)); |
| 59 | + } |
| 60 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); |
| 61 | + return {}; |
| 62 | +} |
| 63 | + |
| 64 | +Status AppendList(ArrowArray* array, const ArrayLike& value) { |
| 65 | + if (array->n_children != 1) { |
| 66 | + return InvalidArrowData("List array must have exactly 1 child"); |
| 67 | + } |
| 68 | + for (size_t i = 0; i < value.size(); ++i) { |
| 69 | + ICEBERG_ASSIGN_OR_RAISE(auto element, value.GetElement(i)); |
| 70 | + ICEBERG_RETURN_UNEXPECTED(AppendScalar(array->children[0], element)); |
| 71 | + } |
| 72 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); |
| 73 | + return {}; |
| 74 | +} |
| 75 | + |
| 76 | +Status AppendMap(ArrowArray* array, const MapLike& value) { |
| 77 | + // A map array is a list of struct<key, value>. |
| 78 | + if (array->n_children != 1 || array->children[0]->n_children != 2) { |
| 79 | + return InvalidArrowData("Map array must have one entries child with 2 children"); |
| 80 | + } |
| 81 | + ArrowArray* entries = array->children[0]; |
| 82 | + for (size_t i = 0; i < value.size(); ++i) { |
| 83 | + ICEBERG_ASSIGN_OR_RAISE(auto key, value.GetKey(i)); |
| 84 | + ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[0], key)); |
| 85 | + ICEBERG_ASSIGN_OR_RAISE(auto item, value.GetValue(i)); |
| 86 | + ICEBERG_RETURN_UNEXPECTED(AppendScalar(entries->children[1], item)); |
| 87 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(entries)); |
| 88 | + } |
| 89 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); |
| 90 | + return {}; |
| 91 | +} |
| 92 | + |
| 93 | +// The Arrow array built from the schema decides the physical encoding, so the scalar |
| 94 | +// alternative alone selects the append call. For example, int64_t serves long, time, |
| 95 | +// and timestamp columns, and std::string_view serves string, binary, fixed, and uuid. |
| 96 | +Status AppendScalar(ArrowArray* array, const Scalar& value) { |
| 97 | + return std::visit( |
| 98 | + [array]<typename T>(const T& v) -> Status { |
| 99 | + if constexpr (std::is_same_v<T, std::monostate>) { |
| 100 | + return AppendNull(array); |
| 101 | + } else if constexpr (std::is_same_v<T, bool>) { |
| 102 | + return AppendBoolean(array, v); |
| 103 | + } else if constexpr (std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t>) { |
| 104 | + return AppendInt(array, v); |
| 105 | + } else if constexpr (std::is_same_v<T, float> || std::is_same_v<T, double>) { |
| 106 | + return AppendDouble(array, v); |
| 107 | + } else if constexpr (std::is_same_v<T, std::string_view>) { |
| 108 | + return AppendBytes(array, |
| 109 | + std::span<const uint8_t>( |
| 110 | + reinterpret_cast<const uint8_t*>(v.data()), // NOLINT |
| 111 | + v.size())); |
| 112 | + } else if constexpr (std::is_same_v<T, Decimal>) { |
| 113 | + return AppendDecimal(array, v); |
| 114 | + } else if constexpr (std::is_same_v<T, std::shared_ptr<StructLike>>) { |
| 115 | + return v == nullptr ? AppendNull(array) : AppendStruct(array, *v); |
| 116 | + } else if constexpr (std::is_same_v<T, std::shared_ptr<ArrayLike>>) { |
| 117 | + return v == nullptr ? AppendNull(array) : AppendList(array, *v); |
| 118 | + } else { |
| 119 | + static_assert(std::is_same_v<T, std::shared_ptr<MapLike>>); |
| 120 | + return v == nullptr ? AppendNull(array) : AppendMap(array, *v); |
| 121 | + } |
| 122 | + }, |
| 123 | + value); |
| 124 | +} |
| 125 | + |
| 126 | +Status AppendRow(ArrowRowBuilder& builder, const StructLike& row) { |
| 127 | + if (std::cmp_not_equal(row.num_fields(), builder.num_columns())) { |
| 128 | + return InvalidArgument("Data task row has {} fields but the schema has {} columns", |
| 129 | + row.num_fields(), builder.num_columns()); |
| 130 | + } |
| 131 | + for (int64_t i = 0; i < builder.num_columns(); ++i) { |
| 132 | + ICEBERG_ASSIGN_OR_RAISE(auto field, row.GetField(static_cast<size_t>(i))); |
| 133 | + ICEBERG_RETURN_UNEXPECTED(AppendScalar(builder.column(i), field)); |
| 134 | + } |
| 135 | + return builder.FinishRow(); |
| 136 | +} |
| 137 | + |
| 138 | +class DataTaskArrowSource { |
| 139 | + public: |
| 140 | + static Result<std::unique_ptr<DataTaskArrowSource>> Make( |
| 141 | + const iceberg::Schema& schema, std::vector<std::shared_ptr<DataTask>> tasks, |
| 142 | + int64_t batch_size) { |
| 143 | + ArrowSchema arrow_schema{}; |
| 144 | + ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(schema, &arrow_schema)); |
| 145 | + return std::unique_ptr<DataTaskArrowSource>( |
| 146 | + new DataTaskArrowSource(std::move(tasks), arrow_schema, batch_size)); |
| 147 | + } |
| 148 | + |
| 149 | + ~DataTaskArrowSource() { std::ignore = Close(); } |
| 150 | + |
| 151 | + Status Close() { |
| 152 | + rows_.reset(); |
| 153 | + tasks_.clear(); |
| 154 | + if (arrow_schema_.release != nullptr) { |
| 155 | + ArrowSchemaRelease(&arrow_schema_); |
| 156 | + } |
| 157 | + return {}; |
| 158 | + } |
| 159 | + |
| 160 | + Result<std::optional<ArrowArray>> Next() { |
| 161 | + if (arrow_schema_.release == nullptr) [[unlikely]] { |
| 162 | + return InvalidArgument("Cannot read from a closed data task stream"); |
| 163 | + } |
| 164 | + |
| 165 | + // Fetch a row before allocating a builder, so an exhausted stream allocates none. |
| 166 | + ICEBERG_ASSIGN_OR_RAISE(auto row, NextRow()); |
| 167 | + if (row == nullptr) { |
| 168 | + return std::nullopt; |
| 169 | + } |
| 170 | + |
| 171 | + ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(&arrow_schema_)); |
| 172 | + while (true) { |
| 173 | + ICEBERG_RETURN_UNEXPECTED(AppendRow(builder, *row)); |
| 174 | + if (builder.num_rows() == batch_size_) { |
| 175 | + break; |
| 176 | + } |
| 177 | + ICEBERG_ASSIGN_OR_RAISE(row, NextRow()); |
| 178 | + if (row == nullptr) { |
| 179 | + break; |
| 180 | + } |
| 181 | + } |
| 182 | + |
| 183 | + ICEBERG_ASSIGN_OR_RAISE(auto array, std::move(builder).Finish()); |
| 184 | + return array; |
| 185 | + } |
| 186 | + |
| 187 | + Result<ArrowSchema> Schema() { |
| 188 | + if (arrow_schema_.release == nullptr) [[unlikely]] { |
| 189 | + return InvalidArgument("Cannot read schema from a closed data task stream"); |
| 190 | + } |
| 191 | + ArrowSchema schema_copy{}; |
| 192 | + ICEBERG_NANOARROW_RETURN_UNEXPECTED( |
| 193 | + ArrowSchemaDeepCopy(&arrow_schema_, &schema_copy)); |
| 194 | + return schema_copy; |
| 195 | + } |
| 196 | + |
| 197 | + private: |
| 198 | + DataTaskArrowSource(std::vector<std::shared_ptr<DataTask>> tasks, |
| 199 | + ArrowSchema arrow_schema, int64_t batch_size) |
| 200 | + : tasks_(std::move(tasks)), arrow_schema_(arrow_schema), batch_size_(batch_size) {} |
| 201 | + |
| 202 | + // Return the next row across all tasks, or nullptr once every task is exhausted. |
| 203 | + Result<std::shared_ptr<StructLike>> NextRow() { |
| 204 | + while (true) { |
| 205 | + if (rows_ == nullptr) { |
| 206 | + if (next_task_ == tasks_.size()) { |
| 207 | + return nullptr; |
| 208 | + } |
| 209 | + ICEBERG_ASSIGN_OR_RAISE(rows_, tasks_[next_task_++]->Rows()); |
| 210 | + } |
| 211 | + ICEBERG_ASSIGN_OR_RAISE(auto row, rows_->Next()); |
| 212 | + if (!row.has_value()) { |
| 213 | + rows_.reset(); |
| 214 | + continue; |
| 215 | + } |
| 216 | + if (*row == nullptr) [[unlikely]] { |
| 217 | + return InvalidArgument("Data task returned a null row"); |
| 218 | + } |
| 219 | + return std::move(*row); |
| 220 | + } |
| 221 | + } |
| 222 | + |
| 223 | + std::vector<std::shared_ptr<DataTask>> tasks_; |
| 224 | + size_t next_task_ = 0; |
| 225 | + std::unique_ptr<StructLikeStream> rows_; |
| 226 | + ArrowSchema arrow_schema_{}; |
| 227 | + int64_t batch_size_; |
| 228 | +}; |
| 229 | + |
| 230 | +} // namespace |
| 231 | + |
| 232 | +Result<ArrowArrayStream> ReadDataTasks(std::shared_ptr<Schema> schema, |
| 233 | + std::vector<std::shared_ptr<DataTask>> tasks, |
| 234 | + int64_t batch_size) { |
| 235 | + ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null"); |
| 236 | + ICEBERG_PRECHECK(batch_size > 0, "Batch size must be positive, got {}", batch_size); |
| 237 | + for (const auto& task : tasks) { |
| 238 | + ICEBERG_PRECHECK(task != nullptr, "Data task cannot be null"); |
| 239 | + ICEBERG_PRECHECK(task->schema() != nullptr, "Data task schema cannot be null"); |
| 240 | + ICEBERG_PRECHECK(task->schema() == schema || *task->schema() == *schema, |
| 241 | + "Data task schema does not match the requested schema"); |
| 242 | + } |
| 243 | + |
| 244 | + ICEBERG_ASSIGN_OR_RAISE( |
| 245 | + auto source, DataTaskArrowSource::Make(*schema, std::move(tasks), batch_size)); |
| 246 | + return MakeArrowArrayStream(std::move(source)); |
| 247 | +} |
| 248 | + |
| 249 | +} // namespace iceberg |
0 commit comments