Skip to content

Commit a810516

Browse files
feat(data): enforce read.arrow.use-large-list by file readers (#1062)
1 parent eb929a7 commit a810516

14 files changed

Lines changed: 764 additions & 485 deletions

‎src/iceberg/arrow_c_data_util.cc‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -233,9 +233,7 @@ Status AppendValue(const ArrowSchema& input_schema, const ArrowArray& input_arra
233233
} // namespace
234234

235235
ProjectionContext::ProjectionContext(ProjectionContext&& other) noexcept
236-
: input_schema_(std::exchange(other.input_schema_, nullptr)),
237-
output_schema_(std::exchange(other.output_schema_, nullptr)),
238-
selected_field_indices_(std::move(other.selected_field_indices_)),
236+
: selected_field_indices_(std::move(other.selected_field_indices_)),
239237
input_arrow_schema_(other.input_arrow_schema_),
240238
output_arrow_schema_(other.output_arrow_schema_),
241239
project_batch_function_(std::exchange(other.project_batch_function_, nullptr)),
@@ -257,8 +255,6 @@ ProjectionContext& ProjectionContext::operator=(ProjectionContext&& other) noexc
257255
output_arrow_schema_.release(&output_arrow_schema_);
258256
}
259257

260-
input_schema_ = std::exchange(other.input_schema_, nullptr);
261-
output_schema_ = std::exchange(other.output_schema_, nullptr);
262258
selected_field_indices_ = std::move(other.selected_field_indices_);
263259
input_arrow_schema_ = other.input_arrow_schema_;
264260
other.input_arrow_schema_.release = nullptr;
@@ -281,27 +277,22 @@ ProjectionContext::~ProjectionContext() {
281277

282278
Result<ProjectionContext> ProjectionContext::Make(
283279
const Schema& input_schema, const Schema& output_schema,
284-
ProjectionContext::ProjectBatchFunction project_batch_function, bool validate_batch) {
280+
ProjectionContext::ProjectBatchFunction project_batch_function, bool validate_batch,
281+
bool use_large_list) {
285282
ICEBERG_ASSIGN_OR_RAISE(
286283
auto selected_field_indices,
287284
BuildSelectedFieldIndices(input_schema.fields(), output_schema.fields()));
288-
289285
ProjectionContext context;
290-
context.input_schema_ = &input_schema;
291-
context.output_schema_ = &output_schema;
292286
context.selected_field_indices_ = std::move(selected_field_indices);
293-
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(input_schema, &context.input_arrow_schema_));
294-
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(output_schema, &context.output_arrow_schema_));
295287
context.project_batch_function_ = project_batch_function;
296288
context.validate_batch_ = validate_batch;
297-
289+
ICEBERG_RETURN_UNEXPECTED(
290+
ToArrowSchema(input_schema, &context.input_arrow_schema_, use_large_list));
291+
ICEBERG_RETURN_UNEXPECTED(
292+
ToArrowSchema(output_schema, &context.output_arrow_schema_, use_large_list));
298293
return context;
299294
}
300295

301-
const Schema& ProjectionContext::input_schema() const { return *input_schema_; }
302-
303-
const Schema& ProjectionContext::output_schema() const { return *output_schema_; }
304-
305296
const ArrowSchema& ProjectionContext::input_arrow_schema() const {
306297
return input_arrow_schema_;
307298
}

‎src/iceberg/arrow_c_data_util_internal.h‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ class ICEBERG_EXPORT ProjectionContext {
6161
/// \brief Resolve the registered ProjectBatch implementation.
6262
static auto ResolveProjectBatchFunction() -> ProjectBatchFunction;
6363

64-
/// \brief Build reusable projection state for a validated schema pair.
64+
/// \brief Build reusable projection state using the configured Arrow list type.
6565
///
6666
/// \param input_schema Schema that describes every input batch.
6767
/// \param output_schema Final schema and column order requested by the caller.
@@ -70,14 +70,17 @@ class ICEBERG_EXPORT ProjectionContext {
7070
/// path.
7171
/// \param validate_batch Whether to perform full content validation. Disable only for
7272
/// trusted input.
73+
/// \param use_large_list Whether lists use 64-bit offsets. Must match the reader.
7374
/// \note It validates that output_schema selects or reorders complete top-level fields
74-
/// by field id. Nested pruning and type changes are rejected. The input_schema and
75-
/// output_schema passed to Make must outlive the context. ProjectBatch may lazily
76-
/// initialize backend cache; do not share one context across concurrent calls.
75+
/// by field id. Nested pruning and type changes are rejected. Schema inputs are only
76+
/// used during construction; the context owns its physical Arrow schemas.
77+
/// ProjectBatch may lazily initialize backend cache; do not share one context across
78+
/// concurrent calls.
7779
static Result<ProjectionContext> Make(const Schema& input_schema,
7880
const Schema& output_schema,
7981
ProjectBatchFunction project_batch_function,
80-
bool validate_batch = true);
82+
bool validate_batch = true,
83+
bool use_large_list = false);
8184

8285
ProjectionContext(ProjectionContext&&) noexcept;
8386
ProjectionContext& operator=(ProjectionContext&&) noexcept;
@@ -86,10 +89,6 @@ class ICEBERG_EXPORT ProjectionContext {
8689
ProjectionContext(const ProjectionContext&) = delete;
8790
ProjectionContext& operator=(const ProjectionContext&) = delete;
8891

89-
const Schema& input_schema() const;
90-
91-
const Schema& output_schema() const;
92-
9392
const ArrowSchema& input_arrow_schema() const;
9493

9594
const ArrowSchema& output_arrow_schema() const;
@@ -105,10 +104,6 @@ class ICEBERG_EXPORT ProjectionContext {
105104
private:
106105
ProjectionContext() = default;
107106

108-
// Raw schema pointers are borrowed from caller-owned schemas. FileScanTaskReader
109-
// keeps those schema objects alive in the same stream source that owns this context.
110-
const Schema* input_schema_ = nullptr;
111-
const Schema* output_schema_ = nullptr;
112107
std::vector<int32_t> selected_field_indices_;
113108
ArrowSchema input_arrow_schema_{};
114109
ArrowSchema output_arrow_schema_{};
@@ -244,7 +239,7 @@ ICEBERG_EXPORT Result<ArrowArrayStream> MakeArrowArrayStream(
244239
/// \param row_indices Zero-based row positions to copy from `input_batch`.
245240
/// \param projection Reusable schema/projection state created by
246241
/// ProjectionContext::Make.
247-
/// \return A newly owned ArrowArray matching `projection.output_schema()`.
242+
/// \return A newly owned ArrowArray matching `projection.output_arrow_schema()`.
248243
ICEBERG_EXPORT Result<ArrowArray> ProjectBatch(ArrowArray* input_batch,
249244
std::span<const int32_t> row_indices,
250245
ProjectionContext& projection);

‎src/iceberg/avro/avro_reader.cc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,20 @@ namespace iceberg::avro {
5151

5252
namespace {
5353

54+
bool HasList(const Type& type) {
55+
if (type.is_list()) {
56+
return true;
57+
}
58+
if (type.is_nested()) {
59+
for (const auto& field : static_cast<const NestedType&>(type).fields()) {
60+
if (HasList(*field.type())) {
61+
return true;
62+
}
63+
}
64+
}
65+
return false;
66+
}
67+
5468
Result<std::unique_ptr<AvroInputStream>> CreateInputStream(const ReaderOptions& options,
5569
int64_t buffer_size) {
5670
ICEBERG_ASSIGN_OR_RAISE(
@@ -291,6 +305,12 @@ class AvroReader::Impl {
291305

292306
batch_size_ = options.properties.Get(ReaderProperties::kBatchSize);
293307
read_schema_ = options.projection;
308+
if (options.properties.Get(ReaderProperties::kArrowUseLargeList) &&
309+
HasList(*read_schema_)) {
310+
return NotSupported(
311+
"Avro reader cannot honor read.arrow.use-large-list=true: "
312+
"projected list columns require large_list, but Avro only supports list");
313+
}
294314

295315
// Open the input stream and adapt to the avro interface.
296316
ICEBERG_ASSIGN_OR_RAISE(

‎src/iceberg/data/file_scan_task_reader.cc‎

Lines changed: 16 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,18 @@
2626
#include <utility>
2727
#include <vector>
2828

29+
#include <nanoarrow/nanoarrow.h>
30+
2931
#include "iceberg/arrow_c_data_guard_internal.h"
3032
#include "iceberg/arrow_c_data_util_internal.h"
3133
#include "iceberg/data/delete_filter.h"
3234
#include "iceberg/expression/binder.h"
3335
#include "iceberg/expression/evaluator.h"
3436
#include "iceberg/file_reader.h"
3537
#include "iceberg/manifest/manifest_entry.h"
38+
#include "iceberg/nanoarrow_status_internal.h"
3639
#include "iceberg/row/arrow_array_wrapper.h"
3740
#include "iceberg/schema.h"
38-
#include "iceberg/schema_internal.h"
3941
#include "iceberg/table_scan.h"
4042
#include "iceberg/util/macros.h"
4143

@@ -78,23 +80,12 @@ class TaskStreamSource {
7880
TaskStreamSource(std::unique_ptr<Reader> reader,
7981
std::unique_ptr<DeleteFilter> delete_filter,
8082
std::unique_ptr<Evaluator> residual_evaluator,
81-
std::shared_ptr<::iceberg::Schema> required_schema,
82-
std::shared_ptr<::iceberg::Schema> projected_schema,
83-
ProjectionContext projection_context)
83+
bool project_all_columns, ProjectionContext projection_context)
8484
: reader_(std::move(reader)),
8585
delete_filter_(std::move(delete_filter)),
8686
residual_evaluator_(std::move(residual_evaluator)),
87-
required_schema_(std::move(required_schema)),
88-
projected_schema_(std::move(projected_schema)),
89-
project_all_columns_(required_schema_->SameSchema(*projected_schema_)),
87+
project_all_columns_(project_all_columns),
9088
projection_context_(std::move(projection_context)) {}
91-
92-
~TaskStreamSource() {
93-
if (cached_schema_.has_value() && cached_schema_->release != nullptr) {
94-
cached_schema_->release(&cached_schema_.value());
95-
}
96-
}
97-
9889
Status Close() {
9990
if (reader_ == nullptr) {
10091
return {};
@@ -103,11 +94,9 @@ class TaskStreamSource {
10394
}
10495

10596
Result<std::optional<ArrowArray>> Next() {
106-
if (!cached_schema_.has_value()) {
107-
// File readers expose one stable Arrow schema for every batch in the stream.
108-
ICEBERG_ASSIGN_OR_RAISE(cached_schema_, reader_->Schema());
109-
}
110-
const ArrowSchema& input_arrow_schema = cached_schema_.value();
97+
// The context owns the physical Arrow schema in the layout the reader enforces,
98+
// so every batch is interpreted in the layout the stream declares.
99+
const ArrowSchema& input_arrow_schema = projection_context_.input_arrow_schema();
111100

112101
while (true) {
113102
ICEBERG_ASSIGN_OR_RAISE(auto next_batch, reader_->Next());
@@ -169,7 +158,8 @@ class TaskStreamSource {
169158

170159
Result<ArrowSchema> Schema() {
171160
ArrowSchema schema{};
172-
ICEBERG_RETURN_UNEXPECTED(ToArrowSchema(*projected_schema_, &schema));
161+
ICEBERG_NANOARROW_RETURN_UNEXPECTED(
162+
ArrowSchemaDeepCopy(&projection_context_.output_arrow_schema(), &schema));
173163
return schema;
174164
}
175165

@@ -212,11 +202,8 @@ class TaskStreamSource {
212202
std::unique_ptr<Reader> reader_;
213203
std::unique_ptr<DeleteFilter> delete_filter_;
214204
std::unique_ptr<Evaluator> residual_evaluator_;
215-
std::shared_ptr<::iceberg::Schema> required_schema_;
216-
std::shared_ptr<::iceberg::Schema> projected_schema_;
217205
bool project_all_columns_ = false;
218206
ProjectionContext projection_context_;
219-
std::optional<ArrowSchema> cached_schema_;
220207
};
221208

222209
} // namespace
@@ -306,12 +293,14 @@ class FileScanTaskReader::Impl {
306293
ICEBERG_ASSIGN_OR_RAISE(auto reader,
307294
ReaderFactoryRegistry::Open(data_file->file_format, options));
308295

309-
// ProjectionContext borrows schemas that are kept in TaskStreamSource.
296+
// Readers enforce the configured list type. Use the same setting for projection
297+
// so pass-through and filtered batches match the stream's declared schema.
310298
ICEBERG_ASSIGN_OR_RAISE(
311299
auto projection_context,
312300
ProjectionContext::Make(*required_schema, *projected_schema_,
313301
ProjectionContext::ResolveProjectBatchFunction(),
314-
properties_.Get(ReaderProperties::kArrowValidateBatch)));
302+
properties_.Get(ReaderProperties::kArrowValidateBatch),
303+
properties_.Get(ReaderProperties::kArrowUseLargeList)));
315304

316305
std::unique_ptr<Evaluator> residual_evaluator;
317306
if (apply_residual) {
@@ -320,9 +309,10 @@ class FileScanTaskReader::Impl {
320309
Evaluator::Make(*projected_schema_, task.residual_filter(), case_sensitive_));
321310
}
322311

312+
const bool project_all_columns = required_schema->SameSchema(*projected_schema_);
323313
auto task_source = std::make_unique<TaskStreamSource>(
324314
std::move(reader), std::move(delete_filter), std::move(residual_evaluator),
325-
std::move(required_schema), projected_schema_, std::move(projection_context));
315+
project_all_columns, std::move(projection_context));
326316
return MakeArrowArrayStream(std::move(task_source));
327317
}
328318

‎src/iceberg/data/file_scan_task_reader.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@ namespace iceberg {
3838
///
3939
/// FileScanTaskReader chooses the copy-on-write path for tasks without deletes and
4040
/// the merge-on-read path for tasks with v2 position or equality deletes. The returned
41-
/// stream always exposes `projected_schema`. When `apply_residuals` is enabled, each
42-
/// task's residual filter is enforced after deletes and before final projection.
41+
/// stream exposes the fields of `projected_schema` with list widths determined by
42+
/// `read.arrow.use-large-list`, or fails if the reader cannot honor that setting. When
43+
/// `apply_residuals` is enabled, each task's residual filter is enforced after deletes
44+
/// and before final projection.
4345
///
4446
/// TODO(gangwu): Add a mode that emits a `_deleted` column instead of filtering rows.
4547
class ICEBERG_DATA_EXPORT FileScanTaskReader {

‎src/iceberg/file_reader.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ class ICEBERG_EXPORT Reader {
5555
virtual Result<std::optional<ArrowArray>> Next() = 0;
5656

5757
/// \brief Get the schema of the data.
58+
/// Every batch must match this schema and the configured Arrow list width.
59+
/// Readers must reject unsupported list widths before emitting any batches.
5860
virtual Result<ArrowSchema> Schema() = 0;
5961

6062
/// \brief Get the metadata of the file.
@@ -77,7 +79,10 @@ class ICEBERG_EXPORT ReaderProperties : public ConfigBase<ReaderProperties> {
7779
/// \brief The batch size to read.
7880
inline static Entry<int64_t> kBatchSize{"read.batch-size", 4096};
7981
/// \brief Read list columns as Arrow large_list (64-bit offsets) instead of list.
80-
/// Only the Parquet reader honors this option; other readers ignore it.
82+
/// Applies to every projected list, including nested lists. Readers return an error
83+
/// if they cannot produce the requested type. In particular, valid Parquet files
84+
/// whose stored ARROW:schema conflicts with this setting are rejected; list widths
85+
/// are not converted. Avro currently supports only ordinary lists.
8186
/// Default: false (use 32-bit offset list).
8287
inline static Entry<bool> kArrowUseLargeList{"read.arrow.use-large-list", false};
8388
/// \brief Validate Arrow batches before merge-on-read projection.

0 commit comments

Comments
 (0)