2626#include < utility>
2727#include < vector>
2828
29+ #include < nanoarrow/nanoarrow.h>
30+
2931#include " iceberg/arrow_c_data_guard_internal.h"
3032#include " iceberg/arrow_c_data_util_internal.h"
3133#include " iceberg/data/delete_filter.h"
3234#include " iceberg/expression/binder.h"
3335#include " iceberg/expression/evaluator.h"
3436#include " iceberg/file_reader.h"
3537#include " iceberg/manifest/manifest_entry.h"
38+ #include " iceberg/nanoarrow_status_internal.h"
3639#include " iceberg/row/arrow_array_wrapper.h"
3740#include " iceberg/schema.h"
38- #include " iceberg/schema_internal.h"
3941#include " iceberg/table_scan.h"
4042#include " iceberg/util/macros.h"
4143
@@ -78,23 +80,12 @@ class TaskStreamSource {
7880 TaskStreamSource (std::unique_ptr<Reader> reader,
7981 std::unique_ptr<DeleteFilter> delete_filter,
8082 std::unique_ptr<Evaluator> residual_evaluator,
81- std::shared_ptr<::iceberg::Schema> required_schema,
82- std::shared_ptr<::iceberg::Schema> projected_schema,
83- ProjectionContext projection_context)
83+ bool project_all_columns, ProjectionContext projection_context)
8484 : reader_(std::move(reader)),
8585 delete_filter_ (std::move(delete_filter)),
8686 residual_evaluator_(std::move(residual_evaluator)),
87- required_schema_(std::move(required_schema)),
88- projected_schema_(std::move(projected_schema)),
89- project_all_columns_(required_schema_->SameSchema (*projected_schema_)),
87+ project_all_columns_(project_all_columns),
9088 projection_context_(std::move(projection_context)) {}
91-
92- ~TaskStreamSource () {
93- if (cached_schema_.has_value () && cached_schema_->release != nullptr ) {
94- cached_schema_->release (&cached_schema_.value ());
95- }
96- }
97-
9889 Status Close () {
9990 if (reader_ == nullptr ) {
10091 return {};
@@ -103,11 +94,9 @@ class TaskStreamSource {
10394 }
10495
10596 Result<std::optional<ArrowArray>> Next () {
106- if (!cached_schema_.has_value ()) {
107- // File readers expose one stable Arrow schema for every batch in the stream.
108- ICEBERG_ASSIGN_OR_RAISE (cached_schema_, reader_->Schema ());
109- }
110- const ArrowSchema& input_arrow_schema = cached_schema_.value ();
97+ // The context owns the physical Arrow schema in the layout the reader enforces,
98+ // so every batch is interpreted in the layout the stream declares.
99+ const ArrowSchema& input_arrow_schema = projection_context_.input_arrow_schema ();
111100
112101 while (true ) {
113102 ICEBERG_ASSIGN_OR_RAISE (auto next_batch, reader_->Next ());
@@ -169,7 +158,8 @@ class TaskStreamSource {
169158
170159 Result<ArrowSchema> Schema () {
171160 ArrowSchema schema{};
172- ICEBERG_RETURN_UNEXPECTED (ToArrowSchema (*projected_schema_, &schema));
161+ ICEBERG_NANOARROW_RETURN_UNEXPECTED (
162+ ArrowSchemaDeepCopy (&projection_context_.output_arrow_schema (), &schema));
173163 return schema;
174164 }
175165
@@ -212,11 +202,8 @@ class TaskStreamSource {
212202 std::unique_ptr<Reader> reader_;
213203 std::unique_ptr<DeleteFilter> delete_filter_;
214204 std::unique_ptr<Evaluator> residual_evaluator_;
215- std::shared_ptr<::iceberg::Schema> required_schema_;
216- std::shared_ptr<::iceberg::Schema> projected_schema_;
217205 bool project_all_columns_ = false ;
218206 ProjectionContext projection_context_;
219- std::optional<ArrowSchema> cached_schema_;
220207};
221208
222209} // namespace
@@ -306,12 +293,14 @@ class FileScanTaskReader::Impl {
306293 ICEBERG_ASSIGN_OR_RAISE (auto reader,
307294 ReaderFactoryRegistry::Open (data_file->file_format , options));
308295
309- // ProjectionContext borrows schemas that are kept in TaskStreamSource.
296+ // Readers enforce the configured list type. Use the same setting for projection
297+ // so pass-through and filtered batches match the stream's declared schema.
310298 ICEBERG_ASSIGN_OR_RAISE (
311299 auto projection_context,
312300 ProjectionContext::Make (*required_schema, *projected_schema_,
313301 ProjectionContext::ResolveProjectBatchFunction (),
314- properties_.Get (ReaderProperties::kArrowValidateBatch )));
302+ properties_.Get (ReaderProperties::kArrowValidateBatch ),
303+ properties_.Get (ReaderProperties::kArrowUseLargeList )));
315304
316305 std::unique_ptr<Evaluator> residual_evaluator;
317306 if (apply_residual) {
@@ -320,9 +309,10 @@ class FileScanTaskReader::Impl {
320309 Evaluator::Make (*projected_schema_, task.residual_filter (), case_sensitive_));
321310 }
322311
312+ const bool project_all_columns = required_schema->SameSchema (*projected_schema_);
323313 auto task_source = std::make_unique<TaskStreamSource>(
324314 std::move (reader), std::move (delete_filter), std::move (residual_evaluator),
325- std::move (required_schema), projected_schema_ , std::move (projection_context));
315+ project_all_columns , std::move (projection_context));
326316 return MakeArrowArrayStream (std::move (task_source));
327317 }
328318
0 commit comments