2424#include < memory>
2525#include < optional>
2626#include < string>
27+ #include < unordered_map>
2728#include < utility>
2829#include < vector>
2930
@@ -52,8 +53,41 @@ namespace {
5253struct ManifestRow {
5354 ManifestFile manifest;
5455 std::shared_ptr<PartitionSpec> spec;
56+ std::shared_ptr<StructType> partition_type;
5557};
5658
59+ Result<std::shared_ptr<StructType>> ManifestPartitionType (const TableMetadata& metadata,
60+ const PartitionSpec& spec) {
61+ ICEBERG_ASSIGN_OR_RAISE (auto current_schema, metadata.Schema ());
62+ std::vector<SchemaField> fields;
63+ fields.reserve (spec.fields ().size ());
64+ for (const auto & partition_field : spec.fields ()) {
65+ ICEBERG_ASSIGN_OR_RAISE (auto source,
66+ current_schema->FindFieldById (partition_field.source_id ()));
67+ // A retained spec can reference a dropped column. Recover its most recent
68+ // known type by field ID, including any promotion before the column was dropped.
69+ if (!source) {
70+ int32_t latest_schema_id = -1 ;
71+ for (const auto & schema : metadata.schemas ) {
72+ ICEBERG_PRECHECK (schema != nullptr , " Table schema cannot be null" );
73+ if (schema->schema_id () > latest_schema_id) {
74+ ICEBERG_ASSIGN_OR_RAISE (auto historical,
75+ schema->FindFieldById (partition_field.source_id ()));
76+ if (historical) {
77+ source = historical;
78+ latest_schema_id = schema->schema_id ();
79+ }
80+ }
81+ }
82+ }
83+ auto source_type = source ? source->get ().type () : unknown ();
84+ fields.push_back (SchemaField::MakeOptional (
85+ partition_field.field_id (), std::string (partition_field.name ()),
86+ partition_field.transform ()->ResultType (source_type)));
87+ }
88+ return std::make_shared<StructType>(std::move (fields));
89+ }
90+
5791Result<std::string> HumanReadableBound (const PartitionSpec& spec,
5892 const StructType& partition_type, size_t index,
5993 const std::vector<uint8_t >& bytes) {
@@ -116,8 +150,7 @@ Status AppendManifestCount(ArrowArray* array, const std::optional<int32_t>& coun
116150}
117151
118152Status AppendManifest (ArrowRowBuilder& builder, const Schema& schema,
119- const ManifestRow& row,
120- const std::shared_ptr<Schema>& table_schema) {
153+ const ManifestRow& row) {
121154 const auto & manifest = row.manifest ;
122155 const bool data = manifest.content == ManifestContent::kData ;
123156 for (size_t index = 0 ; index < schema.fields ().size (); ++index) {
@@ -167,10 +200,8 @@ Status AppendManifest(ArrowRowBuilder& builder, const Schema& schema,
167200 AppendManifestCount (array, manifest.deleted_files_count , !data));
168201 break ;
169202 case 8 : {
170- ICEBERG_ASSIGN_OR_RAISE (auto partition_type,
171- row.spec ->PartitionType (*table_schema));
172- ICEBERG_RETURN_UNEXPECTED (AppendPartitionSummaries (
173- array, row, std::shared_ptr<StructType>(std::move (partition_type))));
203+ ICEBERG_RETURN_UNEXPECTED (
204+ AppendPartitionSummaries (array, row, row.partition_type ));
174205 break ;
175206 }
176207 default :
@@ -188,7 +219,7 @@ ManifestsTable::ManifestsTable(std::shared_ptr<const Table> table)
188219
189220ManifestsTable::~ManifestsTable () = default ;
190221
191- const std::shared_ptr<Schema>& ManifestsTable::schema () const {
222+ std::shared_ptr<Schema> ManifestsTable::schema () const {
192223 static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
193224 SchemaField::MakeRequired (14 , " content" , int32 ()),
194225 SchemaField::MakeRequired (1 , " path" , string ()),
@@ -231,23 +262,30 @@ Result<ArrowArrayStream> ManifestsTable::ReadRows(
231262 ICEBERG_ASSIGN_OR_RAISE (auto snapshot, internal::ResolveMetadataTableSnapshot (
232263 *context.table_metadata , context));
233264 std::vector<ManifestRow> rows;
265+ std::unordered_map<int32_t , std::shared_ptr<StructType>> partition_types;
234266 if (snapshot != nullptr ) {
235267 SnapshotReader snapshot_reader (snapshot.get ());
236268 ICEBERG_ASSIGN_OR_RAISE (auto manifests, snapshot_reader.Manifests (io));
237269 rows.reserve (manifests.size ());
238270 for (const auto & manifest : manifests) {
239271 ICEBERG_ASSIGN_OR_RAISE (auto spec, context.table_metadata ->PartitionSpecById (
240272 manifest.partition_spec_id ));
241- rows.push_back (ManifestRow{.manifest = manifest, .spec = std::move (spec)});
273+ auto [entry, inserted] = partition_types.try_emplace (spec->spec_id ());
274+ if (inserted) {
275+ ICEBERG_ASSIGN_OR_RAISE (entry->second ,
276+ ManifestPartitionType (*context.table_metadata , *spec));
277+ }
278+ rows.push_back (ManifestRow{.manifest = manifest,
279+ .spec = std::move (spec),
280+ .partition_type = entry->second });
242281 }
243282 }
244283
245- ICEBERG_ASSIGN_OR_RAISE (auto table_schema, context.table_metadata ->Schema ());
246284 return internal::MakeMetadataBatchStream (
247285 context.projected_schema , std::move (rows), batch_size,
248- [schema = context.projected_schema , table_schema = std::move (table_schema)](
249- ArrowRowBuilder& builder, const ManifestRow& row) {
250- return AppendManifest (builder, *schema, row, table_schema );
286+ [schema = context.projected_schema ](ArrowRowBuilder& builder,
287+ const ManifestRow& row) {
288+ return AppendManifest (builder, *schema, row);
251289 });
252290}
253291
0 commit comments