Skip to content

Commit ace86f4

Browse files
committed
fix(inspect): isolate scan schemas and recover historical bounds
1 parent 3539c08 commit ace86f4

20 files changed

Lines changed: 203 additions & 72 deletions

‎src/iceberg/inspect/files_table.cc‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,35 +37,30 @@ FilesTable::FilesTable(std::shared_ptr<const Table> table)
3737

3838
FilesTable::~FilesTable() = default;
3939

40-
const std::shared_ptr<Schema>& FilesTable::schema() const { return schema_; }
40+
std::shared_ptr<Schema> FilesTable::schema() const { return schema_.load(); }
4141

4242
Result<std::unique_ptr<FilesTable>> FilesTable::Make(std::shared_ptr<const Table> table) {
4343
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
4444
auto files = std::unique_ptr<FilesTable>(new FilesTable(std::move(table)));
45-
ICEBERG_RETURN_UNEXPECTED(files->RefreshSchema());
45+
ICEBERG_RETURN_UNEXPECTED(files->SchemaForScan(*files->source_table()->metadata()));
4646
return files;
4747
}
4848

49-
Status FilesTable::RefreshSchema() const {
50-
ICEBERG_ASSIGN_OR_RAISE(auto table_schema, source_table()->schema());
51-
ICEBERG_ASSIGN_OR_RAISE(auto partition_type,
52-
internal::UnifiedPartitionType(*source_table()->metadata()));
49+
Result<std::shared_ptr<Schema>> FilesTable::SchemaForScan(
50+
const TableMetadata& metadata) const {
51+
ICEBERG_ASSIGN_OR_RAISE(auto table_schema, metadata.Schema());
52+
ICEBERG_ASSIGN_OR_RAISE(auto partition_type, internal::UnifiedPartitionType(metadata));
5353
ICEBERG_ASSIGN_OR_RAISE(auto schema,
5454
internal::FilesTableSchema(*table_schema, partition_type));
55-
schema_ = std::move(schema);
56-
return {};
55+
schema_.store(schema);
56+
return schema;
5757
}
5858

5959
Result<std::unique_ptr<MetadataScan>> FilesTable::MakeScan(
6060
internal::MetadataScanContext context) const {
6161
return internal::MakeSnapshotMetadataScan(std::move(context));
6262
}
6363

64-
Result<std::unique_ptr<MetadataScanBuilder>> FilesTable::NewScan() const {
65-
ICEBERG_RETURN_UNEXPECTED(RefreshSchema());
66-
return MetadataTable::NewScan();
67-
}
68-
6964
Result<ArrowArrayStream> FilesTable::ReadRows(
7065
const internal::MetadataScanContext& context, const std::shared_ptr<FileIO>& io,
7166
int64_t batch_size) {

‎src/iceberg/inspect/files_table.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
/// \file iceberg/inspect/files_table.h
2323
/// \brief Define the files metadata table.
2424

25+
#include <atomic>
2526
#include <memory>
2627

2728
#include "iceberg/arrow_c_data.h"
@@ -43,11 +44,12 @@ class ICEBERG_EXPORT FilesTable : public MetadataTable {
4344

4445
/// \brief Schema from construction or the most recent scan builder.
4546
/// Existing scans and tasks retain their captured schema.
46-
const std::shared_ptr<Schema>& schema() const override;
47-
48-
Result<std::unique_ptr<MetadataScanBuilder>> NewScan() const override;
47+
std::shared_ptr<Schema> schema() const override;
4948

5049
protected:
50+
Result<std::shared_ptr<Schema>> SchemaForScan(
51+
const TableMetadata& metadata) const override;
52+
5153
Result<std::unique_ptr<MetadataScan>> MakeScan(
5254
internal::MetadataScanContext context) const override;
5355

@@ -60,9 +62,7 @@ class ICEBERG_EXPORT FilesTable : public MetadataTable {
6062

6163
explicit FilesTable(std::shared_ptr<const Table> table);
6264

63-
Status RefreshSchema() const;
64-
65-
mutable std::shared_ptr<Schema> schema_;
65+
mutable std::atomic<std::shared_ptr<Schema>> schema_;
6666
};
6767

6868
} // namespace iceberg

‎src/iceberg/inspect/history_table.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ HistoryTable::HistoryTable(std::shared_ptr<const Table> table)
7777

7878
HistoryTable::~HistoryTable() = default;
7979

80-
const std::shared_ptr<Schema>& HistoryTable::schema() const { return CanonicalSchema(); }
80+
std::shared_ptr<Schema> HistoryTable::schema() const { return CanonicalSchema(); }
8181

8282
const std::shared_ptr<Schema>& HistoryTable::CanonicalSchema() {
8383
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{

‎src/iceberg/inspect/history_table.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ class ICEBERG_EXPORT HistoryTable : public MetadataTable {
4646

4747
MetadataTableKind kind() const noexcept override { return MetadataTableKind::kHistory; }
4848

49-
const std::shared_ptr<Schema>& schema() const override;
49+
std::shared_ptr<Schema> schema() const override;
5050

5151
protected:
5252
Result<std::unique_ptr<MetadataScan>> MakeScan(

‎src/iceberg/inspect/manifests_table.cc‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <memory>
2525
#include <optional>
2626
#include <string>
27+
#include <unordered_map>
2728
#include <utility>
2829
#include <vector>
2930

@@ -52,8 +53,41 @@ namespace {
5253
struct ManifestRow {
5354
ManifestFile manifest;
5455
std::shared_ptr<PartitionSpec> spec;
56+
std::shared_ptr<StructType> partition_type;
5557
};
5658

59+
Result<std::shared_ptr<StructType>> ManifestPartitionType(const TableMetadata& metadata,
60+
const PartitionSpec& spec) {
61+
ICEBERG_ASSIGN_OR_RAISE(auto current_schema, metadata.Schema());
62+
std::vector<SchemaField> fields;
63+
fields.reserve(spec.fields().size());
64+
for (const auto& partition_field : spec.fields()) {
65+
ICEBERG_ASSIGN_OR_RAISE(auto source,
66+
current_schema->FindFieldById(partition_field.source_id()));
67+
// A retained spec can reference a dropped column. Recover its most recent
68+
// known type by field ID, including any promotion before the column was dropped.
69+
if (!source) {
70+
int32_t latest_schema_id = -1;
71+
for (const auto& schema : metadata.schemas) {
72+
ICEBERG_PRECHECK(schema != nullptr, "Table schema cannot be null");
73+
if (schema->schema_id() > latest_schema_id) {
74+
ICEBERG_ASSIGN_OR_RAISE(auto historical,
75+
schema->FindFieldById(partition_field.source_id()));
76+
if (historical) {
77+
source = historical;
78+
latest_schema_id = schema->schema_id();
79+
}
80+
}
81+
}
82+
}
83+
auto source_type = source ? source->get().type() : unknown();
84+
fields.push_back(SchemaField::MakeOptional(
85+
partition_field.field_id(), std::string(partition_field.name()),
86+
partition_field.transform()->ResultType(source_type)));
87+
}
88+
return std::make_shared<StructType>(std::move(fields));
89+
}
90+
5791
Result<std::string> HumanReadableBound(const PartitionSpec& spec,
5892
const StructType& partition_type, size_t index,
5993
const std::vector<uint8_t>& bytes) {
@@ -116,8 +150,7 @@ Status AppendManifestCount(ArrowArray* array, const std::optional<int32_t>& coun
116150
}
117151

118152
Status AppendManifest(ArrowRowBuilder& builder, const Schema& schema,
119-
const ManifestRow& row,
120-
const std::shared_ptr<Schema>& table_schema) {
153+
const ManifestRow& row) {
121154
const auto& manifest = row.manifest;
122155
const bool data = manifest.content == ManifestContent::kData;
123156
for (size_t index = 0; index < schema.fields().size(); ++index) {
@@ -167,10 +200,8 @@ Status AppendManifest(ArrowRowBuilder& builder, const Schema& schema,
167200
AppendManifestCount(array, manifest.deleted_files_count, !data));
168201
break;
169202
case 8: {
170-
ICEBERG_ASSIGN_OR_RAISE(auto partition_type,
171-
row.spec->PartitionType(*table_schema));
172-
ICEBERG_RETURN_UNEXPECTED(AppendPartitionSummaries(
173-
array, row, std::shared_ptr<StructType>(std::move(partition_type))));
203+
ICEBERG_RETURN_UNEXPECTED(
204+
AppendPartitionSummaries(array, row, row.partition_type));
174205
break;
175206
}
176207
default:
@@ -188,7 +219,7 @@ ManifestsTable::ManifestsTable(std::shared_ptr<const Table> table)
188219

189220
ManifestsTable::~ManifestsTable() = default;
190221

191-
const std::shared_ptr<Schema>& ManifestsTable::schema() const {
222+
std::shared_ptr<Schema> ManifestsTable::schema() const {
192223
static const auto schema = std::make_shared<Schema>(std::vector<SchemaField>{
193224
SchemaField::MakeRequired(14, "content", int32()),
194225
SchemaField::MakeRequired(1, "path", string()),
@@ -231,23 +262,30 @@ Result<ArrowArrayStream> ManifestsTable::ReadRows(
231262
ICEBERG_ASSIGN_OR_RAISE(auto snapshot, internal::ResolveMetadataTableSnapshot(
232263
*context.table_metadata, context));
233264
std::vector<ManifestRow> rows;
265+
std::unordered_map<int32_t, std::shared_ptr<StructType>> partition_types;
234266
if (snapshot != nullptr) {
235267
SnapshotReader snapshot_reader(snapshot.get());
236268
ICEBERG_ASSIGN_OR_RAISE(auto manifests, snapshot_reader.Manifests(io));
237269
rows.reserve(manifests.size());
238270
for (const auto& manifest : manifests) {
239271
ICEBERG_ASSIGN_OR_RAISE(auto spec, context.table_metadata->PartitionSpecById(
240272
manifest.partition_spec_id));
241-
rows.push_back(ManifestRow{.manifest = manifest, .spec = std::move(spec)});
273+
auto [entry, inserted] = partition_types.try_emplace(spec->spec_id());
274+
if (inserted) {
275+
ICEBERG_ASSIGN_OR_RAISE(entry->second,
276+
ManifestPartitionType(*context.table_metadata, *spec));
277+
}
278+
rows.push_back(ManifestRow{.manifest = manifest,
279+
.spec = std::move(spec),
280+
.partition_type = entry->second});
242281
}
243282
}
244283

245-
ICEBERG_ASSIGN_OR_RAISE(auto table_schema, context.table_metadata->Schema());
246284
return internal::MakeMetadataBatchStream(
247285
context.projected_schema, std::move(rows), batch_size,
248-
[schema = context.projected_schema, table_schema = std::move(table_schema)](
249-
ArrowRowBuilder& builder, const ManifestRow& row) {
250-
return AppendManifest(builder, *schema, row, table_schema);
286+
[schema = context.projected_schema](ArrowRowBuilder& builder,
287+
const ManifestRow& row) {
288+
return AppendManifest(builder, *schema, row);
251289
});
252290
}
253291

‎src/iceberg/inspect/manifests_table.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class ICEBERG_EXPORT ManifestsTable : public MetadataTable {
4141

4242
Kind kind() const noexcept override { return Kind::kManifests; }
4343

44-
const std::shared_ptr<Schema>& schema() const override;
44+
std::shared_ptr<Schema> schema() const override;
4545

4646
protected:
4747
Result<std::unique_ptr<MetadataScan>> MakeScan(

‎src/iceberg/inspect/metadata_scan.cc‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -107,21 +107,26 @@ class TaskArrowStreamSource {
107107

108108
} // namespace
109109

110-
MetadataScanBuilder::MetadataScanBuilder(const MetadataTable& table)
111-
: table_(&table),
112-
context_{.kind = table.kind(),
113-
.table_metadata = table.source_table()->metadata(),
114-
.schema = table.schema(),
115-
.projected_schema = table.schema(),
116-
.io = table.source_table()->io()} {}
110+
MetadataScanBuilder::MetadataScanBuilder(const MetadataTable& table,
111+
internal::MetadataScanContext context)
112+
: table_(&table), context_(std::move(context)) {}
117113

118114
MetadataScanBuilder::~MetadataScanBuilder() = default;
119115

120116
Result<std::unique_ptr<MetadataScanBuilder>> MetadataScanBuilder::Make(
121117
const MetadataTable& table) {
122118
ICEBERG_PRECHECK(table.source_table() != nullptr, "Source table cannot be null");
123-
ICEBERG_PRECHECK(table.schema() != nullptr, "Metadata table schema cannot be null");
124-
return std::unique_ptr<MetadataScanBuilder>(new MetadataScanBuilder(table));
119+
auto metadata = table.source_table()->metadata();
120+
ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null");
121+
ICEBERG_ASSIGN_OR_RAISE(auto schema, table.SchemaForScan(*metadata));
122+
ICEBERG_PRECHECK(schema != nullptr, "Metadata table schema cannot be null");
123+
internal::MetadataScanContext context{.kind = table.kind(),
124+
.table_metadata = std::move(metadata),
125+
.schema = schema,
126+
.projected_schema = std::move(schema),
127+
.io = table.source_table()->io()};
128+
return std::unique_ptr<MetadataScanBuilder>(
129+
new MetadataScanBuilder(table, std::move(context)));
125130
}
126131

127132
MetadataScanBuilder& MetadataScanBuilder::Select(

‎src/iceberg/inspect/metadata_scan.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ class ICEBERG_EXPORT MetadataScanBuilder : public ErrorCollector {
9898
Result<std::unique_ptr<MetadataScan>> Build();
9999

100100
private:
101-
explicit MetadataScanBuilder(const MetadataTable& table);
101+
MetadataScanBuilder(const MetadataTable& table, internal::MetadataScanContext context);
102102

103103
const MetadataTable* table_;
104104
internal::MetadataScanContext context_;

‎src/iceberg/inspect/metadata_table.cc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ const std::shared_ptr<const Table>& MetadataTable::source_table() const {
6262
return source_table_;
6363
}
6464

65+
Result<std::shared_ptr<Schema>> MetadataTable::SchemaForScan(
66+
const TableMetadata& metadata) const {
67+
return schema();
68+
}
69+
6570
Result<std::unique_ptr<MetadataScanBuilder>> MetadataTable::NewScan() const {
6671
return MetadataScanBuilder::Make(*this);
6772
}

‎src/iceberg/inspect/metadata_table.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,8 @@ class ICEBERG_EXPORT MetadataTable {
7878
/// \brief Return this metadata table's kind.
7979
virtual Kind kind() const noexcept = 0;
8080

81-
/// \brief Return the complete canonical schema of this metadata table.
82-
virtual const std::shared_ptr<Schema>& schema() const = 0;
81+
/// \brief Return shared ownership of this metadata table's canonical schema.
82+
virtual std::shared_ptr<Schema> schema() const = 0;
8383

8484
/// \brief Return the source table whose metadata is exposed.
8585
const std::shared_ptr<const Table>& source_table() const;
@@ -93,6 +93,10 @@ class ICEBERG_EXPORT MetadataTable {
9393
protected:
9494
explicit MetadataTable(std::shared_ptr<const Table> source_table);
9595

96+
/// \brief Derive a scan schema from the captured source metadata.
97+
virtual Result<std::shared_ptr<Schema>> SchemaForScan(
98+
const TableMetadata& metadata) const;
99+
96100
/// \brief Plan a scan using the captured metadata, projection, and snapshot selection.
97101
virtual Result<std::unique_ptr<MetadataScan>> MakeScan(
98102
internal::MetadataScanContext context) const = 0;

0 commit comments

Comments
 (0)