Skip to content

Commit 6b12224

Browse files
committed
feat(inspect): add files metadata table
Add the files metadata table and its focused tests. Depends on the shared snapshot metadata scan support prerequisite. Keep readable metric schema and Arrow serialization local to this table; convert promoted date bounds and reject out-of-range timestamp values. Co-authored-by: Codex <codex@openai.com>
1 parent 1957376 commit 6b12224

10 files changed

Lines changed: 988 additions & 1 deletion

‎src/iceberg/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ set(ICEBERG_SOURCES
5858
geospatial.cc
5959
incremental_append_scan.cc
6060
incremental_changelog_scan.cc
61+
inspect/files_table.cc
62+
inspect/files_table_util_internal.cc
6163
inspect/history_table.cc
6264
inspect/metadata_scan.cc
6365
inspect/metadata_table.cc

‎src/iceberg/cpp20_compatibility_internal.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
#include "iceberg/incremental_append_scan.h"
8181
#include "iceberg/incremental_changelog_scan.h"
8282
#include "iceberg/inheritable_metadata.h"
83+
#include "iceberg/inspect/files_table.h"
8384
#include "iceberg/inspect/history_table.h"
8485
#include "iceberg/inspect/metadata_scan.h"
8586
#include "iceberg/inspect/metadata_scan_task.h"

‎src/iceberg/inspect/files_table.cc‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include "iceberg/inspect/files_table.h"
21+
22+
#include <memory>
23+
#include <mutex>
24+
#include <utility>
25+
26+
#include "iceberg/inspect/files_table_util_internal.h"
27+
#include "iceberg/inspect/metadata_scan_util_internal.h"
28+
#include "iceberg/inspect/metadata_table_util_internal.h"
29+
#include "iceberg/schema.h"
30+
#include "iceberg/table.h"
31+
#include "iceberg/table_metadata.h"
32+
#include "iceberg/type.h"
33+
#include "iceberg/util/macros.h"
34+
35+
namespace iceberg {
36+
37+
FilesTable::FilesTable(std::shared_ptr<const Table> table)
38+
: MetadataTable(std::move(table)) {}
39+
40+
FilesTable::~FilesTable() = default;
41+
42+
std::shared_ptr<Schema> FilesTable::schema() const {
43+
std::lock_guard lock(schema_mutex_);
44+
return schema_;
45+
}
46+
47+
Result<std::unique_ptr<FilesTable>> FilesTable::Make(std::shared_ptr<const Table> table) {
48+
ICEBERG_PRECHECK(table != nullptr, "Table cannot be null");
49+
auto files = std::unique_ptr<FilesTable>(new FilesTable(std::move(table)));
50+
ICEBERG_RETURN_UNEXPECTED(files->SchemaForScan(*files->source_table()->metadata()));
51+
return files;
52+
}
53+
54+
Result<std::shared_ptr<Schema>> FilesTable::SchemaForScan(
55+
const TableMetadata& metadata) const {
56+
ICEBERG_ASSIGN_OR_RAISE(auto table_schema, metadata.Schema());
57+
ICEBERG_ASSIGN_OR_RAISE(auto partition_type, internal::UnifiedPartitionType(metadata));
58+
ICEBERG_ASSIGN_OR_RAISE(auto schema,
59+
internal::FilesTableSchema(*table_schema, partition_type));
60+
{
61+
std::lock_guard lock(schema_mutex_);
62+
schema_ = schema;
63+
}
64+
return schema;
65+
}
66+
67+
Result<std::unique_ptr<MetadataScan>> FilesTable::MakeScan(
68+
internal::MetadataScanContext context) const {
69+
return internal::MakeSnapshotMetadataScan(std::move(context), ReadRows);
70+
}
71+
72+
Result<ArrowArrayStream> FilesTable::ReadRows(
73+
const internal::MetadataScanContext& context, const std::shared_ptr<FileIO>& io,
74+
int64_t batch_size) {
75+
ICEBERG_ASSIGN_OR_RAISE(auto partition_type,
76+
internal::UnifiedPartitionType(*context.table_metadata));
77+
ICEBERG_ASSIGN_OR_RAISE(auto table_schema, context.table_metadata->Schema());
78+
ICEBERG_ASSIGN_OR_RAISE(auto snapshot, internal::ResolveMetadataTableSnapshot(
79+
*context.table_metadata, context));
80+
ICEBERG_ASSIGN_OR_RAISE(auto files,
81+
internal::LiveFiles(*context.table_metadata, io, snapshot));
82+
auto schema = context.projected_schema;
83+
const int32_t readable_metrics_id = context.schema->fields().back().field_id();
84+
return internal::MakeMetadataBatchStream(
85+
schema, std::move(files), batch_size,
86+
[schema, readable_metrics_id, table_schema = std::move(table_schema),
87+
partition_type = std::move(partition_type)](ArrowRowBuilder& builder,
88+
const internal::LiveFile& file) {
89+
return internal::AppendDataFile(builder, *schema, *table_schema, *partition_type,
90+
readable_metrics_id, file);
91+
});
92+
}
93+
94+
} // namespace iceberg

‎src/iceberg/inspect/files_table.h‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
/// \file iceberg/inspect/files_table.h
23+
/// \brief Define the files metadata table.
24+
25+
#include <memory>
26+
#include <mutex>
27+
28+
#include "iceberg/arrow_c_data.h"
29+
#include "iceberg/iceberg_export.h"
30+
#include "iceberg/inspect/metadata_table.h"
31+
#include "iceberg/result.h"
32+
#include "iceberg/type_fwd.h"
33+
34+
namespace iceberg {
35+
36+
/// \brief Metadata table containing live data and delete files in a snapshot.
37+
class ICEBERG_EXPORT FilesTable : public MetadataTable {
38+
public:
39+
static Result<std::unique_ptr<FilesTable>> Make(std::shared_ptr<const Table> table);
40+
41+
~FilesTable() override;
42+
43+
Kind kind() const noexcept override { return Kind::kFiles; }
44+
45+
/// \brief Schema from construction or the most recent scan builder.
46+
/// Existing scans and tasks retain their captured schema.
47+
std::shared_ptr<Schema> schema() const override;
48+
49+
protected:
50+
Result<std::shared_ptr<Schema>> SchemaForScan(
51+
const TableMetadata& metadata) const override;
52+
53+
Result<std::unique_ptr<MetadataScan>> MakeScan(
54+
internal::MetadataScanContext context) const override;
55+
56+
private:
57+
static Result<ArrowArrayStream> ReadRows(const internal::MetadataScanContext& context,
58+
const std::shared_ptr<FileIO>& io,
59+
int64_t batch_size);
60+
61+
explicit FilesTable(std::shared_ptr<const Table> table);
62+
63+
mutable std::mutex schema_mutex_;
64+
mutable std::shared_ptr<Schema> schema_;
65+
};
66+
67+
} // namespace iceberg

0 commit comments

Comments
 (0)