Skip to content

Commit 38ef53b

Browse files
committed
fix(avro): honor decimal fixed widths and native byte order
Share decimal encoding across direct and generic writers, preserve signed values at the schema fixed width, and reject representation overflow. Cover exact wire bytes and decimal file round trips in both backends.
1 parent bb00e0a commit 38ef53b

5 files changed

Lines changed: 195 additions & 9 deletions

File tree

‎src/iceberg/avro/avro_data_util.cc‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
* under the License.
1818
*/
1919

20+
#include <algorithm>
21+
#include <bit>
2022
#include <span>
2123

2224
#include <arrow/array/builder_binary.h>
@@ -669,6 +671,27 @@ constexpr int64_t kValueBranch = 1;
669671

670672
} // namespace
671673

674+
Status EncodeDecimalToFixed(const ::arrow::Decimal128& value, size_t fixed_size,
675+
std::vector<uint8_t>* bytes) {
676+
ICEBERG_PRECHECK(fixed_size > 0, "Avro decimal fixed size must be positive");
677+
// Arrow's bytes are native-endian; Avro fixed decimals are big-endian.
678+
auto big_endian = value.ToBytes();
679+
if constexpr (std::endian::native == std::endian::little) {
680+
std::ranges::reverse(big_endian);
681+
}
682+
const uint8_t padding = (big_endian.front() & 0x80) == 0 ? 0x00 : 0xff;
683+
const auto value_size = std::min(fixed_size, big_endian.size());
684+
const auto first = big_endian.end() - value_size;
685+
// Discard only sign extension, without changing the retained sign bit.
686+
ICEBERG_PRECHECK(std::all_of(big_endian.begin(), first,
687+
[padding](uint8_t byte) { return byte == padding; }) &&
688+
((*first ^ padding) & 0x80) == 0,
689+
"Decimal does not fit Avro fixed size {}", fixed_size);
690+
bytes->assign(fixed_size - value_size, padding);
691+
bytes->insert(bytes->end(), first, big_endian.end());
692+
return {};
693+
}
694+
672695
Status ExtractDatumFromArray(const ::arrow::Array& array, int64_t index,
673696
::avro::GenericDatum* datum) {
674697
if (index < 0 || index >= array.length()) {
@@ -756,12 +779,10 @@ Status ExtractDatumFromArray(const ::arrow::Array& array, int64_t index,
756779
case ::arrow::Type::DECIMAL128: {
757780
const auto& decimal_array =
758781
internal::checked_cast<const ::arrow::Decimal128Array&>(array);
759-
std::string_view decimal_value = decimal_array.GetView(index);
760782
auto& fixed_datum = datum->value<::avro::GenericFixed>();
761-
auto& bytes = fixed_datum.value();
762-
bytes.assign(decimal_value.begin(), decimal_value.end());
763-
std::ranges::reverse(bytes);
764-
return {};
783+
return EncodeDecimalToFixed(::arrow::Decimal128(decimal_array.GetValue(index)),
784+
fixed_datum.schema()->fixedSize(),
785+
&fixed_datum.value());
765786
}
766787

767788
case ::arrow::Type::DATE32: {

‎src/iceberg/avro/avro_data_util_internal.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,14 @@
1919

2020
#pragma once
2121

22+
#include <cstddef>
23+
#include <cstdint>
2224
#include <memory>
25+
#include <vector>
2326

2427
#include <arrow/array/builder_base.h>
2528
#include <arrow/scalar.h>
29+
#include <arrow/util/decimal.h>
2630
#include <avro/GenericDatum.hh>
2731

2832
#include "iceberg/arrow/metadata_column_util_internal.h"
@@ -83,6 +87,11 @@ Status AppendDatumToBuilder(const ::avro::NodePtr& avro_node,
8387
const arrow::MetadataColumnContext& metadata_context,
8488
::arrow::ArrayBuilder* array_builder);
8589

90+
/// \brief Encode a decimal as exactly `fixed_size` big-endian signed bytes.
91+
/// Sign-extends smaller values and rejects values that cannot fit the fixed width.
92+
Status EncodeDecimalToFixed(const ::arrow::Decimal128& value, size_t fixed_size,
93+
std::vector<uint8_t>* bytes);
94+
8695
/// \brief Extract an Avro datum from an Arrow array.
8796
///
8897
/// \param array The Arrow array to extract from.

‎src/iceberg/avro/avro_direct_encoder.cc‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <arrow/type.h>
2626
#include <avro/Specific.hh>
2727

28+
#include "iceberg/avro/avro_data_util_internal.h"
2829
#include "iceberg/avro/avro_direct_encoder_internal.h"
2930
#include "iceberg/type.h"
3031
#include "iceberg/util/checked_cast.h"
@@ -194,10 +195,9 @@ Status EncodeArrowToAvro(const ::avro::NodePtr& avro_node, ::avro::Encoder& enco
194195
if (avro_node->logicalType().type() == ::avro::LogicalType::DECIMAL) {
195196
const auto& decimal_array =
196197
internal::checked_cast<const ::arrow::Decimal128Array&>(array);
197-
std::string_view decimal_value = decimal_array.GetView(row_index);
198-
ctx.bytes_scratch.assign(decimal_value.begin(), decimal_value.end());
199-
// Arrow Decimal128 bytes are in little-endian order, Avro requires big-endian
200-
std::ranges::reverse(ctx.bytes_scratch);
198+
ICEBERG_RETURN_UNEXPECTED(
199+
EncodeDecimalToFixed(::arrow::Decimal128(decimal_array.GetValue(row_index)),
200+
avro_node->fixedSize(), &ctx.bytes_scratch));
201201
encoder.encodeFixed(ctx.bytes_scratch.data(), ctx.bytes_scratch.size());
202202
return {};
203203
}

‎src/iceberg/test/avro_data_test.cc‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <ranges>
2222
#include <vector>
2323

24+
#include <arrow/array/builder_decimal.h>
2425
#include <arrow/array/builder_nested.h>
2526
#include <arrow/array/builder_primitive.h>
2627
#include <arrow/c/bridge.h>
@@ -31,21 +32,159 @@
3132
#include <avro/Compiler.hh>
3233
#include <avro/Generic.hh>
3334
#include <avro/Node.hh>
35+
#include <avro/Specific.hh>
36+
#include <avro/Stream.hh>
3437
#include <avro/Types.hh>
3538
#include <gmock/gmock.h>
3639
#include <gtest/gtest.h>
3740

3841
#include "iceberg/avro/avro_data_util_internal.h"
42+
#include "iceberg/avro/avro_direct_encoder_internal.h"
3943
#include "iceberg/avro/avro_schema_util_internal.h"
4044
#include "iceberg/expression/literal.h"
4145
#include "iceberg/schema.h"
4246
#include "iceberg/schema_internal.h"
4347
#include "iceberg/schema_util.h"
4448
#include "iceberg/test/matchers.h"
4549
#include "iceberg/type.h"
50+
#include "iceberg/util/string_util.h"
4651

4752
namespace iceberg::avro {
4853

54+
class AvroDecimalEncodingTest : public ::testing::TestWithParam<bool> {};
55+
56+
TEST_P(AvroDecimalEncodingTest, FixedWidthBigEndianBytes) {
57+
struct TestCase {
58+
int32_t precision;
59+
std::string unscaled;
60+
std::string expected_hex;
61+
};
62+
const std::vector<TestCase> cases{
63+
{.precision = 1, .unscaled = "0", .expected_hex = "00"},
64+
{.precision = 1, .unscaled = "1", .expected_hex = "01"},
65+
{.precision = 1, .unscaled = "-1", .expected_hex = "ff"},
66+
{.precision = 2, .unscaled = "99", .expected_hex = "63"},
67+
{.precision = 2, .unscaled = "-99", .expected_hex = "9d"},
68+
{.precision = 3, .unscaled = "127", .expected_hex = "007f"},
69+
{.precision = 3, .unscaled = "128", .expected_hex = "0080"},
70+
{.precision = 3, .unscaled = "-128", .expected_hex = "ff80"},
71+
{.precision = 3, .unscaled = "-129", .expected_hex = "ff7f"},
72+
{.precision = 5, .unscaled = "32767", .expected_hex = "007fff"},
73+
{.precision = 5, .unscaled = "32768", .expected_hex = "008000"},
74+
{.precision = 5, .unscaled = "-32768", .expected_hex = "ff8000"},
75+
{.precision = 5, .unscaled = "-32769", .expected_hex = "ff7fff"},
76+
{.precision = 10, .unscaled = "1001", .expected_hex = "00000003e9"},
77+
{.precision = 10, .unscaled = "-1001", .expected_hex = "fffffffc17"},
78+
{.precision = 38,
79+
.unscaled = "0",
80+
.expected_hex = "00000000000000000000000000000000"},
81+
{.precision = 38,
82+
.unscaled = "1",
83+
.expected_hex = "00000000000000000000000000000001"},
84+
{.precision = 38,
85+
.unscaled = "-1",
86+
.expected_hex = "ffffffffffffffffffffffffffffffff"},
87+
{.precision = 38,
88+
.unscaled = "128",
89+
.expected_hex = "00000000000000000000000000000080"},
90+
{.precision = 38,
91+
.unscaled = "-129",
92+
.expected_hex = "ffffffffffffffffffffffffffffff7f"},
93+
{.precision = 38,
94+
.unscaled = "1208925819614629174706299",
95+
.expected_hex = "0000000000010000000000000000007b"},
96+
{.precision = 38,
97+
.unscaled = "-1208925819614629174706299",
98+
.expected_hex = "fffffffffffeffffffffffffffffff85"},
99+
{.precision = 38,
100+
.unscaled = "99999999999999999999999999999999999999",
101+
.expected_hex = "4b3b4ca85a86c47a098a223fffffffff"},
102+
{.precision = 38,
103+
.unscaled = "-99999999999999999999999999999999999999",
104+
.expected_hex = "b4c4b357a5793b85f675ddc000000001"},
105+
};
106+
for (const auto& test : cases) {
107+
SCOPED_TRACE(test.unscaled);
108+
auto type = decimal(test.precision, 0);
109+
::avro::NodePtr node;
110+
ASSERT_THAT(ToAvroNodeVisitor{}.Visit(*type, &node), IsOk());
111+
ICEBERG_UNWRAP_OR_FAIL(auto expected,
112+
StringUtils::HexStringToBytes(test.expected_hex));
113+
ASSERT_EQ(node->fixedSize(), expected.size());
114+
::arrow::Decimal128Builder builder(::arrow::decimal128(test.precision, 0),
115+
::arrow::default_memory_pool());
116+
ASSERT_TRUE(builder.Append(::arrow::Decimal128(test.unscaled)).ok());
117+
auto array = builder.Finish().ValueOrDie();
118+
auto stream = ::avro::memoryOutputStream();
119+
auto encoder = ::avro::binaryEncoder();
120+
encoder->init(*stream);
121+
if (GetParam()) {
122+
EncodeContext context;
123+
ASSERT_THAT(EncodeArrowToAvro(node, *encoder, *type, *array, 0, context), IsOk());
124+
} else {
125+
::avro::GenericDatum datum(node);
126+
ASSERT_THAT(ExtractDatumFromArray(*array, 0, &datum), IsOk());
127+
EXPECT_EQ(datum.value<::avro::GenericFixed>().value(), expected);
128+
::avro::encode(*encoder, datum);
129+
}
130+
encoder->flush();
131+
EXPECT_EQ(*::avro::snapshot(*stream), expected);
132+
}
133+
}
134+
135+
TEST_P(AvroDecimalEncodingTest, RejectsFixedWidthOverflow) {
136+
auto type = decimal(2, 0);
137+
::avro::NodePtr node;
138+
ASSERT_THAT(ToAvroNodeVisitor{}.Visit(*type, &node), IsOk());
139+
ASSERT_EQ(node->fixedSize(), 1);
140+
// Arrow's builder accepts these invalid values; do not silently truncate them.
141+
for (const auto& unscaled : {"128", "-129", "256", "-257"}) {
142+
SCOPED_TRACE(unscaled);
143+
::arrow::Decimal128Builder builder(::arrow::decimal128(2, 0),
144+
::arrow::default_memory_pool());
145+
ASSERT_TRUE(builder.Append(::arrow::Decimal128(std::string(unscaled))).ok());
146+
auto array = builder.Finish().ValueOrDie();
147+
auto stream = ::avro::memoryOutputStream();
148+
auto encoder = ::avro::binaryEncoder();
149+
encoder->init(*stream);
150+
if (GetParam()) {
151+
EncodeContext context;
152+
EXPECT_THAT(EncodeArrowToAvro(node, *encoder, *type, *array, 0, context),
153+
IsError(ErrorKind::kInvalidArgument));
154+
} else {
155+
::avro::GenericDatum datum(node);
156+
EXPECT_THAT(ExtractDatumFromArray(*array, 0, &datum),
157+
IsError(ErrorKind::kInvalidArgument));
158+
}
159+
encoder->flush();
160+
EXPECT_TRUE(::avro::snapshot(*stream)->empty());
161+
}
162+
}
163+
164+
TEST(AvroDecimalFixedTest, SignedBoundariesAndPadding) {
165+
std::vector<uint8_t> bytes;
166+
ASSERT_THAT(EncodeDecimalToFixed(::arrow::Decimal128(127), 1, &bytes), IsOk());
167+
EXPECT_EQ(bytes, (std::vector<uint8_t>{0x7f}));
168+
ASSERT_THAT(EncodeDecimalToFixed(::arrow::Decimal128(-128), 1, &bytes), IsOk());
169+
EXPECT_EQ(bytes, (std::vector<uint8_t>{0x80}));
170+
ASSERT_THAT(EncodeDecimalToFixed(::arrow::Decimal128(1), 17, &bytes), IsOk());
171+
ICEBERG_UNWRAP_OR_FAIL(
172+
auto positive, StringUtils::HexStringToBytes("0000000000000000000000000000000001"));
173+
EXPECT_EQ(bytes, positive);
174+
ASSERT_THAT(EncodeDecimalToFixed(::arrow::Decimal128(-1), 17, &bytes), IsOk());
175+
ICEBERG_UNWRAP_OR_FAIL(
176+
auto negative, StringUtils::HexStringToBytes("ffffffffffffffffffffffffffffffffff"));
177+
EXPECT_EQ(bytes, negative);
178+
EXPECT_THAT(EncodeDecimalToFixed(::arrow::Decimal128(0), 0, &bytes),
179+
IsError(ErrorKind::kInvalidArgument));
180+
}
181+
182+
INSTANTIATE_TEST_SUITE_P(WriterBackends, AvroDecimalEncodingTest,
183+
::testing::Values(true, false),
184+
[](const ::testing::TestParamInfo<bool>& info) {
185+
return info.param ? "DirectEncoder" : "GenericDatum";
186+
});
187+
49188
/// \brief Test case structure for parameterized primitive type tests
50189
struct AppendDatumParam {
51190
std::string name;

‎src/iceberg/test/avro_test.cc‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,6 +1042,23 @@ TEST_P(AvroWriterTest, WritePrimitiveTypes) {
10421042
VerifyWrittenData(test_data);
10431043
}
10441044

1045+
TEST_P(AvroWriterTest, WriteDecimalFixedWidths) {
1046+
auto schema = std::make_shared<Schema>(
1047+
std::vector<SchemaField>{SchemaField::MakeRequired(1, "tiny", decimal(1, 0)),
1048+
SchemaField::MakeRequired(2, "narrow", decimal(3, 0)),
1049+
SchemaField::MakeRequired(3, "wide", decimal(38, 0)),
1050+
SchemaField::MakeRequired(4, "sentinel", int32()),
1051+
SchemaField::MakeOptional(5, "optional", decimal(10, 2))});
1052+
const std::string data = R"([
1053+
["0", "127", "99999999999999999999999999999999999999", 7, "10.01"],
1054+
["1", "128", "0", 8, null],
1055+
["-1", "-128", "-99999999999999999999999999999999999999", 9, "-20.02"],
1056+
["0", "-129", "1", 10, "0.00"]
1057+
])";
1058+
ASSERT_NO_FATAL_FAILURE(WriteAvroFile(schema, data));
1059+
ASSERT_NO_FATAL_FAILURE(VerifyWrittenData(data));
1060+
}
1061+
10451062
TEST_P(AvroWriterTest, WriteUuidType) {
10461063
auto schema = std::make_shared<iceberg::Schema>(std::vector<SchemaField>{
10471064
SchemaField::MakeRequired(1, "uuid_col", iceberg::uuid())});

0 commit comments

Comments
 (0)