diff --git a/src/EngineeredWood.Parquet/Parquet/Data/ArrowSchemaConverter.cs b/src/EngineeredWood.Parquet/Parquet/Data/ArrowSchemaConverter.cs index 8fad111..43ea426 100644 --- a/src/EngineeredWood.Parquet/Parquet/Data/ArrowSchemaConverter.cs +++ b/src/EngineeredWood.Parquet/Parquet/Data/ArrowSchemaConverter.cs @@ -303,7 +303,13 @@ public static IArrowType ToArrowType(ColumnDescriptor column, ParquetReadOptions { BitWidth: 64, IsSigned: false } => UInt64Type.Default, _ => null, }, - LogicalType.TimestampType ts => new TimestampType( + // The guard is load-bearing, not defensive. This arm hands back an Arrow TimestampType, and + // the read path then reinterprets the column's value buffer as int64 (ArrowArrayBuilder maps + // `Int64Type or TimestampType or Time64Type` to a long buffer). A TIMESTAMP annotation on any + // other physical width therefore decoded silent garbage instead of failing. Falling through + // to the physical type is lossless. MakeDecimalType has always switched on the physical type + // for the same reason; this arm had not. + LogicalType.TimestampType ts when column.PhysicalType == PhysicalType.Int64 => new TimestampType( ts.Unit switch { Metadata.TimeUnit.Millis => Apache.Arrow.Types.TimeUnit.Millisecond, @@ -344,8 +350,14 @@ public static IArrowType ToArrowType(ColumnDescriptor column, ParquetReadOptions // TimeZoneInfo overload renders the "+00:00" offset. A file old enough to carry only a // converted type is exactly the kind this matters for, since it was written by a tool // whose output is still being read by everything else. - ConvertedType.TimestampMillis => new TimestampType(Apache.Arrow.Types.TimeUnit.Millisecond, "UTC"), - ConvertedType.TimestampMicros => new TimestampType(Apache.Arrow.Types.TimeUnit.Microsecond, "UTC"), + // Guarded on INT64 for the same reason as the logical-type arm: these hand back an Arrow + // TimestampType, whose buffer is read as int64. TIMESTAMP_MILLIS / TIMESTAMP_MICROS are + // INT64-only converted types, so a file carrying one on another width is malformed — and + // reading it as raw bytes beats decoding a plausible-looking wrong date. + ConvertedType.TimestampMillis when column.PhysicalType == PhysicalType.Int64 + => new TimestampType(Apache.Arrow.Types.TimeUnit.Millisecond, "UTC"), + ConvertedType.TimestampMicros when column.PhysicalType == PhysicalType.Int64 + => new TimestampType(Apache.Arrow.Types.TimeUnit.Microsecond, "UTC"), ConvertedType.TimeMillis => new Time32Type(Apache.Arrow.Types.TimeUnit.Millisecond), ConvertedType.TimeMicros => new Time64Type(Apache.Arrow.Types.TimeUnit.Microsecond), ConvertedType.Int8 => Int8Type.Default, diff --git a/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkReader.cs b/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkReader.cs index de084fc..4009598 100644 --- a/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkReader.cs +++ b/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkReader.cs @@ -1183,7 +1183,7 @@ private static void DecodeDeltaByteArrayValues( throw new NotSupportedException( $"Physical type '{column.PhysicalType}' is not supported for DELTA_BYTE_ARRAY decoding."); - DeltaByteArrayDecoder.Decode(data, count, state); + DeltaByteArrayDecoder.Decode(data, count, state, column.TypeLength ?? 0); } private static void DecodeByteStreamSplitValues( diff --git a/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkWriter.cs b/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkWriter.cs index 2e8b32d..22eacec 100644 --- a/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkWriter.cs +++ b/src/EngineeredWood.Parquet/Parquet/Data/ColumnChunkWriter.cs @@ -233,7 +233,7 @@ private static ColumnChunkResult WriteColumnCore( : StatisticsCollector.Compute( array, physicalType, typeLength, valueDefLevels, nonNullCount, rowCount, floatingPointTotalOrder); - result.MetaData.Statistics = DropDeprecatedMinMaxIfMisordered(stats, ValueType(array)); + result.MetaData.Statistics = DropDeprecatedMinMaxIfMisordered(stats, ValueType(array), physicalType); } // Build Bloom filter if enabled for this column. @@ -502,7 +502,7 @@ internal static ColumnChunkResult WriteDictionaryColumnFromResult( dictResult.DictionaryPageData, dictResult.DictionaryCount, physicalType, typeLength, rowCount - nonNullCount); - result.MetaData.Statistics = DropDeprecatedMinMaxIfMisordered(stats, arrowType); + result.MetaData.Statistics = DropDeprecatedMinMaxIfMisordered(stats, arrowType, physicalType); return result; } @@ -1667,9 +1667,9 @@ private static int EstimateColumnSize(int rowCount, PhysicalType physicalType, i /// signed ints incl. date/time/timestamp, floats); drop them elsewhere. /// private static Statistics DropDeprecatedMinMaxIfMisordered( - Statistics stats, Apache.Arrow.Types.IArrowType type) + Statistics stats, Apache.Arrow.Types.IArrowType type, PhysicalType physicalType) { - if (SignedOrderMatchesLogical(type)) + if (SignedOrderMatchesLogical(type, physicalType)) return stats; return new Statistics @@ -1686,12 +1686,31 @@ private static Statistics DropDeprecatedMinMaxIfMisordered( // True when the type's SIGNED byte ordering equals its logical ordering — the precondition for emitting // the deprecated Statistics.min/max fields (see DropDeprecatedMinMaxIfMisordered). - private static bool SignedOrderMatchesLogical(Apache.Arrow.Types.IArrowType type) => type switch + // + // The real precondition is narrower than "this Arrow type is signed": it is that StatisticsCollector + // compared these values with a TYPED comparator. It does that only for BOOLEAN/INT32/INT64/FLOAT/DOUBLE; + // every FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY column goes through SequenceCompareTo, i.e. unsigned + // lexicographic. So the physical type has to be part of the answer wherever an Arrow type can arrive on + // more than one physical width. + // Internal rather than private so the gate can be pinned directly. The FIXED_LEN_BYTE_ARRAY answer is + // latent until an Arrow TimestampType can map to that physical type, so there is no end-to-end write + // that reaches it yet — a unit test is the only thing that keeps the fix from silently regressing. + internal static bool SignedOrderMatchesLogical( + Apache.Arrow.Types.IArrowType type, PhysicalType physicalType) => type switch { BooleanType => true, Int8Type or Int16Type or Int32Type or Int64Type => true, - FloatType or DoubleType or HalfFloatType => true, - Date32Type or Date64Type or Time32Type or Time64Type or TimestampType or DurationType => true, + FloatType or DoubleType => true, + // FLOAT16 is FLBA(2), so this claim is already wrong for it — its bounds come from a lexicographic + // byte comparison over IEEE-754 halves. That is a separately tracked gap (doc/known-issues.md), and + // is left exactly as it was rather than quietly changed under cover of this fix. + HalfFloatType => true, + // Date/Time/Duration only ever arrive on INT32/INT64, so the collector used a typed comparator. + Date32Type or Date64Type or Time32Type or Time64Type or DurationType => true, + // TIMESTAMP is the one that can also arrive on FIXED_LEN_BYTE_ARRAY. There the collector compares + // bytes unsigned-lexicographically, which is not the signed order these deprecated fields promise — + // and a wrong bound in the footer is a wrong prune, not a cosmetic defect. + TimestampType => physicalType == PhysicalType.Int64, Decimal32Type or Decimal64Type => true, // INT32/INT64 physical — signed numeric ordering _ => false, // UTF-8 strings/binary (unsigned lexical), unsigned ints, decimal FLBA, nested, ... }; diff --git a/src/EngineeredWood.Parquet/Parquet/Data/DeltaByteArrayDecoder.cs b/src/EngineeredWood.Parquet/Parquet/Data/DeltaByteArrayDecoder.cs index 7834687..5647687 100644 --- a/src/EngineeredWood.Parquet/Parquet/Data/DeltaByteArrayDecoder.cs +++ b/src/EngineeredWood.Parquet/Parquet/Data/DeltaByteArrayDecoder.cs @@ -19,8 +19,18 @@ internal static class DeltaByteArrayDecoder /// /// Decodes byte array values and appends them to . /// - public static void Decode(ReadOnlySpan data, int count, ColumnBuildState state) + /// + /// Fixed value width for a FIXED_LEN_BYTE_ARRAY column, or 0 for BYTE_ARRAY. The two physical types + /// share this encoding but not their destination buffers, and the state only allocates the pair the + /// column's physical type calls for -- so the width has to reach here rather than be inferred. + /// + public static void Decode(ReadOnlySpan data, int count, ColumnBuildState state, int typeLength = 0) { + bool fixedWidth = state.PhysicalType == PhysicalType.FixedLenByteArray; + if (fixedWidth && typeLength <= 0) + throw new ParquetFormatException( + "A FIXED_LEN_BYTE_ARRAY column decoded as DELTA_BYTE_ARRAY has no type_length."); + // Step 1: Decode prefix lengths var prefixDecoder = new DeltaBinaryPackedDecoder(data); var prefixLengths = new int[count]; @@ -38,14 +48,51 @@ public static void Decode(ReadOnlySpan data, int count, ColumnBuildState s // Step 4: Reconstruct values by combining prefix from previous value + suffix // Compute total output size var valueLengths = new int[count]; - int totalBytes = 0; + long totalBytes = 0; + long totalSuffixBytes = 0; for (int i = 0; i < count; i++) { - valueLengths[i] = prefixLengths[i] + suffixLengths[i]; + int prefixLen = prefixLengths[i]; + int suffixLen = suffixLengths[i]; + + if (prefixLen < 0 || suffixLen < 0) + throw new ParquetFormatException( + $"DELTA_BYTE_ARRAY value at index {i} has a negative prefix ({prefixLen}) or suffix " + + $"({suffixLen}) length."); + + // A value is the first prefixLen bytes of the PREVIOUS value plus a suffix. A prefix that does + // not fit inside the previous value -- including ANY prefix on the first value, which has no + // predecessor -- would be reconstructed from the zero-filled bytes reserved for this value: + // neither what was encoded nor an error. The output buffer is sized from these same lengths, + // so nothing reads out of bounds and nothing throws; it just comes out wrong. + int previousLength = i == 0 ? 0 : valueLengths[i - 1]; + if (prefixLen > previousLength) + throw new ParquetFormatException( + $"DELTA_BYTE_ARRAY value at index {i} claims a {prefixLen}-byte prefix of a value that " + + $"is {previousLength} bytes long."); + + valueLengths[i] = prefixLen + suffixLen; + if (fixedWidth && valueLengths[i] != typeLength) + throw new ParquetFormatException( + $"DELTA_BYTE_ARRAY value at index {i} is {valueLengths[i]} bytes, but the column is " + + $"FIXED_LEN_BYTE_ARRAY({typeLength}). Every value in such a column is exactly that wide."); totalBytes += valueLengths[i]; + totalSuffixBytes += suffixLen; } - var outputData = new byte[totalBytes]; + // Accumulated as long: prefixes let the total grow faster than the page does, so a malformed + // page can describe more output than an int can hold. + if (totalBytes > int.MaxValue) + throw new ParquetFormatException( + $"DELTA_BYTE_ARRAY page describes {totalBytes} bytes of values, which exceeds the maximum " + + "addressable buffer."); + + if (totalSuffixBytes > rawSuffixes.Length) + throw new ParquetFormatException( + $"DELTA_BYTE_ARRAY page declares {totalSuffixBytes} suffix bytes but carries " + + $"{rawSuffixes.Length}."); + + var outputData = new byte[(int)totalBytes]; var offsets = new int[count + 1]; int outputPos = 0; int suffixPos = 0; @@ -73,6 +120,17 @@ public static void Decode(ReadOnlySpan data, int count, ColumnBuildState s } offsets[count] = outputPos; + if (fixedWidth) + { + // Every value is exactly typeLength bytes, so the reconstruction above is already the packed + // layout the fixed-width buffer wants and the offsets are redundant. AddByteArrayValues is NOT + // an option here: it writes through the data/offsets buffer pair, which ColumnBuildState only + // allocates for BYTE_ARRAY columns -- reaching it with a FIXED_LEN_BYTE_ARRAY column threw a + // NullReferenceException. + outputData.AsSpan(0, count * typeLength).CopyTo(state.ReserveFixedBytes(count, typeLength)); + return; + } + state.AddByteArrayValues(offsets, outputData, count); } } diff --git a/src/EngineeredWood.Parquet/Parquet/ParquetStatisticsAccessor.cs b/src/EngineeredWood.Parquet/Parquet/ParquetStatisticsAccessor.cs index 9a19a75..06644c6 100644 --- a/src/EngineeredWood.Parquet/Parquet/ParquetStatisticsAccessor.cs +++ b/src/EngineeredWood.Parquet/Parquet/ParquetStatisticsAccessor.cs @@ -106,13 +106,13 @@ private static Dictionary BuildNameIndex(SchemaDescriptor schema) private static LiteralValue? DecodeMin(ColumnDescriptor desc, Statistics stats) { var bytes = stats.MinValue ?? FallbackBytes(desc, stats.Min); - return bytes is null ? null : Decode(desc, bytes); + return bytes is null ? null : Decode(desc, bytes, isMax: false); } private static LiteralValue? DecodeMax(ColumnDescriptor desc, Statistics stats) { var bytes = stats.MaxValue ?? FallbackBytes(desc, stats.Max); - return bytes is null ? null : Decode(desc, bytes); + return bytes is null ? null : Decode(desc, bytes, isMax: true); } /// @@ -134,7 +134,12 @@ PhysicalType.Int32 or PhysicalType.Int64 }; } - private static LiteralValue? Decode(ColumnDescriptor desc, byte[] bytes) + /// + /// Which end of the range this bound is. It matters wherever the decode cannot be exact: a bound + /// must only ever move OUTWARD. Rounding a max down, or a min up, narrows the range the file claims + /// and lets a row group be pruned that genuinely contains matching rows. + /// + private static LiteralValue? Decode(ColumnDescriptor desc, byte[] bytes, bool isMax) { var logical = desc.SchemaElement.LogicalType; @@ -143,11 +148,11 @@ PhysicalType.Int32 or PhysicalType.Int64 PhysicalType.Boolean => bytes.Length >= 1 ? (LiteralValue?)LiteralValue.Of(bytes[0] != 0) : null, PhysicalType.Int32 => DecodeInt32(bytes, logical), - PhysicalType.Int64 => DecodeInt64(bytes, logical), + PhysicalType.Int64 => DecodeInt64(bytes, logical, isMax), PhysicalType.Float => DecodeFloat(bytes), PhysicalType.Double => DecodeDouble(bytes), PhysicalType.ByteArray => DecodeByteArray(bytes, logical), - PhysicalType.FixedLenByteArray => DecodeFixedLenByteArray(desc, bytes, logical), + PhysicalType.FixedLenByteArray => DecodeFixedLenByteArray(desc, bytes, logical, isMax), // INT96 sort order is undefined per the Parquet spec. PhysicalType.Int96 => null, _ => null, @@ -203,7 +208,7 @@ PhysicalType.Int32 or PhysicalType.Int64 } } - private static LiteralValue? DecodeInt64(byte[] bytes, LogicalType? logical) + private static LiteralValue? DecodeInt64(byte[] bytes, LogicalType? logical, bool isMax) { if (bytes.Length < 8) return null; long v = BinaryPrimitives.ReadInt64LittleEndian(bytes); @@ -215,24 +220,18 @@ PhysicalType.Int32 or PhysicalType.Int64 case LogicalType.DecimalType d: return LiteralValue.HighPrecisionDecimalOf(new BigInteger(v), d.Scale); case LogicalType.TimestampType ts: - long unixMs = ts.Unit switch - { - TimeUnit.Millis => v, - TimeUnit.Micros => v / 1000, - TimeUnit.Nanos => v / 1_000_000, - _ => v, - }; - var dto = ts.IsAdjustedToUtc - ? DateTimeOffset.FromUnixTimeMilliseconds(unixMs) - : new DateTimeOffset( - DateTimeOffset.FromUnixTimeMilliseconds(unixMs).Ticks, - TimeSpan.Zero); - return LiteralValue.Of(dto); + return TimestampLiteral(new BigInteger(v), ts, isMax); #if NET6_0_OR_GREATER case LogicalType.TimeType t when t.Unit == TimeUnit.Micros: - return LiteralValue.Of(new TimeOnly(v * 10)); // micros → ticks (10 ticks per us) + return LiteralValue.Of(new TimeOnly(v * 10)); // micros → ticks (10 ticks per us), exact case LogicalType.TimeType t when t.Unit == TimeUnit.Nanos: - return LiteralValue.Of(new TimeOnly(v / 100)); // 100 ns per tick + // 100 ns per tick, so this is the one TIME unit that cannot be exact. Round outward, then + // clamp: rounding the last nanoseconds of a day up would leave TimeOnly's range, and its + // maximum is still a sound outward bound. + long timeTicks = (long)DivideOutward(v, 100, isMax); + if (timeTicks < 0 || timeTicks > TimeOnly.MaxValue.Ticks) + timeTicks = Math.Clamp(timeTicks, 0, TimeOnly.MaxValue.Ticks); + return LiteralValue.Of(new TimeOnly(timeTicks)); #endif default: return LiteralValue.Of(v); @@ -254,7 +253,7 @@ or LogicalType.JsonType } private static LiteralValue? DecodeFixedLenByteArray( - ColumnDescriptor desc, byte[] bytes, LogicalType? logical) + ColumnDescriptor desc, byte[] bytes, LogicalType? logical, bool isMax) { switch (logical) { @@ -274,6 +273,58 @@ or LogicalType.JsonType } } + /// Ticks (100 ns) from .NET's epoch (0001-01-01) to the Unix epoch. + private const long UnixEpochTicks = 621_355_968_000_000_000L; + + /// + /// Builds a timestamp bound from a count of 's unit since the Unix epoch. + /// + /// + /// Goes through TICKS rather than milliseconds. A holds 100 ns, + /// so MILLIS and MICROS convert exactly and only NANOS has to round -- where the previous + /// millisecond conversion threw away everything below a millisecond for all three. That was not + /// merely imprecise: it truncated toward zero, so a max bound of 1500 us came back as 0 ms, and a + /// row group whose rows genuinely matched `t > 0.5ms` could be pruned on the strength of it. + /// + /// Returns outside 's range. A bound that + /// cannot be represented is not a bound; clamping one would be indistinguishable from a real + /// endpoint and would prune on a value the file never contained. + /// + private static LiteralValue? TimestampLiteral(BigInteger value, LogicalType.TimestampType ts, bool isMax) + { + BigInteger unixTicks = ts.Unit switch + { + TimeUnit.Millis => value * 10_000, + TimeUnit.Micros => value * 10, + TimeUnit.Nanos => DivideOutward(value, 100, isMax), + _ => value * 10, + }; + + BigInteger ticks = unixTicks + UnixEpochTicks; + if (ticks < BigInteger.Zero || ticks > DateTime.MaxValue.Ticks) + return null; + + return LiteralValue.Of(new DateTimeOffset((long)ticks, TimeSpan.Zero)); + } + + /// + /// Divides so the result moves AWAY from zero-error: a max bound rounds up, a min bound rounds down. + /// Both widen the range the bound describes, which is the only safe direction for pruning. + /// + private static BigInteger DivideOutward(BigInteger value, int divisor, bool isMax) + { + // DivRem truncates toward zero, so which adjustment is needed depends on the sign: for a + // positive value the quotient is already the floor, for a negative one it is already the ceiling. + BigInteger quotient = BigInteger.DivRem(value, divisor, out BigInteger remainder); + if (remainder.IsZero) + return quotient; + + if (isMax) + return value.Sign > 0 ? quotient + BigInteger.One : quotient; + + return value.Sign < 0 ? quotient - BigInteger.One : quotient; + } + /// Days from .NET epoch (0001-01-01) to Unix epoch (1970-01-01). private const int EpochDays = 719_162; diff --git a/test/EngineeredWood.Parquet.Tests/Parquet/Data/DeltaByteArrayMalformedTests.cs b/test/EngineeredWood.Parquet.Tests/Parquet/Data/DeltaByteArrayMalformedTests.cs new file mode 100644 index 0000000..b49d9d1 --- /dev/null +++ b/test/EngineeredWood.Parquet.Tests/Parquet/Data/DeltaByteArrayMalformedTests.cs @@ -0,0 +1,117 @@ +// Copyright (c) clast-project. All rights reserved. +// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information. + +using EngineeredWood.Parquet; +using EngineeredWood.Parquet.Data; + +namespace EngineeredWood.Tests.Parquet.Data; + +/// +/// DELTA_BYTE_ARRAY reconstructs each value as the first prefix_length bytes of the PREVIOUS +/// value, followed by a suffix. Nothing checked that the prefix length actually fit inside the +/// previous value. +/// +/// It does not read out of bounds — the output buffer is sized from the same lengths — so it read +/// forward into the zero-filled region reserved for the value being reconstructed, and produced a +/// value that is neither what was encoded nor an error. A first value with a nonzero prefix is the +/// same bug at index 0, where there is no previous value at all. +/// +/// Malformed input has to be rejected rather than reconstructed into something plausible. These +/// payloads are hand-built, because no encoder here can produce them. +/// +public class DeltaByteArrayMalformedTests +{ + /// + /// Builds a DELTA_BYTE_ARRAY page from raw parts: prefix lengths, suffix lengths, suffix bytes. + /// + private static byte[] Page(int[] prefixLengths, int[] suffixLengths, byte[] suffixBytes) + { + var prefixes = new DeltaBinaryPackedEncoder(64); + prefixes.EncodeInt32s(prefixLengths); + var suffixes = new DeltaBinaryPackedEncoder(64); + suffixes.EncodeInt32s(suffixLengths); + + var page = new byte[prefixes.Length + suffixes.Length + suffixBytes.Length]; + prefixes.WrittenSpan.CopyTo(page); + suffixes.WrittenSpan.CopyTo(page.AsSpan(prefixes.Length)); + suffixBytes.CopyTo(page.AsSpan(prefixes.Length + suffixes.Length)); + return page; + } + + private static ParquetFormatException Decode(byte[] page, int count) + { + using var state = new ColumnBuildState(PhysicalType.ByteArray, 0, 0, capacity: 16); + return Assert.Throws(() => DeltaByteArrayDecoder.Decode(page, count, state)); + } + + [Fact] + public void APrefixLongerThanThePreviousValueIsRejected() + { + // Value 0 is two bytes. Value 1 claims to share five of them — there are only two, so the + // reconstruction would take three bytes of whatever follows. + var page = Page([0, 5], [2, 1], [0xAA, 0xBB, 0xCC]); + + var error = Decode(page, 2); + Assert.Contains("prefix", error.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void APrefixOnTheFirstValueIsRejected() + { + // There is no previous value at index 0, so any nonzero prefix is meaningless. + var page = Page([3], [1], [0xAA]); + + var error = Decode(page, 1); + Assert.Contains("prefix", error.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ANegativePrefixIsRejected() + { + var page = Page([0, -1], [2, 2], [0xAA, 0xBB, 0xCC, 0xDD]); + + Decode(page, 2); + } + + [Fact] + public void ANegativeSuffixIsRejected() + { + var page = Page([0, 0], [2, -1], [0xAA, 0xBB]); + + Decode(page, 2); + } + + [Fact] + public void ASuffixRunningPastTheEndOfThePageIsRejected() + { + // The suffix bytes claimed are longer than the page actually carries. + var page = Page([0, 0], [2, 99], [0xAA, 0xBB]); + + Decode(page, 2); + } + + [Fact] + public void APrefixExactlyTheLengthOfThePreviousValueIsFine() + { + // The boundary the check must not reject: sharing the whole previous value is legal, and is + // what an encoder emits for a repeated value. + var page = Page([0, 2], [2, 0], [0xAA, 0xBB]); + + using var state = new ColumnBuildState(PhysicalType.ByteArray, 0, 0, capacity: 16); + DeltaByteArrayDecoder.Decode(page, 2, state); + + Assert.Equal(2, state.ValueCount); + } + + [Fact] + public void AWellFormedPageStillDecodes() + { + // "AB", then "AC" — one shared prefix byte, which is the ordinary case. + var page = Page([0, 1], [2, 1], [0x41, 0x42, 0x43]); + + using var state = new ColumnBuildState(PhysicalType.ByteArray, 0, 0, capacity: 16); + DeltaByteArrayDecoder.Decode(page, 2, state); + + Assert.Equal(2, state.ValueCount); + } +} diff --git a/test/EngineeredWood.Parquet.Tests/Parquet/Data/FlbaDeltaByteArrayTests.cs b/test/EngineeredWood.Parquet.Tests/Parquet/Data/FlbaDeltaByteArrayTests.cs new file mode 100644 index 0000000..7011f63 --- /dev/null +++ b/test/EngineeredWood.Parquet.Tests/Parquet/Data/FlbaDeltaByteArrayTests.cs @@ -0,0 +1,219 @@ +// Copyright (c) clast-project. All rights reserved. +// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information. + +using Apache.Arrow; +using Apache.Arrow.Arrays; +using Apache.Arrow.Types; +using EngineeredWood.IO.Local; +using EngineeredWood.Parquet; +using EngineeredWood.Parquet.Data; + +namespace EngineeredWood.Tests.Parquet.Data; + +/// +/// DELTA_BYTE_ARRAY is legal for FIXED_LEN_BYTE_ARRAY as well as BYTE_ARRAY, and the writer emits it for +/// both whenever is chosen with V2 pages. The reader could +/// not read the result back: DeltaByteArrayDecoder finished by calling +/// ColumnBuildState.AddByteArrayValues, which writes through the data/offsets buffer pair that the +/// state only allocates for BYTE_ARRAY columns. A fixed-width column reached it with both buffers null and +/// the read died on a . +/// +/// So this library wrote files it could not itself read, for every FIXED_LEN_BYTE_ARRAY column there is — +/// DECIMAL(precision > 18), UUID, FLOAT16, and plain fixed binary — with no test covering any of it. +/// +public sealed class FlbaDeltaByteArrayTests : IDisposable +{ + private readonly string _tempDir; + + public FlbaDeltaByteArrayTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), "ew-flba-dba-" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + + GC.SuppressFinalize(this); + } + + private static ParquetWriteOptions DeltaByteArrayOptions => new() + { + ByteArrayEncoding = ByteArrayEncoding.DeltaByteArray, + DataPageVersion = DataPageVersion.V2, + DictionaryEnabled = false, + }; + + private async Task RoundTripAsync(RecordBatch batch) + { + var path = Path.Combine(_tempDir, Guid.NewGuid().ToString("N")[..8] + ".parquet"); + + await using (var outFile = new LocalSequentialFile(path)) + { + await using var writer = new ParquetFileWriter(outFile, options: DeltaByteArrayOptions); + await writer.WriteRowGroupAsync(batch); + } + + await using var inFile = new LocalRandomAccessFile(path); + await using var reader = new ParquetFileReader(inFile, ownsFile: false); + return await reader.ReadRowGroupAsync(0); + } + + private static RecordBatch FixedBinaryBatch(byte[][] values, bool[]? valid = null) + { + int width = values[0].Length; + var type = new FixedSizeBinaryType(width); + var packed = new byte[values.Length * width]; + for (int i = 0; i < values.Length; i++) + { + values[i].CopyTo(packed, i * width); + } + + var validity = new byte[(values.Length + 7) / 8]; + int nullCount = 0; + for (int i = 0; i < values.Length; i++) + { + if (valid is null || valid[i]) + { + validity[i / 8] |= (byte)(1 << (i % 8)); + } + else + { + nullCount++; + } + } + + var data = new ArrayData( + type, values.Length, nullCount, 0, + [new ArrowBuffer(validity), new ArrowBuffer(packed)]); + + var schema = new Apache.Arrow.Schema([new Field("f", type, nullable: valid is not null)], null); + return new RecordBatch(schema, [new FixedSizeBinaryArray(data)], values.Length); + } + + private static byte[][] DistinctValues(int count, int width) + { + var values = new byte[count][]; + for (int i = 0; i < count; i++) + { + values[i] = new byte[width]; + values[i][0] = (byte)i; + values[i][width - 1] = (byte)(255 - i); + } + + return values; + } + + [Theory] + [InlineData(2)] // FLOAT16's width + [InlineData(12)] // the extended-precision timestamp carrier + [InlineData(16)] // DECIMAL128 / UUID + [InlineData(32)] // DECIMAL256 + public async Task FixedSizeBinaryRoundTripsAtEveryWidthTheFormatUses(int width) + { + var values = DistinctValues(8, width); + + var read = await RoundTripAsync(FixedBinaryBatch(values)); + + var array = Assert.IsType(read.Column(0)); + Assert.Equal(values.Length, array.Length); + for (int i = 0; i < values.Length; i++) + { + Assert.Equal(values[i], array.GetBytes(i).ToArray()); + } + } + + [Fact] + public async Task ValuesSharingAPrefixRoundTrip() + { + // The whole point of the encoding is that value N stores only what it does not share with N-1. + // Distinct-first-byte values leave every prefix length at zero and never exercise reconstruction. + var values = new byte[6][]; + for (int i = 0; i < values.Length; i++) + { + values[i] = [0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11, 0x22, 0x33, 0x44, (byte)i]; + } + + var read = await RoundTripAsync(FixedBinaryBatch(values)); + + var array = Assert.IsType(read.Column(0)); + for (int i = 0; i < values.Length; i++) + { + Assert.Equal(values[i], array.GetBytes(i).ToArray()); + } + } + + [Fact] + public async Task NullsRoundTrip() + { + var values = DistinctValues(6, 12); + bool[] valid = [true, false, true, true, false, true]; + + var read = await RoundTripAsync(FixedBinaryBatch(values, valid)); + + var array = Assert.IsType(read.Column(0)); + Assert.Equal(values.Length, array.Length); + for (int i = 0; i < values.Length; i++) + { + Assert.Equal(!valid[i], array.IsNull(i)); + if (valid[i]) + { + Assert.Equal(values[i], array.GetBytes(i).ToArray()); + } + } + } + + [Fact] + public async Task DecimalsRoundTrip() + { + // The blast radius in practice: DECIMAL above precision 18 is carried on FIXED_LEN_BYTE_ARRAY, so + // this is an ordinary column that a caller could already not read back. + var type = new Decimal128Type(30, 4); + var values = new Decimal128Array.Builder(type) + .Append(12.3456m).Append(-99.9999m).Append(0m).Append(1234567.8901m) + .Build(); + + var schema = new Apache.Arrow.Schema([new Field("d", type, nullable: false)], null); + var read = await RoundTripAsync(new RecordBatch(schema, [values], values.Length)); + + var array = Assert.IsType(read.Column(0)); + Assert.Equal(values.Length, array.Length); + for (int i = 0; i < values.Length; i++) + { + Assert.Equal(values.GetValue(i), array.GetValue(i)); + } + } + + [Fact] + public void AValueOfTheWrongWidthIsRejectedRatherThanMisaligned() + { + // A malformed file could carry variable-length values on a fixed-width column. The bulk copy the + // decoder now does would silently shift every later value, so the width is checked per value. + byte[] joined = [1, 2, 3, 4, 5, 6, 7]; + int[] offsets = [0, 3, 7]; // two values, 3 and 4 bytes wide + var encoded = new byte[256]; + int written = DeltaByteArrayEncoder.Encode(offsets, joined, 0, 2, 2, null, encoded); + + using var state = new ColumnBuildState(PhysicalType.FixedLenByteArray, 0, 0, capacity: 8); + + var error = Assert.Throws( + () => DeltaByteArrayDecoder.Decode(encoded.AsSpan(0, written), 2, state, typeLength: 4)); + Assert.Contains("FIXED_LEN_BYTE_ARRAY(4)", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void AMissingTypeLengthIsRejected() + { + var encoded = new byte[256]; + int written = DeltaByteArrayEncoder.EncodeFixed(new byte[8], 4, 0, 2, 2, null, encoded); + + using var state = new ColumnBuildState(PhysicalType.FixedLenByteArray, 0, 0, capacity: 8); + + Assert.Throws( + () => DeltaByteArrayDecoder.Decode(encoded.AsSpan(0, written), 2, state, typeLength: 0)); + } +} diff --git a/test/EngineeredWood.Parquet.Tests/Parquet/Data/TimestampBoundPrecisionTests.cs b/test/EngineeredWood.Parquet.Tests/Parquet/Data/TimestampBoundPrecisionTests.cs new file mode 100644 index 0000000..9a26ede --- /dev/null +++ b/test/EngineeredWood.Parquet.Tests/Parquet/Data/TimestampBoundPrecisionTests.cs @@ -0,0 +1,175 @@ +// Copyright (c) clast-project. All rights reserved. +// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information. + +using Apache.Arrow; +using Apache.Arrow.Types; +using EngineeredWood.Expressions; +using EngineeredWood.IO.Local; +using EngineeredWood.Parquet; +using TimeUnit = Apache.Arrow.Types.TimeUnit; + +namespace EngineeredWood.Tests.Parquet.Data; + +/// +/// Row-group statistics bounds for sub-millisecond timestamps used to be wrong in the direction that +/// loses data. ParquetStatisticsAccessor converted every timestamp bound through +/// DateTimeOffset.FromUnixTimeMilliseconds, so a MICROS or NANOS column had everything below a +/// millisecond truncated toward zero — a max bound of 1500 µs came back as 0 ms. +/// +/// A max bound that is too SMALL is not a rounding blemish: a predicate of t > 0.5ms compares +/// against it, concludes the row group cannot match, and prunes rows that genuinely do. These pin the +/// rule that replaced it — a bound may only ever move OUTWARD, and a bound that cannot be represented +/// at all is dropped rather than clamped. +/// +public sealed class TimestampBoundPrecisionTests : IDisposable +{ + private readonly string _tempDir; + + public TimestampBoundPrecisionTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), "ew-ts-bounds-" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + + GC.SuppressFinalize(this); + } + + private async Task<(LiteralValue? Min, LiteralValue? Max)> BoundsAsync(TimeUnit unit, params long[] values) + { + var path = Path.Combine(_tempDir, Guid.NewGuid().ToString("N")[..8] + ".parquet"); + var type = new TimestampType(unit, "UTC"); + + var buffer = new ArrowBuffer.Builder(); + foreach (long v in values) + { + buffer.Append(v); + } + + var validity = new byte[(values.Length + 7) / 8]; + for (int i = 0; i < values.Length; i++) + { + validity[i / 8] |= (byte)(1 << (i % 8)); + } + + var array = new TimestampArray( + new ArrayData(type, values.Length, 0, 0, [new ArrowBuffer(validity), buffer.Build()])); + var schema = new Apache.Arrow.Schema([new Field("t", type, nullable: false)], null); + + await using (var file = new LocalSequentialFile(path)) + { + await using var writer = new ParquetFileWriter(file, options: new ParquetWriteOptions()); + await writer.WriteRowGroupAsync(new RecordBatch(schema, [array], values.Length)); + } + + await using var input = new LocalRandomAccessFile(path); + await using var reader = new ParquetFileReader(input, ownsFile: false); + var metadata = await reader.ReadMetadataAsync(); + var accessor = new ParquetStatisticsAccessor(await reader.GetSchemaAsync()); + + return (accessor.GetMinValue(metadata.RowGroups[0], "t"), + accessor.GetMaxValue(metadata.RowGroups[0], "t")); + } + + private static DateTimeOffset Utc(long ticksSinceEpoch) => + new DateTimeOffset(621_355_968_000_000_000L + ticksSinceEpoch, TimeSpan.Zero); + + [Fact] + public async Task MicrosecondBoundsAreExact() + { + // 1 µs is 10 ticks, so nothing needs to round at all — the old code lost these entirely. + var (min, max) = await BoundsAsync(TimeUnit.Microsecond, 500L, 1500L); + + Assert.Equal(Utc(5_000), min?.AsDateTimeOffset); + Assert.Equal(Utc(15_000), max?.AsDateTimeOffset); + } + + [Fact] + public async Task MillisecondBoundsAreExact() + { + var (min, max) = await BoundsAsync(TimeUnit.Millisecond, -3L, 7L); + + Assert.Equal(Utc(-30_000), min?.AsDateTimeOffset); + Assert.Equal(Utc(70_000), max?.AsDateTimeOffset); + } + + [Fact] + public async Task NanosecondBoundsRoundOutward() + { + // 1 tick is 100 ns, so 150 ns and 250 ns both fall between ticks. The min must round DOWN and the + // max UP; rounding either the other way would exclude a value the file actually holds. + var (min, max) = await BoundsAsync(TimeUnit.Nanosecond, 150L, 250L); + + Assert.Equal(Utc(1), min?.AsDateTimeOffset); // floor(150/100) = 1 + Assert.Equal(Utc(3), max?.AsDateTimeOffset); // ceil(250/100) = 3 + } + + [Fact] + public async Task NegativeNanosecondBoundsRoundOutwardToo() + { + // Pre-epoch, where truncation toward zero rounds the opposite way and the signs matter. + var (min, max) = await BoundsAsync(TimeUnit.Nanosecond, -250L, -150L); + + Assert.Equal(Utc(-3), min?.AsDateTimeOffset); // floor(-250/100) = -3 + Assert.Equal(Utc(-1), max?.AsDateTimeOffset); // ceil(-150/100) = -1 + } + + [Fact] + public async Task BoundsThatCannotBeRepresentedAreDroppedRatherThanClamped() + { + // MILLIS spans far past year 9999. A clamped bound is indistinguishable from a real endpoint and + // would prune on a value the file never contained, so there must be no bound at all. + var (min, max) = await BoundsAsync(TimeUnit.Millisecond, long.MinValue / 2, long.MaxValue / 2); + + Assert.Null(min); + Assert.Null(max); + } + + [Fact] + public async Task ARepresentableBoundSurvivesEvenWhenItsPartnerDoesNot() + { + var (min, max) = await BoundsAsync(TimeUnit.Millisecond, 0L, long.MaxValue / 2); + + Assert.Equal(Utc(0), min?.AsDateTimeOffset); + Assert.Null(max); + } + + [Theory] + [InlineData(TimeUnit.Millisecond)] + [InlineData(TimeUnit.Microsecond)] + [InlineData(TimeUnit.Nanosecond)] + public async Task TheBoundsAlwaysContainEveryValue(TimeUnit unit) + { + // The invariant the whole fix exists for, stated directly: whatever rounding happens, every value + // in the column must still fall inside the range the footer advertises. + long[] values = [-1_234_567L, -1L, 0L, 1L, 999L, 1_000L, 1_001L, 7_654_321L]; + var (min, max) = await BoundsAsync(unit, values); + + Assert.NotNull(min); + Assert.NotNull(max); + + long ticksPerUnit = unit switch + { + TimeUnit.Millisecond => 10_000L, + TimeUnit.Microsecond => 10L, + _ => 1L, + }; + + foreach (long v in values) + { + // Nanoseconds are the only unit that cannot land on a tick, so compare in nanoseconds. + long valueNanos = unit == TimeUnit.Nanosecond ? v : v * ticksPerUnit * 100L; + long minNanos = ((min!.Value.AsDateTimeOffset).Ticks - 621_355_968_000_000_000L) * 100L; + long maxNanos = ((max!.Value.AsDateTimeOffset).Ticks - 621_355_968_000_000_000L) * 100L; + + Assert.True(minNanos <= valueNanos, $"min bound excludes {v}"); + Assert.True(maxNanos >= valueNanos, $"max bound excludes {v}"); + } + } +} diff --git a/test/EngineeredWood.Parquet.Tests/Parquet/Data/TimestampCarrierGateTests.cs b/test/EngineeredWood.Parquet.Tests/Parquet/Data/TimestampCarrierGateTests.cs new file mode 100644 index 0000000..70e4894 --- /dev/null +++ b/test/EngineeredWood.Parquet.Tests/Parquet/Data/TimestampCarrierGateTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) clast-project. All rights reserved. +// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information. + +using Apache.Arrow.Types; +using EngineeredWood.Parquet; +using EngineeredWood.Parquet.Data; +using EngineeredWood.Parquet.Metadata; +using EngineeredWood.Parquet.Schema; +using ParquetTimeUnit = EngineeredWood.Parquet.Metadata.TimeUnit; +using TimeUnit = Apache.Arrow.Types.TimeUnit; + +namespace EngineeredWood.Tests.Parquet.Data; + +/// +/// The TIMESTAMP annotation was mapped to an Arrow without ever looking at +/// the column's physical type. That is fine while INT64 is the only carrier the spec allows, and stops +/// being fine the moment a file arrives carrying TIMESTAMP on something else: the read path maps +/// Int64Type or TimestampType or Time64Type onto a long value buffer, so a 12-byte column +/// was reinterpreted eight bytes at a time and decoded plausible-looking wrong dates instead of failing. +/// +/// parquet-format is in the middle of allowing exactly that (apache/parquet-format#601 puts TIMESTAMP on +/// FIXED_LEN_BYTE_ARRAY(12)), so files in this shape are about to exist. These tests pin the gate: an +/// unrecognised carrier falls through to the physical type, which is lossless, rather than being decoded +/// as a timestamp. The FLBA(12) carrier gets a real decode in its own change; this is only the guard. +/// +public class TimestampCarrierGateTests +{ + private static ColumnDescriptor Describe( + PhysicalType physicalType, + LogicalType? logicalType = null, + ConvertedType? convertedType = null, + int? typeLength = null) + { + var element = new SchemaElement + { + Name = "ts", + Type = physicalType, + TypeLength = typeLength, + RepetitionType = FieldRepetitionType.Optional, + LogicalType = logicalType, + ConvertedType = convertedType, + }; + + return new ColumnDescriptor + { + Path = ["ts"], + PhysicalType = physicalType, + TypeLength = typeLength, + MaxDefinitionLevel = 1, + MaxRepetitionLevel = 0, + SchemaElement = element, + SchemaNode = new SchemaNode { Element = element, Children = [] }, + }; + } + + public static TheoryData TimestampUnits => new() + { + { ParquetTimeUnit.Millis, TimeUnit.Millisecond }, + { ParquetTimeUnit.Micros, TimeUnit.Microsecond }, + { ParquetTimeUnit.Nanos, TimeUnit.Nanosecond }, + }; + + [Theory] + [MemberData(nameof(TimestampUnits))] + public void Int64IsStillDecodedAsATimestamp(ParquetTimeUnit parquetUnit, TimeUnit arrowUnit) + { + var column = Describe(PhysicalType.Int64, new LogicalType.TimestampType(true, parquetUnit)); + + var type = Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + Assert.Equal(arrowUnit, type.Unit); + Assert.Equal("UTC", type.Timezone); + } + + [Fact] + public void ANonUtcInt64TimestampIsStillNaive() + { + var column = Describe( + PhysicalType.Int64, + new LogicalType.TimestampType(false, ParquetTimeUnit.Micros)); + + var type = Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + Assert.Null(type.Timezone); + } + + [Theory] + [MemberData(nameof(TimestampUnits))] + public void FixedLenByteArrayFallsThroughToItsPhysicalType(ParquetTimeUnit parquetUnit, TimeUnit _) + { + // The carrier proposed by apache/parquet-format#601. Until it is decoded, twelve honest bytes + // beat a wrong date. + var column = Describe( + PhysicalType.FixedLenByteArray, + new LogicalType.TimestampType(true, parquetUnit), + typeLength: 12); + + var type = Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + Assert.Equal(12, type.ByteWidth); + } + + [Fact] + public void AMalformedTimestampCarrierIsNotDecodedAsATimestampEither() + { + // TIMESTAMP on FLBA(8) is not legal under any proposal. The point is that the width being + // coincidentally right for an int64 must not be what saves us. + var column = Describe( + PhysicalType.FixedLenByteArray, + new LogicalType.TimestampType(true, ParquetTimeUnit.Micros), + typeLength: 8); + + var type = Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + Assert.Equal(8, type.ByteWidth); + } + + [Fact] + public void ByteArrayCarryingTimestampFallsThroughToBinary() + { + var column = Describe( + PhysicalType.ByteArray, + new LogicalType.TimestampType(true, ParquetTimeUnit.Micros)); + + Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + } + + [Theory] + [InlineData(ConvertedType.TimestampMillis, TimeUnit.Millisecond)] + [InlineData(ConvertedType.TimestampMicros, TimeUnit.Microsecond)] + public void Int64ConvertedTimestampsAreStillDecoded(ConvertedType converted, TimeUnit arrowUnit) + { + var column = Describe(PhysicalType.Int64, convertedType: converted); + + var type = Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + Assert.Equal(arrowUnit, type.Unit); + Assert.Equal("UTC", type.Timezone); + } + + [Theory] + [InlineData(ConvertedType.TimestampMillis)] + [InlineData(ConvertedType.TimestampMicros)] + public void ConvertedTimestampsOnTheWrongCarrierFallThrough(ConvertedType converted) + { + // TIMESTAMP_MILLIS / TIMESTAMP_MICROS are INT64-only converted types, and the FLBA(12) proposal + // deliberately does NOT give the new carrier one -- parquet-java suppresses converted_type for it + // precisely so a converted-type-only reader cannot misparse the column. A file carrying one + // anyway is malformed, and the same reinterpretation bug applies. + var column = Describe(PhysicalType.FixedLenByteArray, convertedType: converted, typeLength: 12); + + var type = Assert.IsType(ArrowSchemaConverter.ToArrowType(column)); + Assert.Equal(12, type.ByteWidth); + } + + [Fact] + public void AnInt64TimestampStillEmitsTheDeprecatedBounds() + { + // The deprecated Statistics.min/max may only carry values whose SIGNED ordering is their logical + // ordering. StatisticsCollector compares INT64 columns with a typed comparator, so it does. + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical( + new TimestampType(TimeUnit.Nanosecond, "UTC"), PhysicalType.Int64)); + } + + [Fact] + public void AFixedLenByteArrayTimestampDoesNotEmitTheDeprecatedBounds() + { + // Latent until an Arrow TimestampType can map to FLBA, which is what the FLBA(12) writer will do. + // StatisticsCollector compares every FLBA column with SequenceCompareTo -- unsigned lexicographic + // -- which is not the signed order these fields promise. A wrong bound in the footer is a wrong + // prune, so the deprecated pair has to be dropped rather than filled in from the wrong comparator. + Assert.False(ColumnChunkWriter.SignedOrderMatchesLogical( + new TimestampType(TimeUnit.Nanosecond, "UTC"), PhysicalType.FixedLenByteArray)); + } + + [Fact] + public void TheOtherSignedTemporalTypesAreUnaffected() + { + // Date/Time/Duration only ever arrive on INT32/INT64, so narrowing the timestamp answer must not + // have narrowed theirs. + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical(Date32Type.Default, PhysicalType.Int32)); + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical(Date64Type.Default, PhysicalType.Int64)); + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical( + new Time32Type(TimeUnit.Millisecond), PhysicalType.Int32)); + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical( + new Time64Type(TimeUnit.Microsecond), PhysicalType.Int64)); + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical(Int64Type.Default, PhysicalType.Int64)); + Assert.True(ColumnChunkWriter.SignedOrderMatchesLogical(DoubleType.Default, PhysicalType.Double)); + Assert.False(ColumnChunkWriter.SignedOrderMatchesLogical( + StringType.Default, PhysicalType.ByteArray)); + } +}