diff --git a/variant/README.md b/variant/README.md index c09c49a..6988cb4 100644 --- a/variant/README.md +++ b/variant/README.md @@ -39,7 +39,17 @@ Each example consists of 2 files: 3. `object_empty` -- Example of object (`basic_type` = 3) with no fields 3. `object_primitive` -- Example of object with only primitive fields 4. `object_nested` -- Example of object with other objects in fields -5. `array_empty` -- Example of array (`basic_type` = 4) with no elements +5. `object_unicode_keys_sorted` / `object_unicode_keys_unsorted` -- Examples of an object + whose field names include code points on both sides of the UTF-16 surrogate range + (e.g. `U+E000`, `U+FFFF`, `U+10000`, `U+1F600`, `U+10FFFF`), where the unsigned + lexicographic UTF-8 byte order required by the spec differs from UTF-16 code-unit + order (e.g. Java's `String.compareTo`). Both encode the same object; the `_sorted` + file uses a sorted metadata dictionary (`sorted_strings = 1`) while the `_unsorted` + file uses a deliberately scrambled dictionary, so its object field ids are + non-monotonic and readers must compare the referenced field name bytes. Each field + value is a short string spelling the field name's code point (e.g. `"U+FFFF"`), so + a lookup that lands on the wrong field is self-evident. +6. `array_empty` -- Example of array (`basic_type` = 4) with no elements 5. `array_primitive` -- Example of array with only primitive elements 6. `array_nested` -- Example of an with objects and other arrays in the elements @@ -73,5 +83,14 @@ echo -n 'a' | tr a '\0' > primitive_null.value Currently, Spark [does not support](https://github.com/apache/spark/blob/master/common/variant/README.md) Variant values containing UUID, Time, or nanosecond-precision Timestamp. the `primitive_time.[metadata/value]`, `primitive_timestamp_nanos.[metadata/value]`, `primitive_timestampntz_nanos.[metadata/value]` and `primitive_uuid.[metadata/data]` was generated by [Iceberg test code](https://github.com/apache/iceberg/blob/3a4215dbb714477c89681ab94f1197b6ebcbdfff/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantReaders.java#L355) +### Modification 3: Created `object_unicode_keys_sorted` and `object_unicode_keys_unsorted` with [`regen_unicode_keys.py`](regen_unicode_keys.py) + +These examples exercise the unsigned lexicographic UTF-8 byte ordering of object +field names required by the spec, which differs from UTF-16 code-unit order for +supplementary characters — an easy mistake in languages whose native string +comparison operates on UTF-16 code units (e.g. Java, JavaScript, C#). They are +generated directly from the spec by the standalone +[`regen_unicode_keys.py`](regen_unicode_keys.py) script rather than `regen.py`. + [Variant]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md [primitive types listed in the spec]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#value-data-for-primitive-type-basic_type0 diff --git a/variant/data_dictionary.json b/variant/data_dictionary.json index 8a1faae..c548142 100644 --- a/variant/data_dictionary.json +++ b/variant/data_dictionary.json @@ -52,6 +52,34 @@ "string_field": "Apache Parquet", "timestamp_field": "2025-04-16T12:34:56.78" }, + "object_unicode_keys_sorted": { + "$": "U+0024", + "0": "U+0030", + "A": "U+0041", + "a": "U+0061", + "~": "U+007E", + "\u00a2": "U+00A2", + "\u20ac": "U+20AC", + "\ue000": "U+E000", + "\uffff": "U+FFFF", + "\ud800\udc00": "U+10000", + "\ud83d\ude00": "U+1F600", + "\udbff\udfff": "U+10FFFF" + }, + "object_unicode_keys_unsorted": { + "$": "U+0024", + "0": "U+0030", + "A": "U+0041", + "a": "U+0061", + "~": "U+007E", + "\u00a2": "U+00A2", + "\u20ac": "U+20AC", + "\ue000": "U+E000", + "\uffff": "U+FFFF", + "\ud800\udc00": "U+10000", + "\ud83d\ude00": "U+1F600", + "\udbff\udfff": "U+10FFFF" + }, "primitive_binary": "AxM33q2+78r+", "primitive_boolean_false": false, "primitive_boolean_true": true, @@ -73,5 +101,5 @@ "primitive_timestamp_nanos": "2024-11-07T12:33:54.123456789+00:00", "primitive_timestampntz_nanos": "2024-11-07T12:33:54.123456789", "primitive_uuid": "f24f9b64-81fa-49d1-b74e-8c09a6e31c56", - "short_string": "Less than 64 bytes (\u2764\ufe0f with utf8)", + "short_string": "Less than 64 bytes (\u2764\ufe0f with utf8)" } diff --git a/variant/object_unicode_keys_sorted.metadata b/variant/object_unicode_keys_sorted.metadata new file mode 100644 index 0000000..337d4b2 Binary files /dev/null and b/variant/object_unicode_keys_sorted.metadata differ diff --git a/variant/object_unicode_keys_sorted.value b/variant/object_unicode_keys_sorted.value new file mode 100644 index 0000000..8cc0ce5 Binary files /dev/null and b/variant/object_unicode_keys_sorted.value differ diff --git a/variant/object_unicode_keys_unsorted.metadata b/variant/object_unicode_keys_unsorted.metadata new file mode 100644 index 0000000..445fc76 Binary files /dev/null and b/variant/object_unicode_keys_unsorted.metadata differ diff --git a/variant/object_unicode_keys_unsorted.value b/variant/object_unicode_keys_unsorted.value new file mode 100644 index 0000000..607908b Binary files /dev/null and b/variant/object_unicode_keys_unsorted.value differ diff --git a/variant/regen_unicode_keys.py b/variant/regen_unicode_keys.py new file mode 100644 index 0000000..692f544 --- /dev/null +++ b/variant/regen_unicode_keys.py @@ -0,0 +1,119 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generates the `object_unicode_keys_sorted` and `object_unicode_keys_unsorted` +# example files. +# +# These examples contain object field names on both sides of the UTF-16 +# surrogate range, where unsigned lexicographic UTF-8 byte order (required by +# the Variant spec for object field ordering) disagrees with UTF-16 code-unit +# order (e.g. Java's String.compareTo). For example, UTF-16 orders U+10000 +# (surrogate pair D800 DC00) before U+E000, U+FFFF, while UTF-8 byte order +# places U+10000 (F0 90 80 80) after both (EE 80 80, EF BF BF). An +# implementation that sorts or binary-searches field names with a UTF-16 +# string comparison will silently mishandle these files. +# +# Both examples encode the same logical object. Each field's value is a short +# string spelling the field name's code point, so a lookup that lands on the +# wrong field is self-evident: +# +# * `object_unicode_keys_sorted` -- metadata dictionary is sorted +# (`sorted_strings = 1`) and field ids appear in dictionary order +# * `object_unicode_keys_unsorted` -- metadata dictionary is deliberately +# scrambled (`sorted_strings = 0`), so object field ids are non-monotonic +# and a reader must compare the referenced key bytes, not the ids + +import json + +# Field names in unsigned UTF-8 byte order (equal to code point order) +KEYS = [ + "$", # $ 24 + "0", # 0 30 + "A", # A 41 + "a", # a 61 + "~", # ~ 7E + "¢", # U+00A2 C2 A2 + "€", # U+20AC E2 82 AC + "\ue000", # private EE 80 80 (first BMP char after the surrogates) + "\uffff", # U+FFFF EF BF BF (last BMP char) + "\U00010000", # U+10000 F0 90 80 80 (first supplementary char) + "\U0001f600", # U+1F600 F0 9F 98 80 + "\U0010ffff", # U+10FFFF F4 8F BF BF (last valid code point) +] + +OBJ = {k: "U+%04X" % ord(k) for k in KEYS} + +assert KEYS == sorted(KEYS), "KEYS must be listed in code point order" +assert [k.encode("utf-8") for k in KEYS] == sorted(k.encode("utf-8") for k in KEYS) + + +def encode_metadata(dictionary, sorted_strings): + key_bytes = [k.encode("utf-8") for k in dictionary] + total = sum(len(b) for b in key_bytes) + assert len(dictionary) <= 0xFF and total <= 0xFF, "offset_size 1 is assumed" + header = 0x01 | (0x10 if sorted_strings else 0x00) # version 1, offset_size 1 + out = bytearray([header, len(dictionary)]) + offset = 0 + for b in key_bytes: + out.append(offset) + offset += len(b) + out.append(offset) + for b in key_bytes: + out += b + return bytes(out) + + +def encode_short_string(s): + b = s.encode("utf-8") + assert len(b) < 64 + return bytes([(len(b) << 2) | 0b01]) + b + + +def encode_object(obj, dictionary): + # field_offset_size 1, field_id_size 1, is_large 0 + fields = sorted(obj, key=lambda k: k.encode("utf-8")) + values = [encode_short_string(obj[k]) for k in fields] + assert len(fields) <= 0xFF and sum(len(v) for v in values) <= 0xFF + out = bytearray([0b10, len(fields)]) + for k in fields: + out.append(dictionary.index(k)) + offset = 0 + for v in values: + out.append(offset) + offset += len(v) + out.append(offset) + for v in values: + out += v + return bytes(out) + + +def write_example(name, dictionary, sorted_strings): + with open(f"{name}.metadata", "wb") as f: + f.write(encode_metadata(dictionary, sorted_strings)) + with open(f"{name}.value", "wb") as f: + f.write(encode_object(OBJ, dictionary)) + + +write_example("object_unicode_keys_sorted", KEYS, sorted_strings=True) + +# Scramble the dictionary with a fixed permutation so field ids are +# non-monotonic in the object +scrambled = [KEYS[i] for i in [9, 3, 11, 0, 8, 5, 1, 10, 4, 7, 2, 6]] +assert sorted(scrambled) == KEYS +write_example("object_unicode_keys_unsorted", scrambled, sorted_strings=False) + +print(json.dumps(OBJ, sort_keys=True, indent=4))