From 392ce836cd29ac2c844023937b9bf9c54cf0c442 Mon Sep 17 00:00:00 2001
From: peterxcli <peterxcli@gmail.com>
Date: Tue, 25 Aug 2026 08:22:00 +0800
Subject: [PATCH] Add Variant objects with supplementary-character field names

The Variant spec orders object fields by unsigned lexicographic UTF-8
bytes, which differs from UTF-16 code-unit order for supplementary
characters: UTF-16 sorts U+10000 (surrogate pair D800 DC00) before
U+E000 and U+FFFF, while UTF-8 byte order sorts it after both. This is
an easy mistake in languages whose native string comparison operates on
UTF-16 code units (e.g. Java, JavaScript, C#), and an implementation
that sorts or binary-searches object fields that way will silently
mishandle such objects.

Add two examples encoding the same object, whose field names straddle
the surrogate range ($ 0 A a ~ U+00A2 U+20AC U+E000 U+FFFF U+10000
U+1F600 U+10FFFF), so implementations can verify they order and look up
object fields consistently:

* object_unicode_keys_sorted: sorted metadata dictionary
  (sorted_strings = 1), field ids in dictionary order
* object_unicode_keys_unsorted: deliberately scrambled dictionary, so
  field ids are non-monotonic and readers must compare key bytes

Each field value is a short string spelling the field name's code point,
making a lookup that lands on the wrong field self-evident. The files
are generated directly from the spec by the new standalone
regen_unicode_keys.py script.

Also fix a pre-existing trailing comma that made data_dictionary.json
invalid JSON.
---
 variant/README.md                             |  21 +++-
 variant/data_dictionary.json                  |  30 ++++-
 variant/object_unicode_keys_sorted.metadata   | Bin 0 -> 43 bytes
 variant/object_unicode_keys_sorted.value      | Bin 0 -> 115 bytes
 variant/object_unicode_keys_unsorted.metadata | Bin 0 -> 43 bytes
 variant/object_unicode_keys_unsorted.value    | Bin 0 -> 115 bytes
 variant/regen_unicode_keys.py                 | 119 ++++++++++++++++++
 7 files changed, 168 insertions(+), 2 deletions(-)
 create mode 100644 variant/object_unicode_keys_sorted.metadata
 create mode 100644 variant/object_unicode_keys_sorted.value
 create mode 100644 variant/object_unicode_keys_unsorted.metadata
 create mode 100644 variant/object_unicode_keys_unsorted.value
 create mode 100644 variant/regen_unicode_keys.py

diff --git a/variant/README.md b/variant/README.md
index c09c49a..6988cb4 100644
--- a/variant/README.md
+++ b/variant/README.md
@@ -39,7 +39,17 @@ Each example consists of 2 files:
 3. `object_empty` -- Example of object (`basic_type` = 3) with no fields
 3. `object_primitive` -- Example of object with only primitive fields
 4. `object_nested` -- Example of object with other objects in fields 
-5. `array_empty` -- Example of array (`basic_type` = 4) with no elements
+5. `object_unicode_keys_sorted` / `object_unicode_keys_unsorted` -- Examples of an object
+   whose field names include code points on both sides of the UTF-16 surrogate range
+   (e.g. `U+E000`, `U+FFFF`, `U+10000`, `U+1F600`, `U+10FFFF`), where the unsigned
+   lexicographic UTF-8 byte order required by the spec differs from UTF-16 code-unit
+   order (e.g. Java's `String.compareTo`). Both encode the same object; the `_sorted`
+   file uses a sorted metadata dictionary (`sorted_strings = 1`) while the `_unsorted`
+   file uses a deliberately scrambled dictionary, so its object field ids are
+   non-monotonic and readers must compare the referenced field name bytes. Each field
+   value is a short string spelling the field name's code point (e.g. `"U+FFFF"`), so
+   a lookup that lands on the wrong field is self-evident.
+6. `array_empty` -- Example of array (`basic_type` = 4) with no elements
 5. `array_primitive` -- Example of array with only primitive elements
 6. `array_nested` -- Example of an with objects and other arrays in the elements
 
@@ -73,5 +83,14 @@ echo -n 'a' | tr a '\0' > primitive_null.value
 
 Currently, Spark [does not support](https://github.com/apache/spark/blob/master/common/variant/README.md) Variant values containing UUID, Time, or nanosecond-precision Timestamp. the `primitive_time.[metadata/value]`, `primitive_timestamp_nanos.[metadata/value]`, `primitive_timestampntz_nanos.[metadata/value]` and `primitive_uuid.[metadata/data]` was generated by [Iceberg test code](https://github.com/apache/iceberg/blob/3a4215dbb714477c89681ab94f1197b6ebcbdfff/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantReaders.java#L355)
 
+### Modification 3: Created `object_unicode_keys_sorted` and `object_unicode_keys_unsorted` with [`regen_unicode_keys.py`](regen_unicode_keys.py)
+
+These examples exercise the unsigned lexicographic UTF-8 byte ordering of object
+field names required by the spec, which differs from UTF-16 code-unit order for
+supplementary characters — an easy mistake in languages whose native string
+comparison operates on UTF-16 code units (e.g. Java, JavaScript, C#). They are
+generated directly from the spec by the standalone
+[`regen_unicode_keys.py`](regen_unicode_keys.py) script rather than `regen.py`.
+
 [Variant]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md
 [primitive types listed in the spec]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#value-data-for-primitive-type-basic_type0
diff --git a/variant/data_dictionary.json b/variant/data_dictionary.json
index 8a1faae..c548142 100644
--- a/variant/data_dictionary.json
+++ b/variant/data_dictionary.json
@@ -52,6 +52,34 @@
         "string_field": "Apache Parquet",
         "timestamp_field": "2025-04-16T12:34:56.78"
     },
+    "object_unicode_keys_sorted": {
+        "$": "U+0024",
+        "0": "U+0030",
+        "A": "U+0041",
+        "a": "U+0061",
+        "~": "U+007E",
+        "\u00a2": "U+00A2",
+        "\u20ac": "U+20AC",
+        "\ue000": "U+E000",
+        "\uffff": "U+FFFF",
+        "\ud800\udc00": "U+10000",
+        "\ud83d\ude00": "U+1F600",
+        "\udbff\udfff": "U+10FFFF"
+    },
+    "object_unicode_keys_unsorted": {
+        "$": "U+0024",
+        "0": "U+0030",
+        "A": "U+0041",
+        "a": "U+0061",
+        "~": "U+007E",
+        "\u00a2": "U+00A2",
+        "\u20ac": "U+20AC",
+        "\ue000": "U+E000",
+        "\uffff": "U+FFFF",
+        "\ud800\udc00": "U+10000",
+        "\ud83d\ude00": "U+1F600",
+        "\udbff\udfff": "U+10FFFF"
+    },
     "primitive_binary": "AxM33q2+78r+",
     "primitive_boolean_false": false,
     "primitive_boolean_true": true,
@@ -73,5 +101,5 @@
     "primitive_timestamp_nanos": "2024-11-07T12:33:54.123456789+00:00",
     "primitive_timestampntz_nanos": "2024-11-07T12:33:54.123456789",
     "primitive_uuid": "f24f9b64-81fa-49d1-b74e-8c09a6e31c56",
-    "short_string": "Less than 64 bytes (\u2764\ufe0f with utf8)",
+    "short_string": "Less than 64 bytes (\u2764\ufe0f with utf8)"
 }
diff --git a/variant/object_unicode_keys_sorted.metadata b/variant/object_unicode_keys_sorted.metadata
new file mode 100644
index 0000000000000000000000000000000000000000..337d4b26ded6ee43ae5073c980089da94588536a
GIT binary patch
literal 43
zcmV+`0M!2x3;+QF0|W&J3Jnkx7#t)pL1BKvqT+(A?tp;rzrXO1fPnCyn1J+;zrXu5
B5)J?W

literal 0
HcmV?d00001

diff --git a/variant/object_unicode_keys_sorted.value b/variant/object_unicode_keys_sorted.value
new file mode 100644
index 0000000000000000000000000000000000000000..8cc0ce566e321d5f5d5ef3e2b05a6a81356de991
GIT binary patch
literal 115
zcmZSLVPIrpW?^Mx=iubxW?<(Nl~L9*w6J&gkB|)2HZU+U0aL~XV9LZ0OqoF_b5}6W
m(FjNx88|uvDOUpnkV-cokPX!~1o8|(w40fMfg*?xk^le!5){M$

literal 0
HcmV?d00001

diff --git a/variant/object_unicode_keys_unsorted.metadata b/variant/object_unicode_keys_unsorted.metadata
new file mode 100644
index 0000000000000000000000000000000000000000..445fc7682cc27978d3a2096f1dc62ef1ac62018d
GIT binary patch
literal 43
zcmV+`0M!2h3;+ZL2?`Al5EK;{865DCfPi82kH5bp@4vsoqA>8En1FunfPg{bf~*`U
B5;p(<

literal 0
HcmV?d00001

diff --git a/variant/object_unicode_keys_unsorted.value b/variant/object_unicode_keys_unsorted.value
new file mode 100644
index 0000000000000000000000000000000000000000..607908b96db838a4005a4673a0be85f75daa84dc
GIT binary patch
literal 115
zcmZSLVP@lE<Y49IWMN=uVqoVJl~L9*w6J&gkB|)2HZU+U0aL~XV9LZ0OqoF_b5}6W
m(FjNx88|uvDOUpnkV-cokPX!~1o8|(w40fMfg*?xk^le!xfH|z

literal 0
HcmV?d00001

diff --git a/variant/regen_unicode_keys.py b/variant/regen_unicode_keys.py
new file mode 100644
index 0000000..692f544
--- /dev/null
+++ b/variant/regen_unicode_keys.py
@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generates the `object_unicode_keys_sorted` and `object_unicode_keys_unsorted`
+# example files.
+#
+# These examples contain object field names on both sides of the UTF-16
+# surrogate range, where unsigned lexicographic UTF-8 byte order (required by
+# the Variant spec for object field ordering) disagrees with UTF-16 code-unit
+# order (e.g. Java's String.compareTo). For example, UTF-16 orders U+10000
+# (surrogate pair D800 DC00) before U+E000, U+FFFF, while UTF-8 byte order
+# places U+10000 (F0 90 80 80) after both (EE 80 80, EF BF BF). An
+# implementation that sorts or binary-searches field names with a UTF-16
+# string comparison will silently mishandle these files.
+#
+# Both examples encode the same logical object. Each field's value is a short
+# string spelling the field name's code point, so a lookup that lands on the
+# wrong field is self-evident:
+#
+#   * `object_unicode_keys_sorted` -- metadata dictionary is sorted
+#     (`sorted_strings = 1`) and field ids appear in dictionary order
+#   * `object_unicode_keys_unsorted` -- metadata dictionary is deliberately
+#     scrambled (`sorted_strings = 0`), so object field ids are non-monotonic
+#     and a reader must compare the referenced key bytes, not the ids
+
+import json
+
+# Field names in unsigned UTF-8 byte order (equal to code point order)
+KEYS = [
+    "$",      # $        24
+    "0",      # 0        30
+    "A",      # A        41
+    "a",      # a        61
+    "~",      # ~        7E
+    "¢",      # U+00A2        C2 A2
+    "€",      # U+20AC        E2 82 AC
+    "\ue000",  # private  EE 80 80  (first BMP char after the surrogates)
+    "\uffff",  # U+FFFF   EF BF BF  (last BMP char)
+    "\U00010000",  # U+10000  F0 90 80 80  (first supplementary char)
+    "\U0001f600",  # U+1F600  F0 9F 98 80
+    "\U0010ffff",  # U+10FFFF F4 8F BF BF  (last valid code point)
+]
+
+OBJ = {k: "U+%04X" % ord(k) for k in KEYS}
+
+assert KEYS == sorted(KEYS), "KEYS must be listed in code point order"
+assert [k.encode("utf-8") for k in KEYS] == sorted(k.encode("utf-8") for k in KEYS)
+
+
+def encode_metadata(dictionary, sorted_strings):
+    key_bytes = [k.encode("utf-8") for k in dictionary]
+    total = sum(len(b) for b in key_bytes)
+    assert len(dictionary) <= 0xFF and total <= 0xFF, "offset_size 1 is assumed"
+    header = 0x01 | (0x10 if sorted_strings else 0x00)  # version 1, offset_size 1
+    out = bytearray([header, len(dictionary)])
+    offset = 0
+    for b in key_bytes:
+        out.append(offset)
+        offset += len(b)
+    out.append(offset)
+    for b in key_bytes:
+        out += b
+    return bytes(out)
+
+
+def encode_short_string(s):
+    b = s.encode("utf-8")
+    assert len(b) < 64
+    return bytes([(len(b) << 2) | 0b01]) + b
+
+
+def encode_object(obj, dictionary):
+    # field_offset_size 1, field_id_size 1, is_large 0
+    fields = sorted(obj, key=lambda k: k.encode("utf-8"))
+    values = [encode_short_string(obj[k]) for k in fields]
+    assert len(fields) <= 0xFF and sum(len(v) for v in values) <= 0xFF
+    out = bytearray([0b10, len(fields)])
+    for k in fields:
+        out.append(dictionary.index(k))
+    offset = 0
+    for v in values:
+        out.append(offset)
+        offset += len(v)
+    out.append(offset)
+    for v in values:
+        out += v
+    return bytes(out)
+
+
+def write_example(name, dictionary, sorted_strings):
+    with open(f"{name}.metadata", "wb") as f:
+        f.write(encode_metadata(dictionary, sorted_strings))
+    with open(f"{name}.value", "wb") as f:
+        f.write(encode_object(OBJ, dictionary))
+
+
+write_example("object_unicode_keys_sorted", KEYS, sorted_strings=True)
+
+# Scramble the dictionary with a fixed permutation so field ids are
+# non-monotonic in the object
+scrambled = [KEYS[i] for i in [9, 3, 11, 0, 8, 5, 1, 10, 4, 7, 2, 6]]
+assert sorted(scrambled) == KEYS
+write_example("object_unicode_keys_unsorted", scrambled, sorted_strings=False)
+
+print(json.dumps(OBJ, sort_keys=True, indent=4))