diff --git a/variant/README.md b/variant/README.md index c09c49a..cdb2424 100644 --- a/variant/README.md +++ b/variant/README.md @@ -39,10 +39,28 @@ Each example consists of 2 files: 3. `object_empty` -- Example of object (`basic_type` = 3) with no fields 3. `object_primitive` -- Example of object with only primitive fields 4. `object_nested` -- Example of object with other objects in fields +4. `object_wide_offsets` -- Example of object with 1-byte field IDs and 2-byte field offsets +4. `object_wide_field_ids` -- Example of object with 2-byte field IDs and 1-byte field offsets 5. `array_empty` -- Example of array (`basic_type` = 4) with no elements 5. `array_primitive` -- Example of array with only primitive elements 6. `array_nested` -- Example of an with objects and other arrays in the elements +## Objects whose field ID and field offset widths differ + +`object_wide_offsets` and `object_wide_field_ids` are the only examples in which +an object's `field_id_size` and `field_offset_size` differ. In every other object +example both are 1 byte, so the two `value_header` fields hold the same value and +a reader that reads them from each other's bit positions still decodes correctly. + +* `object_wide_offsets` -- the top-level object has two fields (1-byte field IDs) + and more than 256 bytes of field data (2-byte field offsets), giving header + byte `0x06`. +* `object_wide_field_ids` -- 262 distinct field names force 2-byte field IDs, + while the object under the `nested` field holds a single small value, so its own + field offsets still fit in 1 byte, giving header byte `0x12`. Its enclosing + object has 261 fields, which also makes this the only example with `is_large` + set to 1 (a 4-byte `num_elements`). + ## Regenerating these files The files in this directory were initially generated by running the [`regen.py`](regen.py) diff --git a/variant/data_dictionary.json b/variant/data_dictionary.json index 8a1faae..6a50de6 100644 --- a/variant/data_dictionary.json +++ b/variant/data_dictionary.json @@ -52,6 +52,275 @@ "string_field": "Apache Parquet", "timestamp_field": "2025-04-16T12:34:56.78" }, + "object_wide_field_ids": { + "f000": 0, + "f001": 1, + "f002": 2, + "f003": 3, + "f004": 4, + "f005": 5, + "f006": 6, + "f007": 7, + "f008": 8, + "f009": 9, + "f010": 10, + "f011": 11, + "f012": 12, + "f013": 13, + "f014": 14, + "f015": 15, + "f016": 16, + "f017": 17, + "f018": 18, + "f019": 19, + "f020": 20, + "f021": 21, + "f022": 22, + "f023": 23, + "f024": 24, + "f025": 25, + "f026": 26, + "f027": 27, + "f028": 28, + "f029": 29, + "f030": 30, + "f031": 31, + "f032": 32, + "f033": 33, + "f034": 34, + "f035": 35, + "f036": 36, + "f037": 37, + "f038": 38, + "f039": 39, + "f040": 40, + "f041": 41, + "f042": 42, + "f043": 43, + "f044": 44, + "f045": 45, + "f046": 46, + "f047": 47, + "f048": 48, + "f049": 49, + "f050": 50, + "f051": 51, + "f052": 52, + "f053": 53, + "f054": 54, + "f055": 55, + "f056": 56, + "f057": 57, + "f058": 58, + "f059": 59, + "f060": 60, + "f061": 61, + "f062": 62, + "f063": 63, + "f064": 64, + "f065": 65, + "f066": 66, + "f067": 67, + "f068": 68, + "f069": 69, + "f070": 70, + "f071": 71, + "f072": 72, + "f073": 73, + "f074": 74, + "f075": 75, + "f076": 76, + "f077": 77, + "f078": 78, + "f079": 79, + "f080": 80, + "f081": 81, + "f082": 82, + "f083": 83, + "f084": 84, + "f085": 85, + "f086": 86, + "f087": 87, + "f088": 88, + "f089": 89, + "f090": 90, + "f091": 91, + "f092": 92, + "f093": 93, + "f094": 94, + "f095": 95, + "f096": 96, + "f097": 97, + "f098": 98, + "f099": 99, + "f100": 100, + "f101": 101, + "f102": 102, + "f103": 103, + "f104": 104, + "f105": 105, + "f106": 106, + "f107": 107, + "f108": 108, + "f109": 109, + "f110": 110, + "f111": 111, + "f112": 112, + "f113": 113, + "f114": 114, + "f115": 115, + "f116": 116, + "f117": 117, + "f118": 118, + "f119": 119, + "f120": 120, + "f121": 121, + "f122": 122, + "f123": 123, + "f124": 124, + "f125": 125, + "f126": 126, + "f127": 127, + "f128": 0, + "f129": 1, + "f130": 2, + "f131": 3, + "f132": 4, + "f133": 5, + "f134": 6, + "f135": 7, + "f136": 8, + "f137": 9, + "f138": 10, + "f139": 11, + "f140": 12, + "f141": 13, + "f142": 14, + "f143": 15, + "f144": 16, + "f145": 17, + "f146": 18, + "f147": 19, + "f148": 20, + "f149": 21, + "f150": 22, + "f151": 23, + "f152": 24, + "f153": 25, + "f154": 26, + "f155": 27, + "f156": 28, + "f157": 29, + "f158": 30, + "f159": 31, + "f160": 32, + "f161": 33, + "f162": 34, + "f163": 35, + "f164": 36, + "f165": 37, + "f166": 38, + "f167": 39, + "f168": 40, + "f169": 41, + "f170": 42, + "f171": 43, + "f172": 44, + "f173": 45, + "f174": 46, + "f175": 47, + "f176": 48, + "f177": 49, + "f178": 50, + "f179": 51, + "f180": 52, + "f181": 53, + "f182": 54, + "f183": 55, + "f184": 56, + "f185": 57, + "f186": 58, + "f187": 59, + "f188": 60, + "f189": 61, + "f190": 62, + "f191": 63, + "f192": 64, + "f193": 65, + "f194": 66, + "f195": 67, + "f196": 68, + "f197": 69, + "f198": 70, + "f199": 71, + "f200": 72, + "f201": 73, + "f202": 74, + "f203": 75, + "f204": 76, + "f205": 77, + "f206": 78, + "f207": 79, + "f208": 80, + "f209": 81, + "f210": 82, + "f211": 83, + "f212": 84, + "f213": 85, + "f214": 86, + "f215": 87, + "f216": 88, + "f217": 89, + "f218": 90, + "f219": 91, + "f220": 92, + "f221": 93, + "f222": 94, + "f223": 95, + "f224": 96, + "f225": 97, + "f226": 98, + "f227": 99, + "f228": 100, + "f229": 101, + "f230": 102, + "f231": 103, + "f232": 104, + "f233": 105, + "f234": 106, + "f235": 107, + "f236": 108, + "f237": 109, + "f238": 110, + "f239": 111, + "f240": 112, + "f241": 113, + "f242": 114, + "f243": 115, + "f244": 116, + "f245": 117, + "f246": 118, + "f247": 119, + "f248": 120, + "f249": 121, + "f250": 122, + "f251": 123, + "f252": 124, + "f253": 125, + "f254": 126, + "f255": 127, + "f256": 0, + "f257": 1, + "f258": 2, + "f259": 3, + "nested": { + "last": 1 + } + }, + "object_wide_offsets": { + "id": 1, + "padding": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, "primitive_binary": "AxM33q2+78r+", "primitive_boolean_false": false, "primitive_boolean_true": true, diff --git a/variant/object_wide_field_ids.metadata b/variant/object_wide_field_ids.metadata new file mode 100644 index 0000000..a22a296 Binary files /dev/null and b/variant/object_wide_field_ids.metadata differ diff --git a/variant/object_wide_field_ids.value b/variant/object_wide_field_ids.value new file mode 100644 index 0000000..a83dff1 Binary files /dev/null and b/variant/object_wide_field_ids.value differ diff --git a/variant/object_wide_offsets.metadata b/variant/object_wide_offsets.metadata new file mode 100644 index 0000000..a5c6116 Binary files /dev/null and b/variant/object_wide_offsets.metadata differ diff --git a/variant/object_wide_offsets.value b/variant/object_wide_offsets.value new file mode 100644 index 0000000..0bee127 Binary files /dev/null and b/variant/object_wide_offsets.value differ diff --git a/variant/regen.py b/variant/regen.py index 8de435d..d0c9954 100644 --- a/variant/regen.py +++ b/variant/regen.py @@ -43,6 +43,22 @@ os.rmdir(os.path.join(root, name)) +# Some object examples need more distinct field names, or more field data, than +# is readable as an inline JSON literal, so their JSON is built here and +# substituted into the SQL below. +# +# 260 fields plus a nested object give 262 distinct field names, which forces +# 2-byte field IDs, while the nested object holds a single small value so its +# own field offsets still fit in 1 byte. The outer object's 261 fields also make +# it the only example with is_large = 1 (a 4-byte num_elements). +object_wide_field_ids_json = "{" + ", ".join( + f'"f{i:03d}": {i % 128}' for i in range(260) +) + ', "nested": {"last": 1}}' + +# Two fields keep the field IDs at 1 byte, while more than 256 bytes of field +# data forces the field offsets to 2 bytes. +object_wide_offsets_json = '{"padding": "' + "x" * 300 + '", "id": 1}' + # Create a table with variant and insert various types into it # # This writes data files into spark-warehouse/output @@ -90,13 +106,21 @@ INSERT INTO T VALUES ('object_primitive', parse_json('{"int_field" : 1, "double_field": 1.23456789, "boolean_true_field": true, "boolean_false_field": false, "string_field": "Apache Parquet", "null_field": null, "timestamp_field": "2025-04-16T12:34:56.78"}')::Variant); INSERT INTO T VALUES ('object_nested', parse_json('{ "id" : 1, "species" : { "name": "lava monster", "population": 6789}, "observation" : { "time": "12:34:56", "location": "In the Volcano", "value" : { "temperature": 123, "humidity": 456 } } }')::Variant); +-- The two examples below are the only objects whose field_id_size and +-- field_offset_size differ. Every other object example has 1-byte field IDs and +-- 1-byte field offsets, where the two value_header fields hold the same value, +-- so a reader that transposes them still decodes correctly. +INSERT INTO T VALUES ('object_wide_field_ids', parse_json('__OBJECT_WIDE_FIELD_IDS_JSON__')::Variant); +INSERT INTO T VALUES ('object_wide_offsets', parse_json('__OBJECT_WIDE_OFFSETS_JSON__')::Variant); + -- https://github.com/apache/parquet-testing/issues/77 -- TODO create example variant objects with fields that non-json types (like timestamp, date, etc) -- Casting from "STRUCT<...>" to "VARIANT"" is not yet supported -- INSERT INTO T VALUES ('object_primitive', struct(1234.56::Double as double_field, true as boolean_true_field, false as boolean_false_field, '2025-04-16T12:34:56.78'::Timestamp as timestamp_field, 'Apache Parquet' as string_field, null as null_field)::Variant); ---TODO objects with more than 2**8 distinct fields (that require using more than one byte for field offset) ---TODO objects with more than 2**16 distinct fields (that require using more than 2 bytes for field offset) ---TODO objects with more than 2**24 distinct fields (that require using more than 3 bytes for field offset) + +-- https://github.com/apache/parquet-testing/issues/78 +--TODO objects with more than 2**16 distinct fields (that require using more than 2 bytes for field IDs) +--TODO objects with more than 2**24 distinct fields (that require using more than 3 bytes for field IDs) ------------------------------- -- Array (basic_type=3) @@ -115,6 +139,9 @@ DROP TABLE IF EXISTS output; CREATE TABLE output AS SELECT name, variant_col, to_json(variant_col) as json_col FROM T; """ +sql = sql.replace("__OBJECT_WIDE_FIELD_IDS_JSON__", object_wide_field_ids_json) +sql = sql.replace("__OBJECT_WIDE_OFFSETS_JSON__", object_wide_offsets_json) + for statement in sql.split("\n"): statement = statement.strip() if not statement or statement.startswith("--"):