diff --git a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoSchemaConverter.java b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoSchemaConverter.java index ff27b263e9..024b6e5213 100644 --- a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoSchemaConverter.java +++ b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoSchemaConverter.java @@ -312,21 +312,30 @@ private Builder>, GroupBuilder> addM final GroupBuilder builder, ImmutableSetMultimap seen, int depth) { - // Prevent recursion by terminating with optional proto bytes. + // Terminate with proto bytes anything a static parquet schema cannot represent - recursion + // beyond maxRecursion and empty message types (parquet forbids empty groups) - preserving the + // field's repetition so the write path (Array/Repeated/MapWriter) still matches the schema. depth += 1; String typeName = getInnerTypeName(descriptor); LOG.trace("addMessageField: {} type: {} depth: {}", descriptor.getFullName(), typeName, depth); - if (typeName != null) { - if (seen.get(typeName).size() > maxRecursion) { - return builder.primitive(BINARY, Type.Repetition.OPTIONAL).as((LogicalTypeAnnotation) null); - } - } if (descriptor.isMapField() && parquetSpecsCompliant) { - // the old schema style did not include the MAP wrapper around map groups + // the old schema style did not include the MAP wrapper around map groups. + // The MAP structure is always preserved; a recursive or empty value type is truncated to + // proto bytes by the check below when addMapField recurses into the value field. return addMapField(descriptor, builder, seen, depth); } + boolean emptyMessage = descriptor.getMessageType().getFields().isEmpty(); + if (emptyMessage || (typeName != null && seen.get(typeName).size() > maxRecursion)) { + if (descriptor.isRepeated() && parquetSpecsCompliant) { + // LIST-wrap the truncated bytes the same way any repeated primitive is wrapped + return addRepeatedPrimitive(BINARY, null, builder); + } + // optional, required, or repeated in the old schema style + return builder.primitive(BINARY, getRepetition(descriptor)).as((LogicalTypeAnnotation) null); + } + seen = ImmutableSetMultimap.builder() .putAll(seen) .put(typeName, depth) diff --git a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoWriteSupport.java b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoWriteSupport.java index 51e2d7e25b..c6109468f4 100644 --- a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoWriteSupport.java +++ b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoWriteSupport.java @@ -356,42 +356,45 @@ private FieldWriter createMessageWriter(FieldDescriptor fieldDescriptor, Type ty } // This can happen now that recursive schemas get truncated to bytes. Write the bytes. - if (type.isPrimitive() - && type.asPrimitiveType().getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.BINARY) { + // The truncated type keeps the field's shape, so it may sit behind a LIST wrapper + // (repeated field) or be the value inside a MAP's key_value group. + Type contentType = getContentType(type); + if (contentType.isPrimitive() + && contentType.asPrimitiveType().getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.BINARY) { return new BinaryWriter(); } - return new MessageWriter(fieldDescriptor.getMessageType(), getGroupType(type)); + return new MessageWriter(fieldDescriptor.getMessageType(), contentType.asGroupType()); } - private GroupType getGroupType(Type type) { + /** Unwraps the LIST/MAP wrapper groups to the type holding the message content itself. */ + private Type getContentType(Type type) { + if (type.isPrimitive()) { + return type; + } LogicalTypeAnnotation logicalTypeAnnotation = type.getLogicalTypeAnnotation(); if (logicalTypeAnnotation == null) { - return type.asGroupType(); + return type; } return logicalTypeAnnotation - .accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() { + .accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() { @Override - public Optional visit( - LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) { + public Optional visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) { return ofNullable(type.asGroupType() .getType("list") .asGroupType() - .getType("element") - .asGroupType()); + .getType("element")); } @Override - public Optional visit( - LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) { + public Optional visit(LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) { return ofNullable(type.asGroupType() .getType("key_value") .asGroupType() - .getType("value") - .asGroupType()); + .getType("value")); } }) - .orElse(type.asGroupType()); + .orElse(type); } private MapWriter createMapWriter(FieldDescriptor fieldDescriptor, Type type) { diff --git a/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoEmptyMessageTest.java b/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoEmptyMessageTest.java new file mode 100644 index 0000000000..c0eca35fff --- /dev/null +++ b/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoEmptyMessageTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.proto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.protobuf.Message; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.proto.test.Trees; +import org.apache.parquet.schema.InvalidSchemaException; +import org.junit.jupiter.api.Test; + +/** + * Fields typed as an EMPTY proto message cannot map to a parquet group (parquet forbids empty + * groups, so writer construction used to fail with an {@code InvalidSchemaException}). They are + * now terminated as proto bytes, like recursion beyond maxRecursion, which also keeps the field's + * presence observable (null vs an empty byte array). + */ +public class ProtoEmptyMessageTest { + + private static Path write(boolean specsCompliant, Message... messages) throws IOException { + Path file = TestUtils.someTemporaryFilePath(); + Configuration conf = new Configuration(); + ProtoWriteSupport.setWriteSpecsCompliant(conf, specsCompliant); + try (ParquetWriter writer = ProtoParquetWriter.builder(file) + .withMessage(messages[0].getClass()) + .withConf(conf) + .build()) { + for (Message message : messages) { + writer.write(message); + } + } + return file; + } + + private static List read(Path file) throws IOException { + List rows = new ArrayList<>(); + try (ParquetReader reader = + ParquetReader.builder(new GroupReadSupport(), file).build()) { + for (Group group = reader.read(); group != null; group = reader.read()) { + rows.add(group); + } + } + return rows; + } + + @Test + public void emptyMessageFieldsWriteAsBytes() throws Exception { + Trees.StubBox box = Trees.StubBox.newBuilder() + .setStub(Trees.Stub.getDefaultInstance()) + .addStubs(Trees.Stub.getDefaultInstance()) + .addStubs(Trees.Stub.getDefaultInstance()) + .putStubMap("k", Trees.Stub.getDefaultInstance()) + .setName("x") + .build(); + + Group row = read(write(true, box)).get(0); + assertThat(row.getBinary("stub", 0).length()) + .as("optional empty message present as zero bytes") + .isEqualTo(0); + assertThat(row.getGroup("stubs", 0).getFieldRepetitionCount("list")) + .as("repeated empty messages keep their cardinality") + .isEqualTo(2); + Group entry = row.getGroup("stub_map", 0).getGroup("key_value", 0); + assertThat(entry.getString("key", 0)).as("map keys stay typed").isEqualTo("k"); + assertThat(entry.getBinary("value", 0).length()) + .as("map value is zero bytes") + .isEqualTo(0); + assertThat(row.getString("name", 0)).isEqualTo("x"); + } + + @Test + public void emptyMessagePresenceRoundTrips() throws Exception { + Trees.StubBox with = Trees.StubBox.newBuilder() + .setStub(Trees.Stub.getDefaultInstance()) + .build(); + Trees.StubBox without = Trees.StubBox.getDefaultInstance(); + + List rows = read(write(true, with, without)); + assertThat(rows.get(0).getFieldRepetitionCount("stub")) + .as("set empty message is present") + .isEqualTo(1); + assertThat(rows.get(1).getFieldRepetitionCount("stub")) + .as("unset empty message is null") + .isEqualTo(0); + } + + @Test + public void emptyMessageFieldsWriteAsBytesOldStyle() throws Exception { + Trees.StubBox box = Trees.StubBox.newBuilder() + .addStubs(Trees.Stub.getDefaultInstance()) + .addStubs(Trees.Stub.getDefaultInstance()) + .build(); + + Group row = read(write(false, box)).get(0); + assertThat(row.getFieldRepetitionCount("stubs")) + .as("repeated empty messages keep their cardinality") + .isEqualTo(2); + } + + @Test + public void emptyRootMessageStillRejected() { + // the root message itself cannot be terminated as bytes - there is no field to hold them + assertThatThrownBy(() -> write(true, Trees.Stub.getDefaultInstance())) + .isInstanceOf(InvalidSchemaException.class) + .hasMessageContaining("Cannot write a schema with an empty group"); + } +} diff --git a/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoRecursionTruncationTest.java b/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoRecursionTruncationTest.java new file mode 100644 index 0000000000..84cea68d1f --- /dev/null +++ b/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoRecursionTruncationTest.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.proto; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.protobuf.ListValue; +import com.google.protobuf.Message; +import com.google.protobuf.Struct; +import com.google.protobuf.Value; +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.proto.test.Trees; +import org.junit.jupiter.api.Test; + +/** + * End-to-end write tests for recursive messages that get truncated to proto bytes at + * {@code maxRecursion} depth. Unlike the RecordConsumer-mock tests in {@link + * ProtoWriteSupportTest}, these write through a real {@code MessageColumnIO}, which validates the + * emitted record structure against the schema - the case that used to fail with a + * {@code ClassCastException} (specs-compliant mode) or a repetition violation (old style) when the + * truncated recursive field is repeated or a map. + */ +public class ProtoRecursionTruncationTest { + + /** A WideTree of the given depth with {@code branching} children at every level. */ + private static Trees.WideTree wideTree(int depth, int branching) { + Trees.WideTree.Builder node = Trees.WideTree.newBuilder(); + node.getValueBuilder().setTypeUrl("level-" + depth); + if (depth > 0) { + for (int i = 0; i < branching; i++) { + node.addChildren(wideTree(depth - 1, branching)); + } + } + return node.build(); + } + + /** A Struct nested through its map values to the given depth. */ + private static Struct deepStruct(int depth) { + Struct.Builder struct = Struct.newBuilder(); + if (depth > 0) { + struct.putFields( + "level-" + depth, + Value.newBuilder().setStructValue(deepStruct(depth - 1)).build()); + } else { + struct.putFields("leaf", Value.newBuilder().setStringValue("x").build()); + } + return struct.build(); + } + + private static Path write(Message message, boolean specsCompliant, int maxRecursion) throws IOException { + Path file = TestUtils.someTemporaryFilePath(); + Configuration conf = new Configuration(); + ProtoWriteSupport.setWriteSpecsCompliant(conf, specsCompliant); + ProtoSchemaConverter.setMaxRecursion(conf, maxRecursion); + try (ParquetWriter writer = ProtoParquetWriter.builder(file) + .withMessage(message.getClass()) + .withConf(conf) + .build()) { + writer.write(message); + } + return file; + } + + private static Group readSingleRow(Path file) throws IOException { + try (ParquetReader reader = + ParquetReader.builder(new GroupReadSupport(), file).build()) { + Group group = reader.read(); + assertThat(reader.read()).as("expected exactly one record").isNull(); + return group; + } + } + + @Test + public void repeatedRecursionDeeperThanMaxRecursionSpecsCompliant() throws Exception { + Trees.WideTree tree = wideTree(5, 2); + Path file = write(tree, true, 2); + + // depth 0..2 are materialized groups; each node's children at depth 3 are truncated to a + // LIST of proto bytes with one element per repeated message + Group row = readSingleRow(file); + Group level1 = row.getGroup("children", 0).getGroup("list", 0).getGroup("element", 0); + Group level2 = level1.getGroup("children", 0).getGroup("list", 0).getGroup("element", 0); + Group truncated = level2.getGroup("children", 0).getGroup("list", 0); + assertThat(level2.getGroup("children", 0).getFieldRepetitionCount("list")) + .as("both children survive at the truncation level") + .isEqualTo(2); + + Trees.WideTree expectedSubtree = tree.getChildren(0).getChildren(0).getChildren(0); + Trees.WideTree roundTripped = + Trees.WideTree.parseFrom(truncated.getBinary("element", 0).getBytes()); + assertThat(roundTripped) + .as("truncated bytes are the serialized subtree") + .isEqualTo(expectedSubtree); + } + + @Test + public void repeatedRecursionDeeperThanMaxRecursionOldStyle() throws Exception { + Trees.WideTree tree = wideTree(5, 2); + Path file = write(tree, false, 2); + + // old style: repeated group children { ... repeated binary children; } + Group row = readSingleRow(file); + Group level2 = row.getGroup("children", 0).getGroup("children", 0); + assertThat(level2.getFieldRepetitionCount("children")) + .as("both children survive at the truncation level") + .isEqualTo(2); + + Trees.WideTree expectedSubtree = tree.getChildren(0).getChildren(0).getChildren(0); + Trees.WideTree roundTripped = + Trees.WideTree.parseFrom(level2.getBinary("children", 0).getBytes()); + assertThat(roundTripped) + .as("truncated bytes are the serialized subtree") + .isEqualTo(expectedSubtree); + } + + @Test + public void mapValueRecursionDeeperThanMaxRecursionSpecsCompliant() throws Exception { + Struct struct = deepStruct(6); + Path file = write(struct, true, 2); + + // the MAP shape (key_value group with a typed key) is preserved; the recursion budget + // terminates the recursive struct_value as proto bytes. On this main map path the budget + // trips at the singular struct_value, so this pins the (unchanged) truncation shape. + Group row = readSingleRow(file); + Group entry = row.getGroup("fields", 0).getGroup("key_value", 0); + assertThat(entry.getString("key", 0)).isEqualTo("level-6"); + + int materializedLevels = 1; + Group value = entry.getGroup("value", 0); + while (!value.getType().getType("struct_value").isPrimitive()) { + entry = value.getGroup("struct_value", 0).getGroup("fields", 0).getGroup("key_value", 0); + assertThat(entry.getString("key", 0)).isEqualTo("level-" + (6 - materializedLevels)); + value = entry.getGroup("value", 0); + materializedLevels++; + } + + Struct truncated = Struct.parseFrom(value.getBinary("struct_value", 0).getBytes()); + assertThat(truncated) + .as("truncated bytes are the serialized subtree") + .isEqualTo(deepStruct(6 - materializedLevels)); + } + + @Test + public void mapFieldExhaustingRecursionBudgetKeepsTypedKeys() throws Exception { + // Nesting through list_value makes the recursion budget run out AT a map field: the whole + // MAP - keys included - used to collapse into one optional binary in the schema, and writing + // data through it crashed with the PrimitiveColumnIO -> GroupColumnIO ClassCastException. + // Now the MAP structure survives and only its value is stored as proto bytes. + Struct inner = Struct.newBuilder() + .putFields("deep", Value.newBuilder().setStringValue("x").build()) + .build(); + Value nested = Value.newBuilder() + .setListValue(ListValue.newBuilder() + .addValues(Value.newBuilder() + .setListValue(ListValue.newBuilder() + .addValues(Value.newBuilder().setStructValue(inner))))) + .build(); + Struct root = Struct.newBuilder().putFields("k", nested).build(); + Path file = write(root, true, 2); + + Group row = readSingleRow(file); + Group entry = row.getGroup("fields", 0).getGroup("key_value", 0); + assertThat(entry.getString("key", 0)).isEqualTo("k"); + Group element = entry.getGroup("value", 0) + .getGroup("list_value", 0) + .getGroup("values", 0) + .getGroup("list", 0) + .getGroup("element", 0) + .getGroup("list_value", 0) + .getGroup("values", 0) + .getGroup("list", 0) + .getGroup("element", 0); + Group truncatedEntry = + element.getGroup("struct_value", 0).getGroup("fields", 0).getGroup("key_value", 0); + assertThat(truncatedEntry.getString("key", 0)) + .as("keys of the truncated map stay typed and queryable") + .isEqualTo("deep"); + + Value truncated = Value.parseFrom(truncatedEntry.getBinary("value", 0).getBytes()); + assertThat(truncated.getStringValue()) + .as("truncated map value bytes are the serialized proto value") + .isEqualTo("x"); + } + + @Test + public void optionalRecursionStillTruncatesToOptionalBytes() throws Exception { + // regression guard for the already-working optional case (BinaryTree: left/right) + Trees.BinaryTree.Builder tree = Trees.BinaryTree.newBuilder(); + Trees.BinaryTree.Builder cursor = tree; + for (int i = 0; i < 6; i++) { + cursor.getValueBuilder().setTypeUrl("level-" + i); + cursor = cursor.getLeftBuilder(); + } + Path file = write(tree.build(), true, 2); + + Group row = readSingleRow(file); + Group level2 = row.getGroup("left", 0).getGroup("left", 0); + Trees.BinaryTree truncated = + Trees.BinaryTree.parseFrom(level2.getBinary("left", 0).getBytes()); + assertThat(truncated.getValue().getTypeUrl()).isEqualTo("level-3"); + } +} diff --git a/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoSchemaConverterTest.java b/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoSchemaConverterTest.java index a4539e3397..c75aa18f0b 100644 --- a/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoSchemaConverterTest.java +++ b/parquet-protobuf/src/test/java/org/apache/parquet/proto/ProtoSchemaConverterTest.java @@ -456,7 +456,11 @@ public void testWideTreeRecursion() throws Exception { " optional binary type_url (STRING) = 1;", " optional binary value = 2;", " }", - " optional binary children = 2;", + " optional group children (LIST) = 2 {", + " repeated group list {", + " required binary element;", + " }", + " }", " }", " }", " }", @@ -486,10 +490,19 @@ public void testValueRecursion() throws Exception { " optional binary string_value (STRING) = 3;", " optional boolean bool_value = 4;", " optional group struct_value = 5 {", - " optional binary fields = 1;", + " optional group fields (MAP) = 1 {", + " repeated group key_value {", + " required binary key (STRING);", + " optional binary value;", + " }", + " }", " }", " optional group list_value = 6 {", - " optional binary values = 1;", + " optional group values (LIST) = 1 {", + " repeated group list {", + " required binary element;", + " }", + " }", " }", " }", " }", @@ -504,10 +517,19 @@ public void testValueRecursion() throws Exception { " optional binary string_value (STRING) = 3;", " optional boolean bool_value = 4;", " optional group struct_value = 5 {", - " optional binary fields = 1;", + " optional group fields (MAP) = 1 {", + " repeated group key_value {", + " required binary key (STRING);", + " optional binary value;", + " }", + " }", " }", " optional group list_value = 6 {", - " optional binary values = 1;", + " optional group values (LIST) = 1 {", + " repeated group list {", + " required binary element;", + " }", + " }", " }", " }", " }", @@ -544,7 +566,11 @@ public void testStructRecursion() throws Exception { " optional boolean bool_value = 4;", " optional binary struct_value = 5;", " optional group list_value = 6 {", - " optional binary values = 1;", + " optional group values (LIST) = 1 {", + " repeated group list {", + " required binary element;", + " }", + " }", " }", " }", " }", @@ -559,10 +585,19 @@ public void testStructRecursion() throws Exception { " optional binary string_value (STRING) = 3;", " optional boolean bool_value = 4;", " optional group struct_value = 5 {", - " optional binary fields = 1;", + " optional group fields (MAP) = 1 {", + " repeated group key_value {", + " required binary key (STRING);", + " optional binary value;", + " }", + " }", " }", " optional group list_value = 6 {", - " optional binary values = 1;", + " optional group values (LIST) = 1 {", + " repeated group list {", + " required binary element;", + " }", + " }", " }", " }", " }", @@ -579,6 +614,27 @@ public void testStructRecursion() throws Exception { new ProtoSchemaConverter(true, PAR_RECURSION_DEPTH, false)); } + @Test + public void testEmptyMessageFields() throws Exception { + String expectedSchema = JOINER.join( + "message Trees.StubBox {", + " optional binary stub = 1;", + " optional group stubs (LIST) = 2 {", + " repeated group list {", + " required binary element;", + " }", + " }", + " optional group stub_map (MAP) = 3 {", + " repeated group key_value {", + " required binary key (STRING);", + " optional binary value;", + " }", + " }", + " optional binary name (STRING) = 4;", + "}"); + testConversion(Trees.StubBox.class, expectedSchema, new ProtoSchemaConverter(true, 5, false)); + } + @Test public void testDeepRecursion() { // The general idea is to test the fanout of the schema. @@ -596,9 +652,9 @@ public void testDeepRecursion() { expectedBinaryTreeSize = 2 * expectedBinaryTreeSize + 2; deepSchema = new ProtoSchemaConverter(true, i, false).convert(Struct.class); - // 7, 18, 40, 84, 172, 348, 700, 1404, 2812, 5628 + // 7, 19, 43, 91, 187, 379, 763, 1531, 3067, 6139 assertThat(deepSchema.getPaths()).hasSize((int) expectedStructSize); - expectedStructSize = 2 * expectedStructSize + 4; + expectedStructSize = 2 * expectedStructSize + 5; } } } diff --git a/parquet-protobuf/src/test/resources/Struct.par b/parquet-protobuf/src/test/resources/Struct.par index 4ec0171efc..4f216d09a3 100644 --- a/parquet-protobuf/src/test/resources/Struct.par +++ b/parquet-protobuf/src/test/resources/Struct.par @@ -27,7 +27,11 @@ message google.protobuf.Struct { optional boolean bool_value = 4; optional binary struct_value = 5; optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } @@ -42,10 +46,19 @@ message google.protobuf.Struct { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } @@ -73,10 +86,19 @@ message google.protobuf.Struct { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } @@ -91,10 +113,19 @@ message google.protobuf.Struct { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } diff --git a/parquet-protobuf/src/test/resources/Trees.proto b/parquet-protobuf/src/test/resources/Trees.proto index c62754d6b8..342f6b094d 100644 --- a/parquet-protobuf/src/test/resources/Trees.proto +++ b/parquet-protobuf/src/test/resources/Trees.proto @@ -35,3 +35,15 @@ message WideTree { google.protobuf.Any value = 1; repeated WideTree children = 2; } + +// An empty message: parquet cannot represent an empty group, so fields of this +// type are terminated as proto bytes (like recursion beyond maxRecursion). +message Stub { +} + +message StubBox { + Stub stub = 1; + repeated Stub stubs = 2; + map stub_map = 3; + string name = 4; +} diff --git a/parquet-protobuf/src/test/resources/Value.par b/parquet-protobuf/src/test/resources/Value.par index f0381a3134..8a4f1bae98 100644 --- a/parquet-protobuf/src/test/resources/Value.par +++ b/parquet-protobuf/src/test/resources/Value.par @@ -22,10 +22,19 @@ message google.protobuf.Value { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } @@ -40,10 +49,19 @@ message google.protobuf.Value { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } @@ -71,10 +89,19 @@ message google.protobuf.Value { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } @@ -89,10 +116,19 @@ message google.protobuf.Value { optional binary string_value (STRING) = 3; optional boolean bool_value = 4; optional group struct_value = 5 { - optional binary fields = 1; + optional group fields (MAP) = 1 { + repeated group key_value { + required binary key (STRING); + optional binary value; + } + } } optional group list_value = 6 { - optional binary values = 1; + optional group values (LIST) = 1 { + repeated group list { + required binary element; + } + } } } } diff --git a/parquet-protobuf/src/test/resources/WideTree.par b/parquet-protobuf/src/test/resources/WideTree.par index 701b53ebfc..d5ba6e9906 100644 --- a/parquet-protobuf/src/test/resources/WideTree.par +++ b/parquet-protobuf/src/test/resources/WideTree.par @@ -17,7 +17,11 @@ message Trees.WideTree { optional binary type_url (STRING) = 1; optional binary value = 2; } - optional binary children = 2; + optional group children (LIST) = 2 { + repeated group list { + required binary element; + } + } } } }