Create Nested(...) columns as Array(Tuple(...))

The type parser had no entry for Nested, so a column typed
Array(Nested(key String, value String)) was created as Array(Void)
and the native block stream fell out of sync ("Can't cast from Void",
"some data was not read"). A top-level Nested column, which the server
sends with flatten_nested = 0, was rejected as unsupported.

The server defines Nested as a custom name over Array(Tuple(...))
(DataTypeNested.cpp). The parser now recognises Nested as its own
node, like Map and SimpleAggregateFunction, and the column factory
builds it as its Tuple wrapped in an Array.

Closes #571

Author: Hakaou

README.md

@@ -185,6 +185,7 @@ still might break.
 * String
 * LowCardinality(String) or LowCardinality(FixedString(N))
 * Tuple
+* Nested, read and written as Array(Tuple(...))
 * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64
 * UInt128, Int128
 * UUID

clickhouse/columns/factory.cpp

@@ -191,6 +191,15 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSetti
             return std::make_shared<ColumnTuple>(columns);
         }
 
+        case TypeAst::Nested: {
+            TypeAst tuple = ast;
+            tuple.meta = TypeAst::Tuple;
+            if (auto column = CreateColumnFromAst(tuple, settings)) {
+                return std::make_shared<ColumnArray>(column);
+            }
+            return nullptr;
+        }
+
         case TypeAst::Enum: {
             std::vector<Type::EnumItem> enum_items;
             //ast.elements.size() minimum is 1.

clickhouse/types/type_parser.cpp

@@ -146,6 +146,10 @@ static TypeAst::Meta GetTypeMeta(const StringView& name) {
         return TypeAst::Map;
     }
 
+    if (name == "Nested") {
+        return TypeAst::Nested;
+    }
+
     return TypeAst::Terminal;
 }
 

clickhouse/types/type_parser.h

@@ -22,7 +22,8 @@ struct TypeAst {
         Enum,
         LowCardinality,
         SimpleAggregateFunction,
-        Map
+        Map,
+        Nested
     };
 
     /// Type's category.

docs/index.mdx

@@ -164,6 +164,7 @@ client.Select(
 - `Nullable(T)`
 - `Array(T)`
 - `Tuple`
+- `Nested`, read and written as `Array(Tuple(...))`
 - `Map`
 - `IPv4`, `IPv6`
 - `Point`, `Ring`, `Polygon`, `MultiPolygon`

ut/CreateColumnByType_ut.cpp

@@ -280,3 +280,19 @@ INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByTypeWithName, ::testing::Values(
     "Array(Nullable(LowCardinality(FixedString(10000))))",
     "Array(Enum8('ONE' = 1, 'TWO' = 2))"
 ));
+
+class CreateColumnByNestedType : public ::testing::TestWithParam<std::tuple<const char*, const char*>> {};
+
+TEST_P(CreateColumnByNestedType, CreateColumnByType) {
+    const auto & [type_name, expected_name] = GetParam();
+    const auto col = CreateColumnByType(type_name);
+    ASSERT_NE(nullptr, col) << "CreateColumnByType returned nullptr for " << type_name;
+    EXPECT_EQ(expected_name, col->GetType().GetName());
+}
+
+INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByNestedType, ::testing::Values(
+    std::make_tuple("Nested(key String, value UInt8)", "Array(Tuple(key String, value UInt8))"),
+    std::make_tuple("Array(Nested(key String, value UInt8))", "Array(Array(Tuple(key String, value UInt8)))"),
+    std::make_tuple("Map(String, Nested(key String, value UInt8))", "Map(String, Array(Tuple(key String, value UInt8)))"),
+    std::make_tuple("Nested(a Nested(b String))", "Array(Tuple(a Array(Tuple(b String))))")
+));

ut/client_ut.cpp

@@ -1823,6 +1823,117 @@ const auto LocalHostEndpoint = ClientOptions()
         .SetPassword(       getEnvOrDefault("CLICKHOUSE_PASSWORD", ""))
         .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB",       "default"));
 
+namespace {
+
+using KeyValuePairs = std::vector<std::pair<std::string, uint8_t>>;
+
+KeyValuePairs KeyValuePairsOf(const ColumnRef& column) {
+    auto tuples = column->AsStrict<ColumnTuple>();
+    auto keys = (*tuples)[0]->AsStrict<ColumnString>();
+    auto values = (*tuples)[1]->AsStrict<ColumnUInt8>();
+    KeyValuePairs pairs;
+    for (size_t i = 0; i < tuples->Size(); ++i) {
+        pairs.emplace_back(std::string(keys->At(i)), values->At(i));
+    }
+    return pairs;
+}
+
+std::vector<KeyValuePairs> ArraysOfKeyValuePairsOf(const ColumnRef& column) {
+    auto arrays = column->AsStrict<ColumnArray>();
+    std::vector<KeyValuePairs> arrays_of_pairs;
+    for (size_t i = 0; i < arrays->Size(); ++i) {
+        arrays_of_pairs.push_back(KeyValuePairsOf(arrays->GetAsColumn(i)));
+    }
+    return arrays_of_pairs;
+}
+
+}
+
+TEST_P(ClientCase, ArrayOfNested_SelectServerTypedColumn) {
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested "
+                     "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))");
+    client_->Execute("INSERT INTO test_clickhouse_cpp_array_of_nested VALUES "
+                     "(1, [[('a', 1), ('b', 2)], []]), (2, []), (3, [[('c', 3)]])");
+
+    std::vector<std::vector<KeyValuePairs>> rows;
+    std::string type_name;
+    client_->Select("SELECT id, arrays_of_pairs FROM test_clickhouse_cpp_array_of_nested ORDER BY id", [&](const Block& block) {
+        if (block.GetRowCount() == 0) {
+            return;
+        }
+        type_name = block[1]->GetType().GetName();
+        auto arrays_of_pairs = block[1]->AsStrict<ColumnArray>();
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            rows.push_back(ArraysOfKeyValuePairsOf(arrays_of_pairs->GetAsColumn(row)));
+        }
+    });
+
+    EXPECT_EQ("Array(Array(Tuple(key String, value UInt8)))", type_name);
+    const std::vector<std::vector<KeyValuePairs>> expected = {
+        {{{"a", 1}, {"b", 2}}, {}},
+        {},
+        {{{"c", 3}}},
+    };
+    EXPECT_EQ(expected, rows);
+}
+
+TEST_P(ClientCase, ArrayOfNested_ReinsertSelectedRows) {
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested_source");
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_array_of_nested_copy");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested_source "
+                     "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_array_of_nested_copy "
+                     "(id UInt8, arrays_of_pairs Array(Nested(key String, value UInt8)))");
+    client_->Execute("INSERT INTO test_clickhouse_cpp_array_of_nested_source VALUES (1, [[('a', 1), ('b', 2)], []]), (2, [])");
+
+    std::vector<Block> selected_blocks;
+    client_->Select("SELECT id, arrays_of_pairs FROM test_clickhouse_cpp_array_of_nested_source ORDER BY id", [&](const Block& block) {
+        if (block.GetRowCount() > 0) {
+            selected_blocks.push_back(block);
+        }
+    });
+    for (const auto& block : selected_blocks) {
+        client_->Insert("test_clickhouse_cpp_array_of_nested_copy", block);
+    }
+
+    std::vector<std::string> rendered_rows;
+    client_->Select("SELECT toString(arrays_of_pairs) FROM test_clickhouse_cpp_array_of_nested_copy ORDER BY id",
+                    [&](const Block& block) {
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            rendered_rows.emplace_back(block[0]->AsStrict<ColumnString>()->At(row));
+        }
+    });
+
+    const std::vector<std::string> expected = {"[[('a',1),('b',2)],[]]", "[]"};
+    EXPECT_EQ(expected, rendered_rows);
+}
+
+TEST_P(ClientCase, Nested_UnflattenedColumn) {
+    client_->Execute("SET flatten_nested = 0");
+    client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_nested_unflattened");
+    client_->Execute("CREATE TEMPORARY TABLE test_clickhouse_cpp_nested_unflattened "
+                     "(id UInt8, pairs Nested(key String, value UInt8))");
+    client_->Execute("INSERT INTO test_clickhouse_cpp_nested_unflattened VALUES (1, [('a', 1), ('b', 2)]), (2, [])");
+
+    std::vector<KeyValuePairs> rows;
+    std::string type_name;
+    client_->Select("SELECT id, pairs FROM test_clickhouse_cpp_nested_unflattened ORDER BY id", [&](const Block& block) {
+        if (block.GetRowCount() == 0) {
+            return;
+        }
+        type_name = block[1]->GetType().GetName();
+        auto pairs = block[1]->AsStrict<ColumnArray>();
+        for (size_t row = 0; row < block.GetRowCount(); ++row) {
+            rows.push_back(KeyValuePairsOf(pairs->GetAsColumn(row)));
+        }
+    });
+
+    EXPECT_EQ("Array(Tuple(key String, value UInt8))", type_name);
+    const std::vector<KeyValuePairs> expected = {{{"a", 1}, {"b", 2}}, {}};
+    EXPECT_EQ(expected, rows);
+}
+
 INSTANTIATE_TEST_SUITE_P(
     Client, ClientCase,
     ::testing::Values(

ut/type_parser_ut.cpp

@@ -552,3 +552,29 @@ TEST(TypeParser, AggregateFunction) {
         EXPECT_FALSE(TypeParser("AggregateFunction(argMax, LowCardinality(Nullable(FixedString(4))), DateTime(3, 'UTC'))").Parse(&ast));
     }
 }
+
+TEST(TypeParserCase, ParseNested) {
+    TypeAst ast;
+    ASSERT_TRUE(TypeParser("Nested(key String, value UInt8)").Parse(&ast));
+    ASSERT_EQ(ast.meta, TypeAst::Nested);
+    ASSERT_EQ(ast.name, "Nested");
+    ASSERT_EQ(ast.elements.size(), 2u);
+
+    ASSERT_EQ(ast.elements[0].element_name, "key");
+    ASSERT_EQ(ast.elements[0].name, "String");
+    ASSERT_EQ(ast.elements[0].code, Type::String);
+
+    ASSERT_EQ(ast.elements[1].element_name, "value");
+    ASSERT_EQ(ast.elements[1].name, "UInt8");
+    ASSERT_EQ(ast.elements[1].code, Type::UInt8);
+}
+
+TEST(TypeParserCase, ParseNested_FieldNamedNested) {
+    TypeAst ast;
+    ASSERT_TRUE(TypeParser("Nested(Nested String)").Parse(&ast));
+    ASSERT_EQ(ast.meta, TypeAst::Nested);
+    ASSERT_EQ(ast.elements.size(), 1u);
+    ASSERT_EQ(ast.elements[0].meta, TypeAst::Terminal);
+    ASSERT_EQ(ast.elements[0].element_name, "Nested");
+    ASSERT_EQ(ast.elements[0].name, "String");
+}