Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion csharp/test/E2E/ComplexTypesValueTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,25 @@ protected override async System.Threading.Tasks.Task ValidateTestMapData(string
{
if (!CorrectedMapExpectations.TryGetValue(projection, out string? expected))
expected = value;
await base.ValidateTestMapData(projection, expected);

// Databricks MAP key order is unspecified server-side (and JDBC preserves server
// order without sorting), so assert the map's content order-insensitively rather
// than by exact key order — otherwise the comparison flakes with the server's order.
Statement.SqlQuery = $"SELECT {projection};";
QueryResult result = await Statement.ExecuteQueryAsync();

using IArrowArrayStream stream = result.Stream ?? throw new InvalidOperationException("stream is null");
Field field = stream.Schema.GetFieldByIndex(0);
Assert.IsType<StringType>(field.DataType);

RecordBatch? batch = await stream.ReadNextRecordBatchAsync();
Assert.NotNull(batch);
Assert.Equal(1, batch!.Length);

string? actual = ((StringArray)batch.Column(0)).GetString(0);
Assert.Equal(
DatabricksTestEnvironment.NormalizeMapJson(expected),
DatabricksTestEnvironment.NormalizeMapJson(actual));
}

// COMPLEX-001: Simple ARRAY of integers
Expand Down
43 changes: 42 additions & 1 deletion csharp/test/E2E/DatabricksTestEnvironment.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
using System;
using System.Collections.Generic;
using System.Data.SqlTypes;
using System.Linq;
using System.Text;
using System.Text.Json;
using Apache.Arrow.Adbc;
using AdbcDrivers.Databricks.Telemetry;
using AdbcDrivers.HiveServer2;
Expand All @@ -49,6 +51,40 @@ public class DatabricksTestEnvironment : CommonTestEnvironment<DatabricksTestCon
/// </summary>
public const string FixtureSchema = "adbc_testing";

/// <summary>
/// Normalizes a serialized MAP for order-insensitive comparison by sorting the
/// top-level object's keys. Databricks <c>MAP</c> key order is unspecified — the
/// server may return keys in any order — and JDBC preserves that server order
/// without sorting, so a MAP must be compared by its content, not by key order.
/// Only the map's own keys are reordered; each value's raw JSON is preserved
/// verbatim, so nested STRUCT field order (which IS significant) is untouched.
/// Non-object JSON (e.g. a top-level array) is returned unchanged.
///
/// Normalization is intentionally shallow: a MAP nested inside a value keeps the
/// server's key order in its raw JSON and is NOT reordered. Current MAP test data
/// only has scalar values, so this is safe today; if a future case nests a MAP
/// inside a value, this method must be made recursive to avoid reintroducing the
/// key-order flake for that case.
/// </summary>
internal static string? NormalizeMapJson(string? json)
{
if (string.IsNullOrEmpty(json))
{
return json;
}

using JsonDocument doc = JsonDocument.Parse(json);
if (doc.RootElement.ValueKind != JsonValueKind.Object)
Comment thread
peco-review-bot[bot] marked this conversation as resolved.
{
return json;
}

IEnumerable<string> entries = doc.RootElement.EnumerateObject()
.OrderBy(p => p.Name, StringComparer.Ordinal)
.Select(p => JsonSerializer.Serialize(p.Name) + ":" + p.Value.GetRawText());
return "{" + string.Join(",", entries) + "}";
}

public class Factory : Factory<DatabricksTestEnvironment>
{
public override DatabricksTestEnvironment Create(Func<AdbcConnection> getConnection) => new(getConnection);
Expand Down Expand Up @@ -334,7 +370,12 @@ public override SampleDataBuilder GetSampleDataBuilder()
{
new ColumnNetTypeArrowTypeValue("numbers", typeof(string), typeof(StringType), "[1,2,3]"),
new ColumnNetTypeArrowTypeValue("person", typeof(string), typeof(StringType), """{"name":"John Doe","age":30}"""),
new ColumnNetTypeArrowTypeValue("map", typeof(string), typeof(StringType), """{"age":"29","name":"Jane Doe"}"""), // This is unexpected JSON. Expecting 29 to be a numeric and not string.
// MAP key order is unspecified server-side (and JDBC preserves server order
// without sorting), so validate the map's content order-insensitively rather
// than by exact key order. (The "29" being a quoted string rather than numeric
// is a separate, pre-existing quirk of the serialized output.)
new ColumnNetTypeArrowTypeValue("map", typeof(string), typeof(StringType), true,
actual => NormalizeMapJson("""{"age":"29","name":"Jane Doe"}""") == NormalizeMapJson(actual as string)),
});

sampleDataBuilder.Samples.Add(
Expand Down
Loading