diff --git a/csharp/src/StatementExecution/MetadataCommands/MetadataCommandBase.cs b/csharp/src/StatementExecution/MetadataCommands/MetadataCommandBase.cs index 39563b4c2..78695512f 100644 --- a/csharp/src/StatementExecution/MetadataCommands/MetadataCommandBase.cs +++ b/csharp/src/StatementExecution/MetadataCommands/MetadataCommandBase.cs @@ -15,6 +15,7 @@ */ using System.Text; +using System.Text.RegularExpressions; namespace AdbcDrivers.Databricks.StatementExecution.MetadataCommands { @@ -99,5 +100,95 @@ protected static void AppendCatalogScope(StringBuilder sql, string? catalog) else sql.Append(string.Format(InCatalogFormat, QuoteIdentifier(catalog))); } + + /// + /// Returns true when contains a SQL LIKE wildcard + /// (% or _) that is NOT escaped by a preceding backslash. JDBC metadata APIs + /// treat catalog/schema/table arguments as LIKE patterns, but SEA SHOW commands + /// take literal identifiers, so callers must expand wildcards client-side. + /// + internal static bool ContainsUnescapedWildcard(string? pattern) + { + if (string.IsNullOrEmpty(pattern)) + return false; + + bool escapeNext = false; + for (int i = 0; i < pattern!.Length; i++) + { + char c = pattern[i]; + if (c == '\\') + { + // Two backslashes in a row are an escaped backslash literal, not an escape. + if (i + 1 < pattern.Length && pattern[i + 1] == '\\') + { + i++; + continue; + } + escapeNext = !escapeNext; + } + else if (escapeNext) + { + escapeNext = false; + } + else if (c == '%' || c == '_') + { + return true; + } + } + return false; + } + + /// + /// Returns true when is a pure "match anything" + /// pattern: a single unescaped % (or *). These can be optimised to + /// SHOW SCHEMAS IN ALL CATALOGS without enumerating catalogs. + /// + internal static bool IsMatchAnything(string? pattern) + { + return pattern == "%" || pattern == "*"; + } + + /// + /// Compiles a JDBC LIKE pattern (with % / _ wildcards and + /// \ escapes, where \% / \_ / \\ are literal) + /// into a for client-side filtering. Anchored at both + /// ends; case-sensitive. Used when wildcard expansion has to happen on + /// the driver side (e.g. catalog patterns on SEA — PECO-3035). + /// + internal static Regex JdbcLikeToRegex(string pattern) + { + var sb = new StringBuilder("^"); + bool escapeNext = false; + for (int i = 0; i < pattern.Length; i++) + { + char c = pattern[i]; + if (c == '\\') + { + // Two backslashes → literal backslash. + if (i + 1 < pattern.Length && pattern[i + 1] == '\\') + { + sb.Append("\\\\"); + i++; + continue; + } + escapeNext = !escapeNext; + continue; + } + if (escapeNext) + { + sb.Append(Regex.Escape(c.ToString())); + escapeNext = false; + continue; + } + switch (c) + { + case '%': sb.Append(".*"); break; + case '_': sb.Append("."); break; + default: sb.Append(Regex.Escape(c.ToString())); break; + } + } + sb.Append("$"); + return new Regex(sb.ToString()); + } } } diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index bd57dd1de..182a70a35 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -712,53 +712,10 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin async Task> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) { - // Note: catalogPattern comes from GetObjectsResultBuilder which resolves individual - // catalog names before calling this method. Despite the "pattern" name (from the - // IGetObjectsDataProvider interface), the value passed to ShowSchemasCommand is used - // as a literal catalog identifier (backtick-quoted), not a wildcard pattern. - string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); - - List batches; - try - { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return System.Array.Empty<(string, string)>(); - } - - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog - // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showSchemasInAllCatalogs = catalogPattern == null; - - var result = new List<(string, string)>(); - foreach (var batch in batches) - { - StringArray? catalogArray = null; - StringArray? schemaArray = null; - - if (showSchemasInAllCatalogs) - { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; - } - else - { - schemaArray = batch.Column(0) as StringArray; - } - - if (schemaArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (schemaArray.IsNull(i)) continue; - string catalog = catalogArray != null && !catalogArray.IsNull(i) - ? catalogArray.GetString(i) - : catalogPattern ?? ""; - result.Add((catalog, schemaArray.GetString(i))); - } - } - return result; + // PECO-3035: catalogPattern follows JDBC LIKE semantics (% / _ / \_ ). The SEA + // backend treats SHOW SCHEMAS IN `` as a literal lookup, so we resolve + // wildcards client-side here (see ListSchemasAsync for details). + return await ListSchemasAsync(catalogPattern, schemaPattern, cancellationToken).ConfigureAwait(false); } async Task> IGetObjectsDataProvider.GetTablesAsync( @@ -903,6 +860,91 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc return ExecuteMetadataSqlAsync(sql, cancellationToken).GetAwaiter().GetResult(); } + /// + /// Executes SHOW SCHEMAS with JDBC-style catalog pattern semantics. The SEA + /// backend treats SHOW SCHEMAS IN `` as a literal identifier + /// lookup — it does not expand % / _ wildcards. To match Thrift + /// behaviour (PECO-3035), this helper resolves wildcards client-side: + /// + /// null or "%"/"*" → SHOW SCHEMAS IN ALL CATALOGS (single round-trip). + /// A pattern containing unescaped % or _ → still a single SHOW SCHEMAS IN ALL CATALOGS, then filter the returned catalog column against the JDBC pattern client-side. + /// A literal name → single SHOW SCHEMAS IN `<catalog>` call (avoids fetching everything just to throw most of it away). + /// + /// Returns a flat list of (catalog, schema) pairs in the order produced + /// by the backend (no client-side sorting). + /// + internal async Task> ListSchemasAsync( + string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) + { + // Fast path: null or pure "match anything" → SHOW SCHEMAS IN ALL CATALOGS. + if (catalogPattern == null || MetadataCommands.MetadataCommandBase.IsMatchAnything(catalogPattern)) + { + return await ExecuteShowSchemasAsync(null, schemaPattern, cancellationToken).ConfigureAwait(false); + } + + // Wildcard pattern: fetch all (catalog, schema) pairs in one round-trip and + // filter by the catalog pattern client-side. Avoids the N+1 round-trip cost + // of enumerating catalogs and querying per-catalog. + if (MetadataCommands.MetadataCommandBase.ContainsUnescapedWildcard(catalogPattern)) + { + var all = await ExecuteShowSchemasAsync(null, schemaPattern, cancellationToken).ConfigureAwait(false); + var catalogRegex = MetadataCommands.MetadataCommandBase.JdbcLikeToRegex(catalogPattern); + var filtered = new List<(string, string)>(all.Count); + foreach (var row in all) + { + if (catalogRegex.IsMatch(row.catalog)) + filtered.Add(row); + } + return filtered; + } + + // Literal catalog name. + return await ExecuteShowSchemasAsync(catalogPattern, schemaPattern, cancellationToken).ConfigureAwait(false); + } + + /// + /// Issues a single SHOW SCHEMAS command (with the given literal catalog or + /// IN ALL CATALOGS) and decodes the result. Caller is responsible for + /// resolving wildcard patterns before calling this method. + /// + private async Task> ExecuteShowSchemasAsync( + string? catalog, string? schemaPattern, CancellationToken cancellationToken) + { + string sql = new ShowSchemasCommand(catalog, schemaPattern).Build(); + List batches; + try + { + batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + } + catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) + { + return new List<(string, string)>(); + } + + // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog + // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName + var result = new List<(string, string)>(); + foreach (var batch in batches) + { + var schemaArray = TryGetColumn(batch, "databaseName"); + if (schemaArray == null) continue; + + // catalog column is only present in the IN ALL CATALOGS shape; + // for the literal-catalog shape we synthesize it from the parameter. + var catalogArray = TryGetColumn(batch, "catalog"); + + for (int i = 0; i < batch.Length; i++) + { + if (schemaArray.IsNull(i)) continue; + string cat = catalogArray != null && !catalogArray.IsNull(i) + ? catalogArray.GetString(i) + : catalog ?? ""; + result.Add((cat, schemaArray.GetString(i))); + } + } + return result; + } + /// /// Executes a SHOW COLUMNS command. When catalog is null, iterates over all catalogs /// since SHOW COLUMNS IN ALL CATALOGS is not yet supported by the backend. diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index b910097af..01831e907 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1076,62 +1076,25 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo && MetadataUtilities.NormalizeSparkCatalog(_metadataCatalogName) != null) return MetadataSchemaFactory.CreateEmptySchemasResult(); - string sql = new ShowSchemasCommand( + // PECO-3035: catalog follows JDBC LIKE semantics. ListSchemasAsync expands + // wildcards client-side (SHOW SCHEMAS IN ALL CATALOGS or per-catalog dispatch) + // and returns a flat list of (catalog, schema) pairs. + var rows = await _connection.ListSchemasAsync( catalog, - EscapePatternWildcardsInName(_metadataSchemaName)).Build(); - activity?.SetTag("sql_query", sql); - - List batches; - try - { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_schemas.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptySchemasResult(); - } - - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog - // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showAllCatalogs = catalog == null; + EscapePatternWildcardsInName(_metadataSchemaName), + cancellationToken).ConfigureAwait(false); var tableSchemaBuilder = new StringArray.Builder(); var tableCatalogBuilder = new StringArray.Builder(); - int count = 0; - foreach (var batch in batches) + foreach (var (cat, schemaName) in rows) { - StringArray? catalogArray = null; - StringArray? schemaArray = null; - - if (showAllCatalogs) - { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; - } - else - { - schemaArray = batch.Column(0) as StringArray; - } - - if (schemaArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (schemaArray.IsNull(i)) continue; - tableSchemaBuilder.Append(schemaArray.GetString(i)); - string catalogValue = catalogArray != null && !catalogArray.IsNull(i) - ? catalogArray.GetString(i) - : catalog ?? ""; - tableCatalogBuilder.Append(catalogValue); - count++; - } + tableSchemaBuilder.Append(schemaName); + tableCatalogBuilder.Append(cat); } - activity?.SetTag("result_count", count); + activity?.SetTag("result_count", rows.Count); var schema = MetadataSchemaFactory.CreateSchemasSchema(); - return new QueryResult(count, new HiveInfoArrowStream(schema, new IArrowArray[] + return new QueryResult(rows.Count, new HiveInfoArrowStream(schema, new IArrowArray[] { tableSchemaBuilder.Build(), tableCatalogBuilder.Build() })); diff --git a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs index a7405ff2b..f9e81a51d 100644 --- a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs +++ b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs @@ -20,6 +20,7 @@ using Apache.Arrow; using Apache.Arrow.Adbc; using Apache.Arrow.Adbc.Tests; +using Apache.Arrow.Ipc; using AdbcDrivers.HiveServer2; using Xunit; using Xunit.Abstractions; @@ -369,6 +370,75 @@ public void GetTableSchema_ThriftAndSEA_SameFieldNames() } } + // --- GetObjects: catalog wildcard pattern (PECO-3035) --- + + /// + /// Returns the total number of (catalog, schema) pairs in a + /// GetObjects(depth=DbSchemas) result stream. The result schema is + /// [catalog_name (string), catalog_db_schemas (list<struct{db_schema_name,...}>)]. + /// + private static async Task CountSchemasInGetObjects(IArrowArrayStream stream) + { + int total = 0; + while (true) + { + using var batch = await stream.ReadNextRecordBatchAsync(); + if (batch == null) break; + // Column 1 is the list of per-catalog schemas. Each list entry + // is a struct array; its Length tells us how many schemas the catalog has. + if (batch.Column(1) is not ListArray schemasList) continue; + var schemasStruct = schemasList.Values as StructArray; + if (schemasStruct == null) continue; + for (int i = 0; i < batch.Length; i++) + { + if (schemasList.IsNull(i)) continue; + int start = schemasList.ValueOffsets[i]; + int end = schemasList.ValueOffsets[i + 1]; + total += end - start; + } + } + return total; + } + + [SkippableFact] + public async Task SEA_GetObjects_CatalogPercentWildcard_ReturnsSchemasFromAllCatalogs() + { + // PECO-3035: SEA must treat "%" in the catalog argument as a wildcard, + // matching Thrift / JDBC behavior. Before the fix, SEA wraps "%" in + // backticks and looks for a catalog literally named "%", finding no + // schemas → schema count = 0. With the fix, "%" expands to all catalogs + // and we get the full set of schemas (matching catalogPattern=null). + // + // Note: counting schemas (column 1, the list) rather than just + // catalogs (column 0) is what surfaces the bug — GetObjects always + // populates catalogs via GetCatalogsAsync (which handles "%" via + // SHOW CATALOGS LIKE), but GetSchemasAsync was passing "%" literally. + SkipIfNotConfigured(); + using var conn = CreateSeaConnection(); + + using var baselineStream = conn.GetObjects( + depth: AdbcConnection.GetObjectsDepth.DbSchemas, + catalogPattern: null, + dbSchemaPattern: null, + tableNamePattern: null, + tableTypes: null, + columnNamePattern: null); + int baselineSchemaCount = await CountSchemasInGetObjects(baselineStream); + + using var wildcardStream = conn.GetObjects( + depth: AdbcConnection.GetObjectsDepth.DbSchemas, + catalogPattern: "%", + dbSchemaPattern: null, + tableNamePattern: null, + tableTypes: null, + columnNamePattern: null); + int wildcardSchemaCount = await CountSchemasInGetObjects(wildcardStream); + + Assert.True(baselineSchemaCount > 0, + "Baseline (catalogPattern=null) must return at least one schema"); + Assert.Equal(baselineSchemaCount, wildcardSchemaCount); + } + // --- GetTableTypes --- [SkippableFact] diff --git a/csharp/test/Unit/StatementExecution/ShowCommandTests.cs b/csharp/test/Unit/StatementExecution/ShowCommandTests.cs index 80efb8da1..a2debdd77 100644 --- a/csharp/test/Unit/StatementExecution/ShowCommandTests.cs +++ b/csharp/test/Unit/StatementExecution/ShowCommandTests.cs @@ -188,5 +188,86 @@ public void ShowCatalogs_EmptyPattern() { Assert.Equal("SHOW CATALOGS LIKE ''", new ShowCatalogsCommand("").Build()); } + + // MetadataCommandBase wildcard helpers (PECO-3035). These back the client-side + // catalog-wildcard expansion in StatementExecutionConnection.ListSchemasAsync, + // so the backslash-escape semantics must be locked in. + + [Theory] + [InlineData(null, false)] + [InlineData("", false)] + [InlineData("abc", false)] + [InlineData("prod", false)] + [InlineData("*", false)] // not a JDBC LIKE wildcard + [InlineData("%", true)] + [InlineData("_", true)] + [InlineData("abc%", true)] + [InlineData("_abc", true)] + [InlineData("a%b", true)] + [InlineData("a_b", true)] + [InlineData("\\%", false)] // escaped % + [InlineData("\\_", false)] // escaped _ + [InlineData("\\\\%", true)] // literal backslash + unescaped % + [InlineData("\\\\_", true)] // literal backslash + unescaped _ + [InlineData("\\\\\\%", false)] // literal backslash + escaped % + [InlineData("\\\\\\_", false)] // literal backslash + escaped _ + [InlineData("\\", false)] // lone trailing backslash + [InlineData("foo\\", false)] // trailing backslash, no wildcard + [InlineData("foo\\%bar", false)] // escaped % in the middle + [InlineData("foo\\%bar%baz", true)] // escaped % then unescaped % + public void ContainsUnescapedWildcard_HandlesEscapeSemantics(string? input, bool expected) + { + Assert.Equal(expected, MetadataCommandBase.ContainsUnescapedWildcard(input)); + } + + [Theory] + [InlineData(null, false)] + [InlineData("", false)] + [InlineData("%", true)] + [InlineData("*", true)] // Spark/Hive convention, also fast-pathed + [InlineData("%%", false)] + [InlineData("_", false)] + [InlineData("prod", false)] + [InlineData("\\%", false)] + public void IsMatchAnything_TreatsOnlyBareWildcardsAsMatchAnything(string? input, bool expected) + { + Assert.Equal(expected, MetadataCommandBase.IsMatchAnything(input)); + } + + [Theory] + // Literals (the helper is anchored, so "prod" only matches "prod" exactly). + [InlineData("prod", "prod", true)] + [InlineData("prod", "prod_2", false)] + [InlineData("prod", "production", false)] + // % wildcard — any sequence including empty. + [InlineData("%", "anything", true)] + [InlineData("%", "", true)] + [InlineData("comp%", "compute", true)] + [InlineData("comp%", "comp", true)] + [InlineData("comp%", "system", false)] + [InlineData("%comp", "mycomp", true)] + [InlineData("%comp", "myComp", false)] // case-sensitive: comp ≠ Comp + [InlineData("%comp", "compsomething", false)] + // _ wildcard — exactly one char. + [InlineData("a_c", "abc", true)] + [InlineData("a_c", "ac", false)] + [InlineData("a_c", "abbc", false)] + // Escapes — \% / \_ must match the literal character. + [InlineData("comp\\%", "comp%", true)] + [InlineData("comp\\%", "compute", false)] + [InlineData("a\\_b", "a_b", true)] + [InlineData("a\\_b", "axb", false)] + // \\ → literal backslash. + [InlineData("a\\\\b", "a\\b", true)] + // Regex metacharacters in the literal portion must be escaped. + [InlineData("a.b", "a.b", true)] + [InlineData("a.b", "axb", false)] + [InlineData("a+b", "a+b", true)] + [InlineData("a+b", "ab", false)] + public void JdbcLikeToRegex_MatchesPatternSemantics(string pattern, string input, bool expectedMatch) + { + var regex = MetadataCommandBase.JdbcLikeToRegex(pattern); + Assert.Equal(expectedMatch, regex.IsMatch(input)); + } } }