From 4729818597e1988ae1699fd58d818ba8a2a6a099 Mon Sep 17 00:00:00 2001
From: eric-wang-1990 <e.wang@databricks.com>
Date: Mon, 3 Aug 2026 02:26:20 -0700
Subject: [PATCH] test(csharp): retry EnableMultipleCatalogSupport metadata
 query on transient server 500
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

EnableMultipleCatalogSupportAffectsMetadataQueries intermittently fails the
merge-queue E2E run with a server-side HTTP 500 INTERNAL_ERROR "The result files
are not available in the result metadata" on a metadata query (GetTables /
GetColumns via ExecuteMetadataSqlAsync). This is a SEA backend result-staging
race, not a driver or test-logic defect: it is always the API-level 500 (never an
assertion mismatch), the identical query succeeds on retry, and it hit whichever
PR happened to be live on the shared warehouse (observed on both #614 and #604
merge_group runs at the same time).

Fix: wrap the metadata query in ExecuteMetadataQueryWithRetryAsync — up to 4
attempts with exponential backoff (0.5/1/2s), each on a fresh statement, retrying
ONLY on that specific transient error string. Every other exception and all the
schema/row assertions propagate unchanged, so genuine regressions still fail.

Note: the more general fix is a driver-side retry of this transient 500 in
StatementExecutionClient (PowerBI users hit the same server race); that is a
separate, higher-risk driver change and is left as a follow-up. This PR de-flakes
the test so the merge queue stabilizes.

Local: EnableMultipleCatalogSupportAffectsMetadataQueries 2/2 passed.

Co-authored-by: Isaac
---
 csharp/test/E2E/StatementTests.cs | 54 ++++++++++++++++++++++++++-----
 1 file changed, 46 insertions(+), 8 deletions(-)

diff --git a/csharp/test/E2E/StatementTests.cs b/csharp/test/E2E/StatementTests.cs
index 6e5e0bf85..67629489e 100644
--- a/csharp/test/E2E/StatementTests.cs
+++ b/csharp/test/E2E/StatementTests.cs
@@ -1025,14 +1025,14 @@ private async Task TestMetadataQuery(AdbcConnection connection, string queryType
         {
             OutputHelper?.WriteLine($"Testing {queryType} with EnableMultipleCatalogSupport={shouldAllowMultipleCatalogs}, CatalogName={catalogName}");
 
-            var statement = connection.CreateStatement();
-            statement.SetOption(ApacheParameters.IsMetadataCommand, "true");
-            statement.SetOption(ApacheParameters.CatalogName, catalogName);
-            // Use default as schema name, it is the default schema name
-            statement.SetOption(ApacheParameters.SchemaName, "default");
-            statement.SqlQuery = queryType;
-
-            QueryResult queryResult = await statement.ExecuteQueryAsync();
+            // Retry the metadata query on a transient server-side 500. The SEA backend
+            // intermittently returns HTTP 500 INTERNAL_ERROR "The result files are not available
+            // in the result metadata" for metadata statements — a server-side result-staging race,
+            // not a driver or test defect (the identical query succeeds on the next attempt with a
+            // fresh statement). Without this, the shared-warehouse merge-queue run flakes on that
+            // 500. Only this specific transient error is retried; any other exception (and all the
+            // schema/row assertions below) propagate unchanged, so real regressions still fail.
+            QueryResult queryResult = await ExecuteMetadataQueryWithRetryAsync(connection, queryType, catalogName);
             Assert.NotNull(queryResult.Stream);
 
             // Store SPARK catalog schema for comparison
@@ -1164,6 +1164,44 @@ private async Task TestMetadataQuery(AdbcConnection connection, string queryType
             }
         }
 
+        /// <summary>
+        /// Executes a metadata query, retrying only on the transient server-side 500
+        /// "The result files are not available in the result metadata" (a SEA backend
+        /// result-staging race). Uses a fresh statement per attempt. Any other exception —
+        /// and success — returns/propagates immediately, so genuine failures are not masked.
+        /// </summary>
+        private async Task<QueryResult> ExecuteMetadataQueryWithRetryAsync(
+            AdbcConnection connection, string queryType, string catalogName)
+        {
+            const int maxAttempts = 4;
+            const string transientMarker = "result files are not available in the result metadata";
+
+            for (int attempt = 1; ; attempt++)
+            {
+                var statement = connection.CreateStatement();
+                statement.SetOption(ApacheParameters.IsMetadataCommand, "true");
+                statement.SetOption(ApacheParameters.CatalogName, catalogName);
+                // Use default as schema name, it is the default schema name
+                statement.SetOption(ApacheParameters.SchemaName, "default");
+                statement.SqlQuery = queryType;
+
+                try
+                {
+                    return await statement.ExecuteQueryAsync();
+                }
+                catch (Exception ex) when (attempt < maxAttempts &&
+                    ex.ToString().IndexOf(transientMarker, StringComparison.OrdinalIgnoreCase) >= 0)
+                {
+                    // Exponential backoff: 500ms, 1s, 2s — gives the server time to stage results.
+                    int delayMs = 500 * (1 << (attempt - 1));
+                    OutputHelper?.WriteLine(
+                        $"{queryType} (catalog={catalogName}) hit transient server 500 on attempt {attempt}/{maxAttempts}; " +
+                        $"retrying in {delayMs}ms");
+                    await Task.Delay(delayMs);
+                }
+            }
+        }
+
         private void AssertField(Schema schema, int index, string expectedName, IArrowType expectedType, bool expectedNullable)
         {
             var field = schema.FieldsList[index];