DO NOT MERGE - Test Bench

.github/workflows/continuous-integration.yaml

@@ -94,7 +94,7 @@ jobs:
 
       - name: Build & Test
         run: |
-          ./mvnw -B -ntp clean test
+          ./mvnw -B -ntp -Dstyle.color=always clean test
 
       # Publish baseline coverage for main branch (for future PR comparisons)
       - name: Publish baseline coverage
@@ -382,7 +382,7 @@ jobs:
         # -Dquarkus.package.write-transformed-bytecode-to-build-output=true is required for JaCoCo IT coverage
         # Note: Container image build is disabled for coverage (JaCoCo only works with JAR artifacts)
         run: |
-          ./mvnw -B -ntp clean verify -DskipUnitTests -DRERANKING_CONFIG_RESOURCE=test-reranking-providers-config.yaml -DEMBEDDING_CONFIG_RESOURCE=test-embedding-providers-config.yaml -Drun-create-index-parallel=true -Pjacoco-it -Dquarkus.package.write-transformed-bytecode-to-build-output=true ${{ matrix.profile }}
+          ./mvnw -B -ntp clean verify -Dstyle.color=always -DskipUnitTests -DRERANKING_CONFIG_RESOURCE=test-reranking-providers-config.yaml -DEMBEDDING_CONFIG_RESOURCE=test-embedding-providers-config.yaml -Drun-create-index-parallel=true -Pjacoco-it -Dquarkus.package.write-transformed-bytecode-to-build-output=true ${{ matrix.profile }}
 
       # Publish baseline IT coverage for main branch (for future PR comparisons)
       - name: Publish baseline IT coverage

.github/workflows/test-bench-run.yaml

@@ -0,0 +1,110 @@
+name: API Test Bench Run
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to test against'
+        required: true
+        type: environment
+  pull_request:
+
+# needed when a workflow wants to use OIDC (OpenID Connect) to authenticate to cloud
+permissions:
+  id-token: write
+  contents: write
+  packages: write
+  pull-requests: write
+
+# global env vars, available in all jobs and steps
+env:
+  MAVEN_OPTS: '-Xmx4g'
+  DS_ARTIFACTORY_USERNAME: ${{ secrets.DS_ARTIFACTORY_USERNAME }}
+  DS_ARTIFACTORY_PASSWORD: ${{ secrets.DS_ARTIFACTORY_PASSWORD }}
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.environment || 'DEV' }}
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }}
+    steps:
+      - id: set-matrix
+        env:
+          TEST_TARGETS: ${{ vars.TEST_TARGETS }}
+        run: |
+          {
+            echo "TEST_TARGETS<<EOF"
+            echo "$TEST_TARGETS"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+  # runs unit tests
+  build:
+    name: Unit tests (${{ matrix.name }})
+    needs: setup
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.environment || 'DEV' }}
+
+    # max run time 12 minutes
+    timeout-minutes: 12
+
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.setup.outputs.matrix) }}
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: maven
+
+      - name: Setup Maven
+        run: |
+          mkdir -p ~/.m2
+          cat <<EOF > ~/.m2/settings.xml
+          <settings>
+            <servers>
+              <server>
+                <id>stargate-central</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+              </server>
+              <server>
+                <id>stargate-snapshots</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+             </server>
+              <server>
+                <id>artifactory</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+              </server>
+              <server>
+                <id>artifactory-snapshots</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+              </server>
+              <server>
+                <id>artifactory-releases</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+             </server>
+           </servers>
+          </settings>
+          EOF
+      - name: Build & Test
+        env:
+          TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml
+          Token: ${{ secrets[matrix.token_secret] }}
+          TARGET_NAME: ${{ matrix.name }}
+          ENDPOINT: ${{ matrix.endpoint }}
+          openAi_KEY: ${{ secrets.OPEN_AI_KEY }}
+
+        run: |
+          ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dit.test=TestBenchTests

pom.xml

@@ -54,7 +54,11 @@
     <!-- Testing -->
     <skipTests>false</skipTests>
     <skipUnitTests>${skipTests}</skipUnitTests>
-    <surefire-plugin.version>3.5.4</surefire-plugin.version>
+    <!-- 09-Apr-2026, tatu: Wrt [data-api#2447] need to use not-latest
+        (specifically, 3.5.3; latest being 3.5.5)
+        See: https://github.com/fabriciorby/maven-surefire-junit5-tree-reporter?tab=readme-ov-file#important
+      -->
+    <surefire-plugin.version>3.5.3</surefire-plugin.version>
     <skipITs>false</skipITs>
   </properties>
   <dependencyManagement>
@@ -98,6 +102,11 @@
     </repository>
   </repositories>
   <dependencies>
+    <dependency>
+      <groupId>com.jayway.jsonpath</groupId>
+      <artifactId>json-path</artifactId>
+      <version>2.10.0</version>
+    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-arc</artifactId>
@@ -341,6 +350,18 @@
       <artifactId>quarkus-mcp-server-test</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-surefire-plugin</artifactId>
+      <version>${surefire-plugin.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>me.fabriciorby</groupId>
+      <artifactId>maven-surefire-junit5-tree-reporter</artifactId>
+      <version>1.5.1</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
@@ -356,13 +377,28 @@
       <plugin>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>${surefire-plugin.version}</version>
+        <dependencies>
+          <dependency>
+            <groupId>me.fabriciorby</groupId>
+            <artifactId>maven-surefire-junit5-tree-reporter</artifactId>
+            <version>1.5.1</version>
+          </dependency>
+        </dependencies>
         <configuration>
           <!-- 17-Oct-2025, tatu: agent settings needed to get rid of "inline mock" warning by Mockito;
               leading to duplication of "mockito.version".
               See: https://github.com/stargate/data-api/pull/2233
             -->
           <!-- 02-Dec-2025, tatu: [data-api#2270] Use @{argLine} to preserve JaCoCo agent config -->
           <argLine>@{argLine} -Xmx4g -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar</argLine>
+          <!-- 09-Apr-2026, tatu: Try to prettify output -->
+          <reportFormat>plain</reportFormat>
+          <consoleOutputReporter>
+            <disable>true</disable>
+          </consoleOutputReporter>
+          <statelessTestsetInfoReporter implementation="org.apache.maven.plugin.surefire.extensions.junit5.JUnit5StatelessTestsetInfoTreeReporter">
+            <theme>UNICODE</theme>
+          </statelessTestsetInfoReporter>
           <systemPropertyVariables>
             <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager>
             <maven.home>${maven.home}</maven.home>

src/main/resources/application.yaml

@@ -16,7 +16,7 @@ stargate:
 
   database:
     limits:
-      max-collections: 5
+      max-collections: 15
 
   debug:
     enabled: false

src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java

@@ -87,7 +87,7 @@ public class ResponseAssertions {
             FieldMatcher.errors(hasErrors));
 
     final String msg =
-        "%s: Response fields %s:%s, %s:%s, %s:%s"
+        "%s: %s:%s, %s:%s, %s:%s"
             .formatted(
                 message,
                 Presence.REQUIRED,