feat: report index coverage, segment count and size in SHOW INDEXES

docs/src/operations/ddl/show-indexes.md

@@ -7,7 +7,7 @@ List all indexes defined on a Lance table.
 
 ## Overview
 
-The `SHOW INDEXES` command returns one row for each index on a Lance table. The information is retrieved using the `Dataset.describeIndices` method, and the output columns align with the attributes of `org.lance.index.IndexDescription`, excluding the per-segment metadata list.
+The `SHOW INDEXES` command returns one row for each index on a Lance table. The information is retrieved using the `Dataset.describeIndices` method, and the output columns align with the attributes of `org.lance.index.IndexDescription`. Per-segment metadata is not listed individually; `num_segments` and `size_bytes` summarise it.
 
 This command is useful for inspecting existing indexes, verifying index creation, and understanding the high-level properties of each index.
 
@@ -54,11 +54,26 @@ The `SHOW INDEXES` command returns the following columns:
 | `num_indexed_rows`      | long          | Approximate number of rows covered by the index.                   |
 | `num_unindexed_fragments` | long        | Number of fragments that are not yet indexed.                      |
 | `num_unindexed_rows`    | long          | Approximate number of rows that are not yet covered by the index.  |
+| `indexed_percent`       | double        | Share of rows the index covers, as a percentage truncated to two decimals, so it never overstates coverage. Null for an empty table. |
+| `num_segments`          | long          | Number of physical index segments backing this logical index.       |
+| `size_bytes`            | long          | Total size of all index files across the segments. Null if any segment predates index file size tracking. |
+
+## Interpreting the Output
+
+An `indexed_percent` below 100 means part of the table is not covered — either rows were appended
+since the index was last built, or [OPTIMIZE](./optimize.md) rewrote fragments an index had
+covered. Rebuild with [CREATE INDEX](./create-index.md) to restore full coverage.
+
+`num_segments` reflects how the index was built: a distributed build produces one segment per
+parallel task. Queries search every segment, and Lance can only compact fragments covered by the
+identical set of segments, so a high count costs both query time and `OPTIMIZE`'s ability to
+coalesce. Rebuilding with [CREATE INDEX](./create-index.md) consolidates them.
 
 ## Notes
 
 - The `fields` column returns the logical column names from the Lance schema, ordered according to the index definition.
 - Lance-maintained system indexes, including fragment-reuse and MemWAL indexes, are excluded from the output.
+- Row counts are approximate, so `indexed_percent` is a guide rather than an exact figure.
 
 ## See Also
 

lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ShowIndexes.scala

@@ -47,6 +47,9 @@ object ShowIndexesOutputType {
       StructField("num_indexed_fragments", DataTypes.LongType, nullable = true),
       StructField("num_indexed_rows", DataTypes.LongType, nullable = true),
       StructField("num_unindexed_fragments", DataTypes.LongType, nullable = true),
-      StructField("num_unindexed_rows", DataTypes.LongType, nullable = true)))
+      StructField("num_unindexed_rows", DataTypes.LongType, nullable = true),
+      StructField("indexed_percent", DataTypes.DoubleType, nullable = true),
+      StructField("num_segments", DataTypes.LongType, nullable = true),
+      StructField("size_bytes", DataTypes.LongType, nullable = true)))
     .map(field => AttributeReference(field.name, field.dataType, field.nullable, field.metadata)())
 }

lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowIndexesExec.scala

@@ -53,15 +53,17 @@ case class ShowIndexesExec(
 
     val dataset = Utils.openDatasetBuilder(readOptions).build()
     try {
+      // Group by logical index: one row per name, with every physical segment kept so segment-level
+      // metadata can be aggregated.
       val indexes = dataset.getIndexes.asScala.toSeq
         .filterNot(idx => ShowIndexesExec.isSystemIndex(idx.name()))
         .groupBy(_.name())
         .toSeq
         .sortBy(_._1)
-        .map(_._2.head)
       val lanceSchema = dataset.getLanceSchema()
 
-      indexes.map { idx =>
+      indexes.map { case (_, indexSegments) =>
+        val idx = indexSegments.head
         val fieldIds = idx.fields()
         val fieldNamesArray =
           if (fieldIds == null) {
@@ -98,14 +100,48 @@ case class ShowIndexesExec(
         val numUnindexedFragments = getLong("num_unindexed_fragments")
         val numUnindexedRows = getLong("num_unindexed_rows")
 
+        // Truncated (not rounded) so it never overstates coverage. Null for empty tables.
+        val indexedPercent: java.lang.Double =
+          if (numIndexedRows == null || numUnindexedRows == null) {
+            null
+          } else {
+            val total = numIndexedRows.longValue() + numUnindexedRows.longValue()
+            if (total <= 0L) {
+              null
+            } else {
+              val percent = 100.0 * numIndexedRows.longValue() / total
+              math.floor(percent * 100.0) / 100.0
+            }
+          }
+
+        // Physical segments backing this logical index. Older cores report only `num_indices`.
+        val numSegments = {
+          val reported = getLong("num_segments")
+          if (reported != null) reported else getLong("num_indices")
+        }
+
+        // Total across segments, or null when any segment predates index file size tracking: a
+        // partial sum would understate the index rather than admit it is unknown.
+        val sizeBytes: java.lang.Long = {
+          val perSegment = indexSegments.map(segment => segment.getSizeBytes)
+          if (perSegment.exists(!_.isPresent)) {
+            null
+          } else {
+            perSegment.map(_.get.longValue()).sum
+          }
+        }
+
         new GenericInternalRow(Array[Any](
           UTF8String.fromString(name),
           fieldNamesArray,
           indexTypeUtf8,
           numIndexedFragments,
           numIndexedRows,
           numUnindexedFragments,
-          numUnindexedRows))
+          numUnindexedRows,
+          indexedPercent,
+          numSegments,
+          sizeBytes))
       }
     } finally {
       dataset.close()

lance-spark-base_2.12/src/test/java/org/lance/spark/update/BaseShowIndexesTest.java

@@ -13,6 +13,7 @@
  */
 package org.lance.spark.update;
 
+import org.lance.index.Index;
 import org.lance.index.IndexOptions;
 import org.lance.index.IndexParams;
 import org.lance.index.IndexType;
@@ -112,7 +113,7 @@ public void testShowIndexes() {
     Dataset<Row> result = spark.sql(String.format("show indexes from %s", fullTable));
 
     Assertions.assertEquals(
-        "StructType(StructField(name,StringType,true),StructField(fields,ArrayType(StringType,true),true),StructField(index_type,StringType,true),StructField(num_indexed_fragments,LongType,true),StructField(num_indexed_rows,LongType,true),StructField(num_unindexed_fragments,LongType,true),StructField(num_unindexed_rows,LongType,true))",
+        "StructType(StructField(name,StringType,true),StructField(fields,ArrayType(StringType,true),true),StructField(index_type,StringType,true),StructField(num_indexed_fragments,LongType,true),StructField(num_indexed_rows,LongType,true),StructField(num_unindexed_fragments,LongType,true),StructField(num_unindexed_rows,LongType,true),StructField(indexed_percent,DoubleType,true),StructField(num_segments,LongType,true),StructField(size_bytes,LongType,true))",
         result.schema().toString());
 
     List<Row> rows = result.collectAsList();
@@ -138,6 +139,69 @@ public void testShowIndexes() {
     // num_indexed_rows should be at least 1
     long numIndexedRows = row.getLong(4);
     Assertions.assertTrue(numIndexedRows >= 1L, "num_indexed_rows should be at least 1");
+
+    // a freshly created index covers every row
+    Assertions.assertEquals(100.0d, row.getDouble(7), 1e-9, "indexed_percent should be 100");
+
+    // one logical index backed by at least one physical segment
+    Assertions.assertTrue(row.getLong(8) >= 1L, "num_segments should be at least 1");
+
+    // a built index occupies storage
+    Assertions.assertTrue(row.getLong(9) > 0L, "size_bytes should be positive");
+  }
+
+  @Test
+  public void testShowIndexesAggregatesAcrossSegments() {
+    spark.sql(String.format("create table %s (id int, name string) using lance", fullTable));
+    for (int batch = 0; batch < 4; batch++) {
+      spark.sql(
+          String.format(
+              "insert into %s values (%d, 'n%d'), (%d, 'n%d')",
+              fullTable, batch * 2, batch * 2, batch * 2 + 1, batch * 2 + 1));
+    }
+    spark.sql(
+        String.format(
+            "alter table %s create index test_index using btree (id) with (num_segments = 2)",
+            fullTable));
+
+    long expectedSize = 0L;
+    int segmentCount;
+    try (org.lance.Dataset dataset =
+        Utils.openDatasetBuilder(LanceSparkReadOptions.builder().datasetUri(tableDir).build())
+            .build()) {
+      List<Index> segments =
+          dataset.getIndexes().stream()
+              .filter(index -> "test_index".equals(index.name()))
+              .collect(Collectors.toList());
+      segmentCount = segments.size();
+      for (Index segment : segments) {
+        expectedSize += segment.getSizeBytes().orElse(0L);
+      }
+    }
+    Assertions.assertEquals(
+        2, segmentCount, "num_segments = 2 should have produced two physical segments");
+
+    Row row = spark.sql(String.format("show indexes from %s", fullTable)).collectAsList().get(0);
+
+    Assertions.assertEquals(2L, row.getLong(8), "num_segments must count every physical segment");
+    Assertions.assertEquals(
+        expectedSize,
+        row.getLong(9),
+        "size_bytes must sum every segment's files, not just the first");
+    Assertions.assertEquals(
+        100.0d, row.getDouble(7), 1e-9, "a freshly built index covers every row");
+  }
+
+  @Test
+  public void testShowIndexesReportsNullPercentForEmptyTable() {
+    spark.sql(String.format("create table %s (id int, text string) using lance;", fullTable));
+    spark.sql(String.format("alter table %s create index test_index using btree (id)", fullTable));
+
+    Row row = spark.sql(String.format("show indexes from %s", fullTable)).collectAsList().get(0);
+
+    Assertions.assertTrue(row.isNullAt(7), "indexed_percent should be null for an empty table");
+    Assertions.assertEquals(0L, row.getLong(4), "no rows can be indexed");
+    Assertions.assertEquals(0L, row.getLong(6), "no rows can be unindexed");
   }
 
   @Test