diff --git a/docs/src/operations/ddl/show-indexes.md b/docs/src/operations/ddl/show-indexes.md index 7e47e7682..6fa995a17 100755 --- a/docs/src/operations/ddl/show-indexes.md +++ b/docs/src/operations/ddl/show-indexes.md @@ -7,7 +7,7 @@ List all indexes defined on a Lance table. ## Overview -The `SHOW INDEXES` command returns one row for each index on a Lance table. The information is retrieved using the `Dataset.describeIndices` method, and the output columns align with the attributes of `org.lance.index.IndexDescription`, excluding the per-segment metadata list. +The `SHOW INDEXES` command returns one row for each index on a Lance table. The information is retrieved using the `Dataset.describeIndices` method, and the output columns align with the attributes of `org.lance.index.IndexDescription`. Per-segment metadata is not listed individually; `num_segments` and `size_bytes` summarise it. This command is useful for inspecting existing indexes, verifying index creation, and understanding the high-level properties of each index. @@ -54,11 +54,26 @@ The `SHOW INDEXES` command returns the following columns: | `num_indexed_rows` | long | Approximate number of rows covered by the index. | | `num_unindexed_fragments` | long | Number of fragments that are not yet indexed. | | `num_unindexed_rows` | long | Approximate number of rows that are not yet covered by the index. | +| `indexed_percent` | double | Share of rows the index covers, as a percentage truncated to two decimals, so it never overstates coverage. Null for an empty table. | +| `num_segments` | long | Number of physical index segments backing this logical index. | +| `size_bytes` | long | Total size of all index files across the segments. Null if any segment predates index file size tracking. | + +## Interpreting the Output + +An `indexed_percent` below 100 means part of the table is not covered — either rows were appended +since the index was last built, or [OPTIMIZE](./optimize.md) rewrote fragments an index had +covered. Rebuild with [CREATE INDEX](./create-index.md) to restore full coverage. + +`num_segments` reflects how the index was built: a distributed build produces one segment per +parallel task. Queries search every segment, and Lance can only compact fragments covered by the +identical set of segments, so a high count costs both query time and `OPTIMIZE`'s ability to +coalesce. Rebuilding with [CREATE INDEX](./create-index.md) consolidates them. ## Notes - The `fields` column returns the logical column names from the Lance schema, ordered according to the index definition. - Lance-maintained system indexes, including fragment-reuse and MemWAL indexes, are excluded from the output. +- Row counts are approximate, so `indexed_percent` is a guide rather than an exact figure. ## See Also diff --git a/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ShowIndexes.scala b/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ShowIndexes.scala index 938912bec..b4f28c29a 100755 --- a/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ShowIndexes.scala +++ b/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ShowIndexes.scala @@ -47,6 +47,9 @@ object ShowIndexesOutputType { StructField("num_indexed_fragments", DataTypes.LongType, nullable = true), StructField("num_indexed_rows", DataTypes.LongType, nullable = true), StructField("num_unindexed_fragments", DataTypes.LongType, nullable = true), - StructField("num_unindexed_rows", DataTypes.LongType, nullable = true))) + StructField("num_unindexed_rows", DataTypes.LongType, nullable = true), + StructField("indexed_percent", DataTypes.DoubleType, nullable = true), + StructField("num_segments", DataTypes.LongType, nullable = true), + StructField("size_bytes", DataTypes.LongType, nullable = true))) .map(field => AttributeReference(field.name, field.dataType, field.nullable, field.metadata)()) } diff --git a/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowIndexesExec.scala b/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowIndexesExec.scala index 997778458..3015b584e 100755 --- a/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowIndexesExec.scala +++ b/lance-spark-base_2.12/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowIndexesExec.scala @@ -58,10 +58,10 @@ case class ShowIndexesExec( .groupBy(_.name()) .toSeq .sortBy(_._1) - .map(_._2.head) val lanceSchema = dataset.getLanceSchema() - indexes.map { idx => + indexes.map { case (_, indexSegments) => + val idx = indexSegments.head val fieldIds = idx.fields() val fieldNamesArray = if (fieldIds == null) { @@ -98,6 +98,33 @@ case class ShowIndexesExec( val numUnindexedFragments = getLong("num_unindexed_fragments") val numUnindexedRows = getLong("num_unindexed_rows") + val indexedPercent: java.lang.Double = + if (numIndexedRows == null || numUnindexedRows == null) { + null + } else { + val total = numIndexedRows.longValue() + numUnindexedRows.longValue() + if (total <= 0L) { + null + } else { + val percent = 100.0 * numIndexedRows.longValue() / total + math.floor(percent * 100.0) / 100.0 + } + } + + val numSegments = { + val reported = getLong("num_segments") + if (reported != null) reported else getLong("num_indices") + } + + val sizeBytes: java.lang.Long = { + val perSegment = indexSegments.map(segment => segment.getSizeBytes) + if (perSegment.exists(!_.isPresent)) { + null + } else { + perSegment.map(_.get.longValue()).sum + } + } + new GenericInternalRow(Array[Any]( UTF8String.fromString(name), fieldNamesArray, @@ -105,7 +132,10 @@ case class ShowIndexesExec( numIndexedFragments, numIndexedRows, numUnindexedFragments, - numUnindexedRows)) + numUnindexedRows, + indexedPercent, + numSegments, + sizeBytes)) } } finally { dataset.close() diff --git a/lance-spark-base_2.12/src/test/java/org/lance/spark/update/BaseShowIndexesTest.java b/lance-spark-base_2.12/src/test/java/org/lance/spark/update/BaseShowIndexesTest.java index 0c2ff0bfd..6988a2bbf 100755 --- a/lance-spark-base_2.12/src/test/java/org/lance/spark/update/BaseShowIndexesTest.java +++ b/lance-spark-base_2.12/src/test/java/org/lance/spark/update/BaseShowIndexesTest.java @@ -13,6 +13,7 @@ */ package org.lance.spark.update; +import org.lance.index.Index; import org.lance.index.IndexOptions; import org.lance.index.IndexParams; import org.lance.index.IndexType; @@ -112,7 +113,7 @@ public void testShowIndexes() { Dataset result = spark.sql(String.format("show indexes from %s", fullTable)); Assertions.assertEquals( - "StructType(StructField(name,StringType,true),StructField(fields,ArrayType(StringType,true),true),StructField(index_type,StringType,true),StructField(num_indexed_fragments,LongType,true),StructField(num_indexed_rows,LongType,true),StructField(num_unindexed_fragments,LongType,true),StructField(num_unindexed_rows,LongType,true))", + "StructType(StructField(name,StringType,true),StructField(fields,ArrayType(StringType,true),true),StructField(index_type,StringType,true),StructField(num_indexed_fragments,LongType,true),StructField(num_indexed_rows,LongType,true),StructField(num_unindexed_fragments,LongType,true),StructField(num_unindexed_rows,LongType,true),StructField(indexed_percent,DoubleType,true),StructField(num_segments,LongType,true),StructField(size_bytes,LongType,true))", result.schema().toString()); List rows = result.collectAsList(); @@ -138,6 +139,64 @@ public void testShowIndexes() { // num_indexed_rows should be at least 1 long numIndexedRows = row.getLong(4); Assertions.assertTrue(numIndexedRows >= 1L, "num_indexed_rows should be at least 1"); + + Assertions.assertEquals(100.0d, row.getDouble(7), 1e-9, "indexed_percent should be 100"); + Assertions.assertTrue(row.getLong(8) >= 1L, "num_segments should be at least 1"); + Assertions.assertTrue(row.getLong(9) > 0L, "size_bytes should be positive"); + } + + @Test + public void testShowIndexesAggregatesAcrossSegments() { + spark.sql(String.format("create table %s (id int, name string) using lance", fullTable)); + for (int batch = 0; batch < 4; batch++) { + spark.sql( + String.format( + "insert into %s values (%d, 'n%d'), (%d, 'n%d')", + fullTable, batch * 2, batch * 2, batch * 2 + 1, batch * 2 + 1)); + } + spark.sql( + String.format( + "alter table %s create index test_index using btree (id) with (num_segments = 2)", + fullTable)); + + long expectedSize = 0L; + int segmentCount; + try (org.lance.Dataset dataset = + Utils.openDatasetBuilder(LanceSparkReadOptions.builder().datasetUri(tableDir).build()) + .build()) { + List segments = + dataset.getIndexes().stream() + .filter(index -> "test_index".equals(index.name())) + .collect(Collectors.toList()); + segmentCount = segments.size(); + for (Index segment : segments) { + expectedSize += segment.getSizeBytes().orElse(0L); + } + } + Assertions.assertEquals( + 2, segmentCount, "num_segments = 2 should have produced two physical segments"); + + Row row = spark.sql(String.format("show indexes from %s", fullTable)).collectAsList().get(0); + + Assertions.assertEquals(2L, row.getLong(8), "num_segments must count every physical segment"); + Assertions.assertEquals( + expectedSize, + row.getLong(9), + "size_bytes must sum every segment's files, not just the first"); + Assertions.assertEquals( + 100.0d, row.getDouble(7), 1e-9, "a freshly built index covers every row"); + } + + @Test + public void testShowIndexesReportsNullPercentForEmptyTable() { + spark.sql(String.format("create table %s (id int, text string) using lance;", fullTable)); + spark.sql(String.format("alter table %s create index test_index using btree (id)", fullTable)); + + Row row = spark.sql(String.format("show indexes from %s", fullTable)).collectAsList().get(0); + + Assertions.assertTrue(row.isNullAt(7), "indexed_percent should be null for an empty table"); + Assertions.assertEquals(0L, row.getLong(4), "no rows can be indexed"); + Assertions.assertEquals(0L, row.getLong(6), "no rows can be unindexed"); } @Test