support deletes for leading segment merges

gradle.properties

@@ -5,7 +5,7 @@
 
 version=1.0.0
 systemProp.bwc.version=1.3.4
-jvector_version=4.0.0-rc.4
+jvector_version=4.0.0-rc.6-SNAPSHOT
 java_release_version=21
 
 # org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \

src/main/java/org/opensearch/knn/index/codec/jvector/GraphNodeIdToDocMap.java

@@ -46,7 +46,10 @@ public GraphNodeIdToDocMap(IndexInput in) throws IOException {
         for (int ord = 0; ord < size; ord++) {
             final int docId = in.readVInt();
             graphNodeIdsToDocIds[ord] = docId;
-            docIdsToGraphNodeIds[docId] = ord;
+            if (docId != -1) {
+                // ignore deleted documents
+                docIdsToGraphNodeIds[docId] = ord;
+            }
         }
     }
 
@@ -68,7 +71,11 @@ public GraphNodeIdToDocMap(int[] graphNodeIdsToDocIds) {
         // We are going to assume that the number of ordinals is roughly the same as the number of documents in the segment, therefore,
         // the mapping will not be sparse.
         if (maxDocs < graphNodeIdsToDocIds.length) {
-            throw new IllegalStateException("Max docs " + maxDocs + " is less than the number of ordinals " + graphNodeIdsToDocIds.length);
+            log.info(
+                "Max docs {} is less than the number of ordinals {}, this implies a lot of deleted documents. Or that some documents are missing vectors. Wasting a lot of memory",
+                maxDocs,
+                graphNodeIdsToDocIds.length
+            );
         }
         if (maxDocId > graphNodeIdsToDocIds.length) {
             log.warn(
@@ -80,6 +87,10 @@ public GraphNodeIdToDocMap(int[] graphNodeIdsToDocIds) {
         this.docIdsToGraphNodeIds = new int[maxDocs];
         Arrays.fill(this.docIdsToGraphNodeIds, -1); // -1 means no mapping to ordinal
         for (int ord = 0; ord < graphNodeIdsToDocIds.length; ord++) {
+            // -1 means no mapping to docId since document was deleted
+            if (graphNodeIdsToDocIds[ord] == -1) {
+                continue;
+            }
             this.docIdsToGraphNodeIds[graphNodeIdsToDocIds[ord]] = ord;
         }
     }

src/main/java/org/opensearch/knn/index/codec/jvector/JVectorFormat.java

@@ -36,6 +36,7 @@ public class JVectorFormat extends KnnVectorsFormat {
     // Unfortunately, this can't be managed yet by the OpenSearch ThreadPool because it's not supporting {@link ForkJoinPool} types
     public static final ForkJoinPool SIMD_POOL_MERGE = getPhysicalCoreExecutor();
     public static final ForkJoinPool SIMD_POOL_FLUSH = getPhysicalCoreExecutor();
+    public static final ForkJoinPool PARALLELISM_POOL = ForkJoinPool.commonPool();
 
     private final int maxConn;
     private final int beamWidth;

src/main/java/org/opensearch/knn/index/codec/jvector/JVectorReader.java

@@ -169,7 +169,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits
             // Logic works as follows: if acceptDocs is null, we accept all ordinals. Otherwise, we check if the jVector ordinal has a
             // corresponding Lucene doc ID accepted by acceptDocs filter.
             io.github.jbellis.jvector.util.Bits compatibleBits = ord -> acceptDocs == null
-                || acceptDocs.get(jvectorLuceneDocMap.getLuceneDocId(ord));
+                || (jvectorLuceneDocMap.getLuceneDocId(ord) != -1 && acceptDocs.get(jvectorLuceneDocMap.getLuceneDocId(ord)));
 
             try (var graphSearcher = new GraphSearcher(index)) {
                 final var searchResults = graphSearcher.search(