Supported values are {@code "text"} and {@code "code"}. The code analyzer selects the code
+ * tokenizer defaults and requires FTS format v3. If unset, the analyzer is inferred from {@link
+ * #baseTokenizer(String)}.
+ *
+ * @param analyzer analyzer preset
+ * @return this builder
+ */
+ public Builder analyzer(String analyzer) {
+ Objects.requireNonNull(analyzer, "analyzer must not be null");
+ if (analyzer.isEmpty()) {
+ throw new IllegalArgumentException("analyzer must not be empty");
+ }
+ this.analyzer = analyzer;
+ return this;
+ }
+
+ /**
+ * Configure the document-level tokenizer used before lexical tokenization.
+ *
+ *
Supported values are {@code "text"} for plain strings and {@code "json"} for JSON strings.
+ * If unset, Lance infers the document tokenizer from the Arrow field type.
+ *
+ * @param lanceTokenizer document-level tokenizer
+ * @return this builder
+ */
+ public Builder lanceTokenizer(String lanceTokenizer) {
+ Objects.requireNonNull(lanceTokenizer, "lanceTokenizer must not be null");
+ if (lanceTokenizer.isEmpty()) {
+ throw new IllegalArgumentException("lanceTokenizer must not be empty");
+ }
+ this.lanceTokenizer = lanceTokenizer;
+ return this;
+ }
+
/**
* Configure the base tokenizer.
*
@@ -76,6 +120,11 @@ public static final class Builder {
*
{@code "jieba/*"}: Jieba tokenizer
*
*
+ * Lindera and Jieba tokenizers load their language models from the directory configured by
+ * {@code LANCE_LANGUAGE_MODEL_HOME}, or from Lance's platform-specific default language model
+ * directory. The tokenizer suffix selects a model directory, for example {@code jieba/default}.
+ * The {@code code} tokenizer requires FTS format v3.
+ *
* @param baseTokenizer tokenizer identifier string
* @return this builder
* @throws IllegalArgumentException
@@ -119,12 +168,16 @@ public Builder withPosition(boolean withPosition) {
/**
* Configure the maximum token length.
*
- * @param maxTokenLength maximum token length, must be positive
+ *
The default is {@code 40}. Set this to {@code null} to disable the maximum token length
+ * filter.
+ *
+ * @param maxTokenLength maximum token length, or {@code null} for no limit; non-null values
+ * must be positive
* @return this builder
- * @throws IllegalArgumentException
+ * @throws IllegalArgumentException if {@code maxTokenLength} is not null and is not positive
*/
public Builder maxTokenLength(Integer maxTokenLength) {
- if (maxTokenLength == null || maxTokenLength <= 0) {
+ if (maxTokenLength != null && maxTokenLength <= 0) {
throw new IllegalArgumentException("maxTokenLength must be positive when specified");
}
this.maxTokenLength = maxTokenLength;
@@ -252,15 +305,105 @@ public Builder blockSize(int blockSize) {
}
/**
- * Configure whether to skip the partition merge stage after indexing. If true, skip the
- * partition merge stage after indexing. This can be useful for distributed indexing where merge
- * is handled separately.
+ * Configure whether code identifiers are split into subwords.
+ *
+ *
This option is valid only with the {@code code} analyzer.
+ *
+ * @param splitIdentifiers whether to split identifiers
+ * @return this builder
+ */
+ public Builder splitIdentifiers(boolean splitIdentifiers) {
+ this.splitIdentifiers = splitIdentifiers;
+ return this;
+ }
+
+ /**
+ * Configure whether code identifier subwords are split at letter-number boundaries.
+ *
+ *
This option is valid only with the {@code code} analyzer.
+ *
+ * @param splitOnNumerics whether to split at numeric boundaries
+ * @return this builder
+ */
+ public Builder splitOnNumerics(boolean splitOnNumerics) {
+ this.splitOnNumerics = splitOnNumerics;
+ return this;
+ }
+
+ /**
+ * Configure whether complete code identifiers are indexed alongside their subwords.
+ *
+ *
This option is valid only with the {@code code} analyzer.
+ *
+ * @param preserveOriginal whether to preserve complete identifiers
+ * @return this builder
+ */
+ public Builder preserveOriginal(boolean preserveOriginal) {
+ this.preserveOriginal = preserveOriginal;
+ return this;
+ }
+
+ /**
+ * Configure whether code operators such as {@code ::}, {@code ->}, and {@code !=} are indexed.
+ *
+ *
This option is valid only with the {@code code} analyzer.
+ *
+ * @param indexOperators whether to index operators
+ * @return this builder
+ */
+ public Builder indexOperators(boolean indexOperators) {
+ this.indexOperators = indexOperators;
+ return this;
+ }
+
+ /**
+ * Configure the total memory limit in MiB for the build stage.
+ *
+ *
The limit is split evenly across FTS workers and is not persisted with the index. If
+ * unset, each worker uses a 2 GiB build-time limit.
+ *
+ *
A value of {@code 0} is passed through to Rust.
+ *
+ * @param memoryLimit total memory limit in MiB, must be non-negative
+ * @return this builder
+ * @throws IllegalArgumentException if {@code memoryLimit} is negative
+ */
+ public Builder memoryLimit(long memoryLimit) {
+ if (memoryLimit < 0) {
+ throw new IllegalArgumentException("memoryLimit must be non-negative");
+ }
+ this.memoryLimit = memoryLimit;
+ return this;
+ }
+
+ /**
+ * Configure the number of workers used for the build stage.
+ *
+ *
The effective value is capped at the available compute-intensive CPU count and is not
+ * persisted with the index. Rust clamps a value of {@code 0} to one worker.
+ *
+ * @param numWorkers requested worker count, must be non-negative
+ * @return this builder
+ * @throws IllegalArgumentException if {@code numWorkers} is negative
+ */
+ public Builder numWorkers(int numWorkers) {
+ if (numWorkers < 0) {
+ throw new IllegalArgumentException("numWorkers must be non-negative");
+ }
+ this.numWorkers = numWorkers;
+ return this;
+ }
+
+ /**
+ * This option has no effect because the Rust inverted-index builder does not support skipping
+ * the partition merge stage.
*
* @param skipMerge whether to skip partition merge
* @return this builder
+ * @deprecated this option has no effect and will be removed in a future release
*/
+ @Deprecated
public Builder skipMerge(boolean skipMerge) {
- this.skipMerge = skipMerge;
return this;
}
@@ -268,9 +411,10 @@ public Builder skipMerge(boolean skipMerge) {
* Configure the on-disk FTS format version to write when creating a new index.
*
*
If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise selects
- * v3 for the code analyzer or {@code blockSize = 256}, and v2 for other indexes. Format v3
- * supports both posting block sizes. Formats v1 and v2 support only {@code blockSize = 128} and
- * cannot be used with the code analyzer.
+ * v3 for the code analyzer, {@code baseTokenizer = "code"}, or {@code blockSize = 256}, and v2
+ * for other indexes. Format v3 supports both posting block sizes. Formats v1 and v2 support
+ * only {@code blockSize = 128} and cannot be used with the code analyzer or code base
+ * tokenizer.
*
* @param formatVersion FTS format version, must be 1, 2, or 3
* @return this builder
@@ -305,10 +449,16 @@ public ScalarIndexParams build() {
Preconditions.checkArgument(
formatVersion == 3 || blockSize == 128, "formatVersion 1 and 2 require blockSize 128");
Preconditions.checkArgument(
- !"code".equals(baseTokenizer) || formatVersion == 3,
- "baseTokenizer 'code' requires formatVersion 3");
+ (!"code".equals(analyzer) && !"code".equals(baseTokenizer)) || formatVersion == 3,
+ "code analyzer and baseTokenizer 'code' require formatVersion 3");
}
Map params = new HashMap<>();
+ if (analyzer != null) {
+ params.put("analyzer", analyzer);
+ }
+ if (lanceTokenizer != null) {
+ params.put("lance_tokenizer", lanceTokenizer);
+ }
if (baseTokenizer != null) {
params.put("base_tokenizer", baseTokenizer);
}
@@ -318,9 +468,7 @@ public ScalarIndexParams build() {
if (withPosition != null) {
params.put("with_position", withPosition);
}
- if (maxTokenLength != null) {
- params.put("max_token_length", maxTokenLength);
- }
+ params.put("max_token_length", maxTokenLength);
if (lowerCase != null) {
params.put("lower_case", lowerCase);
}
@@ -353,8 +501,23 @@ public ScalarIndexParams build() {
if (blockSize != null) {
params.put("block_size", blockSize);
}
- if (skipMerge != null) {
- params.put("skip_merge", skipMerge);
+ if (splitIdentifiers != null) {
+ params.put("split_identifiers", splitIdentifiers);
+ }
+ if (splitOnNumerics != null) {
+ params.put("split_on_numerics", splitOnNumerics);
+ }
+ if (preserveOriginal != null) {
+ params.put("preserve_original", preserveOriginal);
+ }
+ if (indexOperators != null) {
+ params.put("index_operators", indexOperators);
+ }
+ if (memoryLimit != null) {
+ params.put("memory_limit", memoryLimit);
+ }
+ if (numWorkers != null) {
+ params.put("num_workers", numWorkers);
}
if (formatVersion != null) {
params.put("format_version", formatVersion);
diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
index 9fb8de4375b..59ef98db9e1 100644
--- a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
+++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
@@ -21,6 +21,7 @@
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -42,6 +43,7 @@ void defaultBlockSizeIsSerialized() {
Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
assertEquals(128, ((Number) json.get("block_size")).intValue());
assertEquals("row", json.get("document_granularity"));
+ assertEquals(40, ((Number) json.get("max_token_length")).intValue());
}
@Test
@@ -101,5 +103,63 @@ void codeAnalyzerRequiresFormatVersionThreeWhenExplicit() {
assertThrows(
IllegalArgumentException.class,
() -> InvertedIndexParams.builder().baseTokenizer("code").formatVersion(2).build());
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> InvertedIndexParams.builder().analyzer("code").formatVersion(1).build());
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> InvertedIndexParams.builder().analyzer("code").formatVersion(2).build());
+ }
+
+ @Test
+ void additionalBuildParametersAreSerialized() {
+ ScalarIndexParams params =
+ InvertedIndexParams.builder()
+ .analyzer("code")
+ .lanceTokenizer("text")
+ .splitIdentifiers(true)
+ .splitOnNumerics(false)
+ .preserveOriginal(true)
+ .indexOperators(true)
+ .memoryLimit(4096)
+ .numWorkers(4)
+ .formatVersion(3)
+ .build();
+
+ Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+ assertEquals("code", json.get("analyzer"));
+ assertEquals("text", json.get("lance_tokenizer"));
+ assertEquals(true, json.get("split_identifiers"));
+ assertEquals(false, json.get("split_on_numerics"));
+ assertEquals(true, json.get("preserve_original"));
+ assertEquals(true, json.get("index_operators"));
+ assertEquals(4096L, ((Number) json.get("memory_limit")).longValue());
+ assertEquals(4, ((Number) json.get("num_workers")).intValue());
+ }
+
+ @Test
+ void nullMaxTokenLengthIsSerializedAsNull() {
+ ScalarIndexParams params = InvertedIndexParams.builder().maxTokenLength(null).build();
+
+ Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+ assertTrue(json.containsKey("max_token_length"));
+ assertNull(json.get("max_token_length"));
+ }
+
+ @Test
+ void zeroResourceParametersAreSerialized() {
+ ScalarIndexParams params = InvertedIndexParams.builder().memoryLimit(0).numWorkers(0).build();
+
+ Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+ assertEquals(0L, ((Number) json.get("memory_limit")).longValue());
+ assertEquals(0, ((Number) json.get("num_workers")).intValue());
+ }
+
+ @Test
+ void negativeResourceParametersAreRejected() {
+ assertThrows(
+ IllegalArgumentException.class, () -> InvertedIndexParams.builder().memoryLimit(-1));
+ assertThrows(
+ IllegalArgumentException.class, () -> InvertedIndexParams.builder().numWorkers(-1));
}
}
diff --git a/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java b/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java
index 9098084ac9f..1468f52256b 100755
--- a/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java
+++ b/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java
@@ -19,6 +19,7 @@
import org.lance.index.IndexOptions;
import org.lance.index.IndexParams;
import org.lance.index.IndexType;
+import org.lance.index.scalar.InvertedIndexParams;
import org.lance.index.scalar.ScalarIndexParams;
import org.apache.arrow.c.ArrowArrayStream;
@@ -40,6 +41,7 @@
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -55,6 +57,23 @@ void testMatchQuery() throws Exception {
2L);
}
+ @Test
+ void testJiebaTokenizer() throws Exception {
+ ScalarIndexParams indexParams =
+ InvertedIndexParams.builder()
+ .baseTokenizer("jieba/default")
+ .stem(false)
+ .removeStopWords(false)
+ .build();
+
+ runFtsQuery(
+ "memory://fts_java_jieba",
+ FullTextQuery.match("我们", "doc"),
+ 1L,
+ Arrays.asList("我们都有光明的前途", "光明的前途"),
+ indexParams);
+ }
+
@Test
void testExplicitListElementGranularityReachesRustRouting() {
RuntimeException error =
@@ -108,6 +127,25 @@ void testBooleanQuery() throws Exception {
}
private void runFtsQuery(String uri, FullTextQuery query, long expectedTotal) throws Exception {
+ ScalarIndexParams indexParams =
+ ScalarIndexParams.create(
+ "inverted",
+ "{\"base_tokenizer\":\"simple\",\"language\":\"English\",\"with_position\":true}");
+ runFtsQuery(
+ uri,
+ query,
+ expectedTotal,
+ Arrays.asList("hello world", "hello lance", "other text"),
+ indexParams);
+ }
+
+ private void runFtsQuery(
+ String uri,
+ FullTextQuery query,
+ long expectedTotal,
+ List documents,
+ ScalarIndexParams scalarParams)
+ throws Exception {
Schema schema =
new Schema(
@@ -122,16 +160,13 @@ private void runFtsQuery(String uri, FullTextQuery query, long expectedTotal) th
VarCharVector titleVector = (VarCharVector) root.getVector("title");
docVector.allocateNew();
- docVector.setSafe(0, "hello world".getBytes(StandardCharsets.UTF_8));
- docVector.setSafe(1, "hello lance".getBytes(StandardCharsets.UTF_8));
- docVector.setSafe(2, "other text".getBytes(StandardCharsets.UTF_8));
-
titleVector.allocateNew();
- titleVector.setSafe(0, "bye world".getBytes(StandardCharsets.UTF_8));
- titleVector.setSafe(1, "bye lance".getBytes(StandardCharsets.UTF_8));
- titleVector.setSafe(2, "say hello".getBytes(StandardCharsets.UTF_8));
-
- root.setRowCount(3);
+ List titles = Arrays.asList("bye world", "bye lance", "say hello");
+ for (int i = 0; i < documents.size(); i++) {
+ docVector.setSafe(i, documents.get(i).getBytes(StandardCharsets.UTF_8));
+ titleVector.setSafe(i, titles.get(i).getBytes(StandardCharsets.UTF_8));
+ }
+ root.setRowCount(documents.size());
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) {
@@ -150,10 +185,6 @@ private void runFtsQuery(String uri, FullTextQuery query, long expectedTotal) th
new WriteParams.Builder().withMode(WriteParams.WriteMode.CREATE).build();
try (Dataset dataset = Dataset.create(allocator, stream, uri, writeParams)) {
- ScalarIndexParams scalarParams =
- ScalarIndexParams.create(
- "inverted",
- "{\"base_tokenizer\":\"simple\",\"language\":\"English\",\"with_position\":true}");
IndexParams indexParams =
IndexParams.builder().setScalarIndexParams(scalarParams).build();
diff --git a/java/src/test/resources/language_models/jieba/default/dict.txt b/java/src/test/resources/language_models/jieba/default/dict.txt
new file mode 100644
index 00000000000..d127091108a
--- /dev/null
+++ b/java/src/test/resources/language_models/jieba/default/dict.txt
@@ -0,0 +1,4 @@
+我们 1000 r
+都有 1000 v
+光明 1000 a
+前途 1000 n