refactor(fts): narrow sdk support and test coverage

This commit is contained in:
Yang Cen
2026-07-29 02:13:55 +08:00
parent 34d5c1175a
commit 1a50b697c5
18 changed files with 163 additions and 1963 deletions
-11
View File
@@ -23,17 +23,6 @@
<artifactId>lance-core</artifactId>
</dependency>
<!--
The JNI-backed RestNamespace cannot forward extension fields that are
absent from its generated Rust request model. The official Apache
client is used by LanceDbFtsIndexClient to send the resolved
custom_stop_words snapshot without that lossy JNI round trip.
-->
<dependency>
<groupId>org.lance</groupId>
<artifactId>lance-namespace-apache-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
@@ -1,118 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.lancedb;
import org.lance.namespace.client.apache.ApiClient;
import org.lance.namespace.client.apache.ApiException;
import org.lance.namespace.client.apache.api.IndexApi;
import org.lance.namespace.model.CreateTableIndexResponse;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
/**
* HTTP client for creating LanceDB FTS indexes with custom stop words.
*
* <p>This deliberately bypasses the current JNI-backed namespace index method. That method
* deserializes requests into an upstream generated Rust model which does not yet have {@code
* custom_stop_words}, silently dropping the resolved snapshot before the HTTP request.
*
* <p>Prefer {@link LanceDbNamespaceClientBuilder#buildFtsIndexClient()}. The public constructor is
* available for callers that need to supply a custom Apache HTTP stack, such as mTLS. This client
* owns the supplied {@link ApiClient}'s HTTP client and closes it from {@link #close()}.
*/
public final class LanceDbFtsIndexClient implements Closeable {
private final ApiClient apiClient;
private final IndexApi indexApi;
private final String delimiter;
/**
* Create a client using a fully configured official Lance Namespace Apache client.
*
* @param apiClient configured HTTP client; ownership transfers to this instance
* @param delimiter namespace identifier delimiter, normally {@code $}
*/
public LanceDbFtsIndexClient(ApiClient apiClient, String delimiter) {
this(apiClient, new IndexApi(Objects.requireNonNull(apiClient, "apiClient")), delimiter);
}
LanceDbFtsIndexClient(ApiClient apiClient, IndexApi indexApi, String delimiter) {
this.apiClient = Objects.requireNonNull(apiClient, "apiClient");
this.indexApi = Objects.requireNonNull(indexApi, "indexApi");
if (delimiter == null || delimiter.isEmpty()) {
throw new IllegalArgumentException("delimiter cannot be null or empty");
}
this.delimiter = delimiter;
}
/**
* Resolve any client-local stop-word file and create the FTS index.
*
* <p>The outgoing JSON contains only the concrete {@code custom_stop_words} list (including a
* meaningful distinction between {@code null} and an empty list), never a client-local path.
*
* @param request FTS index request
* @return asynchronous index-creation transaction response
* @throws IOException if a local stop-word file cannot be read as strict UTF-8
* @throws ApiException if the LanceDB HTTP request fails
*/
public CreateTableIndexResponse createTableIndex(LanceDbFtsIndexRequest request)
throws IOException, ApiException {
validateRequest(request);
request.resolveCustomStopWordsSnapshot();
String id = joinIdentifier(request.getId());
return indexApi.createTableIndex(id, request, delimiter);
}
private void validateRequest(LanceDbFtsIndexRequest request) {
if (request == null) {
throw new IllegalArgumentException("FTS index request cannot be null");
}
if (!"FTS".equals(request.getIndexType())) {
throw new IllegalArgumentException("FTS index request index type must be FTS");
}
if (request.getColumn() == null || request.getColumn().trim().isEmpty()) {
throw new IllegalArgumentException("FTS index request requires a non-empty column");
}
List<String> id = request.getId();
if (id == null || id.isEmpty()) {
throw new IllegalArgumentException("FTS index request requires a non-empty table id");
}
for (int index = 0; index < id.size(); index++) {
String part = id.get(index);
if (part == null || part.isEmpty()) {
throw new IllegalArgumentException(
"FTS index request table id part at index " + index + " cannot be null or empty");
}
}
}
private String joinIdentifier(List<String> id) {
StringBuilder result = new StringBuilder();
for (String part : id) {
if (result.length() > 0) {
result.append(delimiter);
}
result.append(part);
}
return result.toString();
}
@Override
public void close() throws IOException {
apiClient.getHttpClient().close();
}
}
@@ -1,315 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.lancedb;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.lance.namespace.model.CreateTableIndexRequest;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
/**
* FTS index request with a resolved custom stop-word snapshot.
*
* <p>Custom stop words replace the built-in list for {@link #getLanguage()} and only affect
* tokenization when {@link #getRemoveStopWords()} is {@code true}. A {@code null} list keeps the
* built-in language list. An empty list explicitly replaces it with no stop words.
*
* <p>Use exactly one source: {@link #setCustomStopWords(List)} for an inline list or {@link
* #setCustomStopWordsFile(Path)} for a newline-delimited UTF-8 file on the client. Exact empty
* strings are removed and exact duplicates retain their first occurrence. Other content, including
* case and surrounding whitespace, is preserved verbatim. File lines use LF or CRLF terminators; a
* lone carriage return is part of the stop word.
*
* <p>The file is read when {@link LanceDbFtsIndexClient#createTableIndex(LanceDbFtsIndexRequest)}
* is first called. The resulting owned list is then reused, so retries cannot silently pick up a
* different file. Client-local paths are never serialized.
*
* <p>Do not pass this request to the JNI-backed {@code LanceNamespace.createTableIndex} method.
* Its upstream generated request model does not yet contain {@code custom_stop_words}, and the JNI
* layer would discard the extension field. Use {@link LanceDbFtsIndexClient}.
*
* <p>A LanceDB table source is not exposed here because the current Java SDK has no embedded/local
* Table API from which it could prove and materialize a complete snapshot. Remote table references
* are intentionally not sent to the server.
*/
public final class LanceDbFtsIndexRequest extends CreateTableIndexRequest {
public static final String JSON_PROPERTY_CUSTOM_STOP_WORDS = "custom_stop_words";
private enum StopWordsSource {
NONE,
INLINE,
FILE
}
private List<String> customStopWords;
@JsonIgnore private transient StopWordsSource stopWordsSource = StopWordsSource.NONE;
@JsonIgnore private transient Path customStopWordsFile;
@JsonIgnore private transient boolean fileSnapshotResolved;
/** Create an FTS request. The index type is always {@code FTS}. */
public LanceDbFtsIndexRequest() {
super.setIndexType("FTS");
}
/**
* Return the resolved custom stop-word snapshot.
*
* <p>For a file source this remains {@code null} until the FTS index client resolves the file.
*
* @return {@code null} for the built-in language list, or the immutable custom snapshot
*/
@javax.annotation.Nullable
@JsonProperty(JSON_PROPERTY_CUSTOM_STOP_WORDS)
@JsonInclude(value = JsonInclude.Include.ALWAYS)
public List<String> getCustomStopWords() {
return customStopWords;
}
/**
* Set inline custom stop words.
*
* @param words custom stop words, {@code null} to use the built-in language list, or an empty
* list to explicitly use no stop words
* @throws IllegalArgumentException if an element is null or is not a string at runtime
* @throws IllegalStateException if a file source is already configured
*/
@JsonProperty(JSON_PROPERTY_CUSTOM_STOP_WORDS)
@JsonInclude(value = JsonInclude.Include.ALWAYS)
public void setCustomStopWords(@javax.annotation.Nullable List<String> words) {
if (words == null) {
clearCustomStopWords();
return;
}
requireCompatibleSource(StopWordsSource.INLINE);
customStopWords = snapshot(words);
customStopWordsFile = null;
fileSnapshotResolved = false;
stopWordsSource = StopWordsSource.INLINE;
}
/**
* Fluent form of {@link #setCustomStopWords(List)}.
*
* @param words inline stop words
* @return this request
*/
public LanceDbFtsIndexRequest customStopWords(@javax.annotation.Nullable List<String> words) {
setCustomStopWords(words);
return this;
}
/**
* Configure a newline-delimited UTF-8 file source.
*
* <p>The path is client-local and is not serialized. It is read exactly once, on the first call
* to the FTS index client. Pass {@code null} to clear the source and use the built-in language
* list.
*
* @param path client-local stop-word file
* @throws IllegalArgumentException if the path is empty
* @throws IllegalStateException if an inline source is already configured
*/
@JsonIgnore
public void setCustomStopWordsFile(@javax.annotation.Nullable Path path) {
if (path == null) {
clearCustomStopWords();
return;
}
if (path.toString().isEmpty()) {
throw new IllegalArgumentException(
"custom stop words file source requires a non-empty path");
}
requireCompatibleSource(StopWordsSource.FILE);
customStopWords = null;
customStopWordsFile = path;
fileSnapshotResolved = false;
stopWordsSource = StopWordsSource.FILE;
}
/**
* Fluent form of {@link #setCustomStopWordsFile(Path)}.
*
* @param path client-local stop-word file
* @return this request
*/
@JsonIgnore
public LanceDbFtsIndexRequest customStopWordsFile(@javax.annotation.Nullable Path path) {
setCustomStopWordsFile(path);
return this;
}
/**
* Keep this specialized request constrained to FTS.
*
* @param indexType must be {@code FTS}
*/
@Override
public void setIndexType(String indexType) {
if (!"FTS".equals(indexType)) {
throw new IllegalArgumentException("LanceDbFtsIndexRequest index type must be FTS");
}
super.setIndexType(indexType);
}
/**
* Keep this specialized request constrained to FTS.
*
* @param indexType must be {@code FTS}
* @return this request
*/
@Override
public LanceDbFtsIndexRequest indexType(String indexType) {
setIndexType(indexType);
return this;
}
void resolveCustomStopWordsSnapshot() throws IOException {
if (stopWordsSource != StopWordsSource.FILE || fileSnapshotResolved) {
return;
}
byte[] bytes;
try {
bytes = Files.readAllBytes(customStopWordsFile);
} catch (IOException e) {
throw new IOException(
"failed to read custom stop words file `" + customStopWordsFile + "`: " + e.getMessage(),
e);
}
String contents;
try {
contents =
StandardCharsets.UTF_8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
.decode(ByteBuffer.wrap(bytes))
.toString();
} catch (CharacterCodingException e) {
throw new IOException(
"custom stop words file `" + customStopWordsFile + "` is not valid UTF-8", e);
}
customStopWords = snapshot(splitLines(contents));
fileSnapshotResolved = true;
}
/**
* Match Rust {@code str::lines()}: split only on LF and remove a CR only when it belongs to a
* CRLF terminator. A lone CR is stop-word content and must be preserved.
*/
private static List<String> splitLines(String contents) {
List<String> lines = new ArrayList<>();
int lineStart = 0;
for (int index = 0; index < contents.length(); index++) {
if (contents.charAt(index) != '\n') {
continue;
}
int lineEnd = index;
if (lineEnd > lineStart && contents.charAt(lineEnd - 1) == '\r') {
lineEnd--;
}
lines.add(contents.substring(lineStart, lineEnd));
lineStart = index + 1;
}
if (lineStart < contents.length()) {
lines.add(contents.substring(lineStart));
}
return lines;
}
private void clearCustomStopWords() {
customStopWords = null;
customStopWordsFile = null;
fileSnapshotResolved = false;
stopWordsSource = StopWordsSource.NONE;
}
private void requireCompatibleSource(StopWordsSource requested) {
if (stopWordsSource != StopWordsSource.NONE && stopWordsSource != requested) {
throw new IllegalStateException(
"custom stop words inline and file sources are mutually exclusive; clear the "
+ stopWordsSource.name().toLowerCase(Locale.ROOT)
+ " source before configuring a "
+ requested.name().toLowerCase(Locale.ROOT)
+ " source");
}
}
private static List<String> snapshot(List<String> words) {
Set<String> seen = new LinkedHashSet<>();
for (int index = 0; index < words.size(); index++) {
Object value = words.get(index);
if (value == null) {
throw new IllegalArgumentException(
"custom stop words inline value at index " + index + " cannot be null");
}
if (!(value instanceof String)) {
throw new IllegalArgumentException(
"custom stop words inline value at index "
+ index
+ " must be a string, but was "
+ value.getClass().getName());
}
String word = (String) value;
if (!word.isEmpty()) {
seen.add(word);
}
}
return Collections.unmodifiableList(new ArrayList<>(seen));
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
if (!(other instanceof LanceDbFtsIndexRequest)) {
return false;
}
LanceDbFtsIndexRequest that = (LanceDbFtsIndexRequest) other;
return super.equals(other)
&& Objects.equals(customStopWords, that.customStopWords)
&& stopWordsSource == that.stopWordsSource
&& Objects.equals(customStopWordsFile, that.customStopWordsFile)
&& fileSnapshotResolved == that.fileSnapshotResolved;
}
@Override
public int hashCode() {
return Objects.hash(
super.hashCode(),
customStopWords,
stopWordsSource,
customStopWordsFile,
fileSnapshotResolved);
}
}
@@ -14,7 +14,6 @@
package com.lancedb;
import org.lance.namespace.LanceNamespace;
import org.lance.namespace.client.apache.ApiClient;
import java.util.HashMap;
import java.util.Map;
@@ -137,130 +136,29 @@ public class LanceDbNamespaceClientBuilder {
* @throws IllegalStateException if required parameters are missing
*/
public LanceNamespace build() {
validateRequiredFields();
// Build configuration map
Map<String, String> config = new HashMap<>(additionalConfig);
config.put("header.x-lancedb-database", database);
config.put("header.x-api-key", apiKey);
config.put("uri", resolveUri());
return LanceNamespace.connect("rest", config, null);
}
/**
* Build a client for creating FTS indexes with custom stop words.
*
* <p>The generated Lance Namespace request model currently has no {@code custom_stop_words}
* field. Passing a subclass through the JNI-backed {@link LanceNamespace} client would silently
* discard that field. This client uses the official HTTP transport directly so the resolved
* client-side snapshot reaches LanceDB Cloud or Enterprise.
*
* <p>The helper supports the same endpoint, API key, database, identifier delimiter, and {@code
* header.*}/{@code headers.*} configuration as {@link #build()}. TLS and other transport
* configuration cannot be copied safely to the Apache HTTP client and is rejected with an
* actionable error instead of being ignored. Callers needing a custom HTTP stack can construct
* {@link LanceDbFtsIndexClient} with a preconfigured {@link ApiClient}.
*
* @return A configured custom-stop-word FTS index client
* @throws IllegalStateException if required parameters are missing or an unsupported
* configuration key is present
*/
public LanceDbFtsIndexClient buildFtsIndexClient() {
validateRequiredFields();
String delimiter = additionalConfig.getOrDefault("delimiter", "$");
if (delimiter == null || delimiter.isEmpty()) {
throw new IllegalStateException("FTS index client delimiter cannot be null or empty");
}
String uri = resolveUri();
if (uri.trim().isEmpty()) {
throw new IllegalStateException("FTS index client endpoint cannot be empty");
}
Map<String, String> headers = new HashMap<>();
for (Map.Entry<String, String> entry : additionalConfig.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
if (key == null) {
throw new IllegalStateException("FTS index client configuration key cannot be null");
}
if ("delimiter".equals(key)) {
continue;
}
if ("uri".equals(key)) {
throw new IllegalStateException(
"FTS index client configuration key 'uri' is not accepted; use endpoint() instead");
}
String headerName = headerName(key);
if (headerName != null) {
if (headerName.isEmpty()) {
throw new IllegalStateException(
"FTS index client header configuration requires a non-empty header name");
}
if (value == null) {
throw new IllegalStateException(
"FTS index client header '" + headerName + "' cannot have a null value");
}
if (isReservedHeader(headerName)) {
String builderMethod =
"x-api-key".equalsIgnoreCase(headerName) ? "apiKey()" : "database()";
throw new IllegalStateException(
"FTS index client header '"
+ headerName
+ "' is reserved; configure it with "
+ builderMethod
+ " instead");
}
headers.put(headerName, value);
continue;
}
throw new IllegalStateException(
"FTS index client cannot safely apply configuration key '"
+ key
+ "'. Configure an org.lance.namespace.client.apache.ApiClient explicitly and "
+ "pass it to LanceDbFtsIndexClient instead.");
}
ApiClient apiClient = new ApiClient().setBasePath(uri).setApiKey(apiKey);
for (Map.Entry<String, String> header : headers.entrySet()) {
apiClient.addDefaultHeader(header.getKey(), header.getValue());
}
apiClient.addDefaultHeader("x-lancedb-database", database);
return new LanceDbFtsIndexClient(apiClient, delimiter);
}
private void validateRequiredFields() {
// Validate required fields
if (apiKey == null) {
throw new IllegalStateException("API key is required");
}
if (database == null) {
throw new IllegalStateException("Database is required");
}
}
private String resolveUri() {
// Build configuration map
Map<String, String> config = new HashMap<>(additionalConfig);
config.put("header.x-lancedb-database", database);
config.put("header.x-api-key", apiKey);
// Determine base URL
String uri;
if (endpoint.isPresent()) {
return endpoint.get();
uri = endpoint.get();
} else {
String effectiveRegion = region.orElse(DEFAULT_REGION);
uri = String.format(CLOUD_URL_PATTERN, database, effectiveRegion);
}
String effectiveRegion = region.orElse(DEFAULT_REGION);
return String.format(CLOUD_URL_PATTERN, database, effectiveRegion);
}
config.put("uri", uri);
private static String headerName(String key) {
if (key.startsWith("header.")) {
return key.substring("header.".length());
}
if (key.startsWith("headers.")) {
return key.substring("headers.".length());
}
return null;
}
private static boolean isReservedHeader(String headerName) {
return "x-api-key".equalsIgnoreCase(headerName)
|| "x-lancedb-database".equalsIgnoreCase(headerName);
return LanceNamespace.connect("rest", config, null);
}
}
@@ -1,133 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.lancedb;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.lance.namespace.client.apache.ApiClient;
import org.lance.namespace.client.apache.ApiException;
import org.lance.namespace.client.apache.api.IndexApi;
import org.lance.namespace.model.CreateTableIndexRequest;
import org.lance.namespace.model.CreateTableIndexResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Unit tests for {@link LanceDbFtsIndexClient}. */
public class LanceDbFtsIndexClientTest {
@TempDir Path tempDir;
@Test
public void testCreateIndexResolvesFileAndReusesSnapshotOnRetry() throws Exception {
ApiClient apiClient = new ApiClient();
CreateTableIndexResponse response = new CreateTableIndexResponse();
CapturingIndexApi indexApi = new CapturingIndexApi(apiClient, response);
LanceDbFtsIndexRequest request = request();
Path path = tempDir.resolve("stop-words.txt");
Files.write(path, "cat\ncat\n dog \n".getBytes(StandardCharsets.UTF_8));
request.setCustomStopWordsFile(path);
try (LanceDbFtsIndexClient client = new LanceDbFtsIndexClient(apiClient, indexApi, "$")) {
CreateTableIndexResponse actual = client.createTableIndex(request);
Files.write(path, "changed\n".getBytes(StandardCharsets.UTF_8));
CreateTableIndexResponse retry = client.createTableIndex(request);
assertSame(response, actual);
assertSame(response, retry);
assertEquals(Arrays.asList("cat", " dog "), request.getCustomStopWords());
assertEquals("namespace$table", indexApi.id);
assertSame(request, indexApi.request);
assertEquals("$", indexApi.delimiter);
assertEquals(2, indexApi.calls);
}
}
@Test
public void testCreateIndexValidatesRequiredFieldsBeforeNetworkCall() throws Exception {
ApiClient apiClient = new ApiClient();
CapturingIndexApi indexApi =
new CapturingIndexApi(apiClient, new CreateTableIndexResponse());
try (LanceDbFtsIndexClient client = new LanceDbFtsIndexClient(apiClient, indexApi, "$")) {
LanceDbFtsIndexRequest noColumn = new LanceDbFtsIndexRequest();
noColumn.setId(Collections.singletonList("table"));
IllegalArgumentException columnError =
assertThrows(IllegalArgumentException.class, () -> client.createTableIndex(noColumn));
assertTrue(columnError.getMessage().contains("non-empty column"));
LanceDbFtsIndexRequest noId = new LanceDbFtsIndexRequest();
noId.setColumn("text");
IllegalArgumentException idError =
assertThrows(IllegalArgumentException.class, () -> client.createTableIndex(noId));
assertTrue(idError.getMessage().contains("non-empty table id"));
assertEquals(0, indexApi.calls);
}
}
@Test
public void testDelimiterMustBeNonEmpty() throws Exception {
ApiClient apiClient = new ApiClient();
try {
CapturingIndexApi indexApi =
new CapturingIndexApi(apiClient, new CreateTableIndexResponse());
IllegalArgumentException error =
assertThrows(
IllegalArgumentException.class,
() -> new LanceDbFtsIndexClient(apiClient, indexApi, ""));
assertTrue(error.getMessage().contains("delimiter"));
} finally {
apiClient.getHttpClient().close();
}
}
private static LanceDbFtsIndexRequest request() {
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setId(Arrays.asList("namespace", "table"));
request.setColumn("text");
request.setRemoveStopWords(true);
return request;
}
private static final class CapturingIndexApi extends IndexApi {
private final CreateTableIndexResponse response;
private String id;
private CreateTableIndexRequest request;
private String delimiter;
private int calls;
private CapturingIndexApi(ApiClient apiClient, CreateTableIndexResponse response) {
super(apiClient);
this.response = response;
}
@Override
public CreateTableIndexResponse createTableIndex(
String id, CreateTableIndexRequest request, String delimiter) throws ApiException {
this.id = id;
this.request = request;
this.delimiter = delimiter;
calls++;
return response;
}
}
}
@@ -1,239 +0,0 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.lancedb;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.lance.namespace.client.apache.ApiClient;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Unit tests for {@link LanceDbFtsIndexRequest}. */
public class LanceDbFtsIndexRequestTest {
@TempDir Path tempDir;
@Test
public void testInlineSnapshotPreservesContentAndCanonicalizesExactValues() {
List<String> input =
new ArrayList<>(Arrays.asList("cat", "", " dog ", "cat", "CAT", " dog "));
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setCustomStopWords(input);
input.set(0, "mutated");
assertEquals(Arrays.asList("cat", " dog ", "CAT"), request.getCustomStopWords());
assertThrows(
UnsupportedOperationException.class,
() -> request.getCustomStopWords().add("later mutation"));
}
@Test
public void testNullAndEmptySnapshotsRemainDistinctInApacheClientJson() throws Exception {
ApiClient apiClient = new ApiClient();
try {
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setColumn("text");
JsonNode nullJson = serialize(apiClient, request);
assertTrue(nullJson.has("custom_stop_words"));
assertTrue(nullJson.get("custom_stop_words").isNull());
request.setCustomStopWords(Collections.emptyList());
JsonNode emptyJson = serialize(apiClient, request);
assertTrue(emptyJson.get("custom_stop_words").isArray());
assertEquals(0, emptyJson.get("custom_stop_words").size());
} finally {
apiClient.getHttpClient().close();
}
}
@Test
public void testInlineNullAndNonStringValuesAreRejected() {
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
IllegalArgumentException nullError =
assertThrows(
IllegalArgumentException.class,
() -> request.setCustomStopWords(Arrays.asList("cat", null)));
assertTrue(nullError.getMessage().contains("index 1"));
assertTrue(nullError.getMessage().contains("cannot be null"));
@SuppressWarnings({"rawtypes", "unchecked"})
List<String> invalid = (List) Arrays.asList("cat", 42);
IllegalArgumentException typeError =
assertThrows(IllegalArgumentException.class, () -> request.setCustomStopWords(invalid));
assertTrue(typeError.getMessage().contains("index 1"));
assertTrue(typeError.getMessage().contains("must be a string"));
}
@Test
public void testInlineAndFileSourcesAreMutuallyExclusive() {
Path path = tempDir.resolve("stop-words.txt");
LanceDbFtsIndexRequest inline = new LanceDbFtsIndexRequest();
inline.setCustomStopWords(Collections.emptyList());
IllegalStateException inlineError =
assertThrows(IllegalStateException.class, () -> inline.setCustomStopWordsFile(path));
assertTrue(inlineError.getMessage().contains("mutually exclusive"));
LanceDbFtsIndexRequest file = new LanceDbFtsIndexRequest();
file.setCustomStopWordsFile(path);
IllegalStateException fileError =
assertThrows(
IllegalStateException.class,
() -> file.setCustomStopWords(Collections.singletonList("cat")));
assertTrue(fileError.getMessage().contains("mutually exclusive"));
inline.setCustomStopWords(null);
inline.setCustomStopWordsFile(path);
}
@Test
public void testFileSnapshotUsesStrictUtf8AndIsStable() throws Exception {
Path path = tempDir.resolve("stop-words.txt");
Files.write(path, "cat\n\n dog \r\ncat\nCAT\n".getBytes(StandardCharsets.UTF_8));
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setCustomStopWordsFile(path);
assertNull(request.getCustomStopWords());
request.resolveCustomStopWordsSnapshot();
assertEquals(Arrays.asList("cat", " dog ", "CAT"), request.getCustomStopWords());
Files.write(path, "changed\n".getBytes(StandardCharsets.UTF_8));
request.resolveCustomStopWordsSnapshot();
assertEquals(Arrays.asList("cat", " dog ", "CAT"), request.getCustomStopWords());
}
@Test
public void testFilePathIsNeverSerialized() throws Exception {
Path path = tempDir.resolve("private-client-path.txt");
Files.write(path, "cat\n".getBytes(StandardCharsets.UTF_8));
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setColumn("text");
request.setCustomStopWordsFile(path);
request.resolveCustomStopWordsSnapshot();
ApiClient apiClient = new ApiClient();
try {
JsonNode json = serialize(apiClient, request);
assertEquals(Collections.singletonList("cat"), request.getCustomStopWords());
assertEquals("cat", json.get("custom_stop_words").get(0).asText());
assertFalse(json.has("customStopWordsFile"));
assertFalse(json.has("custom_stop_words_file"));
assertFalse(json.toString().contains(path.toString()));
} finally {
apiClient.getHttpClient().close();
}
}
@Test
public void testEmptyFileProducesExplicitEmptySnapshot() throws Exception {
Path path = tempDir.resolve("empty.txt");
Files.write(path, new byte[0]);
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setCustomStopWordsFile(path);
request.resolveCustomStopWordsSnapshot();
assertEquals(Collections.emptyList(), request.getCustomStopWords());
}
@Test
public void testLoneCarriageReturnIsPreservedAsContent() throws Exception {
Path path = tempDir.resolve("lone-cr.txt");
Files.write(path, "cat\rdog\ncat\r\nbird\r".getBytes(StandardCharsets.UTF_8));
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setCustomStopWordsFile(path);
request.resolveCustomStopWordsSnapshot();
assertEquals(Arrays.asList("cat\rdog", "cat", "bird\r"), request.getCustomStopWords());
}
@Test
public void testMissingFileIsRejectedWithPath() {
Path path = tempDir.resolve("missing.txt");
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setCustomStopWordsFile(path);
IOException error =
assertThrows(IOException.class, request::resolveCustomStopWordsSnapshot);
assertTrue(error.getMessage().contains("failed to read custom stop words file"));
assertTrue(error.getMessage().contains(path.toString()));
}
@Test
public void testEmptyFilePathIsRejectedDescriptively() {
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
IllegalArgumentException error =
assertThrows(
IllegalArgumentException.class,
() -> request.setCustomStopWordsFile(Paths.get("")));
assertTrue(error.getMessage().contains("file source requires a non-empty path"));
}
@Test
public void testMalformedUtf8IsRejected() throws Exception {
Path path = tempDir.resolve("invalid.txt");
Files.write(path, new byte[] {(byte) 0xc3, (byte) 0x28});
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
request.setCustomStopWordsFile(path);
IOException error =
assertThrows(IOException.class, request::resolveCustomStopWordsSnapshot);
assertTrue(error.getMessage().contains("not valid UTF-8"));
}
@Test
public void testRequestCannotBeChangedToAnotherIndexType() {
LanceDbFtsIndexRequest request = new LanceDbFtsIndexRequest();
assertEquals("FTS", request.getIndexType());
IllegalArgumentException error =
assertThrows(IllegalArgumentException.class, () -> request.setIndexType("BTREE"));
assertTrue(error.getMessage().contains("must be FTS"));
}
private static JsonNode serialize(ApiClient apiClient, LanceDbFtsIndexRequest request)
throws Exception {
HttpEntity entity =
apiClient.serialize(request, Collections.emptyMap(), ContentType.APPLICATION_JSON);
String json = EntityUtils.toString(entity);
return new ObjectMapper().readTree(json);
}
}
@@ -93,91 +93,4 @@ public class LanceDbNamespaceClientBuilderTest {
assertNotSame(builder1, builder2);
}
@Test
public void testFtsIndexClientRequiresApiKey() {
LanceDbNamespaceClientBuilder builder =
LanceDbNamespaceClientBuilder.newBuilder().database("test-db");
IllegalStateException exception =
assertThrows(IllegalStateException.class, builder::buildFtsIndexClient);
assertEquals("API key is required", exception.getMessage());
}
@Test
public void testFtsIndexClientRequiresDatabase() {
LanceDbNamespaceClientBuilder builder =
LanceDbNamespaceClientBuilder.newBuilder().apiKey("test-api-key");
IllegalStateException exception =
assertThrows(IllegalStateException.class, builder::buildFtsIndexClient);
assertEquals("Database is required", exception.getMessage());
}
@Test
public void testFtsIndexClientRejectsUnsupportedTransportConfiguration() {
LanceDbNamespaceClientBuilder builder =
LanceDbNamespaceClientBuilder.newBuilder()
.apiKey("test-api-key")
.database("test-db")
.config("tls.cert_file", "/tmp/client.pem");
IllegalStateException exception =
assertThrows(IllegalStateException.class, builder::buildFtsIndexClient);
assertTrue(exception.getMessage().contains("tls.cert_file"));
assertTrue(exception.getMessage().contains("ApiClient"));
}
@Test
public void testFtsIndexClientRejectsEmptyDelimiter() {
LanceDbNamespaceClientBuilder builder =
LanceDbNamespaceClientBuilder.newBuilder()
.apiKey("test-api-key")
.database("test-db")
.config("delimiter", "");
IllegalStateException exception =
assertThrows(IllegalStateException.class, builder::buildFtsIndexClient);
assertTrue(exception.getMessage().contains("delimiter"));
}
@Test
public void testFtsIndexClientRejectsUriConfigurationAlias() {
LanceDbNamespaceClientBuilder builder =
LanceDbNamespaceClientBuilder.newBuilder()
.apiKey("test-api-key")
.database("test-db")
.config("uri", "https://example.invalid");
IllegalStateException exception =
assertThrows(IllegalStateException.class, builder::buildFtsIndexClient);
assertTrue(exception.getMessage().contains("uri"));
assertTrue(exception.getMessage().contains("endpoint()"));
}
@Test
public void testFtsIndexClientRejectsReservedHeaders() {
LanceDbNamespaceClientBuilder apiKeyBuilder =
LanceDbNamespaceClientBuilder.newBuilder()
.apiKey("test-api-key")
.database("test-db")
.config("header.x-api-key", "other-key");
IllegalStateException apiKeyError =
assertThrows(IllegalStateException.class, apiKeyBuilder::buildFtsIndexClient);
assertTrue(apiKeyError.getMessage().contains("reserved"));
assertTrue(apiKeyError.getMessage().contains("apiKey()"));
LanceDbNamespaceClientBuilder databaseBuilder =
LanceDbNamespaceClientBuilder.newBuilder()
.apiKey("test-api-key")
.database("test-db")
.config("headers.X-LanceDB-Database", "other-db");
IllegalStateException databaseError =
assertThrows(IllegalStateException.class, databaseBuilder::buildFtsIndexClient);
assertTrue(databaseError.getMessage().contains("reserved"));
assertTrue(databaseError.getMessage().contains("database()"));
}
}