rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-14 14:46:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(fts): narrow sdk support and test coverage

Browse Source
This commit is contained in:
Yang Cen
2026-07-29 02:13:55 +08:00
parent 34d5c1175a
commit 1a50b697c5
18 changed files with 163 additions and 1963 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/tests/docs/test_search.py
+13 -26
View File
@@ -358,38 +358,25 @@ def test_fts_custom_stop_word_sources(tmp_path):
"documents",
data=[{"text": "the lance database"}, {"text": "searchable data"}],
)
# Inline values
documents.create_index(
"text",
config=FTS(stem=False, custom_stop_words=["lance", "searchable"]),
)
# A strict UTF-8 file with one stop word per line
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_text("lance\nsearchable\n", encoding="utf-8")
documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
),
replace=True,
)
# A string column from another open LanceDB table
stop_words = db.create_table(
"stop_words",
data=[{"word": "lance"}, {"word": "searchable"}],
)
documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
replace=True,
)
# Inline values, a strict UTF-8 newline-delimited file, or a local table column.
sources = [
["lance", "searchable"],
FtsStopWordsFile(stop_words_path),
FtsStopWordsTable(stop_words, "word"),
]
for source in sources:
documents.create_index(
"text",
config=FTS(stem=False, custom_stop_words=source),
replace=True,
)
# --8<-- [end:fts_custom_stop_words]
tokens = documents.tokenize("the lance database", column="text")
python/python/tests/test_fts.py
+23 -263
View File
@@ -244,21 +244,19 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
@pytest.mark.parametrize(
("custom_stop_words", "expected_the", "expected_lance"),
("custom_stop_words", "expected"),
[
(None, 0, 2),
([], 1, 2),
(["lance"], 1, 0),
(None, ["lance", "data"]),
([], ["the", "lance", "data"]),
(["lance"], ["the", "data"]),
],
ids=["builtin", "explicit-empty", "custom"],
)
def test_custom_stop_words_inline_none_vs_empty(
tmp_path, custom_stop_words, expected_the, expected_lance
):
def test_custom_stop_words_inline_none_vs_empty(tmp_path, custom_stop_words, expected):
db = ldb.connect(tmp_path)
table = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
data=pa.table({"text": ["the lance data"]}),
)
table.create_index(
"text",
@@ -266,92 +264,23 @@ def test_custom_stop_words_inline_none_vs_empty(
)
assert (
len(table.search("the", query_type="fts").limit(10).to_list()) == expected_the
table.list_indices()[0].index_details["custom_stop_words"] == custom_stop_words
)
assert (
len(table.search("lance", query_type="fts").limit(10).to_list())
== expected_lance
)
details = table.list_indices()[0].index_details
assert details["custom_stop_words"] == custom_stop_words
def test_custom_stop_words_are_inactive_when_removal_is_disabled(tmp_path):
db = ldb.connect(tmp_path)
table = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
table.create_index(
"text",
config=FTS(
stem=False,
remove_stop_words=False,
custom_stop_words=["lance"],
),
)
assert len(table.search("lance", query_type="fts").limit(10).to_list()) == 2
tokens = table.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the", "lance", "data"]
assert [token.text for token in tokens] == expected
def test_custom_stop_words_file_is_snapshotted_and_reopened(tmp_path):
def test_tokenize_custom_stop_words_file(tmp_path):
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_bytes(
"lance\r\n\r\nlance\r\nKeep Case\r\n spaced \r\n".encode()
)
db = ldb.connect(tmp_path / "db")
table = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
table.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
),
stop_words_path.write_text("lance\n", encoding="utf-8")
tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
)
stop_words_path.write_text("the\n", encoding="utf-8")
reopened = db.open_table("documents")
details = reopened.list_indices()[0].index_details
assert details["custom_stop_words"] == ["lance", "Keep Case", " spaced "]
tokens = reopened.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the", "data"]
assert len(reopened.search("the", query_type="fts").limit(10).to_list()) == 1
assert len(reopened.search("lance", query_type="fts").limit(10).to_list()) == 0
def test_custom_stop_words_table_is_snapshotted(tmp_path):
db = ldb.connect(tmp_path)
stop_words = db.create_table(
"stop_words",
data=pa.table({"word": ["lance", "", "lance", "data"]}),
)
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
)
documents.create_index(
"text",
config=FTS(
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
)
db.drop_table("stop_words")
reopened = db.open_table("documents")
assert reopened.list_indices()[0].index_details["custom_stop_words"] == [
"lance",
"data",
]
tokens = reopened.tokenize("the lance data", column="text")
assert [token.text for token in tokens] == ["the"]
@pytest.mark.asyncio
@@ -359,11 +288,11 @@ async def test_custom_stop_words_table_async(tmp_path):
db = await ldb.connect_async(tmp_path)
stop_words = await db.create_table(
"stop_words",
data=pa.table({"word": ["lance", "", "lance"]}),
data=pa.table({"word": ["lance"]}),
)
documents = await db.create_table(
"documents",
data=pa.table({"text": ["the lance database", "lance data"]}),
data=pa.table({"text": ["the lance data"]}),
)
await documents.create_index(
@@ -378,18 +307,7 @@ async def test_custom_stop_words_table_async(tmp_path):
assert [token.text for token in tokens] == ["the", "data"]
def test_custom_stop_words_deprecated_create_fts_index(table):
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index(
"text",
stem=False,
custom_stop_words=["puppy"],
)
assert len(table.search("puppy").limit(5).to_list()) == 0
def test_custom_stop_words_tokenizer_name_compatibility_is_persisted(table):
def test_custom_stop_words_legacy_tokenizer_name(table):
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index(
"text",
@@ -400,117 +318,13 @@ def test_custom_stop_words_tokenizer_name_compatibility_is_persisted(table):
details = table.list_indices()[0].index_details
assert details["custom_stop_words"] == ["puppy"]
assert details["remove_stop_words"] is False
assert len(table.search("puppy").limit(5).to_list()) == 5
assert [token.text for token in table.tokenize("puppy runs", column="text")] == [
"puppy",
"runs",
]
@pytest.mark.parametrize(
("source_factory", "match"),
[
(lambda _tmp, _table: "not-a-sequence", "not a string"),
(lambda _tmp, _table: ["valid", 42], r"custom_stop_words\[1\]"),
(
lambda tmp, _table: FtsStopWordsFile(tmp / "missing.txt"),
"failed to read custom stop words file",
),
(
lambda _tmp, _table: FtsStopWordsFile(""),
"path must be a non-empty string path",
),
(
lambda _tmp, table: FtsStopWordsTable(table, "missing"),
"failed to resolve custom stop words column",
),
(
lambda _tmp, table: FtsStopWordsTable(table, ""),
"column must be a non-empty string",
),
(
lambda _tmp, table: FtsStopWordsTable(table, "number"),
"must have type Utf8",
),
],
)
def test_custom_stop_words_validation_errors(tmp_path, source_factory, match):
db = ldb.connect(tmp_path / "db")
documents = db.create_table(
"documents",
data=pa.table(
{
"text": ["the lance database"],
"number": [1],
}
),
)
with pytest.raises((TypeError, ValueError), match=match):
documents.create_index(
"text",
config=FTS(custom_stop_words=source_factory(tmp_path, documents)),
)
def test_custom_stop_words_file_rejects_invalid_utf8(tmp_path):
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_bytes(b"valid\n\xff")
db = ldb.connect(tmp_path / "db")
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database"]}),
)
with pytest.raises(ValueError, match="not valid UTF-8"):
documents.create_index(
"text",
config=FTS(custom_stop_words=FtsStopWordsFile(stop_words_path)),
)
@pytest.mark.parametrize(
("values", "match"),
[
(pa.array(["valid", None]), "contains NULL"),
(pa.array([1, 2]), "must have type Utf8"),
],
)
def test_custom_stop_words_table_rejects_invalid_values(tmp_path, values, match):
db = ldb.connect(tmp_path)
stop_words = db.create_table("stop_words", data=pa.table({"word": values}))
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database"]}),
)
with pytest.raises(ValueError, match=match):
documents.create_index(
"text",
config=FTS(
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
)
def test_custom_stop_words_table_rejects_closed_table(tmp_path):
db = ldb.connect(tmp_path)
stop_words = db.create_table(
"stop_words",
data=pa.table({"word": ["lance"]}),
)
documents = db.create_table(
"documents",
data=pa.table({"text": ["the lance database"]}),
)
stop_words._table.close()
with pytest.raises(RuntimeError, match="Table stop_words is closed"):
documents.create_index(
"text",
config=FTS(
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
),
def test_custom_stop_words_rejects_non_string_binding_value():
with pytest.raises(TypeError, match=r"custom_stop_words\[1\]"):
ldb.tokenize(
"the lance data",
custom_stop_words=["lance", 42],
)
@@ -1165,60 +979,6 @@ def test_tokenize_uses_explicit_icu_tokenizer():
]
@pytest.mark.parametrize(
("custom_stop_words", "expected"),
[
(None, ["lance", "data"]),
([], ["the", "lance", "data"]),
(["lance"], ["the", "data"]),
],
ids=["builtin", "explicit-empty", "custom"],
)
def test_tokenize_custom_stop_words_none_vs_empty(custom_stop_words, expected):
tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=custom_stop_words,
)
assert [token.text for token in tokens] == expected
def test_tokenize_custom_stop_words_file_and_table(tmp_path):
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_text("lance\nlance\n\n", encoding="utf-8")
file_tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=FtsStopWordsFile(stop_words_path),
)
db = ldb.connect(tmp_path / "db")
stop_words = db.create_table(
"stop_words",
data=pa.table({"word": ["lance", "", "lance"]}),
)
table_tokens = ldb.tokenize(
"the lance data",
stem=False,
custom_stop_words=FtsStopWordsTable(stop_words, "word"),
)
assert [token.text for token in file_tokens] == ["the", "data"]
assert [token.text for token in table_tokens] == ["the", "data"]
def test_tokenize_custom_stop_words_disabled():
tokens = ldb.tokenize(
"the lance data",
stem=False,
remove_stop_words=False,
custom_stop_words=["lance"],
)
assert [token.text for token in tokens] == ["the", "lance", "data"]
def test_fts_on_list(mem_db: DBConnection):
data = pa.table(
{
python/python/tests/test_remote_db.py
+6 -84
View File
@@ -17,7 +17,7 @@ from packaging.version import Version
import lancedb
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.query import ColumnOrdering, MatchQuery
from lancedb.query import ColumnOrdering
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
import pytest
@@ -881,11 +881,6 @@ def test_remote_create_index_new_api(tmp_path):
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
local_db = lancedb.connect(tmp_path / "local-source")
local_stop_words = local_db.create_table(
"stop_words",
data=[{"word": "table-word"}, {"word": ""}, {"word": "table-word"}],
)
# New API: column-first, config= kwarg. Should NOT emit DeprecationWarning.
import warnings as _warnings
@@ -895,25 +890,11 @@ def test_remote_create_index_new_api(tmp_path):
table.create_index("category", config=BTree())
table.create_index("text", config=FTS(block_size=256))
table.create_index("text", config=FTS(custom_stop_words=[]))
table.create_index(
"text",
config=FTS(
custom_stop_words=["cloud", "", "cloud", "Keep Case"],
),
)
stop_words_path = tmp_path / "stop-words.txt"
stop_words_path.write_text("file-word\n\nfile-word\n", encoding="utf-8")
stop_words_path.write_text("file-word\n", encoding="utf-8")
table.create_index(
"text",
config=FTS(
custom_stop_words=FtsStopWordsFile(stop_words_path),
),
)
table.create_index(
"text",
config=FTS(
custom_stop_words=FtsStopWordsTable(local_stop_words, "word"),
),
config=FTS(custom_stop_words=FtsStopWordsFile(stop_words_path)),
)
# IvfRq via new API
table.create_index("vector", config=IvfRq(distance_type="l2"))
@@ -936,51 +917,21 @@ def test_remote_create_index_new_api(tmp_path):
custom_stop_words=FtsStopWordsTable(table, "text"),
),
)
with pytest.raises(
ValueError,
match="remote table sources cannot guarantee a complete snapshot",
):
lancedb.tokenize(
"the cloud",
custom_stop_words=FtsStopWordsTable(table, "text"),
)
assert len(received_requests) == 9
assert len(received_requests) == 7
assert [req["column"] for req in received_requests] == [
"vector",
"category",
"text",
"text",
"text",
"text",
"text",
"vector",
"vector",
]
assert received_requests[2]["block_size"] == 256
assert received_requests[2].get("custom_stop_words") is None
assert received_requests[3]["custom_stop_words"] == []
assert received_requests[4]["custom_stop_words"] == ["cloud", "Keep Case"]
assert received_requests[5]["custom_stop_words"] == ["file-word"]
assert received_requests[6]["custom_stop_words"] == ["table-word"]
assert "path" not in json.dumps(received_requests[5])
assert "table" not in json.dumps(received_requests[5])
assert "table-word" in json.dumps(received_requests[6])
assert not {"source", "table", "path"} & received_requests[6].keys()
# A forked/unpickled RemoteTable may have no live handle. Stop-word
# validation must fail without trying to reopen through LOOP.run from
# the LOOP thread (which would deadlock).
table._table_handle = None
with pytest.raises(
ValueError,
match="remote table sources cannot guarantee a complete snapshot",
):
lancedb.tokenize(
"the cloud",
custom_stop_words=FtsStopWordsTable(table, "text"),
)
assert table._table_handle is None
assert received_requests[4]["custom_stop_words"] == ["file-word"]
assert str(stop_words_path) not in json.dumps(received_requests[4])
def test_table_wait_for_index_timeout():
@@ -1377,35 +1328,6 @@ def test_query_sync_fts():
)
def test_query_sync_fts_fuzziness_boundary():
query_requests = 0
def handler(body):
nonlocal query_requests
query_requests += 1
assert body["full_text_query"]["query"]["match"]["fuzziness"] == 0
return pa.table({"id": [1]})
with query_test_table(handler, server_version=Version("0.3.0")) as table:
table.search(
MatchQuery("puppy", "text", fuzziness=0),
query_type="fts",
).to_list()
assert query_requests == 1
with pytest.raises(
ValueError,
match="explicit fuzzy FTS queries are not supported.*fuzziness=0",
):
table.search(
MatchQuery("puppy", "text", fuzziness=1),
query_type="fts",
).to_list()
# Positive fuzziness fails before issuing another remote query.
assert query_requests == 1
def test_query_sync_hybrid():
def handler(body):
if "full_text_query" in body: