mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 20:18:37 +00:00
Yang Cen
f7feed48c3
## What Expose custom FTS stop-word lists in the Python and TypeScript public APIs, including their standalone tokenize helpers and remote index creation. This PR supports concrete string lists only. It does not add file or LanceDB-table stop-word sources. ## Why Rust already exposes Lance's custom stop-word list option. The Python and TypeScript APIs did not pass it through, and local index details did not retain the full tokenizer parameters needed by index-backed tokenization after reopening a table. ## How - Add `custom_stop_words` / `customStopWords` to the Python and TypeScript FTS and tokenize options. - Preserve `None` / `undefined`, empty lists, and list contents without normalization. - Load the persisted FTS segment parameters when returning local index details. - Serialize the concrete list in remote create-index requests. - Keep Python and TypeScript tests thin; behavior, persistence, query tokenization, and remote JSON coverage live primarily in Rust. ## Validation - `cargo check --quiet --features remote --tests --examples` - `cargo clippy --quiet --features remote --tests --examples` - `cargo test --quiet --features remote --tests` - Python extension rebuild with `uv` and `maturin` - Targeted Python tests: 4 passed - Python `ruff format --check` and `ruff check` - TypeScript build, typecheck, Biome lint, generated docs, and targeted tests --------- Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>
52 lines
1.3 KiB
TypeScript
52 lines
1.3 KiB
TypeScript
// SPDX-License-Identifier: Apache-2.0
|
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
import { expect, test } from "@jest/globals";
|
|
import * as lancedb from "@lancedb/lancedb";
|
|
import { withTempDirectory } from "./util.ts";
|
|
|
|
test("full text search", async () => {
|
|
await withTempDirectory(async (databaseDir) => {
|
|
const db = await lancedb.connect(databaseDir);
|
|
|
|
const words = [
|
|
"apple",
|
|
"banana",
|
|
"cherry",
|
|
"date",
|
|
"elderberry",
|
|
"fig",
|
|
"grape",
|
|
];
|
|
|
|
const data = Array.from({ length: 10_000 }, (_, i) => ({
|
|
vector: Array(1536).fill(i),
|
|
id: i,
|
|
item: `item ${i}`,
|
|
strId: `${i}`,
|
|
doc: words[i % words.length],
|
|
}));
|
|
|
|
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
|
|
|
|
await tbl.createIndex("doc", {
|
|
config: lancedb.Index.fts({
|
|
stem: false,
|
|
removeStopWords: true,
|
|
customStopWords: ["banana"],
|
|
}),
|
|
});
|
|
const tokens = await tbl.tokenize("apple banana", { column: "doc" });
|
|
expect(tokens.map((token) => token.text)).toEqual(["apple"]);
|
|
|
|
// --8<-- [start:full_text_search]
|
|
const result = await tbl
|
|
.query()
|
|
.nearestToText("apple")
|
|
.select(["id", "doc"])
|
|
.limit(10)
|
|
.toArray();
|
|
expect(result.length).toBe(10);
|
|
// --8<-- [end:full_text_search]
|
|
});
|
|
}, 10_000);
|