diff --git a/docs/src/js/interfaces/FtsOptions.md b/docs/src/js/interfaces/FtsOptions.md index 1d3c9ff82..654fa138f 100644 --- a/docs/src/js/interfaces/FtsOptions.md +++ b/docs/src/js/interfaces/FtsOptions.md @@ -37,6 +37,8 @@ The following tokenizers are available: "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token. +"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens. + "icu" - ICU dictionary-based word segmentation. "icu/split" - ICU segmentation with simple-style delimiter splitting. diff --git a/docs/src/js/type-aliases/BaseTokenizer.md b/docs/src/js/type-aliases/BaseTokenizer.md index 75b377f27..5fd1b55d4 100644 --- a/docs/src/js/type-aliases/BaseTokenizer.md +++ b/docs/src/js/type-aliases/BaseTokenizer.md @@ -12,6 +12,7 @@ type BaseTokenizer: | "whitespace" | "raw" | "ngram" + | "code" | "icu" | "icu/split" | `jieba/${string}` diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 837d9b608..eee40fb9c 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -4202,6 +4202,20 @@ describe.each(arrowVersions)( expect(mustNotResults.length).toBe(1); }); + test("full text search code tokenizer", async () => { + const db = await connect(tmpDir.name); + const table = await db.createTable("test", [ + { id: 1, text: "def getUserName(user_id): return user_id" }, + { id: 2, text: "def parseFile(file_path): return file_path" }, + ]); + await table.createIndex("text", { + config: Index.fts({ baseTokenizer: "code" }), + }); + + const results = await table.search("getUserName", "fts").toArray(); + expect(results.map((row) => row.id)).toEqual([1]); + }); + test("full text search ngram", async () => { const db = await connect(tmpDir.name); const data = [ diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts index bc9e19300..43d1cf1b5 100644 --- a/nodejs/lancedb/indices.ts +++ b/nodejs/lancedb/indices.ts @@ -491,6 +491,7 @@ export type BaseTokenizer = | "whitespace" | "raw" | "ngram" + | "code" | "icu" | "icu/split" | `jieba/${string}` @@ -520,6 +521,8 @@ export interface FtsOptions { * * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token. * + * "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens. + * * "icu" - ICU dictionary-based word segmentation. * * "icu/split" - ICU segmentation with simple-style delimiter splitting.