From 3348087d098b6688082455ee6ad336865ac4ed20 Mon Sep 17 00:00:00 2001
From: "lancedb-gatefixer[bot]"
 <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Date: Thu, 8 Oct 2026 19:46:02 -0700
Subject: [PATCH] fix(node): accept code as a base tokenizer (#4368)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

TypeScript rejected `Index.fts({ baseTokenizer: "code" })` because the
public `BaseTokenizer` union omitted `"code"`, although the native
tokenizer already supports it.

Add `"code"` to the shared union, document its identifier behavior, and
regenerate the API reference. The regression test creates and searches a
code-tokenized FTS index with a typed configuration across Arrow
versions 15–18.

Validation:

- The regression failed before the fix with TS2322: `"code"` was not
assignable to `BaseTokenizer`.
- After the fix, all four cases pass: `pnpm test __test__/table.test.ts
--runInBand -t 'full text search code tokenizer' --verbose=false`.
- `pnpm build`, `pnpm lint`, `pnpm run docs`, and formatting of the
changed TypeScript files pass.

Fixes #4366

<!-- lance-gatekeeper-fix:v1 agent=8c37ac529cec8862bd5dbc218c5aaa8d
generation=1 -->

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
---
 docs/src/js/interfaces/FtsOptions.md      |  2 ++
 docs/src/js/type-aliases/BaseTokenizer.md |  1 +
 nodejs/__test__/table.test.ts             | 14 ++++++++++++++
 nodejs/lancedb/indices.ts                 |  3 +++
 4 files changed, 20 insertions(+)

diff --git a/docs/src/js/interfaces/FtsOptions.md b/docs/src/js/interfaces/FtsOptions.md
index 1d3c9ff82..654fa138f 100644
--- a/docs/src/js/interfaces/FtsOptions.md
+++ b/docs/src/js/interfaces/FtsOptions.md
@@ -37,6 +37,8 @@ The following tokenizers are available:
 
 "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
 
+"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
+
 "icu" - ICU dictionary-based word segmentation.
 
 "icu/split" - ICU segmentation with simple-style delimiter splitting.
diff --git a/docs/src/js/type-aliases/BaseTokenizer.md b/docs/src/js/type-aliases/BaseTokenizer.md
index 75b377f27..5fd1b55d4 100644
--- a/docs/src/js/type-aliases/BaseTokenizer.md
+++ b/docs/src/js/type-aliases/BaseTokenizer.md
@@ -12,6 +12,7 @@ type BaseTokenizer:
   | "whitespace"
   | "raw"
   | "ngram"
+  | "code"
   | "icu"
   | "icu/split"
   | `jieba/${string}`
diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts
index 837d9b608..eee40fb9c 100644
--- a/nodejs/__test__/table.test.ts
+++ b/nodejs/__test__/table.test.ts
@@ -4202,6 +4202,20 @@ describe.each(arrowVersions)(
       expect(mustNotResults.length).toBe(1);
     });
 
+    test("full text search code tokenizer", async () => {
+      const db = await connect(tmpDir.name);
+      const table = await db.createTable("test", [
+        { id: 1, text: "def getUserName(user_id): return user_id" },
+        { id: 2, text: "def parseFile(file_path): return file_path" },
+      ]);
+      await table.createIndex("text", {
+        config: Index.fts({ baseTokenizer: "code" }),
+      });
+
+      const results = await table.search("getUserName", "fts").toArray();
+      expect(results.map((row) => row.id)).toEqual([1]);
+    });
+
     test("full text search ngram", async () => {
       const db = await connect(tmpDir.name);
       const data = [
diff --git a/nodejs/lancedb/indices.ts b/nodejs/lancedb/indices.ts
index bc9e19300..43d1cf1b5 100644
--- a/nodejs/lancedb/indices.ts
+++ b/nodejs/lancedb/indices.ts
@@ -491,6 +491,7 @@ export type BaseTokenizer =
   | "whitespace"
   | "raw"
   | "ngram"
+  | "code"
   | "icu"
   | "icu/split"
   | `jieba/${string}`
@@ -520,6 +521,8 @@ export interface FtsOptions {
    *
    * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
    *
+   * "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
+   *
    * "icu" - ICU dictionary-based word segmentation.
    *
    * "icu/split" - ICU segmentation with simple-style delimiter splitting.