fix(node): accept code as a base tokenizer (#4368)

TypeScript rejected `Index.fts({ baseTokenizer: "code" })` because the
public `BaseTokenizer` union omitted `"code"`, although the native
tokenizer already supports it.

Add `"code"` to the shared union, document its identifier behavior, and
regenerate the API reference. The regression test creates and searches a
code-tokenized FTS index with a typed configuration across Arrow
versions 15–18.

Validation:

- The regression failed before the fix with TS2322: `"code"` was not
assignable to `BaseTokenizer`.
- After the fix, all four cases pass: `pnpm test __test__/table.test.ts
--runInBand -t 'full text search code tokenizer' --verbose=false`.
- `pnpm build`, `pnpm lint`, `pnpm run docs`, and formatting of the
changed TypeScript files pass.

Fixes #4366

<!-- lance-gatekeeper-fix:v1 agent=8c37ac529cec8862bd5dbc218c5aaa8d
generation=1 -->

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>

docs/src/js/interfaces/FtsOptions.md

@@ -37,6 +37,8 @@ The following tokenizers are available:
 
 "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
 
+"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
+
 "icu" - ICU dictionary-based word segmentation.
 
 "icu/split" - ICU segmentation with simple-style delimiter splitting.

docs/src/js/type-aliases/BaseTokenizer.md

@@ -12,6 +12,7 @@ type BaseTokenizer:
   | "whitespace"
   | "raw"
   | "ngram"
+  | "code"
   | "icu"
   | "icu/split"
   | `jieba/${string}`

nodejs/__test__/table.test.ts

@@ -4202,6 +4202,20 @@ describe.each(arrowVersions)(
       expect(mustNotResults.length).toBe(1);
     });
 
+    test("full text search code tokenizer", async () => {
+      const db = await connect(tmpDir.name);
+      const table = await db.createTable("test", [
+        { id: 1, text: "def getUserName(user_id): return user_id" },
+        { id: 2, text: "def parseFile(file_path): return file_path" },
+      ]);
+      await table.createIndex("text", {
+        config: Index.fts({ baseTokenizer: "code" }),
+      });
+
+      const results = await table.search("getUserName", "fts").toArray();
+      expect(results.map((row) => row.id)).toEqual([1]);
+    });
+
     test("full text search ngram", async () => {
       const db = await connect(tmpDir.name);
       const data = [

nodejs/lancedb/indices.ts

@@ -491,6 +491,7 @@ export type BaseTokenizer =
   | "whitespace"
   | "raw"
   | "ngram"
+  | "code"
   | "icu"
   | "icu/split"
   | `jieba/${string}`
@@ -520,6 +521,8 @@ export interface FtsOptions {
    *
    * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
    *
+   * "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
+   *
    * "icu" - ICU dictionary-based word segmentation.
    *
    * "icu/split" - ICU segmentation with simple-style delimiter splitting.