From 4dd31b420770353d43d32905f2feb0a79721c978 Mon Sep 17 00:00:00 2001 From: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:21:46 +0000 Subject: [PATCH] fix(node): defer built-in embedding registration --- .../embedding/functions/getRegistry.md | 14 +++--- nodejs/__test__/embedding_registry.test.ts | 44 ++++++++++++++++--- nodejs/__test__/fixtures/auto_fts_search.cjs | 31 +++++++++++++ nodejs/lancedb/embedding/index.ts | 34 ++++++++++++-- nodejs/lancedb/embedding/registry.ts | 12 ++++- 5 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 nodejs/__test__/fixtures/auto_fts_search.cjs diff --git a/docs/src/js/namespaces/embedding/functions/getRegistry.md b/docs/src/js/namespaces/embedding/functions/getRegistry.md index 331dbd60b..b149a4a88 100644 --- a/docs/src/js/namespaces/embedding/functions/getRegistry.md +++ b/docs/src/js/namespaces/embedding/functions/getRegistry.md @@ -10,16 +10,12 @@ function getRegistry(): EmbeddingFunctionRegistry ``` -Utility function to get the global instance of the registry +Get the global embedding function registry. + +LanceDB built-in providers are initialized when this public API is first +used, so importing the root package does not change automatic search +selection for tables without embedding metadata. ## Returns [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md) - -`EmbeddingFunctionRegistry` The global instance of the registry - -## Example - -```ts -const registry = getRegistry(); -const openai = registry.get("openai").create(); diff --git a/nodejs/__test__/embedding_registry.test.ts b/nodejs/__test__/embedding_registry.test.ts index 48ba48f96..83933399a 100644 --- a/nodejs/__test__/embedding_registry.test.ts +++ b/nodejs/__test__/embedding_registry.test.ts @@ -1,9 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; + +import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai"; import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry"; type EmbeddingModule = typeof import("../lancedb/embedding"); +type OpenAIModule = typeof import("../lancedb/embedding/openai"); type RegistryModule = typeof import("../lancedb/embedding/registry"); describe("embedding function registry", () => { @@ -16,20 +21,32 @@ describe("embedding function registry", () => { registries.length = 0; }); - it("registers built-in providers through the public embedding API", () => { + it("defers built-in providers until the public registry API is used", () => { jest.isolateModules(() => { - const { getRegistry } = - require("../lancedb/embedding") as EmbeddingModule; - const registry = getRegistry(); + const embedding = require("../lancedb/embedding") as EmbeddingModule; + const { getRegistry: getInternalRegistry } = + require("../lancedb/embedding/registry") as RegistryModule; + const registry = getInternalRegistry(); registries.push(registry); + expect(registry.length()).toBe(0); + expect(embedding.getRegistry()).toBe(registry); expect(registry.get("openai")).toBeDefined(); expect(registry.get("huggingface")).toBeDefined(); }); }); + it("preserves automatic FTS search in a fresh process", () => { + execFileSync( + process.execPath, + [resolve(__dirname, "fixtures", "auto_fts_search.cjs")], + { stdio: "pipe" }, + ); + }); + it("shares registrations across duplicated provider module graphs", () => { let registeringRegistry: EmbeddingFunctionRegistry | undefined; + let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined; jest.isolateModules(() => { require("../lancedb/embedding/openai"); @@ -42,13 +59,30 @@ describe("embedding function registry", () => { expect(() => { jest.isolateModules(() => { - require("../lancedb/embedding/openai"); + const { OpenAIEmbeddingFunction } = + require("../lancedb/embedding/openai") as OpenAIModule; + latestOpenAIConstructor = OpenAIEmbeddingFunction; const { getRegistry } = require("../lancedb/embedding/registry") as RegistryModule; registries.push(getRegistry()); }); }).not.toThrow(); + const previousApiKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = "test"; + try { + const latestOpenAI = registeringRegistry! + .get("openai")! + .create(); + expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!); + } finally { + if (previousApiKey === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = previousApiKey; + } + } + jest.isolateModules(() => { const { getRegistry } = require("../lancedb/embedding") as EmbeddingModule; diff --git a/nodejs/__test__/fixtures/auto_fts_search.cjs b/nodejs/__test__/fixtures/auto_fts_search.cjs new file mode 100644 index 000000000..23bf6e3f0 --- /dev/null +++ b/nodejs/__test__/fixtures/auto_fts_search.cjs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +const assert = require("node:assert/strict"); +const tmp = require("tmp"); +const { connect, embedding, Index } = require("../../dist"); +const { getRegistry } = require("../../dist/embedding/registry"); + +async function main() { + assert.equal(typeof embedding.getRegistry, "function"); + assert.equal(getRegistry().length(), 0); + + const dir = tmp.dirSync({ unsafeCleanup: true }); + let db; + try { + db = await connect(dir.name); + const table = await db.createTable("docs", [{ text: "hello world" }]); + await table.createIndex("text", { config: Index.fts() }); + + const rows = await table.search("hello").toArray(); + assert.equal(rows[0].text, "hello world"); + } finally { + db?.close(); + dir.removeCallback(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/nodejs/lancedb/embedding/index.ts b/nodejs/lancedb/embedding/index.ts index f8b453211..09bc0ff8d 100644 --- a/nodejs/lancedb/embedding/index.ts +++ b/nodejs/lancedb/embedding/index.ts @@ -4,9 +4,15 @@ import { Field, Schema } from "../arrow"; import { sanitizeType } from "../sanitize"; import { EmbeddingFunction } from "./embedding_function"; -import { EmbeddingFunctionConfig, getRegistry } from "./registry"; -import "./openai"; -import "./transformers"; +import { + EmbeddingFunctionConfig, + EmbeddingFunctionRegistry, + getRegistry as getGlobalRegistry, + registerBuiltIn, +} from "./registry"; + +type OpenAIModule = typeof import("./openai"); +type TransformersModule = typeof import("./transformers"); export { FieldOptions, @@ -18,7 +24,6 @@ export { export { EmbeddingFunctionRegistry, - getRegistry, register, } from "./registry"; export type { @@ -27,6 +32,27 @@ export type { EmbeddingFunctionCreate, } from "./registry"; +function initializeBuiltInProviders() { + const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule; + const { TransformersEmbeddingFunction } = + require("./transformers") as TransformersModule; + + registerBuiltIn("openai", OpenAIEmbeddingFunction); + registerBuiltIn("huggingface", TransformersEmbeddingFunction); +} + +/** + * Get the global embedding function registry. + * + * LanceDB built-in providers are initialized when this public API is first + * used, so importing the root package does not change automatic search + * selection for tables without embedding metadata. + */ +export function getRegistry(): EmbeddingFunctionRegistry { + initializeBuiltInProviders(); + return getGlobalRegistry(); +} + /** * Create a schema with embedding functions. * diff --git a/nodejs/lancedb/embedding/registry.ts b/nodejs/lancedb/embedding/registry.ts index 774ac879d..a8c37a6b6 100644 --- a/nodejs/lancedb/embedding/registry.ts +++ b/nodejs/lancedb/embedding/registry.ts @@ -63,6 +63,15 @@ export class EmbeddingFunctionRegistry { }; } + /** @ignore */ + setBuiltIn< + T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor, + >(name: string, ctor: T): T { + this.#functions.set(name, ctor); + Reflect.defineMetadata("lancedb::embedding::name", name, ctor); + return ctor; + } + get>( name: string, ): EmbeddingFunctionCreate | undefined; @@ -243,8 +252,7 @@ export function registerBuiltIn< >(name: string, ctor: T): T { const builtInFunctions = getBuiltInFunctions(_REGISTRY); if (builtInFunctions.has(name)) { - Reflect.defineMetadata("lancedb::embedding::name", name, ctor); - return ctor; + return _REGISTRY.setBuiltIn(name, ctor); } _REGISTRY.register(name)(ctor); builtInFunctions.add(name);