mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-03 12:08:52 +00:00
fix(node): preserve embedding registration in server bundles (#3806)
## Summary
- lazily initialize built-in OpenAI and Hugging Face providers when
consumers call the public embedding registry API
- choose automatic vector versus FTS search from embedding metadata on a
fresh pinned table revision for every execution
- expose automatic string searches as an `AutoQuery` with only
operations common to both native query families
- keep the registry shared and built-in registration safe across
duplicated module graphs
## Root cause
Nitro treats dependency modules as side-effect-free and removes the bare
OpenAI provider import from its generated route. Registration therefore
never runs, so `getRegistry().get("openai")` remains undefined even when
the registry itself is shared globally. Bundlers may also duplicate the
provider and registry module graphs.
The public embedding entry point now initializes built-in providers only
when `getRegistry()` is explicitly called, keeping initialization on a
live path that Nitro retains. Each terminal automatic-search execution
pins the exact table revision visible at dispatch, reads embedding
metadata and computes an embedding from that snapshot, replays the
builder operations, and constructs and executes the selected native
query against the same snapshot. Pinned native snapshots execute locally
when namespace pushdown cannot carry their revision, while remote
snapshots are seeded directly from one version-and-schema response. The
public `AutoQuery` builder exposes only the operations shared by FTS and
vector search, so runtime class narrowing cannot expose invalid
vector-only methods. Repeated built-in registration replaces stale
constructors from duplicated module graphs while public `register()`
retains its duplicate-alias error.
## Validation
- `cargo fmt --all`
- `cargo check --quiet --features remote --tests --examples`
- `cargo clippy --quiet --features remote --tests --examples`
- `pnpm build`
- `pnpm lint`
- `pnpm run docs`
- `pnpm test --runInBand` (783 passed, 5 skipped)
- serial examples suite with a local OpenAI mock (11 passed), including
`sentence-transformers.test.ts`
- packaged Nitro 2.13.4 server route using the reported imports returned
`{"registered":true}`
- fresh-process FTS fixture initialized both public built-ins and
confirmed automatic string search still returned the indexed row
- schema-consistency regressions cover read-consistency refresh,
checkout, checkoutLatest, restore, runtime class narrowing, concurrent
overwrite during embedding computation, and reused automatic-search
builders
- focused regressions confirm pinned native snapshots bypass unversioned
namespace pushdown and remote snapshots use one describe request
Fixes #2429
<!-- lance-gatekeeper-fix:v1 agent=2adf0f21b8bfb634606ed8897a849e30
generation=1 -->
---------
Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
committed by
GitHub
parent
a57fb68891
commit
35b5d015ac
@@ -0,0 +1,95 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
|
||||
import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
|
||||
|
||||
type EmbeddingModule = typeof import("../lancedb/embedding");
|
||||
type OpenAIModule = typeof import("../lancedb/embedding/openai");
|
||||
type RegistryModule = typeof import("../lancedb/embedding/registry");
|
||||
|
||||
describe("embedding function registry", () => {
|
||||
const registries: EmbeddingFunctionRegistry[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const registry of registries) {
|
||||
registry.reset();
|
||||
}
|
||||
registries.length = 0;
|
||||
});
|
||||
|
||||
it("defers built-in providers until the public registry API is used", () => {
|
||||
jest.isolateModules(() => {
|
||||
const embedding = require("../lancedb/embedding") as EmbeddingModule;
|
||||
const { getRegistry: getInternalRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
const registry = getInternalRegistry();
|
||||
registries.push(registry);
|
||||
|
||||
expect(registry.length()).toBe(0);
|
||||
expect(embedding.getRegistry()).toBe(registry);
|
||||
expect(registry.get("openai")).toBeDefined();
|
||||
expect(registry.get("huggingface")).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves automatic FTS search in a fresh process", () => {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
});
|
||||
|
||||
it("shares registrations across duplicated provider module graphs", () => {
|
||||
let registeringRegistry: EmbeddingFunctionRegistry | undefined;
|
||||
let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
|
||||
|
||||
jest.isolateModules(() => {
|
||||
require("../lancedb/embedding/openai");
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registeringRegistry = getRegistry();
|
||||
registries.push(registeringRegistry);
|
||||
expect(registeringRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
jest.isolateModules(() => {
|
||||
const { OpenAIEmbeddingFunction } =
|
||||
require("../lancedb/embedding/openai") as OpenAIModule;
|
||||
latestOpenAIConstructor = OpenAIEmbeddingFunction;
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registries.push(getRegistry());
|
||||
});
|
||||
}).not.toThrow();
|
||||
|
||||
const previousApiKey = process.env.OPENAI_API_KEY;
|
||||
process.env.OPENAI_API_KEY = "test";
|
||||
try {
|
||||
const latestOpenAI = registeringRegistry!
|
||||
.get<OpenAIEmbeddingFunction>("openai")!
|
||||
.create();
|
||||
expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
|
||||
} finally {
|
||||
if (previousApiKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = previousApiKey;
|
||||
}
|
||||
}
|
||||
|
||||
jest.isolateModules(() => {
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding") as EmbeddingModule;
|
||||
const publicRegistry = getRegistry();
|
||||
registries.push(publicRegistry);
|
||||
expect(publicRegistry).toBe(registeringRegistry);
|
||||
expect(publicRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
const assert = require("node:assert/strict");
|
||||
const tmp = require("tmp");
|
||||
const { connect, embedding, Index } = require("../../dist");
|
||||
const { getRegistry } = require("../../dist/embedding/registry");
|
||||
|
||||
async function main() {
|
||||
assert.equal(typeof embedding.getRegistry, "function");
|
||||
assert.equal(getRegistry().length(), 0);
|
||||
assert.equal(embedding.getRegistry(), getRegistry());
|
||||
assert.equal(getRegistry().length(), 2);
|
||||
|
||||
const dir = tmp.dirSync({ unsafeCleanup: true });
|
||||
let db;
|
||||
try {
|
||||
db = await connect(dir.name);
|
||||
const table = await db.createTable("docs", [{ text: "hello world" }]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const rows = await table.search("hello").toArray();
|
||||
assert.equal(rows[0].text, "hello world");
|
||||
} finally {
|
||||
db?.close();
|
||||
dir.removeCallback();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -11,10 +11,13 @@ import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
AutoQuery,
|
||||
Connection,
|
||||
MatchQuery,
|
||||
PhraseQuery,
|
||||
Query,
|
||||
Table,
|
||||
VectorQuery,
|
||||
connect,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
@@ -1777,6 +1780,194 @@ describe("Read consistency interval", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("automatic search schema consistency", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
|
||||
class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) => [value.length, 1]);
|
||||
}
|
||||
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
function embeddingSchema() {
|
||||
const func = new SchemaRefreshEmbedding();
|
||||
return LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
getRegistry().reset();
|
||||
register("schema-refresh")(SchemaRefreshEmbedding);
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
getRegistry().reset();
|
||||
tmpDir.removeCallback();
|
||||
});
|
||||
|
||||
it("uses the schema refreshed from another connection", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const stale = await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const search = stale.search("hello");
|
||||
expect(search).toBeInstanceOf(AutoQuery);
|
||||
expect(search).not.toBeInstanceOf(Query);
|
||||
expect(search).not.toBeInstanceOf(VectorQuery);
|
||||
expect("nprobes" in search).toBe(false);
|
||||
|
||||
const rows = await search.toArray();
|
||||
expect(rows[0].text).toBe("after hello");
|
||||
expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("tracks embedding metadata across checkout and restore", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const table = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
await table.checkout(1);
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
|
||||
await table.checkoutLatest();
|
||||
expect((await table.search("hello").toArray())[0].text).toBe(
|
||||
"after hello",
|
||||
);
|
||||
|
||||
await table.checkout(1);
|
||||
await table.restore();
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("pins automatic search while computing an embedding", async () => {
|
||||
let markStarted!: () => void;
|
||||
let releaseEmbedding!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const released = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
class BlockingEmbedding extends SchemaRefreshEmbedding {
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
markStarted();
|
||||
await released;
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
register("schema-refresh-blocking")(BlockingEmbedding);
|
||||
const func = new BlockingEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable(
|
||||
"docs",
|
||||
[{ text: "hello before" }],
|
||||
{ schema },
|
||||
);
|
||||
const pending = table.search("hello").toArray();
|
||||
await started;
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
expect((await pending)[0].text).toBe("hello before");
|
||||
} finally {
|
||||
releaseEmbedding();
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("refreshes a reused automatic search for every execution", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable("docs", [
|
||||
{ text: "hello before", marker: "before" },
|
||||
]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
const search = table.search("hello").select(["text"]);
|
||||
|
||||
const before = (await search.toArray())[0];
|
||||
expect(before.text).toBe("hello before");
|
||||
expect(before.marker).toBeUndefined();
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after", marker: "after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const after = (await search.toArray())[0];
|
||||
expect(after.text).toBe("hello after");
|
||||
expect(after.marker).toBeUndefined();
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema evolution", function () {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
|
||||
@@ -4,7 +4,15 @@
|
||||
import { Field, Schema } from "../arrow";
|
||||
import { sanitizeType } from "../sanitize";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./registry";
|
||||
import {
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionRegistry,
|
||||
getRegistry as getGlobalRegistry,
|
||||
registerBuiltIn,
|
||||
} from "./registry";
|
||||
|
||||
type OpenAIModule = typeof import("./openai");
|
||||
type TransformersModule = typeof import("./transformers");
|
||||
|
||||
export {
|
||||
FieldOptions,
|
||||
@@ -14,7 +22,39 @@ export {
|
||||
EmbeddingFunctionConstructor,
|
||||
} from "./embedding_function";
|
||||
|
||||
export * from "./registry";
|
||||
export {
|
||||
EmbeddingFunctionRegistry,
|
||||
parseEmbeddingMetadata,
|
||||
register,
|
||||
} from "./registry";
|
||||
export type {
|
||||
CreateReturnType,
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionCreate,
|
||||
EmbeddingMetadataEntry,
|
||||
ResolvedEmbeddingFunctionConfig,
|
||||
} from "./registry";
|
||||
|
||||
function initializeBuiltInProviders() {
|
||||
const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
|
||||
const { TransformersEmbeddingFunction } =
|
||||
require("./transformers") as TransformersModule;
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the global embedding function registry.
|
||||
*
|
||||
* LanceDB built-in providers are initialized when this public API is first
|
||||
* used, so importing the root package does not change automatic search
|
||||
* selection for tables without embedding metadata.
|
||||
*/
|
||||
export function getRegistry(): EmbeddingFunctionRegistry {
|
||||
initializeBuiltInProviders();
|
||||
return getGlobalRegistry();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a schema with embedding functions.
|
||||
|
||||
@@ -5,14 +5,13 @@ import type OpenAI from "openai";
|
||||
import type { EmbeddingCreateParams } from "openai/resources/index";
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { register } from "./registry";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
|
||||
export type OpenAIOptions = {
|
||||
apiKey: string;
|
||||
model: EmbeddingCreateParams["model"];
|
||||
};
|
||||
|
||||
@register("openai")
|
||||
export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<OpenAIOptions>
|
||||
@@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
|
||||
@@ -7,6 +7,10 @@ import {
|
||||
} from "./embedding_function";
|
||||
import "reflect-metadata";
|
||||
|
||||
const builtInFunctionsKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-built-in-functions::v1",
|
||||
);
|
||||
|
||||
export type CreateReturnType<T> = T extends { init: () => Promise<void> }
|
||||
? Promise<T>
|
||||
: T;
|
||||
@@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
setBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
this.#functions.set(name, ctor);
|
||||
Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
get<T extends EmbeddingFunction<unknown>>(
|
||||
name: string,
|
||||
): EmbeddingFunctionCreate<T> | undefined;
|
||||
@@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry {
|
||||
*/
|
||||
reset(this: EmbeddingFunctionRegistry) {
|
||||
this.#functions.clear();
|
||||
getBuiltInFunctions(this).clear();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -183,12 +197,56 @@ export class EmbeddingFunctionRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
const _REGISTRY = new EmbeddingFunctionRegistry();
|
||||
function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
|
||||
const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
|
||||
[key: symbol]: Set<string> | undefined;
|
||||
};
|
||||
let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
|
||||
if (builtInFunctions === undefined) {
|
||||
builtInFunctions = new Set<string>();
|
||||
registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
|
||||
}
|
||||
return builtInFunctions;
|
||||
}
|
||||
|
||||
// Server bundlers can load the side-effect embedding entry points and the public
|
||||
// embedding API from separate module graphs. Keep their registry shared.
|
||||
const registryKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-function-registry::v1",
|
||||
);
|
||||
const registryGlobal = globalThis as typeof globalThis & {
|
||||
[key: symbol]: EmbeddingFunctionRegistry | undefined;
|
||||
};
|
||||
|
||||
function getGlobalRegistry(): EmbeddingFunctionRegistry {
|
||||
const existingRegistry = registryGlobal[registryKey];
|
||||
if (existingRegistry !== undefined) {
|
||||
return existingRegistry;
|
||||
}
|
||||
const registry = new EmbeddingFunctionRegistry();
|
||||
registryGlobal[registryKey] = registry;
|
||||
return registry;
|
||||
}
|
||||
|
||||
const _REGISTRY = getGlobalRegistry();
|
||||
|
||||
export function register(name?: string) {
|
||||
return _REGISTRY.register(name);
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
export function registerBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
const builtInFunctions = getBuiltInFunctions(_REGISTRY);
|
||||
if (builtInFunctions.has(name)) {
|
||||
return _REGISTRY.setBuiltIn(name, ctor);
|
||||
}
|
||||
_REGISTRY.register(name)(ctor);
|
||||
builtInFunctions.add(name);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to get the global instance of the registry
|
||||
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { register } from "./registry";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
|
||||
export type XenovaTransformerOptions = {
|
||||
/** The wasm compatible model to use */
|
||||
@@ -31,7 +31,6 @@ export type XenovaTransformerOptions = {
|
||||
};
|
||||
};
|
||||
|
||||
@register("huggingface")
|
||||
export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<XenovaTransformerOptions>
|
||||
@@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
|
||||
const tensorDiv = (
|
||||
src: import("@huggingface/transformers").Tensor,
|
||||
divBy: number,
|
||||
|
||||
@@ -103,6 +103,7 @@ export {
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
ExecutableQuery,
|
||||
Query,
|
||||
QueryBase,
|
||||
|
||||
+102
-37
@@ -111,13 +111,15 @@ export class QueryBase<
|
||||
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
|
||||
> implements AsyncIterable<RecordBatch>
|
||||
{
|
||||
protected inner!: NativeQueryType | Promise<NativeQueryType>;
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected constructor(
|
||||
protected inner: NativeQueryType | Promise<NativeQueryType>,
|
||||
) {
|
||||
// intentionally empty
|
||||
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
if (inner !== undefined) {
|
||||
this.inner = inner;
|
||||
}
|
||||
}
|
||||
|
||||
// call a function on the inner (either a promise or the actual object)
|
||||
@@ -135,6 +137,15 @@ export class QueryBase<
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the native query used by the next terminal operation.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
protected async getInner(): Promise<NativeQueryType> {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return only the specified columns.
|
||||
*
|
||||
@@ -207,16 +218,11 @@ export class QueryBase<
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected nativeExecute(
|
||||
protected async nativeExecute(
|
||||
options?: Partial<QueryExecutionOptions>,
|
||||
): Promise<NativeBatchIterator> {
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.execute(options?.maxBatchLength, options?.timeoutMs),
|
||||
);
|
||||
} else {
|
||||
return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -245,12 +251,7 @@ export class QueryBase<
|
||||
/** Collect the results as an Arrow @see {@link ArrowTable}. */
|
||||
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
|
||||
const batches = [];
|
||||
let inner;
|
||||
if (this.inner instanceof Promise) {
|
||||
inner = await this.inner;
|
||||
} else {
|
||||
inner = this.inner;
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
for await (const batch of new RecordBatchIterable(inner, options)) {
|
||||
batches.push(batch);
|
||||
}
|
||||
@@ -279,11 +280,8 @@ export class QueryBase<
|
||||
* @returns A Promise that resolves to a string containing the query execution plan explanation.
|
||||
*/
|
||||
async explainPlan(verbose = false): Promise<string> {
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) => inner.explainPlan(verbose));
|
||||
} else {
|
||||
return this.inner.explainPlan(verbose);
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
return inner.explainPlan(verbose);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -321,13 +319,8 @@ export class QueryBase<
|
||||
distributedMetrics?: AnalyzePlanDistributedMetrics,
|
||||
): Promise<string> {
|
||||
const distributedMetricsMode = distributedMetrics ?? "aggregate";
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.analyzePlan(distributedMetricsMode),
|
||||
);
|
||||
} else {
|
||||
return this.inner.analyzePlan(distributedMetricsMode);
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
return inner.analyzePlan(distributedMetricsMode);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -339,12 +332,8 @@ export class QueryBase<
|
||||
* @returns An Arrow Schema describing the output columns.
|
||||
*/
|
||||
async outputSchema(): Promise<import("./arrow").Schema> {
|
||||
let schemaBuffer: Buffer;
|
||||
if (this.inner instanceof Promise) {
|
||||
schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
|
||||
} else {
|
||||
schemaBuffer = await this.inner.outputSchema();
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
const schemaBuffer = await inner.outputSchema();
|
||||
const schema = tableFromIPC(schemaBuffer).schema;
|
||||
return schema;
|
||||
}
|
||||
@@ -356,7 +345,7 @@ export class StandardQueryBase<
|
||||
extends QueryBase<NativeQueryType>
|
||||
implements ExecutableQuery
|
||||
{
|
||||
constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
|
||||
constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
@@ -788,6 +777,51 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A builder for automatic string searches.
|
||||
*
|
||||
* Automatic search determines whether to use full-text or vector search from
|
||||
* the table revision selected for each execution. This builder exposes the
|
||||
* common operations supported by both query families.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class AutoQuery extends StandardQueryBase<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
private readonly calls: Array<
|
||||
(inner: NativeQuery | NativeVectorQuery) => void
|
||||
> = [];
|
||||
|
||||
/** @hidden */
|
||||
constructor(
|
||||
private readonly createInner: () => Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
>,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override doCall(
|
||||
fn: (inner: NativeQuery | NativeVectorQuery) => void,
|
||||
) {
|
||||
this.calls.push(fn);
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override async getInner(): Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
const calls = [...this.calls];
|
||||
const inner = await this.createInner();
|
||||
for (const call of calls) {
|
||||
call(inner);
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
}
|
||||
|
||||
/** A builder for LanceDB queries.
|
||||
*
|
||||
* @see {@link Table#query}, {@link Table#search}
|
||||
@@ -802,6 +836,37 @@ export class Query extends StandardQueryBase<NativeQuery> {
|
||||
super(tbl.query());
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
static autoSearch(
|
||||
tbl: () => Promise<NativeTable>,
|
||||
query: string,
|
||||
vector: (tbl: NativeTable) => Promise<Awaited<IntoVector> | undefined>,
|
||||
columns?: string[],
|
||||
): AutoQuery {
|
||||
const nativeQuery = async () => {
|
||||
const snapshot = await Promise.resolve(tbl());
|
||||
const resolved = await vector(snapshot);
|
||||
const inner = snapshot.query();
|
||||
if (resolved === undefined) {
|
||||
inner.fullTextSearch({
|
||||
query,
|
||||
columns: columns ?? null,
|
||||
});
|
||||
return inner;
|
||||
}
|
||||
|
||||
const raw = Array.isArray(resolved)
|
||||
? null
|
||||
: extractVectorBuffer(resolved);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(resolved as number[]));
|
||||
};
|
||||
|
||||
return new AutoQuery(nativeQuery);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the nearest vectors to the given query vector.
|
||||
*
|
||||
|
||||
+32
-12
@@ -43,6 +43,7 @@ import {
|
||||
Table as _NativeTable,
|
||||
} from "./native";
|
||||
import {
|
||||
AutoQuery,
|
||||
FullTextQuery,
|
||||
Query,
|
||||
TakeQuery,
|
||||
@@ -523,7 +524,7 @@ export abstract class Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType?: string,
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query;
|
||||
): VectorQuery | Query | AutoQuery;
|
||||
/**
|
||||
* Search the table with a given query vector.
|
||||
*
|
||||
@@ -975,10 +976,11 @@ export class LocalTable extends Table {
|
||||
return this.inner.display();
|
||||
}
|
||||
|
||||
private async getEmbeddingFunctions(): Promise<
|
||||
Map<string, EmbeddingFunctionConfig>
|
||||
> {
|
||||
const schema = await this.schema();
|
||||
private async getEmbeddingFunctions(
|
||||
inner: _NativeTable = this.inner,
|
||||
): Promise<Map<string, EmbeddingFunctionConfig>> {
|
||||
const schemaBuf = await inner.schema();
|
||||
const schema = tableFromIPC(schemaBuf).schema;
|
||||
const registry = getRegistry();
|
||||
return registry.parseFunctions(schema.metadata);
|
||||
}
|
||||
@@ -1160,7 +1162,7 @@ export class LocalTable extends Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType: string = "auto",
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query {
|
||||
): VectorQuery | Query | AutoQuery {
|
||||
if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
|
||||
if (queryType === "fts") {
|
||||
throw new Error("Cannot perform full text search on a vector query");
|
||||
@@ -1175,17 +1177,35 @@ export class LocalTable extends Table {
|
||||
});
|
||||
}
|
||||
|
||||
// The query type is auto or vector
|
||||
// fall back to full text search if no embedding functions are defined and the query is a string
|
||||
if (
|
||||
queryType === "auto" &&
|
||||
(getRegistry().length() === 0 || instanceOfFullTextQuery(query))
|
||||
) {
|
||||
if (queryType === "auto" && typeof query !== "string") {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
});
|
||||
}
|
||||
|
||||
if (queryType === "auto" && typeof query === "string") {
|
||||
const vector = async (snapshot: _NativeTable) => {
|
||||
const functions = await this.getEmbeddingFunctions(snapshot);
|
||||
// TODO: Support multiple embedding functions
|
||||
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
|
||||
.values()
|
||||
.next().value;
|
||||
if (embeddingFunc === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return await embeddingFunc.function.computeQueryEmbeddings(query);
|
||||
};
|
||||
|
||||
const columns =
|
||||
typeof ftsColumns === "string" ? [ftsColumns] : ftsColumns;
|
||||
return Query.autoSearch(
|
||||
() => this.inner.checkoutCurrent(),
|
||||
query,
|
||||
vector,
|
||||
columns,
|
||||
);
|
||||
}
|
||||
|
||||
const queryPromise = this.getEmbeddingFunctions().then(
|
||||
async (functions) => {
|
||||
// TODO: Support multiple embedding functions
|
||||
|
||||
@@ -554,6 +554,12 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout_current(&self) -> napi::Result<Self> {
|
||||
let table = self.inner_ref()?.checkout_current().await.default_error()?;
|
||||
Ok(Self::new(table))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout(&self, version: i64) -> napi::Result<()> {
|
||||
self.inner_ref()?
|
||||
|
||||
Reference in New Issue
Block a user