From db3b44891745acdc844b310a902a82efd412ff30 Mon Sep 17 00:00:00 2001 From: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:12:38 +0000 Subject: [PATCH] fix(node): refresh metadata before automatic search --- nodejs/__test__/table.test.ts | 99 +++++++++++++++++++++++++++++++++++ nodejs/lancedb/connection.ts | 8 +-- nodejs/lancedb/permutation.ts | 2 +- nodejs/lancedb/query.ts | 34 ++++++++++++ nodejs/lancedb/table.ts | 53 ++++++++----------- 5 files changed, 159 insertions(+), 37 deletions(-) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 4cad365af..bc131977f 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -1731,6 +1731,105 @@ describe("Read consistency interval", () => { }); }); +describe("automatic search schema consistency", () => { + let tmpDir: tmp.DirResult; + + class SchemaRefreshEmbedding extends EmbeddingFunction { + ndims() { + return 2; + } + + embeddingDataType() { + return new Float32(); + } + + async computeSourceEmbeddings(data: string[]) { + return data.map((value) => [value.length, 1]); + } + + async computeQueryEmbeddings(value: string) { + return [value.length, 1]; + } + } + + function embeddingSchema() { + const func = new SchemaRefreshEmbedding(); + return LanceSchema({ + text: func.sourceField(new Utf8()), + vector: func.vectorField(), + }); + } + + beforeEach(() => { + getRegistry().reset(); + register("schema-refresh")(SchemaRefreshEmbedding); + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + }); + + afterEach(() => { + getRegistry().reset(); + tmpDir.removeCallback(); + }); + + it("uses the schema refreshed from another connection", async () => { + const first = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + const second = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + + try { + const stale = await first.createTable("docs", [{ text: "before" }], { + schema: embeddingSchema(), + }); + const replacement = await second.createTable( + "docs", + [{ text: "after hello" }], + { mode: "overwrite" }, + ); + await replacement.createIndex("text", { config: Index.fts() }); + + const rows = await stale.search("hello").toArray(); + expect(rows[0].text).toBe("after hello"); + expect((await stale.schema()).metadata.has("embedding_functions")).toBe( + false, + ); + } finally { + first.close(); + second.close(); + } + }); + + it("tracks embedding metadata across checkout and restore", async () => { + const first = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + const second = await connect(tmpDir.name, { readConsistencyInterval: 0 }); + + try { + await first.createTable("docs", [{ text: "before" }], { + schema: embeddingSchema(), + }); + const table = await second.createTable( + "docs", + [{ text: "after hello" }], + { mode: "overwrite" }, + ); + await table.createIndex("text", { config: Index.fts() }); + + await table.checkout(1); + expect((await table.search("before").toArray())[0].text).toBe("before"); + + await table.checkoutLatest(); + expect((await table.search("hello").toArray())[0].text).toBe( + "after hello", + ); + + await table.checkout(1); + await table.restore(); + expect((await table.search("before").toArray())[0].text).toBe("before"); + } finally { + first.close(); + second.close(); + } + }); +}); + describe("schema evolution", function () { let tmpDir: tmp.DirResult; beforeEach(() => { diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index 9e930b5b4..e63a7ae65 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -535,7 +535,7 @@ export class LocalConnection extends Connection { options?.indexCacheSize, ); - let table: Table = await LocalTable.create(innerTable); + let table: Table = new LocalTable(innerTable); // "main" is the default branch, so treat it as no branch. On a real branch, // scope and pin in one step (yielding "version V of branch B"); otherwise // pin the version, if any, against main. @@ -570,7 +570,7 @@ export class LocalConnection extends Connection { options?.isShallow ?? true, ); - return LocalTable.create(innerTable); + return new LocalTable(innerTable); } private getStorageOptions( @@ -652,7 +652,7 @@ export class LocalConnection extends Connection { storageOptions, ); - return LocalTable.create(innerTable); + return new LocalTable(innerTable); } async createEmptyTable( @@ -698,7 +698,7 @@ export class LocalConnection extends Connection { namespacePath ?? [], storageOptions, ); - return LocalTable.create(innerTable); + return new LocalTable(innerTable); } async dropTable(name: string, namespacePath?: string[]): Promise { diff --git a/nodejs/lancedb/permutation.ts b/nodejs/lancedb/permutation.ts index 79d9d7ebb..e55b68553 100644 --- a/nodejs/lancedb/permutation.ts +++ b/nodejs/lancedb/permutation.ts @@ -172,7 +172,7 @@ export class PermutationBuilder { */ async execute(): Promise { const nativeTable: NativeTable = await this.inner.execute(); - return LocalTable.create(nativeTable); + return new LocalTable(nativeTable); } } diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 843a1276f..cfd6645ae 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -802,6 +802,40 @@ export class Query extends StandardQueryBase { super(tbl.query()); } + /** @hidden */ + static autoSearch( + tbl: NativeTable, + query: string, + vector: Promise | undefined>, + columns?: string[], + ): VectorQuery { + const nativeQuery = vector.then((resolved) => { + const inner = tbl.query(); + if (resolved === undefined) { + inner.fullTextSearch({ + query, + columns: columns ?? null, + }); + + // Native Query and NativeVectorQuery share all operations exposed by + // Table.search's union return type. Keeping the wrapper as VectorQuery + // also preserves the existing runtime type when auto search selects a + // vector query after the asynchronous schema check. + return inner as unknown as NativeVectorQuery; + } + + const raw = Array.isArray(resolved) + ? null + : extractVectorBuffer(resolved); + if (raw) { + return inner.nearestToRaw(raw.data, raw.dtype); + } + return inner.nearestTo(Float32Array.from(resolved as number[])); + }); + + return new VectorQuery(nativeQuery); + } + /** * Find the nearest vectors to the given query vector. * diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index a3ac3d415..f44fb5476 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -814,31 +814,10 @@ export abstract class Table { export class LocalTable extends Table { private readonly inner: _NativeTable; - private readonly hasEmbeddingFunctions: boolean; - private constructor(inner: _NativeTable, hasEmbeddingFunctions: boolean) { + constructor(inner: _NativeTable) { super(); this.inner = inner; - this.hasEmbeddingFunctions = hasEmbeddingFunctions; - } - - /** @hidden */ - static async create(inner: _NativeTable): Promise { - const schemaBuf = await inner.schema(); - const schema = tableFromIPC(schemaBuf).schema; - const serializedFunctions = schema.metadata.get("embedding_functions"); - if (serializedFunctions === undefined) { - return new LocalTable(inner, false); - } - - let hasEmbeddingFunctions = true; - try { - const functions = JSON.parse(serializedFunctions); - hasEmbeddingFunctions = !Array.isArray(functions) || functions.length > 0; - } catch { - // Preserve the existing parse error when the search is executed. - } - return new LocalTable(inner, hasEmbeddingFunctions); } get name(): string { return this.inner.name; @@ -1055,17 +1034,29 @@ export class LocalTable extends Table { }); } - // The query type is auto or vector - // fall back to full text search if no embedding functions are defined and the query is a string - if ( - queryType === "auto" && - (!this.hasEmbeddingFunctions || instanceOfFullTextQuery(query)) - ) { + if (queryType === "auto" && typeof query !== "string") { return this.query().fullTextSearch(query, { columns: ftsColumns, }); } + if (queryType === "auto" && typeof query === "string") { + const vector = this.getEmbeddingFunctions().then(async (functions) => { + // TODO: Support multiple embedding functions + const embeddingFunc: EmbeddingFunctionConfig | undefined = functions + .values() + .next().value; + if (embeddingFunc === undefined) { + return undefined; + } + return await embeddingFunc.function.computeQueryEmbeddings(query); + }); + + const columns = + typeof ftsColumns === "string" ? [ftsColumns] : ftsColumns; + return Query.autoSearch(this.inner, query, vector, columns); + } + const queryPromise = this.getEmbeddingFunctions().then( async (functions) => { // TODO: Support multiple embedding functions @@ -1482,9 +1473,7 @@ export class Branches { fromRef?: string, fromVersion?: number, ): Promise
{ - return LocalTable.create( - await this.#inner.create(name, fromRef, fromVersion), - ); + return new LocalTable(await this.#inner.create(name, fromRef, fromVersion)); } /** @@ -1495,7 +1484,7 @@ export class Branches { * latest and stays writable. */ async checkout(name: string, version?: number): Promise
{ - return LocalTable.create(await this.#inner.checkout(name, version)); + return new LocalTable(await this.#inner.checkout(name, version)); } /** Delete a branch. */