diff --git a/docs/src/js/interfaces/QueryExecutionOptions.md b/docs/src/js/interfaces/QueryExecutionOptions.md index fb9bada29..cd36343b8 100644 --- a/docs/src/js/interfaces/QueryExecutionOptions.md +++ b/docs/src/js/interfaces/QueryExecutionOptions.md @@ -29,8 +29,10 @@ Selecting a parent struct or directly aliasing a blob or struct also returns bytes. Blobs inside lists keep their descriptors. Arbitrary projection expressions are not materialized as bytes. -The rows and the bytes come from the same table version. If the table -changes while the query runs, the query runs again, up to three times. +The rows and bytes come from the same snapshot. Remote queries require +authoritative query version and snapshot identity headers, verified again +by the blob response. Older servers can still return `"descriptions"`. +Local queries retry up to three times if the table changes while they run. #### Example diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 8031a4c49..facb9bdfe 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -720,6 +720,25 @@ table changes while the query runs, the query runs again, up to three times: +{/* Draft for the Enterprise TypeScript bytes release. Replace the retry guidance above only after SDK and serving-node rollout. + +Local queries retry up to three times if the table changes while they run. +Remote queries instead bind payload fetches to the version and immutable +snapshot identity from the executed query response. They do not rerun the query +against a newer version to recover missing payloads. Both `toArray()` and +`toArrow()` discover Blob fields from that response, including fields added +after the table handle was opened. + +Every serving node must support `x-lancedb-query-version` and +`x-lancedb-query-snapshot`, enforce the requested identity when fetching bytes, +and echo it in the fetch response. Proxies must forward these headers. Missing +or mismatched provenance fails explicitly; use `blobMode: "descriptions"` until +the rollout is complete. Keep queried versions available for deferred reads: +snapshot identities do not prevent version deletion. Dropping and recreating a +table cannot substitute the replacement's bytes, even if its version is reused. +Hybrid byte queries require matching snapshot provenance from both responses. +*/} + #### Read blobs on demand {#multimodal-read-blobs} Query the rows you want and pass their `_rowid` values to a fetch method. diff --git a/nodejs/__test__/remote_blob_snapshot.test.ts b/nodejs/__test__/remote_blob_snapshot.test.ts new file mode 100644 index 000000000..908dfa9de --- /dev/null +++ b/nodejs/__test__/remote_blob_snapshot.test.ts @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { type Table, connect } from "../lancedb"; +import { + Table as ArrowTable, + DataType, + Field, + Float32, + LargeBinary, + RecordBatch, + Schema, + Struct, + Uint8, + Uint32, + Uint64, + Utf8, + makeArrowTable, + tableToIPC, + vectorFromArray, +} from "../lancedb/arrow"; + +const IDENTITY_A = `v1:${"a".repeat(64)}`; +const IDENTITY_B = `v1:${"b".repeat(64)}`; +const image = new Field( + "image", + new Struct([ + new Field("kind", new Uint8(), false), + new Field("position", new Uint64(), true), + new Field("size", new Uint64(), true), + new Field("blob_id", new Uint32(), false), + new Field("blob_uri", new Utf8(), false), + ]), + true, + // Native descriptor scans replace the logical extension metadata. + new Map([ + ["lance-encoding:blob", "true"], + ["lance-encoding:packed", "true"], + ]), +); +const schema = new Schema([new Field("label", new Utf8(), false), image]); +const description = { + fields: [ + { name: "label", type: { type: "string" }, nullable: false }, + { + name: "image", + nullable: true, + metadata: { "ARROW:extension:name": "lance.blob.v2" }, + type: { + type: "struct", + fields: ["position", "size"].map((name) => ({ + name, + type: { type: "uint64" }, + nullable: true, + })), + }, + }, + ], +}; + +type Request = { + path: string; + body: Record; + raw: string; + identity: string | string[] | undefined; +}; + +interface ServerOptions { + queryBlob?: Field; + describeBlobName?: string | null; + nestedBlob?: boolean; + queryVersion?: string | null; + queryIdentity?: string | null; + echo?: string | null; + empty?: boolean; + values?: boolean; + advanceQueries?: boolean; + otherLegIdentity?: string | null; + otherLegVersion?: string; +} + +// A real HTTP service: the query finishes on A, then the logical table changes +// before the deferred blob request. Recreating B reuses both version and row IDs. +async function withSnapshotServer( + change: "recreate" | "overwrite", + run: (table: Table, requests: Request[]) => Promise, + options: ServerOptions = {}, +) { + const requests: Request[] = []; + let changed = false; + const queryBlob = options.queryBlob ?? image; + const queryOutputSchema = options.nestedBlob + ? new Schema([ + schema.fields[0], + new Field("extra", new Struct([queryBlob]), true), + ]) + : new Schema([schema.fields[0], queryBlob]); + const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + try { + const raw = Buffer.concat(chunks).toString() || "{}"; + const body = JSON.parse(raw); + const path = req.url ?? ""; + const identity = req.headers["x-lancedb-query-snapshot"]; + requests.push({ path, body, raw, identity }); + res.setHeader("phalanx-version", "0.5.0"); + if (path.endsWith("/describe/")) { + res.setHeader("content-type", "application/json"); + res.end( + JSON.stringify({ + version: changed && change === "overwrite" ? 2 : 1, + schema: { + fields: description.fields.flatMap((field) => { + if (field.name !== "image") return [field]; + if (options.describeBlobName === null) return []; + return [ + { ...field, name: options.describeBlobName ?? "image" }, + ]; + }), + }, + }), + ); + } else if (path.endsWith("/query/")) { + const vector = (body.vector?.length ?? 0) > 0; + const advanced = options.advanceQueries === true && changed; + const label = advanced ? "B" : "A"; + const columns: [string, string][] = + body.columns === undefined + ? queryOutputSchema.fields.map((field) => [ + field.name, + field.name, + ]) + : Array.isArray(body.columns) + ? body.columns.map((name: string) => [name, name]) + : Object.entries(body.columns); + const queryFields = columns.map(([output, source]) => + queryOutputSchema.fields + .find((field) => field.name === source)! + .clone({ name: output }), + ); + if (body.with_row_id) { + queryFields.push(new Field("_rowid", new Uint64(), false)); + } + if (vector || body.full_text_query) { + queryFields.push( + new Field(vector ? "_distance" : "_score", new Float32(), false), + ); + } + // Forged IPC metadata must never substitute for authoritative headers. + const querySchema = new Schema( + queryFields, + new Map([ + ["lancedb:query_version", "999"], + ["lancedb:query_snapshot", IDENTITY_B], + ]), + ); + const values = options.empty + ? [] + : options.values + ? ["A", null, ""] + : [label]; + const rows = values.map((value, index) => { + const row: Record = {}; + for (const [output, source] of columns) { + const descriptor = + value === null + ? null + : { + kind: 0, + position: 0n, + size: BigInt(value.length), + // biome-ignore lint/style/useNamingConvention: Native Arrow field name. + blob_id: 0, + // biome-ignore lint/style/useNamingConvention: Native Arrow field name. + blob_uri: "", + }; + row[output] = + source === "image" + ? descriptor + : source === "extra" + ? descriptor === null + ? null + : { image: descriptor } + : label; + } + if (body.with_row_id) row._rowid = BigInt(index); + if (vector || body.full_text_query) { + row[vector ? "_distance" : "_score"] = 0.5; + } + return row; + }); + const result = + rows.length === 0 + ? new ArrowTable(querySchema) + : new ArrowTable( + querySchema, + Object.fromEntries( + queryFields.map((field) => [ + field.name, + vectorFromArray( + rows.map((row) => row[field.name]), + field.type, + ), + ]), + ), + ); + res.setHeader("content-type", "application/vnd.apache.arrow.stream"); + const version = + vector && options.otherLegVersion !== undefined + ? options.otherLegVersion + : options.queryVersion; + const snapshot = + vector && options.otherLegIdentity !== undefined + ? options.otherLegIdentity + : options.queryIdentity; + res.setHeader("x-lancedb-version", "1"); + if (version !== null) { + res.setHeader( + "x-lancedb-query-version", + version ?? (advanced ? "2" : "1"), + ); + } + if (snapshot !== null) { + res.setHeader( + "x-lancedb-query-snapshot", + snapshot ?? (advanced ? IDENTITY_B : IDENTITY_A), + ); + } + changed = true; + res.end(tableToIPC(result, "stream")); + } else if (path.endsWith("/fetch_blobs/")) { + const requestedVersion = /"version":([0-9]+)/.exec(raw)?.[1]; + const readsOriginal = + change === "overwrite" && + requestedVersion === (options.queryVersion ?? "1"); + const actualIdentity = readsOriginal ? IDENTITY_A : IDENTITY_B; + // An older serving node may ignore the guard and answer successfully; + // its echo must still be verified before the client uses any bytes. + if (options.echo !== null) { + res.setHeader( + "x-lancedb-query-snapshot", + options.echo ?? actualIdentity, + ); + } + res.setHeader("content-type", "application/vnd.apache.arrow.stream"); + const payloads = options.values + ? ["A", null, ""] + : [readsOriginal ? "A" : "B"]; + res.end( + tableToIPC( + makeArrowTable( + (body.row_ids as number[]).map((id) => ({ + [String(body.column)]: + payloads[id] === null ? null : Buffer.from(payloads[id]!), + })), + { + schema: new Schema([ + new Field(String(body.column), new LargeBinary(), true), + ]), + }, + ), + "stream", + ), + ); + } else { + res.writeHead(404).end(path); + } + } catch (error) { + res.writeHead(500).end(String(error)); + } + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const db = await connect("db://dev", { + apiKey: "fake", + hostOverride: `http://127.0.0.1:${ + (server.address() as AddressInfo).port + }`, + }); + await run(await db.openTable("test"), requests); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +} + +const fetchRequests = (requests: Request[]) => + requests.filter((request) => request.path.endsWith("/fetch_blobs/")); +const queryRequests = (requests: Request[]) => + requests.filter((request) => request.path.endsWith("/query/")); + +const keepFtsReranker = { + async rerankHybrid(_query: string, _vector: RecordBatch, fts: RecordBatch) { + const fields = fts.schema.fields.map((field) => + field.name === "_score" + ? field.clone({ name: "_relevance_score" }) + : field, + ); + return new RecordBatch( + new Schema(fields, fts.schema.metadata), + fts.data.clone(new Struct(fields)), + ); + }, +}; + +describe("remote blob query snapshot", () => { + it.each([ + image.clone({ metadata: new Map() }), + new Field( + "image", + new Struct(image.type.children.slice(1, 3)), + true, + image.metadata, + ), + ])( + "leaves plain structs and v1 descriptors unchanged: %s", + async (queryBlob) => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const result = await table.query().toArrow({ blobMode: "bytes" }); + expect(DataType.isStruct(result.getChild("image")!.type)).toBe(true); + expect(result.toArray()[0].image.size).toBe(1n); + expect(fetchRequests(requests)).toHaveLength(0); + }, + { queryBlob }, + ); + }, + ); + + it("still recognizes extension-marked Blob results", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const rows = await table.query().toArray({ blobMode: "bytes" }); + expect(Buffer.from(rows[0].image).toString()).toBe("A"); + expect(fetchRequests(requests)).toHaveLength(1); + }, + { + queryBlob: image.clone({ + metadata: new Map([["ARROW:extension:name", "lance.blob.v2"]]), + }), + }, + ); + }); + + it.each([ + [null, false], + [null, true], + ["old_image", false], + ["old_image", true], + ] as const)( + "uses the query schema after cached blob %s changes (alias: %s)", + async (describeBlobName, alias) => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + expect(await table.blobColumns()).toEqual( + describeBlobName === null ? [] : [describeBlobName], + ); + const query = table.query(); + if (alias) query.select({ "photo.image": "image" }); + const output = alias ? "photo.image" : "image"; + const result = await query.toArrow({ blobMode: "bytes" }); + expect(DataType.isLargeBinary(result.getChild(output)!.type)).toBe( + true, + ); + expect( + result + .toArray() + .map((row) => + row[output] === null + ? null + : Buffer.from(row[output]).toString(), + ), + ).toEqual(["A", null, ""]); + expect( + result.schema.fields.some((field) => field.name === "_rowid"), + ).toBe(false); + expect(queryRequests(requests)).toHaveLength(1); + expect(fetchRequests(requests)).toHaveLength(1); + expect(fetchRequests(requests)[0].body.column).toBe("image"); + expect(fetchRequests(requests)[0].identity).toBe(IDENTITY_A); + }, + { describeBlobName, values: true }, + ); + }, + ); + + it("maps nested query fields through literal aliases despite a stale schema", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + expect(await table.blobColumns()).toEqual([]); + const rows = await table + .query() + .select({ "container.alias": "extra" }) + .toArray({ blobMode: "bytes" }); + expect( + rows.map((row) => + row["container.alias"] === null + ? null + : Buffer.from(row["container.alias"].image).toString(), + ), + ).toEqual(["A", null, ""]); + expect(queryRequests(requests)).toHaveLength(1); + expect(fetchRequests(requests)[0].body.column).toBe("extra.image"); + expect(fetchRequests(requests)[0].identity).toBe(IDENTITY_A); + }, + { nestedBlob: true, describeBlobName: null, values: true }, + ); + }); + + it.each(["toArrow", "toArray"] as const)( + "%s rejects payloads from a recreated table with reused version and row IDs", + async (terminal) => { + await withSnapshotServer("recreate", async (table, requests) => { + await expect( + table.query()[terminal]({ blobMode: "bytes" }), + ).rejects.toThrow(/snapshot identity/); + const [fetch] = fetchRequests(requests); + expect(fetch.body.version).toBe(1); + expect(fetch.body.row_ids).toEqual([0]); + expect(fetch.body.column).toBe("image"); + expect(fetch.identity).toBe(IDENTITY_A); + }); + }, + ); + + it("keeps the producing snapshot after an overwrite without rerunning the query", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const rows = await table.query().toArray({ blobMode: "bytes" }); + expect(rows[0].label).toBe("A"); + expect(Buffer.from(rows[0].image).toString()).toBe("A"); + expect(queryRequests(requests)).toHaveLength(1); + const [fetch] = fetchRequests(requests); + expect(fetch.body.version).toBe(1); + expect(fetch.identity).toBe(IDENTITY_A); + expect(await table.version()).toBe(2); + const [latest] = await table.query().toArray({ blobMode: "bytes" }); + expect(latest.label).toBe("B"); + expect(Buffer.from(latest.image).toString()).toBe("B"); + expect(fetchRequests(requests)[1].identity).toBe(IDENTITY_B); + }, + { advanceQueries: true }, + ); + }); + + it.each([ + { queryVersion: null }, + { queryVersion: "0" }, + { queryVersion: "18446744073709551616" }, + { queryVersion: "invalid" }, + { queryIdentity: null }, + { queryIdentity: "v2:unsupported" }, + ])("refuses bytes without authoritative capability %j", async (options) => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const [descriptor] = await table.query().toArray(); + expect(descriptor.image.size).toBe(1n); + await expect( + table.query().toArray({ blobMode: "bytes" }), + ).rejects.toThrow(/blobMode.*bytes.*requires.*snapshot/); + expect(fetchRequests(requests)).toHaveLength(0); + }, + options, + ); + }); + + it("rejects a serving node without the snapshot echo", async () => { + await withSnapshotServer( + "overwrite", + async (table) => { + await expect( + table.query().toArray({ blobMode: "bytes" }), + ).rejects.toThrow(/snapshot identity/); + }, + { echo: null }, + ); + }); + + it("keeps aliases, NULL, empty bytes, and explicit row IDs", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const query = table + .query() + .select({ caption: "label", "photo.image": "image" }); + const rows = await query.withRowId().toArray({ blobMode: "bytes" }); + expect(rows.map((row) => row._rowid)).toEqual([0n, 1n, 2n]); + expect( + rows.map((row) => + row["photo.image"] === null + ? null + : Buffer.from(row["photo.image"]).toString(), + ), + ).toEqual(["A", null, ""]); + expect(rows[0].caption).toBe("A"); + expect(fetchRequests(requests)[0].body.column).toBe("image"); + const [descriptor] = await query.toArray(); + expect(descriptor["photo.image"].size).toBe(1n); + }, + { values: true }, + ); + }); + + it("preserves an empty bytes schema without fetching payloads", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const result = await table.query().toArrow({ blobMode: "bytes" }); + expect(result.numRows).toBe(0); + expect(DataType.isLargeBinary(result.getChild("image")!.type)).toBe( + true, + ); + expect(result.schema.fields.map((field) => field.name)).toEqual([ + "label", + "image", + ]); + expect(fetchRequests(requests)).toHaveLength(0); + }, + { empty: true }, + ); + }); + + it("passes the query version without rounding through a JS number", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + await table.query().toArray({ blobMode: "bytes" }); + expect(fetchRequests(requests)[0].raw).toContain( + '"version":9007199254740993', + ); + }, + { queryVersion: "9007199254740993" }, + ); + }); + + it.each(["vector", "hybrid"])( + "materializes the actual %s query entry point", + async (kind) => { + await withSnapshotServer("overwrite", async (table, requests) => { + const query = table.query().nearestTo([0.1, 0.2]); + if (kind === "hybrid") query.fullTextSearch("A"); + const rows = await query.toArray({ blobMode: "bytes" }); + expect(Buffer.from(rows[0].image).toString()).toBe("A"); + expect(fetchRequests(requests)[0].identity).toBe(IDENTITY_A); + expect(queryRequests(requests)).toHaveLength(kind === "hybrid" ? 2 : 1); + }); + }, + ); + + it("allows scalar-only bytes queries on older servers", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const [row] = await table + .query() + .select(["label"]) + .toArray({ blobMode: "bytes" }); + expect(row.label).toBe("A"); + expect(Object.keys(row)).toEqual(["label"]); + expect(fetchRequests(requests)).toHaveLength(0); + // Row IDs are requested before the executed output schema is known, + // then hidden from scalar-only results without requiring capability. + expect(queryRequests(requests)[0].body.with_row_id).toBe(true); + }, + { queryIdentity: null, queryVersion: null }, + ); + }); + + it.each([false, true])( + "preserves snapshot proof across a custom reranker (empty: %s)", + async (empty) => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const result = await table + .query() + .nearestTo([0.1, 0.2]) + .fullTextSearch("A") + .rerank(keepFtsReranker) + .toArrow({ blobMode: "bytes" }); + expect(result.numRows).toBe(empty ? 0 : 1); + expect(DataType.isLargeBinary(result.getChild("image")!.type)).toBe( + true, + ); + expect(fetchRequests(requests)).toHaveLength(empty ? 0 : 1); + }, + { empty }, + ); + }, + ); + + it("does not trust a custom reranker's first-leg snapshot when the legs disagree", async () => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const query = table + .query() + .nearestTo([0.1, 0.2]) + .fullTextSearch("A") + .rerank(keepFtsReranker); + await expect(query.toArray({ blobMode: "bytes" })).rejects.toThrow( + /snapshot/, + ); + expect(fetchRequests(requests)).toHaveLength(0); + }, + { otherLegIdentity: IDENTITY_B }, + ); + }); + + it.each([ + { otherLegIdentity: IDENTITY_B }, + { otherLegIdentity: null }, + { otherLegVersion: "2" }, + ])( + "refuses hybrid bytes when its actual query legs disagree %j", + async (options) => { + await withSnapshotServer( + "overwrite", + async (table, requests) => { + const query = table.query().nearestTo([0.1, 0.2]).fullTextSearch("A"); + await expect(query.toArray({ blobMode: "bytes" })).rejects.toThrow( + /snapshot/, + ); + expect(fetchRequests(requests)).toHaveLength(0); + expect(await query.toArray()).toHaveLength(1); + }, + options, + ); + }, + ); +}); diff --git a/nodejs/lancedb/blob.ts b/nodejs/lancedb/blob.ts index a46484314..be26f6121 100644 --- a/nodejs/lancedb/blob.ts +++ b/nodejs/lancedb/blob.ts @@ -1,7 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { Field, LargeBinary, Struct, Uint64, Utf8 } from "apache-arrow"; +import { + DataType, + Field, + LargeBinary, + Struct, + Uint64, + Utf8, +} from "apache-arrow"; import { BlobFile as NativeBlobFile } from "./native"; const BLOB_V2_EXTENSION_NAME = "lance.blob.v2"; @@ -178,6 +185,39 @@ export function isBlobField(field: Field): boolean { return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME; } +/** @ignore */ +export function isBlobDescriptorField(field: Field): boolean { + // Native descriptor scans replace the logical extension metadata. Require + // both the encoding marker and the v2 layout to exclude v1 and plain structs. + if ( + field.metadata?.get("lance-encoding:blob") !== "true" || + !DataType.isStruct(field.type) + ) { + return false; + } + const children = field.type.children; + const integers = [ + ["kind", 8], + ["position", 64], + ["size", 64], + ["blob_id", 32], + ] as const; + return ( + children.length === 5 && + integers.every(([name, width], index) => { + const child = children[index]; + return ( + child.name === name && + DataType.isInt(child.type) && + !child.type.isSigned && + child.type.bitWidth === width + ); + }) && + children[4].name === "blob_uri" && + DataType.isUtf8(children[4].type) + ); +} + /** * A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}. * diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index bb845276f..6b72f19fd 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -17,6 +17,7 @@ import { tableFromIPC, vectorFromArray, } from "./arrow"; +import { isBlobDescriptorField, isBlobField } from "./blob"; import { Expr } from "./expr"; import { type IvfPqOptions } from "./indices"; import { @@ -148,6 +149,34 @@ async function readBlobBytes( ); } +// Discover descriptors from the executed query, not the live table's cached +// schema. Projection names are literal aliases; only source paths use dots. +function projectedBlobSources( + schema: Schema, + projection: [string, string][] | undefined, +): BlobSource[] { + const sources: BlobSource[] = []; + for (const field of schema.fields) { + const source = + projection?.find(([output]) => output === field.name)?.[1] ?? field.name; + const visit = (child: Field, path: string[]) => { + if (isBlobField(child) || isBlobDescriptorField(child)) { + sources.push({ + output: field.name, + path, + source: [source, ...path].join("."), + }); + } else if (DataType.isStruct(child.type)) { + for (const nested of child.type.children) { + visit(nested, [...path, nested.name]); + } + } + }; + visit(field, []); + } + return sources; +} + function withBinaryColumns( table: ArrowTable, replacements: Map, @@ -257,8 +286,10 @@ export interface QueryExecutionOptions { * bytes. Blobs inside lists keep their descriptors. Arbitrary projection * expressions are not materialized as bytes. * - * The rows and the bytes come from the same table version. If the table - * changes while the query runs, the query runs again, up to three times. + * The rows and bytes come from the same snapshot. Remote queries require + * authoritative query version and snapshot identity headers, verified again + * by the blob response. Older servers can still return `"descriptions"`. + * Local queries retry up to three times if the table changes while they run. * * @example * ```ts @@ -499,6 +530,34 @@ export class QueryBase< if (context === undefined) { return this.collect(options, false); } + if (!context.table.isNative()) { + // Request row IDs in the same query so newly added Blob columns can be + // fetched without rerunning the query against a different snapshot. + const addRowId = !context.withRowId; + const table = await this.collect(options, addRowId); + const sources = projectedBlobSources(table.schema, context.projection); + if (sources.length === 0) { + return addRowId + ? table.select( + table.schema.fields + .map((field) => field.name) + .filter((name) => name !== "_rowid"), + ) + : table; + } + const version = table.schema.metadata.get("lancedb:query_version"); + const identity = table.schema.metadata.get("lancedb:query_snapshot"); + if (version === undefined || identity === undefined) { + throw new Error( + 'blobMode "bytes" requires a trustworthy query snapshot: missing, invalid, or inconsistent x-lancedb-query-version or x-lancedb-query-snapshot. Upgrade every serving node, or use blobMode "descriptions".', + ); + } + const snapshot = await context.table.querySnapshotWithIdentity( + version, + identity, + ); + return readBlobBytes(table, sources, snapshot, addRowId); + } for (let attempt = 1; ; attempt++) { // The snapshot pins the bytes. The query runs on the live table, so an // unchanged version afterwards proves it read the snapshot's version. @@ -533,10 +592,10 @@ export class QueryBase< for await (const batch of RecordBatchIterator(iterator)) { batches.push(batch); } - if (batches.length === 0) { - return tableFromIPC((await iterator).schema()); - } - return new ArrowTable(batches); + // Read metadata from the executed stream, including empty results and + // query types whose individual batches do not carry the stream metadata. + const schema = tableFromIPC((await iterator).schema()).schema; + return new ArrowTable(schema, batches); } private blobBytesContext( diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index f6abeb553..c82c2cd93 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -349,6 +349,31 @@ impl Table { Ok(Query::new(self.inner_ref()?.query())) } + /// Whether this handle reads an in-process dataset. + #[napi] + pub fn is_native(&self) -> napi::Result { + Ok(self.inner_ref()?.as_native().is_some()) + } + + /// Constrain deferred blob reads by the producing query response. + #[napi(catch_unwind)] + pub async fn query_snapshot_with_identity( + &self, + version: String, + identity: String, + ) -> napi::Result { + // Keep u64 query versions exact across the JavaScript boundary. + let version = version + .parse::() + .map_err(|_| napi::Error::from_reason("Invalid query snapshot version"))?; + let snapshot = self + .inner_ref()? + .query_snapshot_with_identity(version, &identity) + .await + .default_error()?; + Ok(Self::new(snapshot)) + } + /// Return a read-only table handle pinned to the current query revision. #[napi(catch_unwind)] pub async fn query_snapshot(&self) -> napi::Result { diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index f96e84e96..157440910 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -42,7 +42,7 @@ use lance_index::vector::DIST_COL; use crate::error::{Error, Result}; use crate::rerankers::rrf::RRFReranker; use crate::rerankers::{NormalizeMethod, Reranker, check_reranker_result}; -use crate::table::BaseTable; +use crate::table::{BaseTable, NativeTableExt}; use crate::utils::{MaxBatchLengthStream, TimeoutStream}; use crate::{ApproxMode, DistanceType}; use crate::{ @@ -1564,14 +1564,32 @@ impl VectorQuery { vector_query.inner_execute_with_options(internal_options) )?; + // A reranker can replace schema metadata. Only the two producing + // streams can establish the snapshot of a remote hybrid result. + let snapshot = |schema: SchemaRef| { + schema + .metadata() + .get("lancedb:query_version") + .zip(schema.metadata().get("lancedb:query_snapshot")) + .map(|(version, identity)| (version.clone(), identity.clone())) + }; + let stream_schemas = (fts_results.schema(), vec_results.schema()); + let fts_snapshot = snapshot(stream_schemas.0.clone()); + let vec_snapshot = snapshot(stream_schemas.1.clone()); + let query_snapshot = fts_snapshot.filter(|value| Some(value) == vec_snapshot.as_ref()); + let (fts_results, vec_results) = try_join!( fts_results.try_collect::>(), vec_results.try_collect::>() )?; - // try to get the schema to use when combining batches. - // if either - let (fts_schema, vec_schema) = hybrid::query_schemas(&fts_results, &vec_results); + // Preserve remote output schemas even when a leg has no batches. + // The WAL primary-key fallback retains its synthesized score schema. + let (fts_schema, vec_schema) = if self.parent.as_native().is_none() && pk_fusion.is_none() { + stream_schemas + } else { + hybrid::query_schemas(&fts_results, &vec_results) + }; // concatenate all the batches together let mut fts_results = concat_batches(&fts_schema, fts_results.iter())?; @@ -1612,6 +1630,16 @@ impl VectorQuery { check_reranker_result(&results)?; + if self.parent.as_native().is_none() { + let metadata = results.schema_metadata_mut(); + metadata.remove("lancedb:query_version"); + metadata.remove("lancedb:query_snapshot"); + if let Some((version, identity)) = query_snapshot { + metadata.insert("lancedb:query_version".into(), version); + metadata.insert("lancedb:query_snapshot".into(), identity); + } + } + let limit = self.request.base.limit.unwrap_or(DEFAULT_TOP_K); if results.num_rows() > limit { results = results.slice(0, limit);