mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
fix(node): bind blob byte queries to their response snapshot (#5019)
TypeScript remote `toArrow` and `toArray` could combine scalar values from query snapshot A with Blob payloads from a later snapshot B. Cached table schemas and native descriptor layouts could also leave descriptors in a `blobMode: "bytes"` result. Discover Blob fields and snapshot provenance from the executed query stream, recognizing canonical native v2 descriptors as well as extension-marked fields. Fetch payloads through an independent Rust snapshot handle bound to the query version and immutable dataset identity, without changing the live table. Preserve aliases, nested fields, NULL/empty values, explicit row IDs, and scalar-only results. Hybrid byte queries require matching provenance from both actual responses. Missing or mismatched server provenance fails explicitly; logical-input validation and local materialization remain unchanged. Manual integration validation at `4daf354df`: all five real SDK/NAPI → Sophon REST scenarios passed. Both `toArrow` and `toArray` retain A bytes across overwrite and reject same-name/same-URI recreation even when version and row IDs are reused. A previously cached handle discovers a subsequently added Blob field. Ten actual payload fetches include the two expected identity rejections; later queries on the same live handle read B. The canonical-descriptor and Blob regression suites pass 119 tests, with independent source and wire-evidence review. Earlier broader SDK and hybrid evidence is retained within its unchanged-source scope. This is local REST interoperability evidence, not deployed Cloud/header-forwarding acceptance. Remote bytes require the authoritative server version/identity and fetch-echo contract. CI and formal review remain merge requirements; this follow-up does not close ENT-3556 or Blob V2 GA.
This commit is contained in:
@@ -0,0 +1,642 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { createServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { type Table, connect } from "../lancedb";
|
||||
import {
|
||||
Table as ArrowTable,
|
||||
DataType,
|
||||
Field,
|
||||
Float32,
|
||||
LargeBinary,
|
||||
RecordBatch,
|
||||
Schema,
|
||||
Struct,
|
||||
Uint8,
|
||||
Uint32,
|
||||
Uint64,
|
||||
Utf8,
|
||||
makeArrowTable,
|
||||
tableToIPC,
|
||||
vectorFromArray,
|
||||
} from "../lancedb/arrow";
|
||||
|
||||
const IDENTITY_A = `v1:${"a".repeat(64)}`;
|
||||
const IDENTITY_B = `v1:${"b".repeat(64)}`;
|
||||
const image = new Field(
|
||||
"image",
|
||||
new Struct([
|
||||
new Field("kind", new Uint8(), false),
|
||||
new Field("position", new Uint64(), true),
|
||||
new Field("size", new Uint64(), true),
|
||||
new Field("blob_id", new Uint32(), false),
|
||||
new Field("blob_uri", new Utf8(), false),
|
||||
]),
|
||||
true,
|
||||
// Native descriptor scans replace the logical extension metadata.
|
||||
new Map([
|
||||
["lance-encoding:blob", "true"],
|
||||
["lance-encoding:packed", "true"],
|
||||
]),
|
||||
);
|
||||
const schema = new Schema([new Field("label", new Utf8(), false), image]);
|
||||
const description = {
|
||||
fields: [
|
||||
{ name: "label", type: { type: "string" }, nullable: false },
|
||||
{
|
||||
name: "image",
|
||||
nullable: true,
|
||||
metadata: { "ARROW:extension:name": "lance.blob.v2" },
|
||||
type: {
|
||||
type: "struct",
|
||||
fields: ["position", "size"].map((name) => ({
|
||||
name,
|
||||
type: { type: "uint64" },
|
||||
nullable: true,
|
||||
})),
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
type Request = {
|
||||
path: string;
|
||||
body: Record<string, unknown>;
|
||||
raw: string;
|
||||
identity: string | string[] | undefined;
|
||||
};
|
||||
|
||||
interface ServerOptions {
|
||||
queryBlob?: Field;
|
||||
describeBlobName?: string | null;
|
||||
nestedBlob?: boolean;
|
||||
queryVersion?: string | null;
|
||||
queryIdentity?: string | null;
|
||||
echo?: string | null;
|
||||
empty?: boolean;
|
||||
values?: boolean;
|
||||
advanceQueries?: boolean;
|
||||
otherLegIdentity?: string | null;
|
||||
otherLegVersion?: string;
|
||||
}
|
||||
|
||||
// A real HTTP service: the query finishes on A, then the logical table changes
|
||||
// before the deferred blob request. Recreating B reuses both version and row IDs.
|
||||
async function withSnapshotServer(
|
||||
change: "recreate" | "overwrite",
|
||||
run: (table: Table, requests: Request[]) => Promise<void>,
|
||||
options: ServerOptions = {},
|
||||
) {
|
||||
const requests: Request[] = [];
|
||||
let changed = false;
|
||||
const queryBlob = options.queryBlob ?? image;
|
||||
const queryOutputSchema = options.nestedBlob
|
||||
? new Schema([
|
||||
schema.fields[0],
|
||||
new Field("extra", new Struct([queryBlob]), true),
|
||||
])
|
||||
: new Schema([schema.fields[0], queryBlob]);
|
||||
const server = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk) => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
try {
|
||||
const raw = Buffer.concat(chunks).toString() || "{}";
|
||||
const body = JSON.parse(raw);
|
||||
const path = req.url ?? "";
|
||||
const identity = req.headers["x-lancedb-query-snapshot"];
|
||||
requests.push({ path, body, raw, identity });
|
||||
res.setHeader("phalanx-version", "0.5.0");
|
||||
if (path.endsWith("/describe/")) {
|
||||
res.setHeader("content-type", "application/json");
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
version: changed && change === "overwrite" ? 2 : 1,
|
||||
schema: {
|
||||
fields: description.fields.flatMap((field) => {
|
||||
if (field.name !== "image") return [field];
|
||||
if (options.describeBlobName === null) return [];
|
||||
return [
|
||||
{ ...field, name: options.describeBlobName ?? "image" },
|
||||
];
|
||||
}),
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else if (path.endsWith("/query/")) {
|
||||
const vector = (body.vector?.length ?? 0) > 0;
|
||||
const advanced = options.advanceQueries === true && changed;
|
||||
const label = advanced ? "B" : "A";
|
||||
const columns: [string, string][] =
|
||||
body.columns === undefined
|
||||
? queryOutputSchema.fields.map((field) => [
|
||||
field.name,
|
||||
field.name,
|
||||
])
|
||||
: Array.isArray(body.columns)
|
||||
? body.columns.map((name: string) => [name, name])
|
||||
: Object.entries(body.columns);
|
||||
const queryFields = columns.map(([output, source]) =>
|
||||
queryOutputSchema.fields
|
||||
.find((field) => field.name === source)!
|
||||
.clone({ name: output }),
|
||||
);
|
||||
if (body.with_row_id) {
|
||||
queryFields.push(new Field("_rowid", new Uint64(), false));
|
||||
}
|
||||
if (vector || body.full_text_query) {
|
||||
queryFields.push(
|
||||
new Field(vector ? "_distance" : "_score", new Float32(), false),
|
||||
);
|
||||
}
|
||||
// Forged IPC metadata must never substitute for authoritative headers.
|
||||
const querySchema = new Schema(
|
||||
queryFields,
|
||||
new Map([
|
||||
["lancedb:query_version", "999"],
|
||||
["lancedb:query_snapshot", IDENTITY_B],
|
||||
]),
|
||||
);
|
||||
const values = options.empty
|
||||
? []
|
||||
: options.values
|
||||
? ["A", null, ""]
|
||||
: [label];
|
||||
const rows = values.map((value, index) => {
|
||||
const row: Record<string, unknown> = {};
|
||||
for (const [output, source] of columns) {
|
||||
const descriptor =
|
||||
value === null
|
||||
? null
|
||||
: {
|
||||
kind: 0,
|
||||
position: 0n,
|
||||
size: BigInt(value.length),
|
||||
// biome-ignore lint/style/useNamingConvention: Native Arrow field name.
|
||||
blob_id: 0,
|
||||
// biome-ignore lint/style/useNamingConvention: Native Arrow field name.
|
||||
blob_uri: "",
|
||||
};
|
||||
row[output] =
|
||||
source === "image"
|
||||
? descriptor
|
||||
: source === "extra"
|
||||
? descriptor === null
|
||||
? null
|
||||
: { image: descriptor }
|
||||
: label;
|
||||
}
|
||||
if (body.with_row_id) row._rowid = BigInt(index);
|
||||
if (vector || body.full_text_query) {
|
||||
row[vector ? "_distance" : "_score"] = 0.5;
|
||||
}
|
||||
return row;
|
||||
});
|
||||
const result =
|
||||
rows.length === 0
|
||||
? new ArrowTable(querySchema)
|
||||
: new ArrowTable(
|
||||
querySchema,
|
||||
Object.fromEntries(
|
||||
queryFields.map((field) => [
|
||||
field.name,
|
||||
vectorFromArray(
|
||||
rows.map((row) => row[field.name]),
|
||||
field.type,
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
res.setHeader("content-type", "application/vnd.apache.arrow.stream");
|
||||
const version =
|
||||
vector && options.otherLegVersion !== undefined
|
||||
? options.otherLegVersion
|
||||
: options.queryVersion;
|
||||
const snapshot =
|
||||
vector && options.otherLegIdentity !== undefined
|
||||
? options.otherLegIdentity
|
||||
: options.queryIdentity;
|
||||
res.setHeader("x-lancedb-version", "1");
|
||||
if (version !== null) {
|
||||
res.setHeader(
|
||||
"x-lancedb-query-version",
|
||||
version ?? (advanced ? "2" : "1"),
|
||||
);
|
||||
}
|
||||
if (snapshot !== null) {
|
||||
res.setHeader(
|
||||
"x-lancedb-query-snapshot",
|
||||
snapshot ?? (advanced ? IDENTITY_B : IDENTITY_A),
|
||||
);
|
||||
}
|
||||
changed = true;
|
||||
res.end(tableToIPC(result, "stream"));
|
||||
} else if (path.endsWith("/fetch_blobs/")) {
|
||||
const requestedVersion = /"version":([0-9]+)/.exec(raw)?.[1];
|
||||
const readsOriginal =
|
||||
change === "overwrite" &&
|
||||
requestedVersion === (options.queryVersion ?? "1");
|
||||
const actualIdentity = readsOriginal ? IDENTITY_A : IDENTITY_B;
|
||||
// An older serving node may ignore the guard and answer successfully;
|
||||
// its echo must still be verified before the client uses any bytes.
|
||||
if (options.echo !== null) {
|
||||
res.setHeader(
|
||||
"x-lancedb-query-snapshot",
|
||||
options.echo ?? actualIdentity,
|
||||
);
|
||||
}
|
||||
res.setHeader("content-type", "application/vnd.apache.arrow.stream");
|
||||
const payloads = options.values
|
||||
? ["A", null, ""]
|
||||
: [readsOriginal ? "A" : "B"];
|
||||
res.end(
|
||||
tableToIPC(
|
||||
makeArrowTable(
|
||||
(body.row_ids as number[]).map((id) => ({
|
||||
[String(body.column)]:
|
||||
payloads[id] === null ? null : Buffer.from(payloads[id]!),
|
||||
})),
|
||||
{
|
||||
schema: new Schema([
|
||||
new Field(String(body.column), new LargeBinary(), true),
|
||||
]),
|
||||
},
|
||||
),
|
||||
"stream",
|
||||
),
|
||||
);
|
||||
} else {
|
||||
res.writeHead(404).end(path);
|
||||
}
|
||||
} catch (error) {
|
||||
res.writeHead(500).end(String(error));
|
||||
}
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const db = await connect("db://dev", {
|
||||
apiKey: "fake",
|
||||
hostOverride: `http://127.0.0.1:${
|
||||
(server.address() as AddressInfo).port
|
||||
}`,
|
||||
});
|
||||
await run(await db.openTable("test"), requests);
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
const fetchRequests = (requests: Request[]) =>
|
||||
requests.filter((request) => request.path.endsWith("/fetch_blobs/"));
|
||||
const queryRequests = (requests: Request[]) =>
|
||||
requests.filter((request) => request.path.endsWith("/query/"));
|
||||
|
||||
const keepFtsReranker = {
|
||||
async rerankHybrid(_query: string, _vector: RecordBatch, fts: RecordBatch) {
|
||||
const fields = fts.schema.fields.map((field) =>
|
||||
field.name === "_score"
|
||||
? field.clone({ name: "_relevance_score" })
|
||||
: field,
|
||||
);
|
||||
return new RecordBatch(
|
||||
new Schema(fields, fts.schema.metadata),
|
||||
fts.data.clone(new Struct(fields)),
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
describe("remote blob query snapshot", () => {
|
||||
it.each([
|
||||
image.clone({ metadata: new Map() }),
|
||||
new Field(
|
||||
"image",
|
||||
new Struct(image.type.children.slice(1, 3)),
|
||||
true,
|
||||
image.metadata,
|
||||
),
|
||||
])(
|
||||
"leaves plain structs and v1 descriptors unchanged: %s",
|
||||
async (queryBlob) => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const result = await table.query().toArrow({ blobMode: "bytes" });
|
||||
expect(DataType.isStruct(result.getChild("image")!.type)).toBe(true);
|
||||
expect(result.toArray()[0].image.size).toBe(1n);
|
||||
expect(fetchRequests(requests)).toHaveLength(0);
|
||||
},
|
||||
{ queryBlob },
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("still recognizes extension-marked Blob results", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const rows = await table.query().toArray({ blobMode: "bytes" });
|
||||
expect(Buffer.from(rows[0].image).toString()).toBe("A");
|
||||
expect(fetchRequests(requests)).toHaveLength(1);
|
||||
},
|
||||
{
|
||||
queryBlob: image.clone({
|
||||
metadata: new Map([["ARROW:extension:name", "lance.blob.v2"]]),
|
||||
}),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[null, false],
|
||||
[null, true],
|
||||
["old_image", false],
|
||||
["old_image", true],
|
||||
] as const)(
|
||||
"uses the query schema after cached blob %s changes (alias: %s)",
|
||||
async (describeBlobName, alias) => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
expect(await table.blobColumns()).toEqual(
|
||||
describeBlobName === null ? [] : [describeBlobName],
|
||||
);
|
||||
const query = table.query();
|
||||
if (alias) query.select({ "photo.image": "image" });
|
||||
const output = alias ? "photo.image" : "image";
|
||||
const result = await query.toArrow({ blobMode: "bytes" });
|
||||
expect(DataType.isLargeBinary(result.getChild(output)!.type)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
result
|
||||
.toArray()
|
||||
.map((row) =>
|
||||
row[output] === null
|
||||
? null
|
||||
: Buffer.from(row[output]).toString(),
|
||||
),
|
||||
).toEqual(["A", null, ""]);
|
||||
expect(
|
||||
result.schema.fields.some((field) => field.name === "_rowid"),
|
||||
).toBe(false);
|
||||
expect(queryRequests(requests)).toHaveLength(1);
|
||||
expect(fetchRequests(requests)).toHaveLength(1);
|
||||
expect(fetchRequests(requests)[0].body.column).toBe("image");
|
||||
expect(fetchRequests(requests)[0].identity).toBe(IDENTITY_A);
|
||||
},
|
||||
{ describeBlobName, values: true },
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("maps nested query fields through literal aliases despite a stale schema", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
expect(await table.blobColumns()).toEqual([]);
|
||||
const rows = await table
|
||||
.query()
|
||||
.select({ "container.alias": "extra" })
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(
|
||||
rows.map((row) =>
|
||||
row["container.alias"] === null
|
||||
? null
|
||||
: Buffer.from(row["container.alias"].image).toString(),
|
||||
),
|
||||
).toEqual(["A", null, ""]);
|
||||
expect(queryRequests(requests)).toHaveLength(1);
|
||||
expect(fetchRequests(requests)[0].body.column).toBe("extra.image");
|
||||
expect(fetchRequests(requests)[0].identity).toBe(IDENTITY_A);
|
||||
},
|
||||
{ nestedBlob: true, describeBlobName: null, values: true },
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["toArrow", "toArray"] as const)(
|
||||
"%s rejects payloads from a recreated table with reused version and row IDs",
|
||||
async (terminal) => {
|
||||
await withSnapshotServer("recreate", async (table, requests) => {
|
||||
await expect(
|
||||
table.query()[terminal]({ blobMode: "bytes" }),
|
||||
).rejects.toThrow(/snapshot identity/);
|
||||
const [fetch] = fetchRequests(requests);
|
||||
expect(fetch.body.version).toBe(1);
|
||||
expect(fetch.body.row_ids).toEqual([0]);
|
||||
expect(fetch.body.column).toBe("image");
|
||||
expect(fetch.identity).toBe(IDENTITY_A);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the producing snapshot after an overwrite without rerunning the query", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const rows = await table.query().toArray({ blobMode: "bytes" });
|
||||
expect(rows[0].label).toBe("A");
|
||||
expect(Buffer.from(rows[0].image).toString()).toBe("A");
|
||||
expect(queryRequests(requests)).toHaveLength(1);
|
||||
const [fetch] = fetchRequests(requests);
|
||||
expect(fetch.body.version).toBe(1);
|
||||
expect(fetch.identity).toBe(IDENTITY_A);
|
||||
expect(await table.version()).toBe(2);
|
||||
const [latest] = await table.query().toArray({ blobMode: "bytes" });
|
||||
expect(latest.label).toBe("B");
|
||||
expect(Buffer.from(latest.image).toString()).toBe("B");
|
||||
expect(fetchRequests(requests)[1].identity).toBe(IDENTITY_B);
|
||||
},
|
||||
{ advanceQueries: true },
|
||||
);
|
||||
});
|
||||
|
||||
it.each<ServerOptions>([
|
||||
{ queryVersion: null },
|
||||
{ queryVersion: "0" },
|
||||
{ queryVersion: "18446744073709551616" },
|
||||
{ queryVersion: "invalid" },
|
||||
{ queryIdentity: null },
|
||||
{ queryIdentity: "v2:unsupported" },
|
||||
])("refuses bytes without authoritative capability %j", async (options) => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const [descriptor] = await table.query().toArray();
|
||||
expect(descriptor.image.size).toBe(1n);
|
||||
await expect(
|
||||
table.query().toArray({ blobMode: "bytes" }),
|
||||
).rejects.toThrow(/blobMode.*bytes.*requires.*snapshot/);
|
||||
expect(fetchRequests(requests)).toHaveLength(0);
|
||||
},
|
||||
options,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a serving node without the snapshot echo", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table) => {
|
||||
await expect(
|
||||
table.query().toArray({ blobMode: "bytes" }),
|
||||
).rejects.toThrow(/snapshot identity/);
|
||||
},
|
||||
{ echo: null },
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps aliases, NULL, empty bytes, and explicit row IDs", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const query = table
|
||||
.query()
|
||||
.select({ caption: "label", "photo.image": "image" });
|
||||
const rows = await query.withRowId().toArray({ blobMode: "bytes" });
|
||||
expect(rows.map((row) => row._rowid)).toEqual([0n, 1n, 2n]);
|
||||
expect(
|
||||
rows.map((row) =>
|
||||
row["photo.image"] === null
|
||||
? null
|
||||
: Buffer.from(row["photo.image"]).toString(),
|
||||
),
|
||||
).toEqual(["A", null, ""]);
|
||||
expect(rows[0].caption).toBe("A");
|
||||
expect(fetchRequests(requests)[0].body.column).toBe("image");
|
||||
const [descriptor] = await query.toArray();
|
||||
expect(descriptor["photo.image"].size).toBe(1n);
|
||||
},
|
||||
{ values: true },
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves an empty bytes schema without fetching payloads", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const result = await table.query().toArrow({ blobMode: "bytes" });
|
||||
expect(result.numRows).toBe(0);
|
||||
expect(DataType.isLargeBinary(result.getChild("image")!.type)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(result.schema.fields.map((field) => field.name)).toEqual([
|
||||
"label",
|
||||
"image",
|
||||
]);
|
||||
expect(fetchRequests(requests)).toHaveLength(0);
|
||||
},
|
||||
{ empty: true },
|
||||
);
|
||||
});
|
||||
|
||||
it("passes the query version without rounding through a JS number", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
await table.query().toArray({ blobMode: "bytes" });
|
||||
expect(fetchRequests(requests)[0].raw).toContain(
|
||||
'"version":9007199254740993',
|
||||
);
|
||||
},
|
||||
{ queryVersion: "9007199254740993" },
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["vector", "hybrid"])(
|
||||
"materializes the actual %s query entry point",
|
||||
async (kind) => {
|
||||
await withSnapshotServer("overwrite", async (table, requests) => {
|
||||
const query = table.query().nearestTo([0.1, 0.2]);
|
||||
if (kind === "hybrid") query.fullTextSearch("A");
|
||||
const rows = await query.toArray({ blobMode: "bytes" });
|
||||
expect(Buffer.from(rows[0].image).toString()).toBe("A");
|
||||
expect(fetchRequests(requests)[0].identity).toBe(IDENTITY_A);
|
||||
expect(queryRequests(requests)).toHaveLength(kind === "hybrid" ? 2 : 1);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("allows scalar-only bytes queries on older servers", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const [row] = await table
|
||||
.query()
|
||||
.select(["label"])
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(row.label).toBe("A");
|
||||
expect(Object.keys(row)).toEqual(["label"]);
|
||||
expect(fetchRequests(requests)).toHaveLength(0);
|
||||
// Row IDs are requested before the executed output schema is known,
|
||||
// then hidden from scalar-only results without requiring capability.
|
||||
expect(queryRequests(requests)[0].body.with_row_id).toBe(true);
|
||||
},
|
||||
{ queryIdentity: null, queryVersion: null },
|
||||
);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"preserves snapshot proof across a custom reranker (empty: %s)",
|
||||
async (empty) => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const result = await table
|
||||
.query()
|
||||
.nearestTo([0.1, 0.2])
|
||||
.fullTextSearch("A")
|
||||
.rerank(keepFtsReranker)
|
||||
.toArrow({ blobMode: "bytes" });
|
||||
expect(result.numRows).toBe(empty ? 0 : 1);
|
||||
expect(DataType.isLargeBinary(result.getChild("image")!.type)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(fetchRequests(requests)).toHaveLength(empty ? 0 : 1);
|
||||
},
|
||||
{ empty },
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not trust a custom reranker's first-leg snapshot when the legs disagree", async () => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const query = table
|
||||
.query()
|
||||
.nearestTo([0.1, 0.2])
|
||||
.fullTextSearch("A")
|
||||
.rerank(keepFtsReranker);
|
||||
await expect(query.toArray({ blobMode: "bytes" })).rejects.toThrow(
|
||||
/snapshot/,
|
||||
);
|
||||
expect(fetchRequests(requests)).toHaveLength(0);
|
||||
},
|
||||
{ otherLegIdentity: IDENTITY_B },
|
||||
);
|
||||
});
|
||||
|
||||
it.each<ServerOptions>([
|
||||
{ otherLegIdentity: IDENTITY_B },
|
||||
{ otherLegIdentity: null },
|
||||
{ otherLegVersion: "2" },
|
||||
])(
|
||||
"refuses hybrid bytes when its actual query legs disagree %j",
|
||||
async (options) => {
|
||||
await withSnapshotServer(
|
||||
"overwrite",
|
||||
async (table, requests) => {
|
||||
const query = table.query().nearestTo([0.1, 0.2]).fullTextSearch("A");
|
||||
await expect(query.toArray({ blobMode: "bytes" })).rejects.toThrow(
|
||||
/snapshot/,
|
||||
);
|
||||
expect(fetchRequests(requests)).toHaveLength(0);
|
||||
expect(await query.toArray()).toHaveLength(1);
|
||||
},
|
||||
options,
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
+41
-1
@@ -1,7 +1,14 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { Field, LargeBinary, Struct, Uint64, Utf8 } from "apache-arrow";
|
||||
import {
|
||||
DataType,
|
||||
Field,
|
||||
LargeBinary,
|
||||
Struct,
|
||||
Uint64,
|
||||
Utf8,
|
||||
} from "apache-arrow";
|
||||
import { BlobFile as NativeBlobFile } from "./native";
|
||||
|
||||
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
|
||||
@@ -178,6 +185,39 @@ export function isBlobField(field: Field): boolean {
|
||||
return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
export function isBlobDescriptorField(field: Field): boolean {
|
||||
// Native descriptor scans replace the logical extension metadata. Require
|
||||
// both the encoding marker and the v2 layout to exclude v1 and plain structs.
|
||||
if (
|
||||
field.metadata?.get("lance-encoding:blob") !== "true" ||
|
||||
!DataType.isStruct(field.type)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const children = field.type.children;
|
||||
const integers = [
|
||||
["kind", 8],
|
||||
["position", 64],
|
||||
["size", 64],
|
||||
["blob_id", 32],
|
||||
] as const;
|
||||
return (
|
||||
children.length === 5 &&
|
||||
integers.every(([name, width], index) => {
|
||||
const child = children[index];
|
||||
return (
|
||||
child.name === name &&
|
||||
DataType.isInt(child.type) &&
|
||||
!child.type.isSigned &&
|
||||
child.type.bitWidth === width
|
||||
);
|
||||
}) &&
|
||||
children[4].name === "blob_uri" &&
|
||||
DataType.isUtf8(children[4].type)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
|
||||
*
|
||||
|
||||
+65
-6
@@ -17,6 +17,7 @@ import {
|
||||
tableFromIPC,
|
||||
vectorFromArray,
|
||||
} from "./arrow";
|
||||
import { isBlobDescriptorField, isBlobField } from "./blob";
|
||||
import { Expr } from "./expr";
|
||||
import { type IvfPqOptions } from "./indices";
|
||||
import {
|
||||
@@ -148,6 +149,34 @@ async function readBlobBytes(
|
||||
);
|
||||
}
|
||||
|
||||
// Discover descriptors from the executed query, not the live table's cached
|
||||
// schema. Projection names are literal aliases; only source paths use dots.
|
||||
function projectedBlobSources(
|
||||
schema: Schema,
|
||||
projection: [string, string][] | undefined,
|
||||
): BlobSource[] {
|
||||
const sources: BlobSource[] = [];
|
||||
for (const field of schema.fields) {
|
||||
const source =
|
||||
projection?.find(([output]) => output === field.name)?.[1] ?? field.name;
|
||||
const visit = (child: Field, path: string[]) => {
|
||||
if (isBlobField(child) || isBlobDescriptorField(child)) {
|
||||
sources.push({
|
||||
output: field.name,
|
||||
path,
|
||||
source: [source, ...path].join("."),
|
||||
});
|
||||
} else if (DataType.isStruct(child.type)) {
|
||||
for (const nested of child.type.children) {
|
||||
visit(nested, [...path, nested.name]);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(field, []);
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
|
||||
function withBinaryColumns(
|
||||
table: ArrowTable,
|
||||
replacements: Map<Field, (Buffer | null)[]>,
|
||||
@@ -257,8 +286,10 @@ export interface QueryExecutionOptions {
|
||||
* bytes. Blobs inside lists keep their descriptors. Arbitrary projection
|
||||
* expressions are not materialized as bytes.
|
||||
*
|
||||
* The rows and the bytes come from the same table version. If the table
|
||||
* changes while the query runs, the query runs again, up to three times.
|
||||
* The rows and bytes come from the same snapshot. Remote queries require
|
||||
* authoritative query version and snapshot identity headers, verified again
|
||||
* by the blob response. Older servers can still return `"descriptions"`.
|
||||
* Local queries retry up to three times if the table changes while they run.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
@@ -499,6 +530,34 @@ export class QueryBase<
|
||||
if (context === undefined) {
|
||||
return this.collect(options, false);
|
||||
}
|
||||
if (!context.table.isNative()) {
|
||||
// Request row IDs in the same query so newly added Blob columns can be
|
||||
// fetched without rerunning the query against a different snapshot.
|
||||
const addRowId = !context.withRowId;
|
||||
const table = await this.collect(options, addRowId);
|
||||
const sources = projectedBlobSources(table.schema, context.projection);
|
||||
if (sources.length === 0) {
|
||||
return addRowId
|
||||
? table.select(
|
||||
table.schema.fields
|
||||
.map((field) => field.name)
|
||||
.filter((name) => name !== "_rowid"),
|
||||
)
|
||||
: table;
|
||||
}
|
||||
const version = table.schema.metadata.get("lancedb:query_version");
|
||||
const identity = table.schema.metadata.get("lancedb:query_snapshot");
|
||||
if (version === undefined || identity === undefined) {
|
||||
throw new Error(
|
||||
'blobMode "bytes" requires a trustworthy query snapshot: missing, invalid, or inconsistent x-lancedb-query-version or x-lancedb-query-snapshot. Upgrade every serving node, or use blobMode "descriptions".',
|
||||
);
|
||||
}
|
||||
const snapshot = await context.table.querySnapshotWithIdentity(
|
||||
version,
|
||||
identity,
|
||||
);
|
||||
return readBlobBytes(table, sources, snapshot, addRowId);
|
||||
}
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
// The snapshot pins the bytes. The query runs on the live table, so an
|
||||
// unchanged version afterwards proves it read the snapshot's version.
|
||||
@@ -533,10 +592,10 @@ export class QueryBase<
|
||||
for await (const batch of RecordBatchIterator(iterator)) {
|
||||
batches.push(batch);
|
||||
}
|
||||
if (batches.length === 0) {
|
||||
return tableFromIPC((await iterator).schema());
|
||||
}
|
||||
return new ArrowTable(batches);
|
||||
// Read metadata from the executed stream, including empty results and
|
||||
// query types whose individual batches do not carry the stream metadata.
|
||||
const schema = tableFromIPC((await iterator).schema()).schema;
|
||||
return new ArrowTable(schema, batches);
|
||||
}
|
||||
|
||||
private blobBytesContext(
|
||||
|
||||
@@ -349,6 +349,31 @@ impl Table {
|
||||
Ok(Query::new(self.inner_ref()?.query()))
|
||||
}
|
||||
|
||||
/// Whether this handle reads an in-process dataset.
|
||||
#[napi]
|
||||
pub fn is_native(&self) -> napi::Result<bool> {
|
||||
Ok(self.inner_ref()?.as_native().is_some())
|
||||
}
|
||||
|
||||
/// Constrain deferred blob reads by the producing query response.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn query_snapshot_with_identity(
|
||||
&self,
|
||||
version: String,
|
||||
identity: String,
|
||||
) -> napi::Result<Self> {
|
||||
// Keep u64 query versions exact across the JavaScript boundary.
|
||||
let version = version
|
||||
.parse::<u64>()
|
||||
.map_err(|_| napi::Error::from_reason("Invalid query snapshot version"))?;
|
||||
let snapshot = self
|
||||
.inner_ref()?
|
||||
.query_snapshot_with_identity(version, &identity)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(Self::new(snapshot))
|
||||
}
|
||||
|
||||
/// Return a read-only table handle pinned to the current query revision.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn query_snapshot(&self) -> napi::Result<Self> {
|
||||
|
||||
Reference in New Issue
Block a user