mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-03 12:08:52 +00:00
Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2057-1
# Conflicts: # rust/lancedb/src/remote/table.rs
This commit is contained in:
Gatefixer
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.38.0-beta.3"
|
||||
version = "0.38.0-beta.10"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -173,6 +173,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
}
|
||||
|
||||
describe("The function makeArrowTable", function () {
|
||||
it("accepts snake_case embedding metadata like camelCase", function () {
|
||||
const spellings = [
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
{ source_column: "text", vector_column: "vector" },
|
||||
{ sourceColumn: "text", vectorColumn: "vector" },
|
||||
];
|
||||
for (const columns of spellings) {
|
||||
const schema = new Schema(
|
||||
[
|
||||
new Field("text", new Utf8(), false),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||
false,
|
||||
),
|
||||
],
|
||||
new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
JSON.stringify([{ name: "mock", model: {}, ...columns }]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
// The vector field is non-nullable and absent from the data; only a
|
||||
// recognized embedding config makes that acceptable.
|
||||
const table = makeArrowTable([{ text: "hello" }], { schema });
|
||||
expect(table.numRows).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("will use data types from a provided schema instead of inference", async function () {
|
||||
const schema = new Schema([
|
||||
new Field("a", new Int32(), false),
|
||||
@@ -485,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
);
|
||||
});
|
||||
|
||||
it("will allow matching inferred types across records", function () {
|
||||
expect(() =>
|
||||
makeArrowTable([{ value: 1 }, { value: 2 }]),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("will reject mismatched inferred types across records", function () {
|
||||
expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.",
|
||||
);
|
||||
});
|
||||
|
||||
it("will ignore generated dictionary IDs when comparing inferred types", function () {
|
||||
const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
|
||||
dictionaryEncodeStrings: true,
|
||||
});
|
||||
|
||||
expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("will preserve null values without treating them as type mismatches", function () {
|
||||
for (const records of [
|
||||
[{ vector: [1, 2, 3] }, { vector: null }],
|
||||
[{ vector: null }, { vector: [1, 2, 3] }],
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
|
||||
expect(table.numRows).toBe(2);
|
||||
expect(table.getChild("vector")?.nullCount).toBe(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("will preserve empty variable-size lists", function () {
|
||||
for (const records of [
|
||||
[{ items: [1] }, { items: [] }],
|
||||
[{ items: [] }, { items: [1] }],
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
expect(
|
||||
table
|
||||
.getChild("items")
|
||||
?.toJSON()
|
||||
.map((value) => value.toJSON()),
|
||||
).toEqual(records.map((record) => record.items));
|
||||
}
|
||||
});
|
||||
|
||||
it("will propagate deferred evidence through nested lists", function () {
|
||||
for (const records of [
|
||||
[{ items: [1] }, { items: [null] }],
|
||||
[{ items: [null] }, { items: [1] }],
|
||||
[{ items: [null, 1] }, { items: [2, null] }],
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
expect(
|
||||
table
|
||||
.getChild("items")
|
||||
?.toJSON()
|
||||
.map((value) => value.toJSON()),
|
||||
).toEqual(records.map((record) => record.items));
|
||||
}
|
||||
|
||||
const nestedRecords = [{ items: [[1]] }, { items: [[null]] }];
|
||||
const nestedTable = makeArrowTable(nestedRecords);
|
||||
expect(
|
||||
nestedTable
|
||||
.getChild("items")
|
||||
?.toJSON()
|
||||
.map((value) =>
|
||||
value
|
||||
.toJSON()
|
||||
.map((nestedValue: { toJSON: () => unknown[] }) =>
|
||||
nestedValue.toJSON(),
|
||||
),
|
||||
),
|
||||
).toEqual(nestedRecords.map((record) => record.items));
|
||||
});
|
||||
|
||||
it("will reject incompatible deferred evidence within a list", function () {
|
||||
for (const items of [
|
||||
[[], 1],
|
||||
[1, []],
|
||||
[[null], 1],
|
||||
[1, [null]],
|
||||
]) {
|
||||
expect(() => makeArrowTable([{ items }])).toThrow(
|
||||
"Failed to infer data type for field items at row 0.",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("will reject empty fixed-size lists", function () {
|
||||
expect(() =>
|
||||
makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]),
|
||||
).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type FixedSizeList[3]<Float32> but found List[0] for field vector at row 1.",
|
||||
);
|
||||
});
|
||||
|
||||
it("will reject inferred leaf and branch shape changes", function () {
|
||||
expect(() =>
|
||||
makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]),
|
||||
).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.",
|
||||
);
|
||||
expect(() =>
|
||||
makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]),
|
||||
).toThrow(
|
||||
"Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.",
|
||||
);
|
||||
});
|
||||
|
||||
it("will allow null values around inferred struct values", function () {
|
||||
for (const { records, nullIndex } of [
|
||||
{
|
||||
records: [{ value: null }, { value: { nested: 2 } }],
|
||||
nullIndex: 0,
|
||||
},
|
||||
{
|
||||
records: [{ value: { nested: 1 } }, { value: null }],
|
||||
nullIndex: 1,
|
||||
},
|
||||
]) {
|
||||
const table = makeArrowTable(records);
|
||||
const values = table.getChild("value");
|
||||
|
||||
expect(values?.nullCount).toBe(1);
|
||||
expect(values?.get(nullIndex)).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it("will allow a schema to be provided", async function () {
|
||||
await checkTableCreation(
|
||||
async (records, _, schema) =>
|
||||
|
||||
@@ -4,7 +4,13 @@
|
||||
import { readdirSync } from "fs";
|
||||
import { Field, Float64, Schema } from "apache-arrow";
|
||||
import * as tmp from "tmp";
|
||||
import { Connection, Table, connect, connectNamespace } from "../lancedb";
|
||||
import {
|
||||
Connection,
|
||||
ListTablesResponse,
|
||||
Table,
|
||||
connect,
|
||||
connectNamespace,
|
||||
} from "../lancedb";
|
||||
import { LocalTable } from "../lancedb/table";
|
||||
|
||||
describe("when connecting", () => {
|
||||
@@ -47,6 +53,7 @@ describe("given a connection", () => {
|
||||
await db.close();
|
||||
expect(db.isOpen()).toBe(false);
|
||||
await expect(db.tableNames()).rejects.toThrow("Connection is closed");
|
||||
await expect(db.listTables()).rejects.toThrow("Connection is closed");
|
||||
await expect(db.renameTable("a", "b")).rejects.toThrow(
|
||||
"Connection is closed",
|
||||
);
|
||||
@@ -129,6 +136,66 @@ describe("given a connection", () => {
|
||||
expect(tables).toEqual(["b", "c"]);
|
||||
});
|
||||
|
||||
it("should respect limit and page token when listing tables", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
|
||||
await db.createTable("b", [{ id: 1 }]);
|
||||
await db.createTable("a", [{ id: 1 }]);
|
||||
await db.createTable("c", [{ id: 1 }]);
|
||||
|
||||
const all = await db.listTables();
|
||||
expect(all.tables).toEqual(["a", "b", "c"]);
|
||||
expect(all.pageToken).toBeUndefined();
|
||||
|
||||
const first = await db.listTables({ limit: 1 });
|
||||
expect(first.tables).toEqual(["a"]);
|
||||
expect(first.pageToken).toBeDefined();
|
||||
|
||||
const second = await db.listTables({
|
||||
limit: 1,
|
||||
pageToken: first.pageToken,
|
||||
});
|
||||
expect(second.tables).toEqual(["b"]);
|
||||
});
|
||||
|
||||
it("should visit every table exactly once when walking pages", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
|
||||
const created = ["a", "b", "c", "d", "e"];
|
||||
for (const name of created) {
|
||||
await db.createTable(name, [{ id: 1 }]);
|
||||
}
|
||||
|
||||
const seen: string[] = [];
|
||||
let pageToken: string | undefined = undefined;
|
||||
do {
|
||||
const page: ListTablesResponse = await db.listTables({
|
||||
limit: 2,
|
||||
pageToken,
|
||||
});
|
||||
seen.push(...page.tables);
|
||||
pageToken = page.pageToken;
|
||||
} while (pageToken);
|
||||
|
||||
expect(seen).toEqual(created);
|
||||
});
|
||||
|
||||
it("should list tables in a namespace", async () => {
|
||||
const db = await connect(tmpDir.name, {
|
||||
// biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
|
||||
namespaceClientProperties: { manifest_enabled: "true" },
|
||||
});
|
||||
await db.createNamespace(["child"]);
|
||||
await db.createTable("nested", [{ id: 1 }], ["child"]);
|
||||
|
||||
await expect(db.listTables(["child"])).resolves.toEqual(
|
||||
expect.objectContaining({ tables: ["nested"] }),
|
||||
);
|
||||
await expect(db.listTables()).resolves.toEqual(
|
||||
expect.objectContaining({ tables: [] }),
|
||||
);
|
||||
});
|
||||
|
||||
it("should create tables in v2 mode", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [...Array(10000).keys()].map((i) => ({ id: i }));
|
||||
|
||||
@@ -487,4 +487,52 @@ describe("embedding functions", () => {
|
||||
expect(stringSchema3).toEqual(stringExpectedSchema);
|
||||
},
|
||||
);
|
||||
test("parses one function writing several vector columns", async () => {
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return Array.from({ length: data.length }).fill([
|
||||
1, 2, 3,
|
||||
]) as number[][];
|
||||
}
|
||||
}
|
||||
const registry = getRegistry();
|
||||
registry.register("multi_output_mock")(MockEmbeddingFunction);
|
||||
|
||||
// A materialized view can project one source vector column under two
|
||||
// names, so a table's configuration names the same function twice.
|
||||
const parsed = await registry.parseFunctions(
|
||||
new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
JSON.stringify([
|
||||
{
|
||||
name: "multi_output_mock",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_a",
|
||||
model: {},
|
||||
},
|
||||
{
|
||||
name: "multi_output_mock",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_b",
|
||||
model: {},
|
||||
},
|
||||
]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
|
||||
expect(
|
||||
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
|
||||
).toEqual(["vector_a", "vector_b"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
|
||||
import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
|
||||
|
||||
type EmbeddingModule = typeof import("../lancedb/embedding");
|
||||
type OpenAIModule = typeof import("../lancedb/embedding/openai");
|
||||
type RegistryModule = typeof import("../lancedb/embedding/registry");
|
||||
|
||||
describe("embedding function registry", () => {
|
||||
const registries: EmbeddingFunctionRegistry[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const registry of registries) {
|
||||
registry.reset();
|
||||
}
|
||||
registries.length = 0;
|
||||
});
|
||||
|
||||
it("defers built-in providers until the public registry API is used", () => {
|
||||
jest.isolateModules(() => {
|
||||
const embedding = require("../lancedb/embedding") as EmbeddingModule;
|
||||
const { getRegistry: getInternalRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
const registry = getInternalRegistry();
|
||||
registries.push(registry);
|
||||
|
||||
expect(registry.length()).toBe(0);
|
||||
expect(embedding.getRegistry()).toBe(registry);
|
||||
expect(registry.get("openai")).toBeDefined();
|
||||
expect(registry.get("huggingface")).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves automatic FTS search in a fresh process", () => {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
});
|
||||
|
||||
it("shares registrations across duplicated provider module graphs", () => {
|
||||
let registeringRegistry: EmbeddingFunctionRegistry | undefined;
|
||||
let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
|
||||
|
||||
jest.isolateModules(() => {
|
||||
require("../lancedb/embedding/openai");
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registeringRegistry = getRegistry();
|
||||
registries.push(registeringRegistry);
|
||||
expect(registeringRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
|
||||
expect(() => {
|
||||
jest.isolateModules(() => {
|
||||
const { OpenAIEmbeddingFunction } =
|
||||
require("../lancedb/embedding/openai") as OpenAIModule;
|
||||
latestOpenAIConstructor = OpenAIEmbeddingFunction;
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding/registry") as RegistryModule;
|
||||
registries.push(getRegistry());
|
||||
});
|
||||
}).not.toThrow();
|
||||
|
||||
const previousApiKey = process.env.OPENAI_API_KEY;
|
||||
process.env.OPENAI_API_KEY = "test";
|
||||
try {
|
||||
const latestOpenAI = registeringRegistry!
|
||||
.get<OpenAIEmbeddingFunction>("openai")!
|
||||
.create();
|
||||
expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
|
||||
} finally {
|
||||
if (previousApiKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = previousApiKey;
|
||||
}
|
||||
}
|
||||
|
||||
jest.isolateModules(() => {
|
||||
const { getRegistry } =
|
||||
require("../lancedb/embedding") as EmbeddingModule;
|
||||
const publicRegistry = getRegistry();
|
||||
registries.push(publicRegistry);
|
||||
expect(publicRegistry).toBe(registeringRegistry);
|
||||
expect(publicRegistry.get("openai")).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
const assert = require("node:assert/strict");
|
||||
const tmp = require("tmp");
|
||||
const { connect, embedding, Index } = require("../../dist");
|
||||
const { getRegistry } = require("../../dist/embedding/registry");
|
||||
|
||||
async function main() {
|
||||
assert.equal(typeof embedding.getRegistry, "function");
|
||||
assert.equal(getRegistry().length(), 0);
|
||||
assert.equal(embedding.getRegistry(), getRegistry());
|
||||
assert.equal(getRegistry().length(), 2);
|
||||
|
||||
const dir = tmp.dirSync({ unsafeCleanup: true });
|
||||
let db;
|
||||
try {
|
||||
db = await connect(dir.name);
|
||||
const table = await db.createTable("docs", [{ text: "hello world" }]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const rows = await table.search("hello").toArray();
|
||||
assert.equal(rows[0].text, "hello world");
|
||||
} finally {
|
||||
db?.close();
|
||||
dir.removeCallback();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,147 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import { Connection, connect } from "../lancedb";
|
||||
import {
|
||||
DEFINITION_META_KEY,
|
||||
definitionFromMetadata,
|
||||
} from "../lancedb/materialized_view";
|
||||
|
||||
describe("materialized views", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let db: Connection;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
db = await connect(tmpDir.name);
|
||||
await db.createTable(
|
||||
"people",
|
||||
[
|
||||
{ name: "ada", age: 36 },
|
||||
{ name: "kid", age: 7 },
|
||||
{ name: "grace", age: 85 },
|
||||
],
|
||||
{ storageOptions: { newTableEnableStableRowIds: "true" } },
|
||||
);
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("rejects a stored limit a number cannot carry", () => {
|
||||
const big = new Map([
|
||||
[
|
||||
DEFINITION_META_KEY,
|
||||
'{"kind":"select","source_table":"people","limit":9007199254740993}',
|
||||
],
|
||||
]);
|
||||
expect(() => definitionFromMetadata(big, "v")).toThrow(
|
||||
/too large to represent exactly/,
|
||||
);
|
||||
|
||||
const safe = new Map([
|
||||
[
|
||||
DEFINITION_META_KEY,
|
||||
'{"kind":"select","source_table":"people","limit":42}',
|
||||
],
|
||||
]);
|
||||
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
|
||||
});
|
||||
|
||||
it("creates, refreshes and queries a view", async () => {
|
||||
const view = await db.createMaterializedView("adults", "people", {
|
||||
select: ["name", ["shout", "upper(name)"]],
|
||||
where: "age >= 18",
|
||||
});
|
||||
expect(view.name).toBe("adults");
|
||||
expect(await view.table().countRows()).toBe(0);
|
||||
|
||||
const result = await view.refresh();
|
||||
expect(result.mode).toBe("rebuild");
|
||||
expect(Number(result.rowsWritten)).toBe(2);
|
||||
|
||||
const rows = await view.table().query().toArray();
|
||||
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
|
||||
});
|
||||
|
||||
it("round-trips the definition", async () => {
|
||||
await db.createMaterializedView("adults", "people", {
|
||||
where: "age >= 18",
|
||||
});
|
||||
const view = await db.openMaterializedView("adults");
|
||||
const definition = await view.definition();
|
||||
expect(definition.sourceTable).toBe("people");
|
||||
expect(definition.filter).toBe("age >= 18");
|
||||
expect(definition.projections).toEqual([
|
||||
["name", "`name`"],
|
||||
["age", "`age`"],
|
||||
]);
|
||||
expect(definition.inputs).toEqual(["age", "name"]);
|
||||
});
|
||||
|
||||
it("refreshes incrementally after an append", async () => {
|
||||
const view = await db.createMaterializedView("copy", "people");
|
||||
await view.refresh();
|
||||
|
||||
const people = await db.openTable("people");
|
||||
await people.add([{ name: "alan", age: 41 }]);
|
||||
const result = await view.refresh();
|
||||
expect(result.mode).toBe("incremental");
|
||||
expect(Number(result.rowsWritten)).toBe(1);
|
||||
expect(await view.table().countRows()).toBe(4);
|
||||
|
||||
expect((await view.refresh()).mode).toBe("no_op");
|
||||
});
|
||||
|
||||
it("lists views and rejects non-views", async () => {
|
||||
await db.createMaterializedView("adults", "people", {
|
||||
where: "age >= 18",
|
||||
});
|
||||
expect(await db.listMaterializedViews()).toEqual(["adults"]);
|
||||
await expect(db.openMaterializedView("people")).rejects.toThrow(
|
||||
"not a materialized view",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an invalid expression at create time", async () => {
|
||||
await expect(
|
||||
db.createMaterializedView("bad", "people", {
|
||||
select: [["x", "missing + 1"]],
|
||||
}),
|
||||
).rejects.toThrow("missing");
|
||||
});
|
||||
|
||||
it("rejects invalid numeric options before creating anything", async () => {
|
||||
for (const limit of [-5, 1.5, Infinity, NaN]) {
|
||||
await expect(
|
||||
db.createMaterializedView("bad", "people", { limit }),
|
||||
).rejects.toThrow("non-negative integer");
|
||||
}
|
||||
expect(await db.listMaterializedViews()).toEqual([]);
|
||||
|
||||
const view = await db.createMaterializedView("copy", "people");
|
||||
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
|
||||
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
|
||||
"non-negative integer",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("quotes bare select names", async () => {
|
||||
await db.createTable("odd_names", [{ "order item": "widget" }], {
|
||||
storageOptions: { newTableEnableStableRowIds: "true" },
|
||||
});
|
||||
const view = await db.createMaterializedView("quoted", "odd_names", {
|
||||
select: ["order item"],
|
||||
});
|
||||
const result = await view.refresh();
|
||||
expect(Number(result.rowsWritten)).toBe(1);
|
||||
});
|
||||
|
||||
it("requires stable row ids on the source", async () => {
|
||||
await db.createTable("plain", [{ x: 1 }]);
|
||||
await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
|
||||
"stable row ids",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -106,6 +106,77 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
|
||||
'Embedding function with alias "mock-embedding" already exists',
|
||||
);
|
||||
});
|
||||
test("parseFunctions keeps entries sharing a function name", async () => {
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32() as apiArrow.Float;
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map(() => [1, 2, 3]);
|
||||
}
|
||||
}
|
||||
register("mock-embedding")(MockEmbeddingFunction);
|
||||
const parsed = await getRegistry().parseFunctions(
|
||||
new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
JSON.stringify([
|
||||
{
|
||||
name: "mock-embedding",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_a",
|
||||
model: {},
|
||||
},
|
||||
{
|
||||
name: "mock-embedding",
|
||||
sourceColumn: "text",
|
||||
vectorColumn: "vector_b",
|
||||
model: {},
|
||||
},
|
||||
]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
expect([...parsed.values()].map((f) => f.vectorColumn)).toEqual([
|
||||
"vector_a",
|
||||
"vector_b",
|
||||
]);
|
||||
|
||||
// The Python bindings write snake_case keys.
|
||||
const snake = await getRegistry().parseFunctions(
|
||||
new Map([
|
||||
[
|
||||
"embedding_functions",
|
||||
JSON.stringify([
|
||||
{
|
||||
name: "mock-embedding",
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
source_column: "text",
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
vector_column: "vector_a",
|
||||
model: {},
|
||||
},
|
||||
{
|
||||
name: "mock-embedding",
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
source_column: "text",
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
vector_column: "vector_b",
|
||||
model: {},
|
||||
},
|
||||
]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
expect([...snake.keys()]).toEqual(["vector_a", "vector_b"]);
|
||||
expect([...snake.values()].map((f) => f.sourceColumn)).toEqual([
|
||||
"text",
|
||||
"text",
|
||||
]);
|
||||
});
|
||||
test("schema should contain correct metadata", async () => {
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
constructor(args: FunctionOptions = {}) {
|
||||
|
||||
@@ -75,6 +75,25 @@ async function withMockDatabase(
|
||||
}
|
||||
|
||||
describe("remote connection", () => {
|
||||
it("refuses materialized views before issuing any request", async () => {
|
||||
const paths: string[] = [];
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
paths.push(req.url ?? "");
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
|
||||
/only on local databases/,
|
||||
);
|
||||
await expect(db.listMaterializedViews()).rejects.toThrow(
|
||||
/only on local databases/,
|
||||
);
|
||||
expect(paths).toEqual([]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should accept partial connection options", async () => {
|
||||
await connect("db://test", {
|
||||
apiKey: "fake",
|
||||
@@ -311,7 +330,7 @@ describe("remote connection", () => {
|
||||
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
|
||||
});
|
||||
|
||||
it("diffs and merges remote branches", async () => {
|
||||
it("diffs and cherry-picks remote branches", async () => {
|
||||
const sampleDiff = {
|
||||
fromBranch: "exp",
|
||||
parentVersion: 1,
|
||||
@@ -333,10 +352,9 @@ describe("remote connection", () => {
|
||||
changedColumns: [],
|
||||
addedIndexes: [],
|
||||
removedIndexes: [],
|
||||
mergeable: true,
|
||||
mergeBlockers: [],
|
||||
errors: [],
|
||||
};
|
||||
const mergeBodies: Record<string, unknown>[] = [];
|
||||
const cherryPickBodies: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
@@ -366,17 +384,16 @@ describe("remote connection", () => {
|
||||
.end(JSON.stringify(sampleDiff));
|
||||
return;
|
||||
}
|
||||
if (path.endsWith("/branches/merge/")) {
|
||||
mergeBodies.push(body);
|
||||
if (path.endsWith("/branches/cherry_pick/")) {
|
||||
cherryPickBodies.push(body);
|
||||
const dryRun = body["dry_run"] === true;
|
||||
const response = {
|
||||
status: dryRun ? "ready" : "rejected",
|
||||
status: dryRun ? "ready" : "failed",
|
||||
diff: dryRun
|
||||
? sampleDiff
|
||||
: {
|
||||
...sampleDiff,
|
||||
mergeable: false,
|
||||
mergeBlockers: [
|
||||
errors: [
|
||||
{ code: "baseMoved", message: "main has advanced" },
|
||||
],
|
||||
},
|
||||
@@ -398,19 +415,19 @@ describe("remote connection", () => {
|
||||
|
||||
await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
|
||||
|
||||
const rejected = await branches.merge("exp");
|
||||
expect(rejected.status).toBe("rejected");
|
||||
expect(rejected.diff.mergeBlockers).toEqual([
|
||||
const failed = await branches.cherryPick("exp");
|
||||
expect(failed.status).toBe("failed");
|
||||
expect(failed.diff.errors).toEqual([
|
||||
{ code: "baseMoved", message: "main has advanced" },
|
||||
]);
|
||||
|
||||
const preview = await branches.merge("exp", true);
|
||||
const preview = await branches.cherryPick("exp", true);
|
||||
expect(preview.status).toBe("ready");
|
||||
expect(preview.preview.promotedColumns).toEqual(["tag"]);
|
||||
},
|
||||
);
|
||||
|
||||
expect(mergeBodies).toEqual([
|
||||
expect(cherryPickBodies).toEqual([
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
{ from_branch: "exp", dry_run: false },
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
|
||||
@@ -11,10 +11,13 @@ import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
|
||||
import {
|
||||
AutoQuery,
|
||||
Connection,
|
||||
MatchQuery,
|
||||
PhraseQuery,
|
||||
Query,
|
||||
Table,
|
||||
VectorQuery,
|
||||
connect,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
@@ -682,6 +685,56 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
},
|
||||
);
|
||||
|
||||
// https://github.com/lancedb/lancedb/issues/1963
|
||||
it("should query documents with LangChain PDF metadata", async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
try {
|
||||
const db = await connect(tmpDir.name);
|
||||
const documents = [
|
||||
{
|
||||
text: "first page",
|
||||
vector: [1, 0],
|
||||
source: "first.pdf",
|
||||
loc: { pageNumber: 1, lines: { from: 1, to: 12 } },
|
||||
pdf: {
|
||||
version: "1.10.100",
|
||||
info: {
|
||||
format: "PDF 1.7",
|
||||
producer: "pdf.js",
|
||||
creator: "Writer",
|
||||
},
|
||||
totalPages: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
text: "second page",
|
||||
vector: [0, 1],
|
||||
source: "second.pdf",
|
||||
loc: { pageNumber: 2, lines: { from: 13, to: 24 } },
|
||||
pdf: {
|
||||
version: "1.10.100",
|
||||
info: {
|
||||
format: "PDF 1.7",
|
||||
producer: "pdf.js",
|
||||
creator: "Writer",
|
||||
},
|
||||
totalPages: 2,
|
||||
},
|
||||
},
|
||||
];
|
||||
const documentsTable = await db.createTable("documents", documents);
|
||||
|
||||
const results = await documentsTable.query().toArray();
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].source).toBe("first.pdf");
|
||||
expect(results[0].pdf.info.producer).toBe("pdf.js");
|
||||
expect(results[1].loc.pageNumber).toBe(2);
|
||||
} finally {
|
||||
tmpDir.removeCallback();
|
||||
}
|
||||
});
|
||||
|
||||
describe("merge insert", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
@@ -1777,6 +1830,194 @@ describe("Read consistency interval", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("automatic search schema consistency", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
|
||||
class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 2;
|
||||
}
|
||||
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map((value) => [value.length, 1]);
|
||||
}
|
||||
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
function embeddingSchema() {
|
||||
const func = new SchemaRefreshEmbedding();
|
||||
return LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
getRegistry().reset();
|
||||
register("schema-refresh")(SchemaRefreshEmbedding);
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
getRegistry().reset();
|
||||
tmpDir.removeCallback();
|
||||
});
|
||||
|
||||
it("uses the schema refreshed from another connection", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const stale = await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const search = stale.search("hello");
|
||||
expect(search).toBeInstanceOf(AutoQuery);
|
||||
expect(search).not.toBeInstanceOf(Query);
|
||||
expect(search).not.toBeInstanceOf(VectorQuery);
|
||||
expect("nprobes" in search).toBe(false);
|
||||
|
||||
const rows = await search.toArray();
|
||||
expect(rows[0].text).toBe("after hello");
|
||||
expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("tracks embedding metadata across checkout and restore", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
await first.createTable("docs", [{ text: "before" }], {
|
||||
schema: embeddingSchema(),
|
||||
});
|
||||
const table = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "after hello" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
await table.checkout(1);
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
|
||||
await table.checkoutLatest();
|
||||
expect((await table.search("hello").toArray())[0].text).toBe(
|
||||
"after hello",
|
||||
);
|
||||
|
||||
await table.checkout(1);
|
||||
await table.restore();
|
||||
expect((await table.search("before").toArray())[0].text).toBe("before");
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("pins automatic search while computing an embedding", async () => {
|
||||
let markStarted!: () => void;
|
||||
let releaseEmbedding!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const released = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
class BlockingEmbedding extends SchemaRefreshEmbedding {
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
markStarted();
|
||||
await released;
|
||||
return [value.length, 1];
|
||||
}
|
||||
}
|
||||
|
||||
register("schema-refresh-blocking")(BlockingEmbedding);
|
||||
const func = new BlockingEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable(
|
||||
"docs",
|
||||
[{ text: "hello before" }],
|
||||
{ schema },
|
||||
);
|
||||
const pending = table.search("hello").toArray();
|
||||
await started;
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
expect((await pending)[0].text).toBe("hello before");
|
||||
} finally {
|
||||
releaseEmbedding();
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("refreshes a reused automatic search for every execution", async () => {
|
||||
const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
|
||||
|
||||
try {
|
||||
const table = await first.createTable("docs", [
|
||||
{ text: "hello before", marker: "before" },
|
||||
]);
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
const search = table.search("hello").select(["text"]);
|
||||
|
||||
const before = (await search.toArray())[0];
|
||||
expect(before.text).toBe("hello before");
|
||||
expect(before.marker).toBeUndefined();
|
||||
|
||||
const replacement = await second.createTable(
|
||||
"docs",
|
||||
[{ text: "hello after", marker: "after" }],
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
await replacement.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const after = (await search.toArray())[0];
|
||||
expect(after.text).toBe("hello after");
|
||||
expect(after.marker).toBeUndefined();
|
||||
} finally {
|
||||
first.close();
|
||||
second.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("schema evolution", function () {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
@@ -2344,7 +2585,24 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
);
|
||||
});
|
||||
|
||||
test("full text search if no embedding function provided", async () => {
|
||||
test("full text search if only an unrelated embedding function is registered", async () => {
|
||||
register("unused")(
|
||||
class extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(_data: string) {
|
||||
return [1, 2, 3];
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map(() => [1, 2, 3]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
|
||||
@@ -2366,6 +2624,306 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(results2[0].text).toBe(data[1].text);
|
||||
});
|
||||
|
||||
test("auto search stays consistent with the active revision", async () => {
|
||||
let initCalls = 0;
|
||||
let queryCalls = 0;
|
||||
let markStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
let releaseEmbedding!: () => void;
|
||||
const embeddingReleased = new Promise<void>((resolve) => {
|
||||
releaseEmbedding = resolve;
|
||||
});
|
||||
|
||||
@register("refresh-test")
|
||||
class TestEmbedding extends EmbeddingFunction<string> {
|
||||
async init() {
|
||||
initCalls += 1;
|
||||
}
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings(value: string) {
|
||||
queryCalls += 1;
|
||||
if (value === "blocked") {
|
||||
markStarted();
|
||||
await embeddingReleased;
|
||||
}
|
||||
return value === "greetings" ? [0.1] : [0.2];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map((value) =>
|
||||
value === "hello world" ? [0.1] : [0.2],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
await writer.createTable("test", [{ text: "plain", vector: [0.0] }]);
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("test");
|
||||
type SnapshotCountingNative = {
|
||||
querySnapshot: () => Promise<unknown>;
|
||||
};
|
||||
const native = (tracked as unknown as { inner: SnapshotCountingNative })
|
||||
.inner;
|
||||
const querySnapshot = native.querySnapshot.bind(native);
|
||||
let snapshotCalls = 0;
|
||||
native.querySnapshot = async () => {
|
||||
snapshotCalls += 1;
|
||||
return await querySnapshot();
|
||||
};
|
||||
const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
|
||||
|
||||
const func = new TestEmbedding();
|
||||
const schema = LanceSchema({
|
||||
text: func.sourceField(new arrow.Utf8()),
|
||||
vector: func.vectorField(),
|
||||
});
|
||||
const data = [{ text: "hello world" }, { text: "goodbye world" }];
|
||||
await writer.createTable("test", data, { mode: "overwrite", schema });
|
||||
const baselineInitCalls = initCalls;
|
||||
|
||||
expect(
|
||||
(await tracked.schema()).metadata.get("embedding_functions"),
|
||||
).toBeDefined();
|
||||
const results = await autoQuery.toArray();
|
||||
expect(results[0].text).toBe(data[0].text);
|
||||
expect(initCalls).toBe(baselineInitCalls + 1);
|
||||
expect(queryCalls).toBe(1);
|
||||
expect(snapshotCalls).toBe(1);
|
||||
|
||||
const repeatedResults = await autoQuery.toArray();
|
||||
expect(repeatedResults[0].text).toBe(data[0].text);
|
||||
expect(initCalls).toBe(baselineInitCalls + 1);
|
||||
expect(queryCalls).toBe(1);
|
||||
expect(snapshotCalls).toBe(2);
|
||||
|
||||
const pending = tracked
|
||||
.search("blocked")
|
||||
.select(["text"])
|
||||
.limit(1)
|
||||
.toArray();
|
||||
await started;
|
||||
|
||||
const ftsData = [
|
||||
{ text: "greetings from full text", vector: [0.0] },
|
||||
{ text: "blocked from full text", vector: [0.0] },
|
||||
];
|
||||
const ftsTable = await writer.createTable("test", ftsData, {
|
||||
mode: "overwrite",
|
||||
});
|
||||
await ftsTable.createIndex("text", { config: Index.fts() });
|
||||
releaseEmbedding();
|
||||
|
||||
const pendingResults = await pending;
|
||||
expect(pendingResults[0].text).toBe(data[1].text);
|
||||
|
||||
expect(
|
||||
(await tracked.schema()).metadata.get("embedding_functions"),
|
||||
).toBeUndefined();
|
||||
const ftsResults = await autoQuery.toArray();
|
||||
expect(ftsResults[0].text).toBe(ftsData[0].text);
|
||||
});
|
||||
|
||||
test("auto search keeps newer preparation during a revision race", async () => {
|
||||
let aCalls = 0;
|
||||
let bCalls = 0;
|
||||
let markAStarted!: () => void;
|
||||
const aStarted = new Promise<void>((resolve) => {
|
||||
markAStarted = resolve;
|
||||
});
|
||||
let releaseA!: () => void;
|
||||
const aReleased = new Promise<void>((resolve) => {
|
||||
releaseA = resolve;
|
||||
});
|
||||
let markBStarted!: () => void;
|
||||
const bStarted = new Promise<void>((resolve) => {
|
||||
markBStarted = resolve;
|
||||
});
|
||||
let releaseB!: () => void;
|
||||
const bReleased = new Promise<void>((resolve) => {
|
||||
releaseB = resolve;
|
||||
});
|
||||
|
||||
@register("race-a")
|
||||
class EmbeddingA extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
aCalls += 1;
|
||||
markAStarted();
|
||||
await aReleased;
|
||||
return [0.1];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.1]);
|
||||
}
|
||||
}
|
||||
|
||||
@register("race-b")
|
||||
class EmbeddingB extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
bCalls += 1;
|
||||
markBStarted();
|
||||
await bReleased;
|
||||
return [0.2];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.2]);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
const embeddingA = new EmbeddingA();
|
||||
const schemaA = LanceSchema({
|
||||
text: embeddingA.sourceField(new arrow.Utf8()),
|
||||
vector: embeddingA.vectorField(),
|
||||
});
|
||||
await writer.createTable("race", [{ text: "revision a" }], {
|
||||
schema: schemaA,
|
||||
});
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("race");
|
||||
const query = tracked.search("query");
|
||||
|
||||
const first = query.toArray();
|
||||
await aStarted;
|
||||
|
||||
const embeddingB = new EmbeddingB();
|
||||
const schemaB = LanceSchema({
|
||||
text: embeddingB.sourceField(new arrow.Utf8()),
|
||||
vector: embeddingB.vectorField(),
|
||||
});
|
||||
await writer.createTable("race", [{ text: "revision b" }], {
|
||||
mode: "overwrite",
|
||||
schema: schemaB,
|
||||
});
|
||||
const second = query.toArray();
|
||||
await bStarted;
|
||||
|
||||
releaseA();
|
||||
releaseB();
|
||||
await Promise.all([first, second]);
|
||||
expect(aCalls).toBe(1);
|
||||
expect(bCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("stale FTS routing keeps newer vector preparation", async () => {
|
||||
let vectorCalls = 0;
|
||||
let markVectorStarted!: () => void;
|
||||
const vectorStarted = new Promise<void>((resolve) => {
|
||||
markVectorStarted = resolve;
|
||||
});
|
||||
let releaseVector!: () => void;
|
||||
const vectorReleased = new Promise<void>((resolve) => {
|
||||
releaseVector = resolve;
|
||||
});
|
||||
|
||||
@register("stale-fts-race")
|
||||
class RaceEmbedding extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 1;
|
||||
}
|
||||
embeddingDataType() {
|
||||
return new arrow.Float32();
|
||||
}
|
||||
async computeQueryEmbeddings() {
|
||||
vectorCalls += 1;
|
||||
markVectorStarted();
|
||||
await vectorReleased;
|
||||
return [0.1];
|
||||
}
|
||||
async computeSourceEmbeddings(values: string[]) {
|
||||
return values.map(() => [0.1]);
|
||||
}
|
||||
}
|
||||
|
||||
const writer = await connect(tmpDir.name);
|
||||
const ftsTable = await writer.createTable("stale_fts", [
|
||||
{ text: "hello", vector: [0.0] },
|
||||
]);
|
||||
await ftsTable.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const reader = await connect(tmpDir.name, {
|
||||
readConsistencyInterval: 0,
|
||||
});
|
||||
const tracked = await reader.openTable("stale_fts");
|
||||
type Snapshot = {
|
||||
schema: () => Promise<Buffer>;
|
||||
};
|
||||
type NativeWithSnapshot = {
|
||||
querySnapshot: () => Promise<Snapshot>;
|
||||
};
|
||||
const native = (tracked as unknown as { inner: NativeWithSnapshot })
|
||||
.inner;
|
||||
const querySnapshot = native.querySnapshot.bind(native);
|
||||
let snapshotCalls = 0;
|
||||
let markStaleSchemaStarted!: () => void;
|
||||
const staleSchemaStarted = new Promise<void>((resolve) => {
|
||||
markStaleSchemaStarted = resolve;
|
||||
});
|
||||
let releaseStaleSchema!: () => void;
|
||||
const staleSchemaReleased = new Promise<void>((resolve) => {
|
||||
releaseStaleSchema = resolve;
|
||||
});
|
||||
native.querySnapshot = async () => {
|
||||
const snapshot = await querySnapshot();
|
||||
snapshotCalls += 1;
|
||||
if (snapshotCalls === 1) {
|
||||
const schema = snapshot.schema.bind(snapshot);
|
||||
snapshot.schema = async () => {
|
||||
markStaleSchemaStarted();
|
||||
await staleSchemaReleased;
|
||||
return await schema();
|
||||
};
|
||||
}
|
||||
return snapshot;
|
||||
};
|
||||
|
||||
const query = tracked.search("hello");
|
||||
const staleFtsExecution = query.toArray();
|
||||
await staleSchemaStarted;
|
||||
|
||||
const embedding = new RaceEmbedding();
|
||||
const vectorSchema = LanceSchema({
|
||||
text: embedding.sourceField(new arrow.Utf8()),
|
||||
vector: embedding.vectorField(),
|
||||
});
|
||||
await writer.createTable("stale_fts", [{ text: "hello" }], {
|
||||
mode: "overwrite",
|
||||
schema: vectorSchema,
|
||||
});
|
||||
|
||||
const vectorExecution = query.toArray();
|
||||
await vectorStarted;
|
||||
releaseStaleSchema();
|
||||
await staleFtsExecution;
|
||||
releaseVector();
|
||||
await vectorExecution;
|
||||
|
||||
await query.toArray();
|
||||
expect(vectorCalls).toBe(1);
|
||||
});
|
||||
|
||||
test("tokenizes FTS queries by column or index name", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
@@ -2916,6 +3474,30 @@ describe("column name options", () => {
|
||||
expect(results[1].query_index).toBe(1);
|
||||
});
|
||||
|
||||
test("observes promised additional vectors while the query is pending", async () => {
|
||||
const initialVector = new Promise<number[]>(() => undefined);
|
||||
const query = table.query().nearestTo(initialVector);
|
||||
const unhandled: unknown[] = [];
|
||||
const onUnhandled = (reason: unknown) => unhandled.push(reason);
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
|
||||
try {
|
||||
query.addQueryVector(Promise.reject(new Error("extra vector failed")));
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(unhandled).toEqual([]);
|
||||
|
||||
const rejectedQuery = table
|
||||
.query()
|
||||
.nearestTo([0.1, 0.2])
|
||||
.addQueryVector(Promise.reject(new Error("consumed vector failed")));
|
||||
await expect(rejectedQuery.toArray()).rejects.toThrow(
|
||||
"consumed vector failed",
|
||||
);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
}
|
||||
});
|
||||
|
||||
test("index and search multivectors", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [];
|
||||
@@ -2953,7 +3535,7 @@ describe("column name options", () => {
|
||||
.limit(10)
|
||||
.toArray();
|
||||
expect(results2.length).toBe(10);
|
||||
});
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
describe("when creating an empty table", () => {
|
||||
|
||||
+41
-312
@@ -5,7 +5,6 @@ import {
|
||||
Data as ArrowData,
|
||||
Table as ArrowTable,
|
||||
Binary,
|
||||
Bool,
|
||||
BufferType,
|
||||
DataType,
|
||||
DateUnit,
|
||||
@@ -18,12 +17,7 @@ import {
|
||||
FixedSizeList,
|
||||
Float,
|
||||
Float32,
|
||||
Float64,
|
||||
Int,
|
||||
Int8,
|
||||
Int16,
|
||||
Int32,
|
||||
Int64,
|
||||
LargeBinary,
|
||||
List,
|
||||
Null,
|
||||
@@ -36,33 +30,29 @@ import {
|
||||
Struct,
|
||||
Timestamp,
|
||||
Type,
|
||||
Uint8,
|
||||
Uint16,
|
||||
Uint32,
|
||||
Utf8,
|
||||
Vector,
|
||||
makeVector as arrowMakeVector,
|
||||
util as arrowUtil,
|
||||
vectorFromArray as badVectorFromArray,
|
||||
makeBuilder,
|
||||
makeData,
|
||||
} from "apache-arrow";
|
||||
import { Buffers } from "apache-arrow/data";
|
||||
import { typedArrayToArrowType } from "./arrow_type";
|
||||
import { type EmbeddingFunction } from "./embedding/embedding_function";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import {
|
||||
EmbeddingFunctionConfig,
|
||||
getRegistry,
|
||||
parseEmbeddingMetadata,
|
||||
} from "./embedding/registry";
|
||||
import {
|
||||
sanitizeField,
|
||||
sanitizeSchema,
|
||||
sanitizeTable,
|
||||
sanitizeType,
|
||||
} from "./sanitize";
|
||||
|
||||
/**
|
||||
* Check if a field name indicates a vector column.
|
||||
*/
|
||||
function nameSuggestsVectorColumn(fieldName: string): boolean {
|
||||
const nameLower = fieldName.toLowerCase();
|
||||
return nameLower.includes("vector") || nameLower.includes("embedding");
|
||||
}
|
||||
import { inferSchema } from "./schema";
|
||||
|
||||
export * from "apache-arrow";
|
||||
export type SchemaLike =
|
||||
@@ -455,110 +445,6 @@ export function makeArrowTable(
|
||||
return new ArrowTable(inferredSchema, finalColumns);
|
||||
}
|
||||
|
||||
function inferSchema(
|
||||
data: Array<Record<string, unknown>>,
|
||||
schema: Schema | undefined,
|
||||
opts: MakeArrowTableOptions,
|
||||
): Schema {
|
||||
// We will collect all fields we see in the data.
|
||||
const pathTree = new PathTree<DataType>();
|
||||
|
||||
for (const [rowI, row] of data.entries()) {
|
||||
for (const [path, value] of rowPathsAndValues(row)) {
|
||||
if (!pathTree.has(path)) {
|
||||
// First time seeing this field.
|
||||
if (schema !== undefined) {
|
||||
const field = getFieldForPath(schema, path);
|
||||
if (field === undefined) {
|
||||
throw new Error(
|
||||
`Found field not in schema: ${path.join(".")} at row ${rowI}`,
|
||||
);
|
||||
} else {
|
||||
pathTree.set(path, field.type);
|
||||
}
|
||||
} else {
|
||||
const inferredType = inferType(value, path, opts);
|
||||
if (inferredType === undefined) {
|
||||
throw new Error(`Failed to infer data type for field ${path.join(
|
||||
".",
|
||||
)} at row ${rowI}. \
|
||||
Consider providing an explicit schema.`);
|
||||
}
|
||||
pathTree.set(path, inferredType);
|
||||
}
|
||||
} else if (schema === undefined) {
|
||||
const currentType = pathTree.get(path);
|
||||
const newType = inferType(value, path, opts);
|
||||
if (currentType !== newType) {
|
||||
new Error(`Failed to infer schema for data. Previously inferred type \
|
||||
${currentType} but found ${newType} at row ${rowI}. Consider \
|
||||
providing an explicit schema.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (schema === undefined) {
|
||||
function fieldsFromPathTree(pathTree: PathTree<DataType>): Field[] {
|
||||
const fields = [];
|
||||
for (const [name, value] of pathTree.map.entries()) {
|
||||
if (value instanceof PathTree) {
|
||||
const children = fieldsFromPathTree(value);
|
||||
fields.push(new Field(name, new Struct(children), true));
|
||||
} else {
|
||||
fields.push(new Field(name, value, true));
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
const fields = fieldsFromPathTree(pathTree);
|
||||
return new Schema(fields);
|
||||
} else {
|
||||
function takeMatchingFields(
|
||||
fields: Field[],
|
||||
pathTree: PathTree<DataType>,
|
||||
): Field[] {
|
||||
const outFields = [];
|
||||
for (const field of fields) {
|
||||
if (pathTree.map.has(field.name)) {
|
||||
const value = pathTree.get([field.name]);
|
||||
if (value instanceof PathTree) {
|
||||
const struct = field.type as Struct;
|
||||
const children = takeMatchingFields(struct.children, value);
|
||||
outFields.push(
|
||||
new Field(field.name, new Struct(children), field.nullable),
|
||||
);
|
||||
} else {
|
||||
outFields.push(
|
||||
new Field(field.name, value as DataType, field.nullable),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return outFields;
|
||||
}
|
||||
const fields = takeMatchingFields(schema.fields, pathTree);
|
||||
return new Schema(fields);
|
||||
}
|
||||
}
|
||||
|
||||
function* rowPathsAndValues(
|
||||
row: Record<string, unknown>,
|
||||
basePath: string[] = [],
|
||||
): Generator<[string[], unknown]> {
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
if (isObject(value)) {
|
||||
yield* rowPathsAndValues(value, [...basePath, key]);
|
||||
} else {
|
||||
// Skip undefined values - they should be treated the same as missing fields
|
||||
// for embedding function purposes
|
||||
if (value !== undefined) {
|
||||
yield [[...basePath, key], value];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isObject(value: unknown): value is Record<string, unknown> {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
@@ -573,146 +459,19 @@ function isObject(value: unknown): value is Record<string, unknown> {
|
||||
);
|
||||
}
|
||||
|
||||
function getFieldForPath(schema: Schema, path: string[]): Field | undefined {
|
||||
let current: Field | Schema = schema;
|
||||
function valueAtPath(datum: Record<string, unknown>, path: string[]): unknown {
|
||||
let current: unknown = datum;
|
||||
for (const key of path) {
|
||||
if (current instanceof Schema) {
|
||||
const field: Field | undefined = current.fields.find(
|
||||
(f) => f.name === key,
|
||||
);
|
||||
if (field === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = field;
|
||||
} else if (current instanceof Field && DataType.isStruct(current.type)) {
|
||||
const struct: Struct = current.type;
|
||||
const field = struct.children.find((f) => f.name === key);
|
||||
if (field === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = field;
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
|
||||
current = current[key];
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
if (current instanceof Field) {
|
||||
return current;
|
||||
} else {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to infer which Arrow type to use for a given value.
|
||||
*
|
||||
* May return undefined if the type cannot be inferred.
|
||||
*/
|
||||
function inferType(
|
||||
value: unknown,
|
||||
path: string[],
|
||||
opts: MakeArrowTableOptions,
|
||||
): DataType | undefined {
|
||||
if (typeof value === "bigint") {
|
||||
return new Int64();
|
||||
} else if (typeof value === "number") {
|
||||
// Even if it's an integer, it's safer to assume Float64. Users can
|
||||
// always provide an explicit schema or use BigInt if they mean integer.
|
||||
return new Float64();
|
||||
} else if (typeof value === "string") {
|
||||
if (opts.dictionaryEncodeStrings) {
|
||||
return new Dictionary(new Utf8(), new Int32());
|
||||
} else {
|
||||
return new Utf8();
|
||||
}
|
||||
} else if (typeof value === "boolean") {
|
||||
return new Bool();
|
||||
} else if (value instanceof Buffer) {
|
||||
return new Binary();
|
||||
} else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
||||
const info = typedArrayToArrowType(value);
|
||||
if (info !== undefined) {
|
||||
const child = new Field("item", info.elementType, true);
|
||||
return new FixedSizeList(info.length, child);
|
||||
}
|
||||
return undefined;
|
||||
} else if (Array.isArray(value)) {
|
||||
if (value.length === 0) {
|
||||
return undefined; // Without any values we can't infer the type
|
||||
}
|
||||
if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
|
||||
const floatType = sanitizeType(opts.vectorColumns[path[0]].type);
|
||||
return new FixedSizeList(
|
||||
value.length,
|
||||
new Field("item", floatType, true),
|
||||
);
|
||||
}
|
||||
const valueType = inferType(value[0], path, opts);
|
||||
if (valueType === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
// Try to automatically detect embedding columns.
|
||||
if (nameSuggestsVectorColumn(path[path.length - 1])) {
|
||||
// Check if value is a Uint8Array for integer vector type determination
|
||||
if (value instanceof Uint8Array) {
|
||||
// For integer vectors, we default to Uint8 (matching Python implementation)
|
||||
const child = new Field("item", new Uint8(), true);
|
||||
return new FixedSizeList(value.length, child);
|
||||
} else {
|
||||
// For float vectors, we default to Float32
|
||||
const child = new Field("item", new Float32(), true);
|
||||
return new FixedSizeList(value.length, child);
|
||||
}
|
||||
} else {
|
||||
const child = new Field("item", valueType, true);
|
||||
return new List(child);
|
||||
}
|
||||
} else {
|
||||
// TODO: timestamp
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
class PathTree<V> {
|
||||
map: Map<string, V | PathTree<V>>;
|
||||
|
||||
constructor(entries?: [string[], V][]) {
|
||||
this.map = new Map();
|
||||
if (entries !== undefined) {
|
||||
for (const [path, value] of entries) {
|
||||
this.set(path, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
has(path: string[]): boolean {
|
||||
let ref: PathTree<V> = this;
|
||||
for (const part of path) {
|
||||
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
||||
return false;
|
||||
}
|
||||
ref = ref.map.get(part) as PathTree<V>;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
get(path: string[]): V | undefined {
|
||||
let ref: PathTree<V> = this;
|
||||
for (const part of path) {
|
||||
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
||||
return undefined;
|
||||
}
|
||||
ref = ref.map.get(part) as PathTree<V>;
|
||||
}
|
||||
return ref as V;
|
||||
}
|
||||
set(path: string[], value: V): void {
|
||||
let ref: PathTree<V> = this;
|
||||
for (const part of path.slice(0, path.length - 1)) {
|
||||
if (!ref.map.has(part)) {
|
||||
ref.map.set(part, new PathTree<V>());
|
||||
}
|
||||
ref = ref.map.get(part) as PathTree<V>;
|
||||
}
|
||||
ref.map.set(path[path.length - 1], value);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function transposeData(
|
||||
@@ -720,37 +479,26 @@ function transposeData(
|
||||
field: Field,
|
||||
path: string[] = [],
|
||||
): Vector {
|
||||
const valuesPath = [...path, field.name];
|
||||
const values = data.map((datum) => valueAtPath(datum, valuesPath));
|
||||
if (field.type instanceof Struct) {
|
||||
const childFields = field.type.children;
|
||||
const fullPath = [...path, field.name];
|
||||
const childVectors = childFields.map((child) => {
|
||||
return transposeData(data, child, fullPath);
|
||||
return transposeData(data, child, valuesPath);
|
||||
});
|
||||
const nullCount = values.filter((value) => value === null).length;
|
||||
const structData = makeData({
|
||||
type: field.type,
|
||||
length: values.length,
|
||||
nullCount,
|
||||
nullBitmap:
|
||||
nullCount > 0
|
||||
? arrowUtil.packBools(values.map((value) => value !== null))
|
||||
: undefined,
|
||||
children: childVectors as unknown as ArrowData<DataType>[],
|
||||
});
|
||||
return arrowMakeVector(structData);
|
||||
} else {
|
||||
const valuesPath = [...path, field.name];
|
||||
const values = data.map((datum) => {
|
||||
let current: unknown = datum;
|
||||
for (const key of valuesPath) {
|
||||
if (current == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
isObject(current) &&
|
||||
(Object.hasOwn(current, key) || key in current)
|
||||
) {
|
||||
current = current[key];
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return current;
|
||||
});
|
||||
return makeVector(values, field.type, undefined, field.nullable);
|
||||
}
|
||||
}
|
||||
@@ -793,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
|
||||
return listBuilder.finish().toVector();
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a JS TypedArray instance to the corresponding Arrow element DataType
|
||||
* and its length. Returns undefined if the value is not a recognized TypedArray.
|
||||
*/
|
||||
function typedArrayToArrowType(
|
||||
value: ArrayBufferView,
|
||||
): { elementType: DataType; length: number } | undefined {
|
||||
if (value instanceof Float32Array)
|
||||
return { elementType: new Float32(), length: value.length };
|
||||
if (value instanceof Float64Array)
|
||||
return { elementType: new Float64(), length: value.length };
|
||||
if (value instanceof Uint8Array)
|
||||
return { elementType: new Uint8(), length: value.length };
|
||||
if (value instanceof Uint16Array)
|
||||
return { elementType: new Uint16(), length: value.length };
|
||||
if (value instanceof Uint32Array)
|
||||
return { elementType: new Uint32(), length: value.length };
|
||||
if (value instanceof Int8Array)
|
||||
return { elementType: new Int8(), length: value.length };
|
||||
if (value instanceof Int16Array)
|
||||
return { elementType: new Int16(), length: value.length };
|
||||
if (value instanceof Int32Array)
|
||||
return { elementType: new Int32(), length: value.length };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Helper function to convert an Array of JS values to an Arrow Vector */
|
||||
function makeVector(
|
||||
values: unknown[],
|
||||
@@ -933,7 +655,7 @@ async function applyEmbeddingsFromMetadata(
|
||||
|
||||
for (const functionEntry of functions.values()) {
|
||||
const sourceColumn = columns[functionEntry.sourceColumn];
|
||||
const destColumn = functionEntry.vectorColumn ?? "vector";
|
||||
const destColumn = functionEntry.vectorColumn;
|
||||
if (sourceColumn === undefined) {
|
||||
throw new Error(
|
||||
`Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`,
|
||||
@@ -1385,11 +1107,10 @@ function validateSchemaEmbeddings(
|
||||
|
||||
// Check schema metadata for embedding functions
|
||||
if (schema.metadata.has("embedding_functions")) {
|
||||
const embeddings = JSON.parse(
|
||||
const entries = parseEmbeddingMetadata(
|
||||
schema.metadata.get("embedding_functions")!,
|
||||
);
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
|
||||
if (embeddings.find((f: any) => f["vectorColumn"] === field.name)) {
|
||||
if (entries.some((f) => f.vectorColumn === field.name)) {
|
||||
hasEmbeddingFunction = true;
|
||||
}
|
||||
}
|
||||
@@ -1459,8 +1180,12 @@ export function ensureNestedFieldsExist(
|
||||
completeRow[field.name] = row[field.name];
|
||||
}
|
||||
} else {
|
||||
// Field is missing from the data - set to null
|
||||
completeRow[field.name] = null;
|
||||
// Keep a missing struct valid while filling each of its children with
|
||||
// null. This is distinct from an explicitly null struct value.
|
||||
completeRow[field.name] =
|
||||
field.type.constructor.name === "Struct"
|
||||
? ensureStructFieldsExist({}, field.type as Struct)
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1495,8 +1220,12 @@ function ensureStructFieldsExist(
|
||||
completeStruct[childField.name] = data[childField.name];
|
||||
}
|
||||
} else {
|
||||
// Field is missing - set to null
|
||||
completeStruct[childField.name] = null;
|
||||
// Keep a missing struct valid while filling each of its children with
|
||||
// null. This is distinct from an explicitly null struct value.
|
||||
completeStruct[childField.name] =
|
||||
childField.type.constructor.name === "Struct"
|
||||
? ensureStructFieldsExist({}, childField.type as Struct)
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
type DataType,
|
||||
Float32,
|
||||
Float64,
|
||||
Int8,
|
||||
Int16,
|
||||
Int32,
|
||||
Uint8,
|
||||
Uint16,
|
||||
Uint32,
|
||||
} from "apache-arrow";
|
||||
|
||||
/**
|
||||
* Map a JS TypedArray instance to the corresponding Arrow element type and
|
||||
* length. Returns undefined when the view is not a supported TypedArray.
|
||||
*/
|
||||
export function typedArrayToArrowType(
|
||||
value: ArrayBufferView,
|
||||
): { elementType: DataType; length: number } | undefined {
|
||||
if (value instanceof Float32Array)
|
||||
return { elementType: new Float32(), length: value.length };
|
||||
if (value instanceof Float64Array)
|
||||
return { elementType: new Float64(), length: value.length };
|
||||
if (value instanceof Uint8Array)
|
||||
return { elementType: new Uint8(), length: value.length };
|
||||
if (value instanceof Uint16Array)
|
||||
return { elementType: new Uint16(), length: value.length };
|
||||
if (value instanceof Uint32Array)
|
||||
return { elementType: new Uint32(), length: value.length };
|
||||
if (value instanceof Int8Array)
|
||||
return { elementType: new Int8(), length: value.length };
|
||||
if (value instanceof Int16Array)
|
||||
return { elementType: new Int16(), length: value.length };
|
||||
if (value instanceof Int32Array)
|
||||
return { elementType: new Int32(), length: value.length };
|
||||
return undefined;
|
||||
}
|
||||
@@ -16,6 +16,12 @@ import {
|
||||
makeEmptyTable,
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import {
|
||||
MaterializedView,
|
||||
MaterializedViewSelect,
|
||||
normalizeSelect,
|
||||
validateNonNegativeInteger,
|
||||
} from "./materialized_view";
|
||||
import { Connection as LanceDbConnection } from "./native";
|
||||
import type {
|
||||
CreateNamespaceResponse,
|
||||
@@ -25,12 +31,14 @@ import type {
|
||||
JobDescription,
|
||||
JobInfo,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
} from "./native";
|
||||
export type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
};
|
||||
import { sanitizeTable } from "./sanitize";
|
||||
import { LocalTable, Table } from "./table";
|
||||
@@ -128,6 +136,10 @@ export interface OpenTableOptions {
|
||||
indexCacheSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
|
||||
* instead.
|
||||
*/
|
||||
export interface TableNamesOptions {
|
||||
/**
|
||||
* If present, only return names that come lexicographically after the
|
||||
@@ -141,6 +153,24 @@ export interface TableNamesOptions {
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ListTablesOptions {
|
||||
/**
|
||||
* Token from a previous response, to resume listing where it left off.
|
||||
*
|
||||
* The token is opaque: it carries whatever the database needs to resume, and
|
||||
* callers should not construct or interpret one.
|
||||
*/
|
||||
pageToken?: string;
|
||||
/**
|
||||
* An upper bound on how many tables to return.
|
||||
*
|
||||
* A page may hold fewer than this and still not be the last one, so keep
|
||||
* going while the response carries a page token rather than while pages are
|
||||
* full.
|
||||
*/
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ListNamespacesOptions {
|
||||
/** Token from a previous response for pagination. */
|
||||
pageToken?: string;
|
||||
@@ -225,6 +255,7 @@ export abstract class Connection {
|
||||
* @param {Partial<TableNamesOptions>} options - options to control the
|
||||
* paging / start point (backwards compatibility)
|
||||
*
|
||||
* @deprecated Use {@link Connection.listTables} instead.
|
||||
*/
|
||||
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
||||
/**
|
||||
@@ -235,18 +266,94 @@ export abstract class Connection {
|
||||
* @param {Partial<TableNamesOptions>} options - options to control the
|
||||
* paging / start point
|
||||
*
|
||||
* @deprecated Use {@link Connection.listTables} instead.
|
||||
*/
|
||||
abstract tableNames(
|
||||
namespacePath?: string[],
|
||||
options?: Partial<TableNamesOptions>,
|
||||
): Promise<string[]>;
|
||||
|
||||
/**
|
||||
* List a page of the tables in this database.
|
||||
*
|
||||
* To retrieve the tables after the page, pass the `pageToken` the response
|
||||
* carries back in. A page can be shorter than `limit` without being the last
|
||||
* one, so walk until a response carries no page token:
|
||||
*
|
||||
* ```ts
|
||||
* const names = [];
|
||||
* let pageToken = undefined;
|
||||
* do {
|
||||
* const page = await conn.listTables({ pageToken, limit: 100 });
|
||||
* names.push(...page.tables);
|
||||
* pageToken = page.pageToken;
|
||||
* } while (pageToken);
|
||||
* ```
|
||||
*
|
||||
* @param {Partial<ListTablesOptions>} options - Pagination options
|
||||
* (`pageToken`, `limit`).
|
||||
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
||||
* optional token for the tables after it.
|
||||
*/
|
||||
abstract listTables(
|
||||
options?: Partial<ListTablesOptions>,
|
||||
): Promise<ListTablesResponse>;
|
||||
/**
|
||||
* List a page of the tables in this database.
|
||||
*
|
||||
* @param {string[]} namespacePath - The namespace path to list tables from
|
||||
* (defaults to root namespace)
|
||||
* @param {Partial<ListTablesOptions>} options - Pagination options
|
||||
* (`pageToken`, `limit`).
|
||||
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
||||
* optional token for the tables after it.
|
||||
*/
|
||||
abstract listTables(
|
||||
namespacePath?: string[],
|
||||
options?: Partial<ListTablesOptions>,
|
||||
): Promise<ListTablesResponse>;
|
||||
|
||||
/**
|
||||
* Open a table in the database.
|
||||
* @param {string} name - The name of the table
|
||||
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
|
||||
* @param {Partial<OpenTableOptions>} options - Additional options
|
||||
*/
|
||||
/**
|
||||
* Define a materialized view named `name` over the table `source`.
|
||||
*
|
||||
* The view is created empty, with the query recorded in its schema
|
||||
* metadata; `view.refresh()` computes the rows. The view is a normal
|
||||
* table: it can be queried, indexed and searched, and it appears in
|
||||
* `tableNames`. The source table must have stable row ids (create it with
|
||||
* the `newTableEnableStableRowIds` storage option); they keep the view's
|
||||
* provenance valid across source compactions and cannot be enabled after
|
||||
* a table exists. Local databases only.
|
||||
*/
|
||||
abstract createMaterializedView(
|
||||
name: string,
|
||||
source: string,
|
||||
options?: {
|
||||
select?: MaterializedViewSelect;
|
||||
where?: string;
|
||||
limit?: number;
|
||||
},
|
||||
): Promise<MaterializedView>;
|
||||
|
||||
/**
|
||||
* Open the materialized view named `name`.
|
||||
*
|
||||
* Rejects a table that exists but is not a materialized view.
|
||||
*/
|
||||
abstract openMaterializedView(name: string): Promise<MaterializedView>;
|
||||
|
||||
/**
|
||||
* The names of the materialized views in this database.
|
||||
*
|
||||
* Found by reading every table's schema, so this costs an open per table.
|
||||
*/
|
||||
abstract listMaterializedViews(): Promise<string[]>;
|
||||
|
||||
abstract openTable(
|
||||
name: string,
|
||||
namespacePath?: string[],
|
||||
@@ -531,6 +638,54 @@ export class LocalConnection extends Connection {
|
||||
);
|
||||
}
|
||||
|
||||
async createMaterializedView(
|
||||
name: string,
|
||||
source: string,
|
||||
options?: {
|
||||
select?: MaterializedViewSelect;
|
||||
where?: string;
|
||||
limit?: number;
|
||||
},
|
||||
): Promise<MaterializedView> {
|
||||
validateNonNegativeInteger(options?.limit, "limit");
|
||||
const innerTable = await this.inner.createMaterializedView(
|
||||
name,
|
||||
source,
|
||||
normalizeSelect(options?.select),
|
||||
options?.where,
|
||||
options?.limit,
|
||||
);
|
||||
return new MaterializedView(new LocalTable(innerTable));
|
||||
}
|
||||
|
||||
async openMaterializedView(name: string): Promise<MaterializedView> {
|
||||
const innerTable = await this.inner.openMaterializedView(name);
|
||||
return new MaterializedView(new LocalTable(innerTable));
|
||||
}
|
||||
|
||||
async listMaterializedViews(): Promise<string[]> {
|
||||
return await this.inner.listMaterializedViews();
|
||||
}
|
||||
|
||||
async listTables(
|
||||
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
|
||||
options?: Partial<ListTablesOptions>,
|
||||
): Promise<ListTablesResponse> {
|
||||
// Detect if first argument is namespacePath array or options object
|
||||
const namespacePath = Array.isArray(namespacePathOrOptions)
|
||||
? namespacePathOrOptions
|
||||
: undefined;
|
||||
const listTablesOptions = Array.isArray(namespacePathOrOptions)
|
||||
? options
|
||||
: namespacePathOrOptions;
|
||||
|
||||
return this.inner.listTables(
|
||||
namespacePath ?? [],
|
||||
listTablesOptions?.pageToken,
|
||||
listTablesOptions?.limit,
|
||||
);
|
||||
}
|
||||
|
||||
async openTable(
|
||||
name: string,
|
||||
namespacePath?: string[],
|
||||
|
||||
@@ -4,7 +4,15 @@
|
||||
import { Field, Schema } from "../arrow";
|
||||
import { sanitizeType } from "../sanitize";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./registry";
|
||||
import {
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionRegistry,
|
||||
getRegistry as getGlobalRegistry,
|
||||
registerBuiltIn,
|
||||
} from "./registry";
|
||||
|
||||
type OpenAIModule = typeof import("./openai");
|
||||
type TransformersModule = typeof import("./transformers");
|
||||
|
||||
export {
|
||||
FieldOptions,
|
||||
@@ -14,7 +22,39 @@ export {
|
||||
EmbeddingFunctionConstructor,
|
||||
} from "./embedding_function";
|
||||
|
||||
export * from "./registry";
|
||||
export {
|
||||
EmbeddingFunctionRegistry,
|
||||
parseEmbeddingMetadata,
|
||||
register,
|
||||
} from "./registry";
|
||||
export type {
|
||||
CreateReturnType,
|
||||
EmbeddingFunctionConfig,
|
||||
EmbeddingFunctionCreate,
|
||||
EmbeddingMetadataEntry,
|
||||
ResolvedEmbeddingFunctionConfig,
|
||||
} from "./registry";
|
||||
|
||||
function initializeBuiltInProviders() {
|
||||
const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
|
||||
const { TransformersEmbeddingFunction } =
|
||||
require("./transformers") as TransformersModule;
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the global embedding function registry.
|
||||
*
|
||||
* LanceDB built-in providers are initialized when this public API is first
|
||||
* used, so importing the root package does not change automatic search
|
||||
* selection for tables without embedding metadata.
|
||||
*/
|
||||
export function getRegistry(): EmbeddingFunctionRegistry {
|
||||
initializeBuiltInProviders();
|
||||
return getGlobalRegistry();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a schema with embedding functions.
|
||||
|
||||
@@ -5,14 +5,13 @@ import type OpenAI from "openai";
|
||||
import type { EmbeddingCreateParams } from "openai/resources/index";
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { register } from "./registry";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
|
||||
export type OpenAIOptions = {
|
||||
apiKey: string;
|
||||
model: EmbeddingCreateParams["model"];
|
||||
};
|
||||
|
||||
@register("openai")
|
||||
export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<OpenAIOptions>
|
||||
@@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
|
||||
return response.data[0].embedding;
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("openai", OpenAIEmbeddingFunction);
|
||||
|
||||
@@ -7,6 +7,10 @@ import {
|
||||
} from "./embedding_function";
|
||||
import "reflect-metadata";
|
||||
|
||||
const builtInFunctionsKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-built-in-functions::v1",
|
||||
);
|
||||
|
||||
export type CreateReturnType<T> = T extends { init: () => Promise<void> }
|
||||
? Promise<T>
|
||||
: T;
|
||||
@@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry {
|
||||
};
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
setBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
this.#functions.set(name, ctor);
|
||||
Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
get<T extends EmbeddingFunction<unknown>>(
|
||||
name: string,
|
||||
): EmbeddingFunctionCreate<T> | undefined;
|
||||
@@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry {
|
||||
*/
|
||||
reset(this: EmbeddingFunctionRegistry) {
|
||||
this.#functions.clear();
|
||||
getBuiltInFunctions(this).clear();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -104,41 +118,29 @@ export class EmbeddingFunctionRegistry {
|
||||
async parseFunctions(
|
||||
this: EmbeddingFunctionRegistry,
|
||||
metadata: Map<string, string>,
|
||||
): Promise<Map<string, EmbeddingFunctionConfig>> {
|
||||
): Promise<Map<string, ResolvedEmbeddingFunctionConfig>> {
|
||||
if (!metadata.has("embedding_functions")) {
|
||||
return new Map();
|
||||
} else {
|
||||
type FunctionConfig = {
|
||||
name: string;
|
||||
sourceColumn: string;
|
||||
vectorColumn: string;
|
||||
model: EmbeddingFunction["TOptions"];
|
||||
};
|
||||
|
||||
const functions = <FunctionConfig[]>(
|
||||
JSON.parse(metadata.get("embedding_functions")!)
|
||||
);
|
||||
|
||||
const items: [string, EmbeddingFunctionConfig][] = await Promise.all(
|
||||
functions.map(async (f) => {
|
||||
const fn = this.get(f.name);
|
||||
if (!fn) {
|
||||
throw new Error(`Function "${f.name}" not found in registry`);
|
||||
}
|
||||
const func = await this.get(f.name)!.create(f.model);
|
||||
return [
|
||||
f.name,
|
||||
{
|
||||
sourceColumn: f.sourceColumn,
|
||||
vectorColumn: f.vectorColumn,
|
||||
function: func,
|
||||
},
|
||||
];
|
||||
}),
|
||||
);
|
||||
|
||||
return new Map(items);
|
||||
}
|
||||
const entries = parseEmbeddingMetadata(
|
||||
metadata.get("embedding_functions")!,
|
||||
);
|
||||
const items = await Promise.all(
|
||||
entries.map(async (f): Promise<ResolvedEmbeddingFunctionConfig> => {
|
||||
const fn = this.get(f.name);
|
||||
if (!fn) {
|
||||
throw new Error(`Function "${f.name}" not found in registry`);
|
||||
}
|
||||
const func = await fn.create(f.model);
|
||||
return {
|
||||
sourceColumn: f.sourceColumn,
|
||||
vectorColumn: f.vectorColumn,
|
||||
function: func,
|
||||
};
|
||||
}),
|
||||
);
|
||||
// Keyed by output column: one function may serve several columns.
|
||||
return new Map(items.map((config) => [config.vectorColumn, config]));
|
||||
}
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
|
||||
@@ -195,12 +197,56 @@ export class EmbeddingFunctionRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
const _REGISTRY = new EmbeddingFunctionRegistry();
|
||||
function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
|
||||
const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
|
||||
[key: symbol]: Set<string> | undefined;
|
||||
};
|
||||
let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
|
||||
if (builtInFunctions === undefined) {
|
||||
builtInFunctions = new Set<string>();
|
||||
registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
|
||||
}
|
||||
return builtInFunctions;
|
||||
}
|
||||
|
||||
// Server bundlers can load the side-effect embedding entry points and the public
|
||||
// embedding API from separate module graphs. Keep their registry shared.
|
||||
const registryKey = Symbol.for(
|
||||
"@lancedb/lancedb::embedding-function-registry::v1",
|
||||
);
|
||||
const registryGlobal = globalThis as typeof globalThis & {
|
||||
[key: symbol]: EmbeddingFunctionRegistry | undefined;
|
||||
};
|
||||
|
||||
function getGlobalRegistry(): EmbeddingFunctionRegistry {
|
||||
const existingRegistry = registryGlobal[registryKey];
|
||||
if (existingRegistry !== undefined) {
|
||||
return existingRegistry;
|
||||
}
|
||||
const registry = new EmbeddingFunctionRegistry();
|
||||
registryGlobal[registryKey] = registry;
|
||||
return registry;
|
||||
}
|
||||
|
||||
const _REGISTRY = getGlobalRegistry();
|
||||
|
||||
export function register(name?: string) {
|
||||
return _REGISTRY.register(name);
|
||||
}
|
||||
|
||||
/** @ignore */
|
||||
export function registerBuiltIn<
|
||||
T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
|
||||
>(name: string, ctor: T): T {
|
||||
const builtInFunctions = getBuiltInFunctions(_REGISTRY);
|
||||
if (builtInFunctions.has(name)) {
|
||||
return _REGISTRY.setBuiltIn(name, ctor);
|
||||
}
|
||||
_REGISTRY.register(name)(ctor);
|
||||
builtInFunctions.add(name);
|
||||
return ctor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility function to get the global instance of the registry
|
||||
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
||||
@@ -218,3 +264,52 @@ export interface EmbeddingFunctionConfig {
|
||||
vectorColumn?: string;
|
||||
function: EmbeddingFunction;
|
||||
}
|
||||
|
||||
/** An [EmbeddingFunctionConfig] read back from table metadata, where the
|
||||
* vector column is always recorded. */
|
||||
export type ResolvedEmbeddingFunctionConfig = EmbeddingFunctionConfig & {
|
||||
vectorColumn: string;
|
||||
};
|
||||
|
||||
/** One entry of the `embedding_functions` schema metadata, with the column
|
||||
* keys normalized across the bindings' spellings. */
|
||||
export type EmbeddingMetadataEntry = {
|
||||
name: string;
|
||||
sourceColumn: string;
|
||||
vectorColumn: string;
|
||||
model: EmbeddingFunction["TOptions"];
|
||||
};
|
||||
|
||||
/** The single parser for `embedding_functions` schema metadata: every reader
|
||||
* goes through here, so the wire contract cannot fork between them. */
|
||||
export function parseEmbeddingMetadata(json: string): EmbeddingMetadataEntry[] {
|
||||
// The wire format, honestly: the Python bindings write snake_case keys.
|
||||
type Raw = {
|
||||
name: string;
|
||||
sourceColumn?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
source_column?: string;
|
||||
vectorColumn?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: the Python wire spelling
|
||||
vector_column?: string;
|
||||
model: EmbeddingFunction["TOptions"];
|
||||
};
|
||||
const entries = <Raw[]>JSON.parse(json);
|
||||
const seen = new Set<string>();
|
||||
return entries.map((f) => {
|
||||
const sourceColumn = f.sourceColumn ?? f.source_column;
|
||||
const vectorColumn = f.vectorColumn ?? f.vector_column;
|
||||
if (sourceColumn === undefined || vectorColumn === undefined) {
|
||||
throw new Error(
|
||||
`Embedding function "${f.name}" metadata names no source or vector column`,
|
||||
);
|
||||
}
|
||||
if (seen.has(vectorColumn)) {
|
||||
throw new Error(
|
||||
`Multiple embedding configs claim vector column "${vectorColumn}"`,
|
||||
);
|
||||
}
|
||||
seen.add(vectorColumn);
|
||||
return { name: f.name, sourceColumn, vectorColumn, model: f.model };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import { Float, Float32 } from "../arrow";
|
||||
import { EmbeddingFunction } from "./embedding_function";
|
||||
import { register } from "./registry";
|
||||
import { registerBuiltIn } from "./registry";
|
||||
|
||||
export type XenovaTransformerOptions = {
|
||||
/** The wasm compatible model to use */
|
||||
@@ -31,7 +31,6 @@ export type XenovaTransformerOptions = {
|
||||
};
|
||||
};
|
||||
|
||||
@register("huggingface")
|
||||
export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
string,
|
||||
Partial<XenovaTransformerOptions>
|
||||
@@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
|
||||
}
|
||||
}
|
||||
|
||||
registerBuiltIn("huggingface", TransformersEmbeddingFunction);
|
||||
|
||||
const tensorDiv = (
|
||||
src: import("@huggingface/transformers").Tensor,
|
||||
divBy: number,
|
||||
|
||||
+12
-3
@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
|
||||
import type { FtsToken } from "./table";
|
||||
|
||||
// Re-export native header provider for use with connectWithHeaderProvider
|
||||
export {
|
||||
MaterializedView,
|
||||
MaterializedViewDefinition,
|
||||
MaterializedViewSelect,
|
||||
} from "./materialized_view";
|
||||
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
||||
|
||||
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
|
||||
@@ -51,6 +56,7 @@ export {
|
||||
AddResult,
|
||||
AddColumnsResult,
|
||||
RefreshColumnResult,
|
||||
RefreshMaterializedViewResult,
|
||||
AlterColumnsResult,
|
||||
UpdateFieldMetadataResult,
|
||||
DeleteResult,
|
||||
@@ -75,11 +81,13 @@ export {
|
||||
Connection,
|
||||
CreateTableOptions,
|
||||
TableNamesOptions,
|
||||
ListTablesOptions,
|
||||
OpenTableOptions,
|
||||
ListNamespacesOptions,
|
||||
CreateNamespaceOptions,
|
||||
DropNamespaceOptions,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
CreateNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
@@ -95,6 +103,7 @@ export {
|
||||
} from "./native.js";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
ExecutableQuery,
|
||||
Query,
|
||||
QueryBase,
|
||||
@@ -135,10 +144,10 @@ export {
|
||||
BranchColumnChange,
|
||||
BranchIndexSummary,
|
||||
BranchRowCountSummary,
|
||||
MergeBlocker,
|
||||
CherryPickError,
|
||||
BranchDiff,
|
||||
MergePreview,
|
||||
MergeBranchResult,
|
||||
CherryPickPreview,
|
||||
CherryPickResult,
|
||||
AddDataOptions,
|
||||
UpdateOptions,
|
||||
OptimizeOptions,
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { RefreshMaterializedViewResult } from "./native";
|
||||
import { Table } from "./table";
|
||||
|
||||
/** Schema metadata key holding a materialized view's definition. */
|
||||
export const DEFINITION_META_KEY = "mv.definition";
|
||||
|
||||
/** The query that defines a materialized view. */
|
||||
export interface MaterializedViewDefinition {
|
||||
/** Name of the source table, in the same database as the view. */
|
||||
sourceTable: string;
|
||||
/** `[output column, SQL expression]` pairs, in view schema order. */
|
||||
projections: [string, string][];
|
||||
/** SQL predicate selecting the source rows the view holds. */
|
||||
filter?: string;
|
||||
/** Cap on the number of rows the view holds. */
|
||||
limit?: number;
|
||||
/** Source columns the projections and filter read. */
|
||||
inputs: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
|
||||
* record of the same. A bare name projects itself.
|
||||
*/
|
||||
export type MaterializedViewSelect =
|
||||
| (string | [string, string])[]
|
||||
| Record<string, string>;
|
||||
|
||||
/**
|
||||
* @internal Reject a numeric option N-API would otherwise silently coerce:
|
||||
* `Infinity` reaches Rust as 0, `1.5` as 1.
|
||||
*/
|
||||
export function validateNonNegativeInteger(
|
||||
value: number | undefined,
|
||||
name: string,
|
||||
): void {
|
||||
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
|
||||
throw new Error(`${name} must be a non-negative integer`);
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
|
||||
function quoteIdentifier(name: string): string {
|
||||
return "`" + name.replace(/`/g, "``") + "`";
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal Normalize a select argument into `[alias, expression]` pairs.
|
||||
* A bare name projects itself and is quoted, so any valid column name works;
|
||||
* pair and record entries are kept verbatim because their right side is an
|
||||
* expression.
|
||||
*/
|
||||
export function normalizeSelect(
|
||||
select?: MaterializedViewSelect,
|
||||
): [string, string][] | undefined {
|
||||
if (select === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
if (Array.isArray(select)) {
|
||||
return select.map((item) =>
|
||||
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
|
||||
);
|
||||
}
|
||||
return Object.entries(select);
|
||||
}
|
||||
|
||||
/** @internal Parse a definition off a table's stored schema metadata. */
|
||||
export function definitionFromMetadata(
|
||||
metadata: Map<string, string>,
|
||||
name: string,
|
||||
): MaterializedViewDefinition {
|
||||
const raw = metadata.get(DEFINITION_META_KEY);
|
||||
if (raw === undefined) {
|
||||
throw new Error(`Table '${name}' is not a materialized view`);
|
||||
}
|
||||
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
|
||||
const value: any = JSON.parse(raw);
|
||||
if (value.kind !== "select") {
|
||||
throw new Error(
|
||||
`materialized view '${name}' is defined by '${value.kind}', which this ` +
|
||||
"version of lancedb cannot refresh",
|
||||
);
|
||||
}
|
||||
const limit = value.limit ?? undefined;
|
||||
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
|
||||
// integer and every rounded one does not, so this rejects precisely the
|
||||
// values a number cannot carry.
|
||||
if (limit !== undefined && !Number.isSafeInteger(limit)) {
|
||||
throw new Error(
|
||||
`materialized view '${name}' has a stored limit too large to represent exactly`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
sourceTable: value.source_table,
|
||||
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
|
||||
projections: (value.projections ?? []).map((p: any) => [
|
||||
p.output,
|
||||
p.expression,
|
||||
]),
|
||||
filter: value.filter ?? undefined,
|
||||
limit,
|
||||
inputs: value.inputs ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A handle on a materialized view: its table plus its definition.
|
||||
*
|
||||
* Obtained from {@link Connection#createMaterializedView} or
|
||||
* {@link Connection#openMaterializedView}. The view is a normal table --
|
||||
* queries, indexes and search all apply through {@link MaterializedView#table}
|
||||
* -- whose contents are maintained by {@link MaterializedView#refresh}.
|
||||
*/
|
||||
export class MaterializedView {
|
||||
private readonly inner: Table;
|
||||
|
||||
constructor(table: Table) {
|
||||
this.inner = table;
|
||||
}
|
||||
|
||||
get name(): string {
|
||||
return this.inner.name;
|
||||
}
|
||||
|
||||
/** The view, as the table it is. */
|
||||
table(): Table {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/** The query that defines the view, read from its stored schema. */
|
||||
async definition(): Promise<MaterializedViewDefinition> {
|
||||
const schema = await this.inner.schema();
|
||||
return definitionFromMetadata(schema.metadata, this.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recompute the view from its source.
|
||||
*
|
||||
* The refresh is incremental when the source's changes can be reconciled
|
||||
* into the view -- rows added, changed or removed since the last one --
|
||||
* and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
|
||||
* refreshes to that source version instead of the latest.
|
||||
*
|
||||
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
||||
* plan the same source rows conflict on commit, and the loser throws
|
||||
* rather than writing them a second time.
|
||||
*/
|
||||
async refresh(options?: {
|
||||
full?: boolean;
|
||||
sourceVersion?: number;
|
||||
}): Promise<RefreshMaterializedViewResult> {
|
||||
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
|
||||
return await this.inner.refreshMaterializedView(
|
||||
options?.full,
|
||||
options?.sourceVersion,
|
||||
);
|
||||
}
|
||||
}
|
||||
+200
-101
@@ -100,6 +100,29 @@ export interface FullTextSearchOptions {
|
||||
columns?: string | string[];
|
||||
}
|
||||
|
||||
function nearestToNative(
|
||||
inner: NativeQuery,
|
||||
vector: Awaited<IntoVector>,
|
||||
): NativeVectorQuery {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(vector as number[]));
|
||||
}
|
||||
|
||||
function addQueryVectorToNative(
|
||||
inner: NativeVectorQuery,
|
||||
vector: Awaited<IntoVector>,
|
||||
) {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
inner.addQueryVectorRaw(raw.data, raw.dtype);
|
||||
} else {
|
||||
inner.addQueryVector(Float32Array.from(vector as number[]));
|
||||
}
|
||||
}
|
||||
|
||||
/** Common methods supported by all query types
|
||||
*
|
||||
* @see {@link Query}
|
||||
@@ -111,13 +134,15 @@ export class QueryBase<
|
||||
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
|
||||
> implements AsyncIterable<RecordBatch>
|
||||
{
|
||||
protected inner!: NativeQueryType | Promise<NativeQueryType>;
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected constructor(
|
||||
protected inner: NativeQueryType | Promise<NativeQueryType>,
|
||||
) {
|
||||
// intentionally empty
|
||||
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
if (inner !== undefined) {
|
||||
this.inner = inner;
|
||||
}
|
||||
}
|
||||
|
||||
// call a function on the inner (either a promise or the actual object)
|
||||
@@ -135,6 +160,15 @@ export class QueryBase<
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the native query used by the next terminal operation.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
protected async getInner(): Promise<NativeQueryType> {
|
||||
return this.inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return only the specified columns.
|
||||
*
|
||||
@@ -207,16 +241,11 @@ export class QueryBase<
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected nativeExecute(
|
||||
protected async nativeExecute(
|
||||
options?: Partial<QueryExecutionOptions>,
|
||||
): Promise<NativeBatchIterator> {
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.execute(options?.maxBatchLength, options?.timeoutMs),
|
||||
);
|
||||
} else {
|
||||
return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -245,12 +274,7 @@ export class QueryBase<
|
||||
/** Collect the results as an Arrow @see {@link ArrowTable}. */
|
||||
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
|
||||
const batches = [];
|
||||
let inner;
|
||||
if (this.inner instanceof Promise) {
|
||||
inner = await this.inner;
|
||||
} else {
|
||||
inner = this.inner;
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
for await (const batch of new RecordBatchIterable(inner, options)) {
|
||||
batches.push(batch);
|
||||
}
|
||||
@@ -279,11 +303,8 @@ export class QueryBase<
|
||||
* @returns A Promise that resolves to a string containing the query execution plan explanation.
|
||||
*/
|
||||
async explainPlan(verbose = false): Promise<string> {
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) => inner.explainPlan(verbose));
|
||||
} else {
|
||||
return this.inner.explainPlan(verbose);
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
return inner.explainPlan(verbose);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -321,13 +342,8 @@ export class QueryBase<
|
||||
distributedMetrics?: AnalyzePlanDistributedMetrics,
|
||||
): Promise<string> {
|
||||
const distributedMetricsMode = distributedMetrics ?? "aggregate";
|
||||
if (this.inner instanceof Promise) {
|
||||
return this.inner.then((inner) =>
|
||||
inner.analyzePlan(distributedMetricsMode),
|
||||
);
|
||||
} else {
|
||||
return this.inner.analyzePlan(distributedMetricsMode);
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
return inner.analyzePlan(distributedMetricsMode);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -339,12 +355,8 @@ export class QueryBase<
|
||||
* @returns An Arrow Schema describing the output columns.
|
||||
*/
|
||||
async outputSchema(): Promise<import("./arrow").Schema> {
|
||||
let schemaBuffer: Buffer;
|
||||
if (this.inner instanceof Promise) {
|
||||
schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
|
||||
} else {
|
||||
schemaBuffer = await this.inner.outputSchema();
|
||||
}
|
||||
const inner = await this.getInner();
|
||||
const schemaBuffer = await inner.outputSchema();
|
||||
const schema = tableFromIPC(schemaBuffer).schema;
|
||||
return schema;
|
||||
}
|
||||
@@ -356,7 +368,7 @@ export class StandardQueryBase<
|
||||
extends QueryBase<NativeQueryType>
|
||||
implements ExecutableQuery
|
||||
{
|
||||
constructor(inner: NativeQueryType | Promise<NativeQueryType>) {
|
||||
constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
@@ -510,6 +522,13 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
super(inner);
|
||||
}
|
||||
|
||||
/**
|
||||
* @hidden
|
||||
*/
|
||||
protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
|
||||
super.doCall(fn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the number of partitions to search (probe)
|
||||
*
|
||||
@@ -537,7 +556,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* the minimum and maximum to the same value.
|
||||
*/
|
||||
nprobes(nprobes: number): VectorQuery {
|
||||
super.doCall((inner) => inner.nprobes(nprobes));
|
||||
this.doVectorCall((inner) => inner.nprobes(nprobes));
|
||||
|
||||
return this;
|
||||
}
|
||||
@@ -551,7 +570,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* but will also increase latency.
|
||||
*/
|
||||
minimumNprobes(minimumNprobes: number): VectorQuery {
|
||||
super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
|
||||
this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -565,7 +584,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* potential false negatives.
|
||||
*/
|
||||
maximumNprobes(maximumNprobes: number): VectorQuery {
|
||||
super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
|
||||
this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -578,7 +597,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* `undefined` means no lower or upper bound.
|
||||
*/
|
||||
distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
|
||||
super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
|
||||
this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -592,7 +611,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* also increase the latency of your query. The default value is 1.5*limit.
|
||||
*/
|
||||
ef(ef: number): VectorQuery {
|
||||
super.doCall((inner) => inner.ef(ef));
|
||||
this.doVectorCall((inner) => inner.ef(ef));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -606,7 +625,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* whose data type is a fixed-size-list of floats.
|
||||
*/
|
||||
column(column: string): VectorQuery {
|
||||
super.doCall((inner) => inner.column(column));
|
||||
this.doVectorCall((inner) => inner.column(column));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -627,7 +646,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
distanceType(
|
||||
distanceType: Required<IvfPqOptions>["distanceType"],
|
||||
): VectorQuery {
|
||||
super.doCall((inner) => inner.distanceType(distanceType));
|
||||
this.doVectorCall((inner) => inner.distanceType(distanceType));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -661,7 +680,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* distance between the query vector and the actual uncompressed vector.
|
||||
*/
|
||||
refineFactor(refineFactor: number): VectorQuery {
|
||||
super.doCall((inner) => inner.refineFactor(refineFactor));
|
||||
this.doVectorCall((inner) => inner.refineFactor(refineFactor));
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -686,7 +705,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* factor can often help restore some of the results lost by post filtering.
|
||||
*/
|
||||
postfilter(): VectorQuery {
|
||||
super.doCall((inner) => inner.postfilter());
|
||||
this.doVectorCall((inner) => inner.postfilter());
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -700,7 +719,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
* calculate your recall to select an appropriate value for nprobes.
|
||||
*/
|
||||
bypassVectorIndex(): VectorQuery {
|
||||
super.doCall((inner) => inner.bypassVectorIndex());
|
||||
this.doVectorCall((inner) => inner.bypassVectorIndex());
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -716,35 +735,31 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
*/
|
||||
addQueryVector(vector: IntoVector): VectorQuery {
|
||||
if (vector instanceof Promise) {
|
||||
// Observe the promise as soon as it is accepted. The existing native
|
||||
// query may still be pending, and delaying observation until it resolves
|
||||
// can otherwise surface a fast rejection as unhandled.
|
||||
const settledVector = vector.then(
|
||||
(value) => ({ status: "fulfilled" as const, value }),
|
||||
(reason) => ({ status: "rejected" as const, reason }),
|
||||
);
|
||||
const res = (async () => {
|
||||
try {
|
||||
const v = await vector;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
|
||||
const value: any = this.addQueryVector(v);
|
||||
const inner = value.inner as
|
||||
| NativeVectorQuery
|
||||
| Promise<NativeVectorQuery>;
|
||||
return inner;
|
||||
} catch (e) {
|
||||
return Promise.reject(e);
|
||||
const inner = await this.getInner();
|
||||
const outcome = await settledVector;
|
||||
if (outcome.status === "rejected") {
|
||||
throw outcome.reason;
|
||||
}
|
||||
addQueryVectorToNative(inner, outcome.value);
|
||||
return inner;
|
||||
})();
|
||||
return new VectorQuery(res);
|
||||
} else {
|
||||
super.doCall((inner) => {
|
||||
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
|
||||
if (raw) {
|
||||
inner.addQueryVectorRaw(raw.data, raw.dtype);
|
||||
} else {
|
||||
inner.addQueryVector(Float32Array.from(vector as number[]));
|
||||
}
|
||||
});
|
||||
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
rerank(reranker: Reranker): VectorQuery {
|
||||
super.doCall((inner) =>
|
||||
this.doVectorCall((inner) =>
|
||||
inner.rerank(async (args) => {
|
||||
const vecResults = await fromBufferToRecordBatch(args.vecResults);
|
||||
const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
|
||||
@@ -763,6 +778,71 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a string query whose vector/FTS routing is resolved against the active
|
||||
* table schema when the query executes.
|
||||
*
|
||||
* @hidden
|
||||
*/
|
||||
export function createAutoQuery(
|
||||
table: NativeTable,
|
||||
query: string,
|
||||
columns: string[] | null,
|
||||
getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
|
||||
): AutoQuery {
|
||||
type RouteSnapshot = {
|
||||
table: NativeTable;
|
||||
embeddingMetadata: string | undefined;
|
||||
};
|
||||
type CachedPreparation = {
|
||||
metadata: string;
|
||||
vector: Promise<Awaited<IntoVector>>;
|
||||
};
|
||||
|
||||
let cachedPreparation: CachedPreparation | undefined;
|
||||
|
||||
const snapshotRoute = async (): Promise<RouteSnapshot> => {
|
||||
const snapshot = await table.querySnapshot();
|
||||
const schema = tableFromIPC(await snapshot.schema()).schema;
|
||||
return {
|
||||
table: snapshot,
|
||||
embeddingMetadata: schema.metadata.get("embedding_functions"),
|
||||
};
|
||||
};
|
||||
|
||||
const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
|
||||
const route = await snapshotRoute();
|
||||
if (route.embeddingMetadata === undefined) {
|
||||
const inner = route.table.query();
|
||||
inner.fullTextSearch({ query, columns });
|
||||
return inner;
|
||||
}
|
||||
|
||||
const metadata = route.embeddingMetadata;
|
||||
if (cachedPreparation?.metadata !== metadata) {
|
||||
cachedPreparation = {
|
||||
metadata,
|
||||
vector: Promise.resolve().then(() => getVector(metadata)),
|
||||
};
|
||||
}
|
||||
|
||||
const preparation = cachedPreparation;
|
||||
let vector: Awaited<IntoVector>;
|
||||
try {
|
||||
vector = await preparation.vector;
|
||||
} catch (error) {
|
||||
if (cachedPreparation === preparation) {
|
||||
cachedPreparation = undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return nearestToNative(route.table.query(), vector);
|
||||
};
|
||||
|
||||
return new AutoQuery(createInner);
|
||||
}
|
||||
|
||||
/**
|
||||
* A query that returns a subset of the rows in the table.
|
||||
*
|
||||
@@ -788,6 +868,51 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A builder for automatic string searches.
|
||||
*
|
||||
* Automatic search determines whether to use full-text or vector search from
|
||||
* the table revision selected for each execution. This builder exposes the
|
||||
* common operations supported by both query families.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class AutoQuery extends StandardQueryBase<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
private readonly calls: Array<
|
||||
(inner: NativeQuery | NativeVectorQuery) => void
|
||||
> = [];
|
||||
|
||||
/** @hidden */
|
||||
constructor(
|
||||
private readonly createInner: () => Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
>,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override doCall(
|
||||
fn: (inner: NativeQuery | NativeVectorQuery) => void,
|
||||
) {
|
||||
this.calls.push(fn);
|
||||
}
|
||||
|
||||
/** @hidden */
|
||||
protected override async getInner(): Promise<
|
||||
NativeQuery | NativeVectorQuery
|
||||
> {
|
||||
const calls = [...this.calls];
|
||||
const inner = await this.createInner();
|
||||
for (const call of calls) {
|
||||
call(inner);
|
||||
}
|
||||
return inner;
|
||||
}
|
||||
}
|
||||
|
||||
/** A builder for LanceDB queries.
|
||||
*
|
||||
* @see {@link Table#query}, {@link Table#search}
|
||||
@@ -840,45 +965,19 @@ export class Query extends StandardQueryBase<NativeQuery> {
|
||||
* a default `limit` of 10 will be used. @see {@link Query#limit}
|
||||
*/
|
||||
nearestTo(vector: IntoVector): VectorQuery {
|
||||
const callNearestTo = (
|
||||
inner: NativeQuery,
|
||||
resolved: Float32Array | Float64Array | Uint8Array | number[],
|
||||
): NativeVectorQuery => {
|
||||
const raw = Array.isArray(resolved)
|
||||
? null
|
||||
: extractVectorBuffer(resolved);
|
||||
if (raw) {
|
||||
return inner.nearestToRaw(raw.data, raw.dtype);
|
||||
}
|
||||
return inner.nearestTo(Float32Array.from(resolved as number[]));
|
||||
};
|
||||
|
||||
if (this.inner instanceof Promise) {
|
||||
const nativeQuery = this.inner.then(async (inner) => {
|
||||
const resolved = vector instanceof Promise ? await vector : vector;
|
||||
return callNearestTo(inner, resolved);
|
||||
});
|
||||
const inner = this.inner;
|
||||
if (inner instanceof Promise) {
|
||||
const nativeQuery = inner.then(async (resolvedInner) =>
|
||||
nearestToNative(resolvedInner, await vector),
|
||||
);
|
||||
return new VectorQuery(nativeQuery);
|
||||
}
|
||||
if (vector instanceof Promise) {
|
||||
const res = (async () => {
|
||||
try {
|
||||
const v = await vector;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
|
||||
const value: any = this.nearestTo(v);
|
||||
const inner = value.inner as
|
||||
| NativeVectorQuery
|
||||
| Promise<NativeVectorQuery>;
|
||||
return inner;
|
||||
} catch (e) {
|
||||
return Promise.reject(e);
|
||||
}
|
||||
})();
|
||||
return new VectorQuery(res);
|
||||
} else {
|
||||
const vectorQuery = callNearestTo(this.inner, vector);
|
||||
return new VectorQuery(vectorQuery);
|
||||
return new VectorQuery(
|
||||
vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
|
||||
);
|
||||
}
|
||||
return new VectorQuery(nearestToNative(inner, vector));
|
||||
}
|
||||
|
||||
nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
|
||||
|
||||
@@ -0,0 +1,566 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
Binary,
|
||||
Bool,
|
||||
DataType,
|
||||
Dictionary,
|
||||
Field,
|
||||
FixedSizeList,
|
||||
Float32,
|
||||
Float64,
|
||||
Int32,
|
||||
Int64,
|
||||
List,
|
||||
Schema,
|
||||
Struct,
|
||||
Utf8,
|
||||
util as arrowUtil,
|
||||
} from "apache-arrow";
|
||||
import { typedArrayToArrowType } from "./arrow_type";
|
||||
import { sanitizeType } from "./sanitize";
|
||||
|
||||
type InferenceOptions = {
|
||||
dictionaryEncodeStrings: boolean;
|
||||
vectorColumns: Record<string, { type: unknown }>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Infer the Arrow schema represented by a set of records.
|
||||
*
|
||||
* This is the intentionally small interface to schema inference. The stateful
|
||||
* details of combining partial type evidence are encapsulated below so callers
|
||||
* only need to provide records, an optional schema, and inference options.
|
||||
*/
|
||||
export function inferSchema(
|
||||
data: Array<Record<string, unknown>>,
|
||||
schema: Schema | undefined,
|
||||
options: InferenceOptions,
|
||||
): Schema {
|
||||
return new SchemaInferrer(schema, options).infer(data);
|
||||
}
|
||||
|
||||
class SchemaInferrer {
|
||||
private readonly fields = new FieldTree();
|
||||
|
||||
constructor(
|
||||
private readonly providedSchema: Schema | undefined,
|
||||
private readonly options: InferenceOptions,
|
||||
) {}
|
||||
|
||||
infer(data: Array<Record<string, unknown>>): Schema {
|
||||
for (const [row, record] of data.entries()) {
|
||||
for (const [path, value] of recordPathsAndValues(record)) {
|
||||
this.observe(path, value, row);
|
||||
}
|
||||
}
|
||||
|
||||
return this.providedSchema === undefined
|
||||
? new Schema(fieldsFromTree(this.fields))
|
||||
: new Schema(matchingFields(this.providedSchema.fields, this.fields));
|
||||
}
|
||||
|
||||
private observe(path: string[], value: unknown, row: number): void {
|
||||
const current = this.fields.get(path);
|
||||
if (current === undefined) {
|
||||
this.addField(path, value, row);
|
||||
} else if (this.providedSchema === undefined) {
|
||||
this.updateInferredField(path, value, row, current);
|
||||
}
|
||||
}
|
||||
|
||||
private addField(path: string[], value: unknown, row: number): void {
|
||||
if (this.providedSchema !== undefined) {
|
||||
this.addSchemaField(this.providedSchema, path, row);
|
||||
return;
|
||||
}
|
||||
|
||||
const evidence =
|
||||
this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
|
||||
if (evidence === undefined) {
|
||||
throw typeInferenceError(path, row);
|
||||
}
|
||||
|
||||
const conflict = this.fields.set(
|
||||
path,
|
||||
evidence,
|
||||
(existing) =>
|
||||
existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
|
||||
);
|
||||
if (conflict !== undefined) {
|
||||
throw branchConflictError(conflict, row, "Struct");
|
||||
}
|
||||
}
|
||||
|
||||
private addSchemaField(schema: Schema, path: string[], row: number): void {
|
||||
const field = fieldAtPath(schema, path);
|
||||
if (field === undefined) {
|
||||
throw new Error(
|
||||
`Found field not in schema: ${path.join(".")} at row ${row}`,
|
||||
);
|
||||
}
|
||||
|
||||
const conflict = this.fields.set(path, field.type);
|
||||
if (conflict !== undefined) {
|
||||
throw branchConflictError(conflict, row, "Struct");
|
||||
}
|
||||
}
|
||||
|
||||
private updateInferredField(
|
||||
path: string[],
|
||||
value: unknown,
|
||||
row: number,
|
||||
current: FieldNode,
|
||||
): void {
|
||||
const newType = this.inferType(value, path);
|
||||
const deferred = DeferredTypeEvidence.from(value, row);
|
||||
|
||||
if (current instanceof FieldTree) {
|
||||
if (deferred?.isOnlyNulls()) {
|
||||
return;
|
||||
}
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
"Struct",
|
||||
describeEvidence(newType ?? deferred),
|
||||
);
|
||||
}
|
||||
|
||||
if (current instanceof DeferredTypeEvidence) {
|
||||
this.resolveDeferredField(path, row, current, newType, deferred);
|
||||
return;
|
||||
}
|
||||
|
||||
if (newType !== undefined) {
|
||||
if (!inferredTypesEqual(current, newType)) {
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
describeEvidence(current),
|
||||
describeEvidence(newType),
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (deferred === undefined || !deferred.matches(current)) {
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
describeEvidence(current),
|
||||
describeEvidence(deferred),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private resolveDeferredField(
|
||||
path: string[],
|
||||
row: number,
|
||||
current: DeferredTypeEvidence,
|
||||
newType: DataType | undefined,
|
||||
deferred: DeferredTypeEvidence | undefined,
|
||||
): void {
|
||||
if (newType !== undefined) {
|
||||
if (!current.matches(newType)) {
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
current.describe(),
|
||||
describeEvidence(newType),
|
||||
);
|
||||
}
|
||||
this.fields.set(path, newType);
|
||||
return;
|
||||
}
|
||||
|
||||
if (deferred !== undefined) {
|
||||
this.fields.set(path, current.merge(deferred));
|
||||
return;
|
||||
}
|
||||
|
||||
throw schemaInferenceError(
|
||||
path,
|
||||
row,
|
||||
current.describe(),
|
||||
describeEvidence(newType),
|
||||
);
|
||||
}
|
||||
|
||||
private inferType(value: unknown, path: string[]): DataType | undefined {
|
||||
if (typeof value === "bigint") {
|
||||
return new Int64();
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
return new Float64();
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return this.options.dictionaryEncodeStrings
|
||||
? new Dictionary(new Utf8(), new Int32())
|
||||
: new Utf8();
|
||||
}
|
||||
if (typeof value === "boolean") {
|
||||
return new Bool();
|
||||
}
|
||||
if (value instanceof Buffer) {
|
||||
return new Binary();
|
||||
}
|
||||
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
||||
const typedArray = typedArrayToArrowType(value);
|
||||
return typedArray === undefined
|
||||
? undefined
|
||||
: new FixedSizeList(
|
||||
typedArray.length,
|
||||
new Field("item", typedArray.elementType, true),
|
||||
);
|
||||
}
|
||||
if (!Array.isArray(value) || value.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const configuredVector =
|
||||
path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
|
||||
if (configuredVector !== undefined) {
|
||||
return new FixedSizeList(
|
||||
value.length,
|
||||
new Field("item", sanitizeType(configuredVector.type), true),
|
||||
);
|
||||
}
|
||||
|
||||
const itemType = this.inferArrayItemType(value, path);
|
||||
if (itemType === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return nameSuggestsVectorColumn(path[path.length - 1])
|
||||
? new FixedSizeList(value.length, new Field("item", new Float32(), true))
|
||||
: new List(new Field("item", itemType, true));
|
||||
}
|
||||
|
||||
private inferArrayItemType(
|
||||
values: unknown[],
|
||||
path: string[],
|
||||
): DataType | undefined {
|
||||
let itemType: DataType | undefined;
|
||||
const deferredItems: unknown[] = [];
|
||||
|
||||
for (const value of values) {
|
||||
const candidate = this.inferType(value, path);
|
||||
if (candidate === undefined) {
|
||||
if (!isDeferredValue(value)) {
|
||||
return undefined;
|
||||
}
|
||||
deferredItems.push(value);
|
||||
} else if (itemType === undefined) {
|
||||
itemType = candidate;
|
||||
} else if (!inferredTypesEqual(itemType, candidate)) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
if (itemType === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return deferredItems.every((value) =>
|
||||
deferredValueMatchesType(value, itemType),
|
||||
)
|
||||
? itemType
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Nulls and empty/all-null lists that do not determine a type by themselves. */
|
||||
class DeferredTypeEvidence {
|
||||
private constructor(
|
||||
private readonly values: Array<{ value: unknown; row: number }>,
|
||||
) {}
|
||||
|
||||
static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
|
||||
return isDeferredValue(value)
|
||||
? new DeferredTypeEvidence([{ value, row }])
|
||||
: undefined;
|
||||
}
|
||||
|
||||
isOnlyNulls(): boolean {
|
||||
return this.values.every(({ value }) => value == null);
|
||||
}
|
||||
|
||||
matches(type: DataType): boolean {
|
||||
return this.values.every(({ value }) =>
|
||||
deferredValueMatchesType(value, type),
|
||||
);
|
||||
}
|
||||
|
||||
merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
|
||||
return new DeferredTypeEvidence([...this.values, ...other.values]);
|
||||
}
|
||||
|
||||
describe(): string {
|
||||
const list = this.values.find(({ value }) => Array.isArray(value));
|
||||
return list === undefined
|
||||
? "null"
|
||||
: `List[${(list.value as unknown[]).length}]`;
|
||||
}
|
||||
|
||||
firstRow(): number {
|
||||
return this.values[0].row;
|
||||
}
|
||||
}
|
||||
|
||||
type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
|
||||
type LeafNode = Exclude<FieldNode, FieldTree>;
|
||||
type FieldConflict = { path: string[]; value: FieldNode };
|
||||
|
||||
/** Nested field state, kept separate from Arrow's eventual Struct types. */
|
||||
class FieldTree {
|
||||
private readonly children = new Map<string, FieldNode>();
|
||||
|
||||
get(path: string[]): FieldNode | undefined {
|
||||
let current: FieldNode = this;
|
||||
for (const part of path) {
|
||||
if (!(current instanceof FieldTree)) {
|
||||
return undefined;
|
||||
}
|
||||
const child = current.children.get(part);
|
||||
if (child === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
current = child;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
set(
|
||||
path: string[],
|
||||
value: LeafNode,
|
||||
canReplaceLeaf: (value: LeafNode) => boolean = () => false,
|
||||
): FieldConflict | undefined {
|
||||
let branch: FieldTree = this;
|
||||
for (const [index, part] of path.slice(0, -1).entries()) {
|
||||
const child = branch.children.get(part);
|
||||
if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
|
||||
const nextBranch = new FieldTree();
|
||||
branch.children.set(part, nextBranch);
|
||||
branch = nextBranch;
|
||||
} else if (child instanceof FieldTree) {
|
||||
branch = child;
|
||||
} else {
|
||||
return { path: path.slice(0, index + 1), value: child };
|
||||
}
|
||||
}
|
||||
|
||||
const name = path[path.length - 1];
|
||||
const current = branch.children.get(name);
|
||||
if (current instanceof FieldTree) {
|
||||
return { path, value: current };
|
||||
}
|
||||
branch.children.set(name, value);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
entries(): IterableIterator<[string, FieldNode]> {
|
||||
return this.children.entries();
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this.children.has(name);
|
||||
}
|
||||
}
|
||||
|
||||
function isLeaf(value: FieldNode): value is LeafNode {
|
||||
return !(value instanceof FieldTree);
|
||||
}
|
||||
|
||||
function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
|
||||
const fields: Field[] = [];
|
||||
for (const [name, value] of tree.entries()) {
|
||||
if (value instanceof FieldTree) {
|
||||
fields.push(
|
||||
new Field(
|
||||
name,
|
||||
new Struct(fieldsFromTree(value, [...path, name])),
|
||||
true,
|
||||
),
|
||||
);
|
||||
} else if (value instanceof DeferredTypeEvidence) {
|
||||
throw typeInferenceError([...path, name], value.firstRow());
|
||||
} else {
|
||||
fields.push(new Field(name, value, true));
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
function matchingFields(fields: Field[], tree: FieldTree): Field[] {
|
||||
const matches: Field[] = [];
|
||||
for (const field of fields) {
|
||||
if (!tree.has(field.name)) {
|
||||
continue;
|
||||
}
|
||||
const value = tree.get([field.name]);
|
||||
if (value instanceof FieldTree) {
|
||||
const struct = field.type as Struct;
|
||||
matches.push(
|
||||
new Field(
|
||||
field.name,
|
||||
new Struct(matchingFields(struct.children, value)),
|
||||
field.nullable,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
matches.push(new Field(field.name, value as DataType, field.nullable));
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
function* recordPathsAndValues(
|
||||
record: Record<string, unknown>,
|
||||
path: string[] = [],
|
||||
): Generator<[string[], unknown]> {
|
||||
for (const [name, value] of Object.entries(record)) {
|
||||
if (isRecord(value)) {
|
||||
yield* recordPathsAndValues(value, [...path, name]);
|
||||
} else if (value !== undefined) {
|
||||
yield [[...path, name], value];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
!Array.isArray(value) &&
|
||||
!(value instanceof RegExp) &&
|
||||
!(value instanceof Date) &&
|
||||
!(value instanceof Set) &&
|
||||
!(value instanceof Map) &&
|
||||
!(value instanceof Buffer) &&
|
||||
!ArrayBuffer.isView(value)
|
||||
);
|
||||
}
|
||||
|
||||
function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
|
||||
let fields = schema.fields;
|
||||
let field: Field | undefined;
|
||||
for (const [index, name] of path.entries()) {
|
||||
field = fields.find((candidate) => candidate.name === name);
|
||||
if (field === undefined || index === path.length - 1) {
|
||||
return field;
|
||||
}
|
||||
if (!DataType.isStruct(field.type)) {
|
||||
return undefined;
|
||||
}
|
||||
fields = field.type.children;
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
function isDeferredValue(value: unknown): boolean {
|
||||
return (
|
||||
value == null || (Array.isArray(value) && value.every(isDeferredValue))
|
||||
);
|
||||
}
|
||||
|
||||
function deferredValueMatchesType(value: unknown, type: DataType): boolean {
|
||||
if (value == null) {
|
||||
return true;
|
||||
}
|
||||
if (!Array.isArray(value)) {
|
||||
return false;
|
||||
}
|
||||
if (DataType.isList(type)) {
|
||||
return value.every((item) =>
|
||||
deferredValueMatchesType(item, type.valueType),
|
||||
);
|
||||
}
|
||||
if (DataType.isFixedSizeList(type)) {
|
||||
return (
|
||||
value.length === type.listSize &&
|
||||
value.every((item) => deferredValueMatchesType(item, type.valueType))
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
|
||||
if (DataType.isDictionary(current)) {
|
||||
return (
|
||||
DataType.isDictionary(candidate) &&
|
||||
current.isOrdered === candidate.isOrdered &&
|
||||
inferredTypesEqual(current.indices, candidate.indices) &&
|
||||
inferredTypesEqual(current.dictionary, candidate.dictionary)
|
||||
);
|
||||
}
|
||||
if (DataType.isList(current)) {
|
||||
return (
|
||||
DataType.isList(candidate) &&
|
||||
current.valueField.name === candidate.valueField.name &&
|
||||
current.valueField.nullable === candidate.valueField.nullable &&
|
||||
inferredTypesEqual(current.valueType, candidate.valueType)
|
||||
);
|
||||
}
|
||||
if (DataType.isFixedSizeList(current)) {
|
||||
return (
|
||||
DataType.isFixedSizeList(candidate) &&
|
||||
current.listSize === candidate.listSize &&
|
||||
current.valueField.name === candidate.valueField.name &&
|
||||
current.valueField.nullable === candidate.valueField.nullable &&
|
||||
inferredTypesEqual(current.valueType, candidate.valueType)
|
||||
);
|
||||
}
|
||||
return arrowUtil.compareTypes(current, candidate);
|
||||
}
|
||||
|
||||
function describeEvidence(
|
||||
evidence: DataType | DeferredTypeEvidence | undefined,
|
||||
): string {
|
||||
if (evidence === undefined) {
|
||||
return "an unsupported value";
|
||||
}
|
||||
return evidence instanceof DeferredTypeEvidence
|
||||
? evidence.describe()
|
||||
: evidence.toString();
|
||||
}
|
||||
|
||||
function branchConflictError(
|
||||
conflict: FieldConflict,
|
||||
row: number,
|
||||
candidate: string,
|
||||
): Error {
|
||||
return schemaInferenceError(
|
||||
conflict.path,
|
||||
row,
|
||||
conflict.value instanceof FieldTree
|
||||
? "Struct"
|
||||
: describeEvidence(conflict.value),
|
||||
candidate,
|
||||
);
|
||||
}
|
||||
|
||||
function schemaInferenceError(
|
||||
path: string[],
|
||||
row: number,
|
||||
currentType: string,
|
||||
newType: string,
|
||||
): Error {
|
||||
return new Error(
|
||||
`Failed to infer schema for data. Previously inferred type ${currentType} ` +
|
||||
`but found ${newType} for field ${path.join(".")} at row ${row}. ` +
|
||||
"Consider providing an explicit schema.",
|
||||
);
|
||||
}
|
||||
|
||||
function typeInferenceError(path: string[], row: number): Error {
|
||||
return new Error(
|
||||
`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
|
||||
"Consider providing an explicit schema.",
|
||||
);
|
||||
}
|
||||
|
||||
function nameSuggestsVectorColumn(name: string): boolean {
|
||||
const normalized = name.toLowerCase();
|
||||
return normalized.includes("vector") || normalized.includes("embedding");
|
||||
}
|
||||
+69
-33
@@ -35,6 +35,7 @@ import {
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
RefreshColumnResult,
|
||||
RefreshMaterializedViewResult,
|
||||
TableStatistics,
|
||||
Tags,
|
||||
UpdateFieldMetadataResult,
|
||||
@@ -42,10 +43,12 @@ import {
|
||||
Table as _NativeTable,
|
||||
} from "./native";
|
||||
import {
|
||||
AutoQuery,
|
||||
FullTextQuery,
|
||||
Query,
|
||||
TakeQuery,
|
||||
VectorQuery,
|
||||
createAutoQuery,
|
||||
instanceOfFullTextQuery,
|
||||
} from "./query";
|
||||
import { sanitizeType } from "./sanitize";
|
||||
@@ -522,7 +525,7 @@ export abstract class Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType?: string,
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query;
|
||||
): VectorQuery | Query | AutoQuery;
|
||||
/**
|
||||
* Search the table with a given query vector.
|
||||
*
|
||||
@@ -602,6 +605,18 @@ export abstract class Table {
|
||||
*/
|
||||
abstract refreshColumnAsync(column: string): Promise<Job>;
|
||||
|
||||
/**
|
||||
* Recompute this table's contents from its materialized-view definition.
|
||||
*
|
||||
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
|
||||
* it: rejects tables that carry no view definition. Local tables only.
|
||||
* @ignore
|
||||
*/
|
||||
abstract refreshMaterializedView(
|
||||
full?: boolean,
|
||||
sourceVersion?: number,
|
||||
): Promise<RefreshMaterializedViewResult>;
|
||||
|
||||
/**
|
||||
* Alter the name or nullability of columns.
|
||||
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
|
||||
@@ -962,10 +977,11 @@ export class LocalTable extends Table {
|
||||
return this.inner.display();
|
||||
}
|
||||
|
||||
private async getEmbeddingFunctions(): Promise<
|
||||
Map<string, EmbeddingFunctionConfig>
|
||||
> {
|
||||
const schema = await this.schema();
|
||||
private async getEmbeddingFunctions(
|
||||
inner: _NativeTable = this.inner,
|
||||
): Promise<Map<string, EmbeddingFunctionConfig>> {
|
||||
const schemaBuf = await inner.schema();
|
||||
const schema = tableFromIPC(schemaBuf).schema;
|
||||
const registry = getRegistry();
|
||||
return registry.parseFunctions(schema.metadata);
|
||||
}
|
||||
@@ -1147,7 +1163,7 @@ export class LocalTable extends Table {
|
||||
query: string | IntoVector | MultiVector | FullTextQuery,
|
||||
queryType: string = "auto",
|
||||
ftsColumns?: string | string[],
|
||||
): VectorQuery | Query {
|
||||
): VectorQuery | Query | AutoQuery {
|
||||
if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
|
||||
if (queryType === "fts") {
|
||||
throw new Error("Cannot perform full text search on a vector query");
|
||||
@@ -1162,14 +1178,28 @@ export class LocalTable extends Table {
|
||||
});
|
||||
}
|
||||
|
||||
// The query type is auto or vector
|
||||
// fall back to full text search if no embedding functions are defined and the query is a string
|
||||
if (
|
||||
queryType === "auto" &&
|
||||
(getRegistry().length() === 0 || instanceOfFullTextQuery(query))
|
||||
) {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
if (queryType === "auto") {
|
||||
if (instanceOfFullTextQuery(query)) {
|
||||
return this.query().fullTextSearch(query, {
|
||||
columns: ftsColumns,
|
||||
});
|
||||
}
|
||||
|
||||
const columns =
|
||||
typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
|
||||
return createAutoQuery(this.inner, query, columns, async (metadata) => {
|
||||
const functions = await getRegistry().parseFunctions(
|
||||
new Map([["embedding_functions", metadata]]),
|
||||
);
|
||||
// TODO: Support multiple embedding functions
|
||||
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
|
||||
.values()
|
||||
.next().value;
|
||||
// The route only calls this callback when embedding metadata exists.
|
||||
// parseFunctions either yields a provider or reports malformed metadata.
|
||||
if (!embeddingFunc)
|
||||
throw new Error("Invalid embedding function metadata");
|
||||
return await embeddingFunc.function.computeQueryEmbeddings(query);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1264,6 +1294,13 @@ export class LocalTable extends Table {
|
||||
return await this.inner.refreshColumnAsync(column);
|
||||
}
|
||||
|
||||
async refreshMaterializedView(
|
||||
full?: boolean,
|
||||
sourceVersion?: number,
|
||||
): Promise<RefreshMaterializedViewResult> {
|
||||
return await this.inner.refreshMaterializedView(full, sourceVersion);
|
||||
}
|
||||
|
||||
async alterColumns(
|
||||
columnAlterations: ColumnAlteration[],
|
||||
): Promise<AlterColumnsResult> {
|
||||
@@ -1557,8 +1594,8 @@ export interface BranchRowCountSummary {
|
||||
deltaAvailable: boolean;
|
||||
}
|
||||
|
||||
/** A reason why a branch cannot currently be merged. */
|
||||
export interface MergeBlocker {
|
||||
/** A reason why a cherry-pick cannot currently land. */
|
||||
export interface CherryPickError {
|
||||
code: string;
|
||||
message: string;
|
||||
}
|
||||
@@ -1578,20 +1615,19 @@ export interface BranchDiff {
|
||||
changedColumns: BranchColumnChange[];
|
||||
addedIndexes: BranchIndexSummary[];
|
||||
removedIndexes: BranchIndexSummary[];
|
||||
mergeable: boolean;
|
||||
mergeBlockers: MergeBlocker[];
|
||||
errors: CherryPickError[];
|
||||
}
|
||||
|
||||
/** Changes that would be, or were, promoted by a branch merge. */
|
||||
export interface MergePreview {
|
||||
/** Changes that would be, or were, promoted by a cherry-pick. */
|
||||
export interface CherryPickPreview {
|
||||
promotedColumns: string[];
|
||||
}
|
||||
|
||||
/** Result of previewing or attempting a branch merge. */
|
||||
export interface MergeBranchResult {
|
||||
status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
|
||||
/** Result of previewing or attempting a cherry-pick. */
|
||||
export interface CherryPickResult {
|
||||
status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
|
||||
diff: BranchDiff;
|
||||
preview: MergePreview;
|
||||
preview: CherryPickPreview;
|
||||
mainVersionAfter?: number;
|
||||
}
|
||||
|
||||
@@ -1654,21 +1690,21 @@ export class Branches {
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a branch into main.
|
||||
* Cherry-pick a branch onto main.
|
||||
*
|
||||
* Set `dryRun` to `true` to preview the merge. A rejected merge resolves
|
||||
* with `status: "rejected"` instead of throwing.
|
||||
* Set `dryRun` to `true` to preview. A failed cherry-pick resolves
|
||||
* with `status: "failed"` instead of throwing.
|
||||
*
|
||||
* @param fromBranch Branch to merge from.
|
||||
* @param dryRun When true, only preview the merge. Defaults to false.
|
||||
* @param fromBranch Branch to cherry-pick from.
|
||||
* @param dryRun When true, only preview. Defaults to false.
|
||||
*/
|
||||
async merge(
|
||||
async cherryPick(
|
||||
fromBranch: string,
|
||||
dryRun: boolean = false,
|
||||
): Promise<MergeBranchResult> {
|
||||
return (await this.#inner.merge(
|
||||
): Promise<CherryPickResult> {
|
||||
return (await this.#inner.cherryPick(
|
||||
fromBranch,
|
||||
dryRun,
|
||||
)) as unknown as MergeBranchResult;
|
||||
)) as unknown as CherryPickResult;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.38.0-beta.3",
|
||||
"version": "0.38.0-beta.10",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
@@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne
|
||||
|
||||
use lance_namespace::models::{
|
||||
CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
|
||||
ListTablesRequest,
|
||||
};
|
||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
|
||||
@@ -36,6 +37,12 @@ pub struct ListNamespacesResponse {
|
||||
pub page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct ListTablesResponse {
|
||||
pub tables: Vec<String>,
|
||||
pub page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct CreateNamespaceResponse {
|
||||
pub properties: Option<HashMap<String, String>>,
|
||||
@@ -206,6 +213,33 @@ impl Connection {
|
||||
op.execute().await.default_error()
|
||||
}
|
||||
|
||||
/// List a page of tables in the database.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn list_tables(
|
||||
&self,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
page_token: Option<String>,
|
||||
limit: Option<u32>,
|
||||
) -> napi::Result<ListTablesResponse> {
|
||||
let request = ListTablesRequest {
|
||||
// The root namespace is an empty path, not an absent one: a namespace-backed
|
||||
// database rejects a request that names no namespace.
|
||||
id: Some(namespace_path.unwrap_or_default()),
|
||||
page_token,
|
||||
limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
|
||||
..Default::default()
|
||||
};
|
||||
let response = self
|
||||
.get_inner()?
|
||||
.list_tables(request)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(ListTablesResponse {
|
||||
tables: response.tables,
|
||||
page_token: response.page_token,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create table from a Apache Arrow IPC (file) buffer.
|
||||
///
|
||||
/// Parameters:
|
||||
@@ -266,6 +300,58 @@ impl Connection {
|
||||
Ok(Table::new(tbl))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn create_materialized_view(
|
||||
&self,
|
||||
name: String,
|
||||
source: String,
|
||||
projections: Option<Vec<Vec<String>>>,
|
||||
filter: Option<String>,
|
||||
limit: Option<i64>,
|
||||
) -> napi::Result<Table> {
|
||||
let mut builder = self.get_inner()?.create_materialized_view(name, source);
|
||||
if let Some(projections) = projections {
|
||||
let mut pairs = Vec::with_capacity(projections.len());
|
||||
for pair in projections {
|
||||
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
|
||||
napi::Error::from_reason("each projection must be an [output, expression] pair")
|
||||
})?;
|
||||
pairs.push((output, expression));
|
||||
}
|
||||
builder = builder.select(pairs);
|
||||
}
|
||||
if let Some(filter) = filter {
|
||||
builder = builder.only_if(filter);
|
||||
}
|
||||
if let Some(limit) = limit {
|
||||
let limit = u64::try_from(limit)
|
||||
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
|
||||
builder = builder.limit(limit);
|
||||
}
|
||||
let view = builder.execute().await.default_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
|
||||
let view = self
|
||||
.get_inner()?
|
||||
.open_materialized_view(&name)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(Table::new(view.table().clone()))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
|
||||
let views = self
|
||||
.get_inner()?
|
||||
.list_materialized_views()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(views.into_iter().map(|v| v.name).collect())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_table(
|
||||
&self,
|
||||
|
||||
+5
-2
@@ -14,9 +14,12 @@ pub struct Job {
|
||||
}
|
||||
|
||||
impl Job {
|
||||
pub(crate) fn new(inner: lancedb::Job) -> Self {
|
||||
pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
|
||||
where
|
||||
T: Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self {
|
||||
inner: Arc::new(inner),
|
||||
inner: Arc::new(inner.map(|_| ())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// The materialized-view refresh future deepens the type graph past the
|
||||
// default trait-recursion depth; same raise as the core crate applies.
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use env_logger::Env;
|
||||
|
||||
+61
-3
@@ -278,6 +278,13 @@ impl Table {
|
||||
Ok(Query::new(self.inner_ref()?.query()))
|
||||
}
|
||||
|
||||
/// Return a read-only table handle pinned to the current query revision.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn query_snapshot(&self) -> napi::Result<Self> {
|
||||
let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?;
|
||||
Ok(Self::new(snapshot))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
|
||||
Ok(TakeQuery::new(
|
||||
@@ -381,6 +388,26 @@ impl Table {
|
||||
Ok(crate::job::Job::new(job))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh_materialized_view(
|
||||
&self,
|
||||
full: Option<bool>,
|
||||
source_version: Option<i64>,
|
||||
) -> napi::Result<RefreshMaterializedViewResult> {
|
||||
let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
|
||||
.await
|
||||
.default_error()?;
|
||||
let mut builder = view.refresh().full(full.unwrap_or(false));
|
||||
if let Some(version) = source_version {
|
||||
let version = u64::try_from(version).map_err(|_| {
|
||||
napi::Error::from_reason("sourceVersion must be a non-negative integer")
|
||||
})?;
|
||||
builder = builder.source_version(version);
|
||||
}
|
||||
let result = builder.execute().await.default_error()?;
|
||||
Ok(result.into())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn add_columns_with_schema(
|
||||
&self,
|
||||
@@ -534,6 +561,12 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout_current(&self) -> napi::Result<Self> {
|
||||
let table = self.inner_ref()?.checkout_current().await.default_error()?;
|
||||
Ok(Self::new(table))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn checkout(&self, version: i64) -> napi::Result<()> {
|
||||
self.inner_ref()?
|
||||
@@ -1387,6 +1420,31 @@ pub struct RefreshColumnResult {
|
||||
pub version: i64,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct RefreshMaterializedViewResult {
|
||||
/// How the view was brought up to date: "rebuild", "incremental" or "no_op".
|
||||
pub mode: String,
|
||||
pub rows_written: i64,
|
||||
pub source_version: i64,
|
||||
pub version: i64,
|
||||
}
|
||||
|
||||
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
|
||||
fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
|
||||
let mode = match value.mode {
|
||||
lancedb::RefreshMode::Rebuild => "rebuild",
|
||||
lancedb::RefreshMode::Incremental => "incremental",
|
||||
lancedb::RefreshMode::NoOp => "no_op",
|
||||
};
|
||||
Self {
|
||||
mode: mode.to_string(),
|
||||
rows_written: value.rows_written as i64,
|
||||
source_version: value.source_version as i64,
|
||||
version: value.version as i64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
|
||||
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
|
||||
Self {
|
||||
@@ -1605,18 +1663,18 @@ impl Branches {
|
||||
}
|
||||
|
||||
#[napi(ts_return_type = "Promise<Record<string, unknown>>")]
|
||||
pub async fn merge(
|
||||
pub async fn cherry_pick(
|
||||
&self,
|
||||
from_branch: String,
|
||||
dry_run: Option<bool>,
|
||||
) -> napi::Result<serde_json::Value> {
|
||||
let result = self
|
||||
.inner
|
||||
.merge_branch(&from_branch, dry_run.unwrap_or(false))
|
||||
.cherry_pick(&from_branch, dry_run.unwrap_or(false))
|
||||
.await
|
||||
.default_error()?;
|
||||
serde_json::to_value(result).map_err(|err| {
|
||||
napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
|
||||
napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user