mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-18 01:32:27 +00:00
Merge branch 'merge-insert-source-stats' into merge-insert-source-stats-bindings
This commit is contained in:
@@ -266,8 +266,41 @@ describe("remote catalog", () => {
|
||||
"limit",
|
||||
);
|
||||
}
|
||||
await expect(catalog.connectDatabase("a$b")).rejects.toThrow(
|
||||
"Invalid database name",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"",
|
||||
".",
|
||||
"..",
|
||||
"/db",
|
||||
"db/",
|
||||
"a//b",
|
||||
"a/./b",
|
||||
"a/../b",
|
||||
"a b",
|
||||
"a:b",
|
||||
"a$b",
|
||||
"a%b",
|
||||
"a?b",
|
||||
"a#b",
|
||||
"a\\b",
|
||||
"café",
|
||||
"a\nb",
|
||||
])(
|
||||
"rejects invalid database name %j before sending requests",
|
||||
async (name) => {
|
||||
await withCatalog([], async (catalog, requests) => {
|
||||
for (const method of [
|
||||
"createDatabase",
|
||||
"connectDatabase",
|
||||
"dropDatabase",
|
||||
] as const) {
|
||||
await expect(catalog[method](name)).rejects.toThrow(
|
||||
"Invalid database name",
|
||||
);
|
||||
}
|
||||
expect(requests).toEqual([]);
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -527,3 +527,106 @@ describe("Query orderBy", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("IVF search effort", () => {
|
||||
let directory: tmp.DirResult;
|
||||
let table: Table;
|
||||
const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = tmp.dirSync({ unsafeCleanup: true });
|
||||
const db = await connect(directory.name);
|
||||
const rows = Array.from({ length: 512 }, (_, id) => ({
|
||||
id,
|
||||
vector: Array.from(
|
||||
{ length: 16 },
|
||||
(_, j) =>
|
||||
(Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
|
||||
),
|
||||
}));
|
||||
table = await db.createTable("effort", rows.slice(0, 256));
|
||||
await table.add(rows.slice(256));
|
||||
await table.createIndex("vector", {
|
||||
config: Index.ivfFlat({ numPartitions: 16 }),
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
directory.removeCallback();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[null, 6],
|
||||
[0, 1],
|
||||
[0.25, 3],
|
||||
[0.5, 6],
|
||||
[0.75, 10],
|
||||
[1, 16],
|
||||
])(
|
||||
"preserves effort %s through execution and probe precedence",
|
||||
async (effort, expected) => {
|
||||
const partitions = async (query: ReturnType<Table["vectorSearch"]>) => {
|
||||
const plan = await query.limit(8).analyzePlan();
|
||||
const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
|
||||
expect(matches.length).toBeGreaterThan(0);
|
||||
return matches.reduce((total, match) => total + Number(match[1]), 0);
|
||||
};
|
||||
expect(
|
||||
await partitions(table.vectorSearch(vector).searchEffort(effort)),
|
||||
).toBe(expected);
|
||||
expect(
|
||||
await partitions(
|
||||
table.vectorSearch(vector).nprobes(1).searchEffort(effort),
|
||||
),
|
||||
).toBe(effort === null ? 1 : expected);
|
||||
expect(
|
||||
await partitions(
|
||||
table.vectorSearch(vector).searchEffort(effort).nprobes(1),
|
||||
),
|
||||
).toBe(effort === null ? 1 : expected);
|
||||
expect(
|
||||
await partitions(
|
||||
table
|
||||
.vectorSearch(vector)
|
||||
.searchEffort(effort)
|
||||
.nprobes(1)
|
||||
.searchEffort(null),
|
||||
),
|
||||
).toBe(1);
|
||||
expect(
|
||||
await partitions(
|
||||
table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
|
||||
),
|
||||
).toBeLessThanOrEqual(2);
|
||||
const actual = await table
|
||||
.vectorSearch(vector)
|
||||
.searchEffort(effort)
|
||||
.limit(8)
|
||||
.toArray();
|
||||
const exact = await table
|
||||
.vectorSearch(vector)
|
||||
.bypassVectorIndex()
|
||||
.limit(8)
|
||||
.toArray();
|
||||
const ids = new Set(exact.map((row) => row.id));
|
||||
expect(
|
||||
actual.filter((row) => ids.has(row.id)).length / 8,
|
||||
).toBeGreaterThanOrEqual(0.5);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
-0.1,
|
||||
1.1,
|
||||
Number.NaN,
|
||||
Number.POSITIVE_INFINITY,
|
||||
Number.NEGATIVE_INFINITY,
|
||||
])("rejects invalid effort %s before execution", (effort) => {
|
||||
expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
|
||||
"search_effort must be finite and in [0, 1]",
|
||||
);
|
||||
expect(() =>
|
||||
table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
|
||||
).toThrow("search_effort must be finite and in [0, 1]");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -125,10 +125,6 @@ describe("remote connection", () => {
|
||||
const cases: [() => Promise<unknown>, string][] = [
|
||||
[() => table.usesV2ManifestPaths(), "uses_v2_manifest_paths"],
|
||||
[() => table.migrateManifestPathsV2(), "migrate_manifest_paths_v2"],
|
||||
[
|
||||
() => table.setUnenforcedPrimaryKey("id"),
|
||||
"set_unenforced_primary_key",
|
||||
],
|
||||
[() => table.optimize(), "optimize"],
|
||||
[
|
||||
() => table.createIndex("vector", { config: Index.hnswPq() }),
|
||||
@@ -480,6 +476,15 @@ describe("remote connection", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a zero blob request concurrency limit", async () => {
|
||||
await expect(
|
||||
connect("db://dev", {
|
||||
apiKey: "fake",
|
||||
clientConfig: { blobRequestConcurrency: 0 },
|
||||
}),
|
||||
).rejects.toThrow("blob_request_concurrency");
|
||||
});
|
||||
|
||||
it("shows the full error messages on retry errors", async () => {
|
||||
await withMockDatabase(
|
||||
(_req, res) => {
|
||||
|
||||
+257
-50
@@ -10,6 +10,7 @@ import * as tmp from "tmp";
|
||||
import {
|
||||
AutoQuery,
|
||||
type BlobInput,
|
||||
type BlobMode,
|
||||
Connection,
|
||||
MatchQuery,
|
||||
PhraseQuery,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
VectorQuery,
|
||||
blob,
|
||||
connect,
|
||||
isBlobField,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
import {
|
||||
@@ -640,11 +642,11 @@ describe.each(arrowVersions)(
|
||||
);
|
||||
});
|
||||
|
||||
it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
|
||||
it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => {
|
||||
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
|
||||
// useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
|
||||
// useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
|
||||
// where take-by-row-id auto-routes to the LSM scanner and is rejected).
|
||||
const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
|
||||
const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray();
|
||||
expect(res.map((r) => r.id)).toEqual([1, 3]);
|
||||
});
|
||||
|
||||
@@ -2827,6 +2829,18 @@ describe("when dealing with blob columns", () => {
|
||||
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
|
||||
});
|
||||
|
||||
it("rejects a whole blob read batch when one row ID was deleted", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
await table.delete("id = 2");
|
||||
|
||||
await expect(table.fetchBlobs("image", rowIds)).rejects.toThrow(
|
||||
`first missing row ids: [${rowIds[1]}]`,
|
||||
);
|
||||
await expect(table.fetchBlobFiles("image", rowIds)).rejects.toThrow(
|
||||
`first missing row ids: [${rowIds[1]}]`,
|
||||
);
|
||||
});
|
||||
|
||||
it("adds a LargeBinary Arrow column to a blob table", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
|
||||
@@ -3014,6 +3028,161 @@ describe("when dealing with blob columns", () => {
|
||||
expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4));
|
||||
});
|
||||
|
||||
it("toArrow returns blob bytes with blobMode bytes", async () => {
|
||||
const { table, alpha, beta } = await openBlobTable();
|
||||
const result = await table.query().toArrow({ blobMode: "bytes" });
|
||||
const field = result.schema.fields.find((f) => f.name === "image")!;
|
||||
expect(arrow.DataType.isLargeBinary(field.type)).toBe(true);
|
||||
expect(isBlobField(field)).toBe(false);
|
||||
expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]);
|
||||
const byId = new Map(
|
||||
result.toArray().map((row) => [Number(row.id), row.image]),
|
||||
);
|
||||
expect(Buffer.from(byId.get(1)!)).toEqual(alpha);
|
||||
expect(Buffer.from(byId.get(2)!)).toEqual(beta);
|
||||
expect(byId.get(3)).toBeNull();
|
||||
});
|
||||
|
||||
it("toArray returns blob bytes and keeps _rowid only when asked", async () => {
|
||||
const { table, rowIds, alpha } = await openBlobTable();
|
||||
const [plain] = await table
|
||||
.query()
|
||||
.where("id = 1")
|
||||
.select(["id", "image"])
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(Object.keys(plain)).toEqual(["id", "image"]);
|
||||
expect(Buffer.from(plain.image)).toEqual(alpha);
|
||||
|
||||
const [withRowId] = await table
|
||||
.query()
|
||||
.where("id = 1")
|
||||
.withRowId()
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(withRowId._rowid).toBe(rowIds[0]);
|
||||
expect(Buffer.from(withRowId.image)).toEqual(alpha);
|
||||
});
|
||||
|
||||
it("fills a renamed blob column with bytes", async () => {
|
||||
const { table, beta } = await openBlobTable();
|
||||
const [row] = await table
|
||||
.query()
|
||||
.where("id = 2")
|
||||
.select(
|
||||
new Map([
|
||||
["key", "id"],
|
||||
["picture", "image"],
|
||||
]),
|
||||
)
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(Number(row.key)).toBe(2);
|
||||
expect(Buffer.from(row.picture)).toEqual(beta);
|
||||
});
|
||||
|
||||
it("returns blob bytes from a take query", async () => {
|
||||
const { table, rowIds, alpha, beta } = await openBlobTable();
|
||||
const rows = await table
|
||||
.takeRowIds([rowIds[1], rowIds[0]])
|
||||
.toArray({ blobMode: "bytes" });
|
||||
const byId = new Map(rows.map((row) => [Number(row.id), row]));
|
||||
expect(Buffer.from(byId.get(1)!.image)).toEqual(alpha);
|
||||
expect(Buffer.from(byId.get(2)!.image)).toEqual(beta);
|
||||
expect(Object.keys(rows[0])).not.toContain("_rowid");
|
||||
});
|
||||
|
||||
it("does not change later executions of the same query", async () => {
|
||||
const { table } = await openBlobTable();
|
||||
const query = table.query().where("id = 1");
|
||||
await query.toArray({ blobMode: "bytes" });
|
||||
const [row] = await query.toArray();
|
||||
expect(Object.keys(row)).not.toContain("_rowid");
|
||||
expect(Number(row.image.size)).toBe(5);
|
||||
});
|
||||
|
||||
it("skips row ids when no blob column is selected", async () => {
|
||||
const { table } = await openBlobTable();
|
||||
const result = await table
|
||||
.query()
|
||||
.select(["id"])
|
||||
.toArrow({ blobMode: "bytes" });
|
||||
expect(result.schema.fields.map((f) => f.name)).toEqual(["id"]);
|
||||
expect(result.numRows).toBe(3);
|
||||
});
|
||||
|
||||
it("returns an empty bytes column for an empty result", async () => {
|
||||
const { table } = await openBlobTable();
|
||||
const result = await table
|
||||
.query()
|
||||
.where("id > 100")
|
||||
.toArrow({ blobMode: "bytes" });
|
||||
expect(result.numRows).toBe(0);
|
||||
expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]);
|
||||
const field = result.schema.fields.find((f) => f.name === "image")!;
|
||||
expect(arrow.DataType.isLargeBinary(field.type)).toBe(true);
|
||||
expect(isBlobField(field)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps the descriptor schema for an empty result", async () => {
|
||||
const { table } = await openBlobTable();
|
||||
const result = await table.query().where("id > 100").toArrow();
|
||||
expect(result.numRows).toBe(0);
|
||||
expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]);
|
||||
const field = result.schema.fields.find((f) => f.name === "image")!;
|
||||
expect(arrow.DataType.isStruct(field.type)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects an unknown blobMode", async () => {
|
||||
const { table } = await openBlobTable();
|
||||
await expect(
|
||||
table.query().toArray({ blobMode: "lazy" as BlobMode }),
|
||||
).rejects.toThrow(/blobMode must be/);
|
||||
});
|
||||
|
||||
it("reruns the query when the table changes before the version check", async () => {
|
||||
const { table, native, update } = await openVersionedBlobTable();
|
||||
const liveVersion = native.version.bind(native);
|
||||
native.version = async () => {
|
||||
native.version = liveVersion;
|
||||
await update(2n, "new");
|
||||
return liveVersion();
|
||||
};
|
||||
const [row] = await table.query().toArray({ blobMode: "bytes" });
|
||||
expect(row.generation).toBe(2n);
|
||||
expect(Buffer.from(row.image).toString()).toBe("new");
|
||||
});
|
||||
|
||||
it("reads bytes from the query's version when a write lands before the fetch", async () => {
|
||||
const { table, native, update } = await openVersionedBlobTable();
|
||||
const liveVersion = native.version.bind(native);
|
||||
native.version = async () => {
|
||||
native.version = liveVersion;
|
||||
const version = await liveVersion();
|
||||
await update(2n, "new");
|
||||
return version;
|
||||
};
|
||||
const [row] = await table.query().toArray({ blobMode: "bytes" });
|
||||
expect(row.generation).toBe(1n);
|
||||
expect(Buffer.from(row.image).toString()).toBe("old");
|
||||
});
|
||||
|
||||
it("fails when the table changes during every attempt", async () => {
|
||||
const { table, native, update } = await openVersionedBlobTable();
|
||||
const liveVersion = native.version.bind(native);
|
||||
let generation = 1n;
|
||||
native.version = async () => {
|
||||
generation += 1n;
|
||||
await update(generation, `generation ${generation}`);
|
||||
return liveVersion();
|
||||
};
|
||||
try {
|
||||
await expect(
|
||||
table.query().toArray({ blobMode: "bytes" }),
|
||||
).rejects.toThrow(/changed during each of 3 attempts/);
|
||||
expect(generation).toBe(4n);
|
||||
} finally {
|
||||
native.version = liveVersion;
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects fetchBlobs on a non-blob column", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
|
||||
@@ -3039,7 +3208,7 @@ describe("when dealing with blob columns", () => {
|
||||
expect(bytes[0]!.equals(payload)).toBe(true);
|
||||
});
|
||||
|
||||
it("creates and adds list blob columns", async () => {
|
||||
it("creates list blobs without advertising unfetchable elements", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
@@ -3058,7 +3227,16 @@ describe("when dealing with blob columns", () => {
|
||||
{ id: 3n, images: [gamma, null] },
|
||||
{ id: 4n, images: [] },
|
||||
]);
|
||||
expect(await table.blobColumns()).toEqual(["images.image"]);
|
||||
expect(await table.blobColumns()).toEqual([]);
|
||||
const [first] = await table.query().withRowId().toArray();
|
||||
for (const column of ["images", "images.image"]) {
|
||||
await expect(
|
||||
table.fetchBlobs(column, [first._rowid as bigint]),
|
||||
).rejects.toThrow(/blobs inside lists cannot be fetched/);
|
||||
await expect(
|
||||
table.fetchBlobFiles(column, [first._rowid as bigint]),
|
||||
).rejects.toThrow(/blobs inside lists cannot be fetched/);
|
||||
}
|
||||
const rows = await table.query().toArray();
|
||||
const byId = new Map(rows.map((row) => [Number(row.id), row]));
|
||||
expect(descriptorSizes(byId.get(1)!.images)).toEqual([
|
||||
@@ -3104,6 +3282,7 @@ describe("when dealing with blob columns", () => {
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(await table.blobColumns()).toEqual([]);
|
||||
const rows = await table.query().toArray();
|
||||
const byId = new Map(rows.map((row) => [Number(row.id), row]));
|
||||
expect(
|
||||
@@ -3186,6 +3365,32 @@ describe("when dealing with blob columns", () => {
|
||||
).map((value) => (value == null ? null : Number(value.size)));
|
||||
}
|
||||
|
||||
async function openVersionedBlobTable() {
|
||||
const db = await connect(tmpDir.name, {
|
||||
storageOptions: { newTableEnableStableRowIds: "true" },
|
||||
});
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), false),
|
||||
new Field("generation", new Int64(), true),
|
||||
blob("image"),
|
||||
]);
|
||||
const table = await db.createTable(
|
||||
"versions",
|
||||
[{ id: 1n, generation: 1n, image: Buffer.from("old") }],
|
||||
{ schema },
|
||||
);
|
||||
const native = (
|
||||
table as unknown as { inner: { version: () => Promise<number> } }
|
||||
).inner;
|
||||
const update = async (generation: bigint, image: string) => {
|
||||
await table
|
||||
.mergeInsert("id")
|
||||
.whenMatchedUpdateAll()
|
||||
.execute([{ id: 1n, generation, image: Buffer.from(image) }]);
|
||||
};
|
||||
return { table, native, update };
|
||||
}
|
||||
|
||||
async function openBlobTable() {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
@@ -4616,7 +4821,7 @@ describe("setUnenforcedPrimaryKey", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
describe("enableMemWal / disableMemWal", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -4636,16 +4841,16 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
const table = await makeTable(conn);
|
||||
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.setLsmWriteSpec({
|
||||
await table.enableMemWal({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 4,
|
||||
});
|
||||
await table.unsetLsmWriteSpec();
|
||||
await table.disableMemWal();
|
||||
// A second unset errors — there is no spec left to remove.
|
||||
await expect(table.unsetLsmWriteSpec()).rejects.toThrow();
|
||||
await expect(table.disableMemWal()).rejects.toThrow();
|
||||
// A fresh spec can be installed after unset.
|
||||
await table.setLsmWriteSpec({
|
||||
await table.enableMemWal({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 8,
|
||||
@@ -4657,8 +4862,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
const table = await makeTable(conn);
|
||||
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.setLsmWriteSpec({ specType: "unsharded" });
|
||||
await table.unsetLsmWriteSpec();
|
||||
await table.enableMemWal({ specType: "unsharded" });
|
||||
await table.disableMemWal();
|
||||
});
|
||||
|
||||
it("installs an identity spec", async () => {
|
||||
@@ -4666,8 +4871,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
const table = await makeTable(conn);
|
||||
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
|
||||
await table.unsetLsmWriteSpec();
|
||||
await table.enableMemWal({ specType: "identity", column: "id" });
|
||||
await table.disableMemWal();
|
||||
});
|
||||
|
||||
it("rejects an invalid spec", async () => {
|
||||
@@ -4677,7 +4882,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
// num_buckets out of range.
|
||||
await expect(
|
||||
table.setLsmWriteSpec({
|
||||
table.enableMemWal({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 0,
|
||||
@@ -4685,7 +4890,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
).rejects.toThrow();
|
||||
// Column mismatch.
|
||||
await expect(
|
||||
table.setLsmWriteSpec({
|
||||
table.enableMemWal({
|
||||
specType: "bucket",
|
||||
column: "missing",
|
||||
numBuckets: 4,
|
||||
@@ -4693,13 +4898,13 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("reads back the installed spec via getLsmWriteSpec", async () => {
|
||||
it("reads back the installed spec via getMemWalSpec", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await makeTable(conn);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
|
||||
// Nothing installed yet.
|
||||
expect(await table.getLsmWriteSpec()).toBeUndefined();
|
||||
expect(await table.getMemWalSpec()).toBeUndefined();
|
||||
|
||||
// A real scalar index is needed to name it as a maintained index.
|
||||
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
|
||||
@@ -4710,14 +4915,14 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
// defaults. Lance writer-config keys are canonically snake_case.
|
||||
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
|
||||
const writerConfigDefaults = { durable_write: "false" };
|
||||
await table.setLsmWriteSpec({
|
||||
await table.enableMemWal({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 4,
|
||||
maintainedIndexes: [indexName],
|
||||
writerConfigDefaults,
|
||||
});
|
||||
const spec = await table.getLsmWriteSpec();
|
||||
const spec = await table.getMemWalSpec();
|
||||
expect(spec).toBeDefined();
|
||||
expect(spec?.specType).toBe("bucket");
|
||||
expect(spec?.column).toBe("id");
|
||||
@@ -4726,19 +4931,19 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
|
||||
|
||||
// After unset, undefined again.
|
||||
await table.unsetLsmWriteSpec();
|
||||
expect(await table.getLsmWriteSpec()).toBeUndefined();
|
||||
await table.disableMemWal();
|
||||
expect(await table.getMemWalSpec()).toBeUndefined();
|
||||
|
||||
// Identity round-trips (column recovered from the schema).
|
||||
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
|
||||
const identity = await table.getLsmWriteSpec();
|
||||
await table.enableMemWal({ specType: "identity", column: "id" });
|
||||
const identity = await table.getMemWalSpec();
|
||||
expect(identity?.specType).toBe("identity");
|
||||
expect(identity?.column).toBe("id");
|
||||
await table.unsetLsmWriteSpec();
|
||||
await table.disableMemWal();
|
||||
|
||||
// Unsharded round-trips (no routing column).
|
||||
await table.setLsmWriteSpec({ specType: "unsharded" });
|
||||
const unsharded = await table.getLsmWriteSpec();
|
||||
await table.enableMemWal({ specType: "unsharded" });
|
||||
const unsharded = await table.getMemWalSpec();
|
||||
expect(unsharded?.specType).toBe("unsharded");
|
||||
expect(unsharded?.column).toBeFalsy();
|
||||
});
|
||||
@@ -4767,7 +4972,7 @@ describe("LSM merge insert", () => {
|
||||
]);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
// numBuckets = 1: every row routes to the single bucket.
|
||||
await table.setLsmWriteSpec({
|
||||
await table.enableMemWal({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 1,
|
||||
@@ -4792,17 +4997,17 @@ describe("LSM merge insert", () => {
|
||||
expect(res.version).toBe(0);
|
||||
expect(res.numInsertedRows).toBe(0);
|
||||
|
||||
await table.closeLsmWriters();
|
||||
await table.closeMemWalWriters();
|
||||
});
|
||||
|
||||
it("falls back to the standard path with useLsm(false)", async () => {
|
||||
it("falls back to the standard path with useMemWal(false)", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await bucketTable(conn);
|
||||
|
||||
const res = await table
|
||||
.mergeInsert("id")
|
||||
.whenNotMatchedInsertAll()
|
||||
.useLsm(false)
|
||||
.useMemWal(false)
|
||||
.execute([
|
||||
{ id: "b", value: 9 },
|
||||
{ id: "e", value: 5 },
|
||||
@@ -4825,7 +5030,7 @@ describe("LSM merge insert", () => {
|
||||
expect(res.numRows).toBe(1);
|
||||
});
|
||||
|
||||
it("rejects a non-upsert merge under an LSM spec", async () => {
|
||||
it("rejects a non-upsert merge under a MemWAL spec", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await bucketTable(conn);
|
||||
|
||||
@@ -4851,22 +5056,24 @@ describe("LSM merge insert", () => {
|
||||
const lsm = await table.query().toArray();
|
||||
expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
|
||||
|
||||
// useLsm(false) bypasses the MemWAL and reads the base table only.
|
||||
const baseOnly = await table.query().useLsm(false).toArray();
|
||||
// useMemWal(false) bypasses the MemWAL and reads the base table only.
|
||||
const baseOnly = await table.query().useMemWal(false).toArray();
|
||||
expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("reads the base table when no LSM spec is installed", async () => {
|
||||
it("reads the base table when no MemWAL spec is installed", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await conn.createEmptyTable(
|
||||
"plain",
|
||||
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
|
||||
);
|
||||
// No spec: default read and useLsm(false) both succeed against the base table.
|
||||
// No spec: default read and useMemWal(false) both succeed against the base table.
|
||||
await expect(table.query().toArray()).resolves.toBeDefined();
|
||||
await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
|
||||
// useLsm(true) demands MemWAL routing; without a spec it errors.
|
||||
await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
|
||||
await expect(
|
||||
table.query().useMemWal(false).toArray(),
|
||||
).resolves.toBeDefined();
|
||||
// useMemWal(true) demands MemWAL routing; without a spec it errors.
|
||||
await expect(table.query().useMemWal(true).toArray()).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4884,7 +5091,7 @@ describe("LSM convergence and stats", () => {
|
||||
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
|
||||
);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.setLsmWriteSpec({ specType: "unsharded" });
|
||||
await table.enableMemWal({ specType: "unsharded" });
|
||||
return table;
|
||||
}
|
||||
|
||||
@@ -4892,34 +5099,34 @@ describe("LSM convergence and stats", () => {
|
||||
// rejects them rather than answering. What is asserted here is that the
|
||||
// bindings reach the core at all; the behavior against a real endpoint is
|
||||
// covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
|
||||
it("rejects flushLsm on a local table", async () => {
|
||||
it("rejects flushMemWal on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.flushMemWal()).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
|
||||
it("rejects compactLsm on a local table", async () => {
|
||||
it("rejects compactMemWal on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.compactMemWal()).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
|
||||
it("rejects getLsmStats on a local table", async () => {
|
||||
it("rejects getMemWalStats on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
|
||||
await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
|
||||
it("rejects checkpointLsm on a local table", async () => {
|
||||
it("rejects checkpointMemWal on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
// checkpointLsm seals first, so it surfaces flushLsm's rejection.
|
||||
await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
|
||||
// checkpointMemWal freezes first, so it surfaces flushMemWal's rejection.
|
||||
await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user