mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-22 11:42:27 +00:00
Merge branch 'merge-insert-source-stats' into merge-insert-source-stats-bindings
This commit is contained in:
Will Jones
@@ -642,11 +642,11 @@ describe.each(arrowVersions)(
|
||||
);
|
||||
});
|
||||
|
||||
it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => {
|
||||
it("should expose useRTE on takeRowIds as the base-only escape hatch", async () => {
|
||||
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
|
||||
// useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
|
||||
// useRTE(false) is reachable on TakeQuery (the escape hatch for RTE tables,
|
||||
// where take-by-row-id auto-routes to the LSM scanner and is rejected).
|
||||
const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray();
|
||||
const res = await table.takeRowIds([0, 2]).useRTE(false).toArray();
|
||||
expect(res.map((r) => r.id)).toEqual([1, 3]);
|
||||
});
|
||||
|
||||
@@ -5008,7 +5008,7 @@ describe("setUnenforcedPrimaryKey", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("enableMemWal / disableMemWal", () => {
|
||||
describe("enableRTE / disableRTE", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -5028,16 +5028,16 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
const table = await makeTable(conn);
|
||||
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.enableMemWal({
|
||||
await table.enableRTE({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 4,
|
||||
});
|
||||
await table.disableMemWal();
|
||||
await table.disableRTE();
|
||||
// A second unset errors — there is no spec left to remove.
|
||||
await expect(table.disableMemWal()).rejects.toThrow();
|
||||
await expect(table.disableRTE()).rejects.toThrow();
|
||||
// A fresh spec can be installed after unset.
|
||||
await table.enableMemWal({
|
||||
await table.enableRTE({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 8,
|
||||
@@ -5049,8 +5049,8 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
const table = await makeTable(conn);
|
||||
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.enableMemWal({ specType: "unsharded" });
|
||||
await table.disableMemWal();
|
||||
await table.enableRTE({ specType: "unsharded" });
|
||||
await table.disableRTE();
|
||||
});
|
||||
|
||||
it("installs an identity spec", async () => {
|
||||
@@ -5058,8 +5058,8 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
const table = await makeTable(conn);
|
||||
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.enableMemWal({ specType: "identity", column: "id" });
|
||||
await table.disableMemWal();
|
||||
await table.enableRTE({ specType: "identity", column: "id" });
|
||||
await table.disableRTE();
|
||||
});
|
||||
|
||||
it("rejects an invalid spec", async () => {
|
||||
@@ -5069,7 +5069,7 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
// num_buckets out of range.
|
||||
await expect(
|
||||
table.enableMemWal({
|
||||
table.enableRTE({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 0,
|
||||
@@ -5077,7 +5077,7 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
).rejects.toThrow();
|
||||
// Column mismatch.
|
||||
await expect(
|
||||
table.enableMemWal({
|
||||
table.enableRTE({
|
||||
specType: "bucket",
|
||||
column: "missing",
|
||||
numBuckets: 4,
|
||||
@@ -5085,13 +5085,13 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("reads back the installed spec via getMemWalSpec", async () => {
|
||||
it("reads back the installed spec via getRTESpec", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await makeTable(conn);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
|
||||
// Nothing installed yet.
|
||||
expect(await table.getMemWalSpec()).toBeUndefined();
|
||||
expect(await table.getRTESpec()).toBeUndefined();
|
||||
|
||||
// A real scalar index is needed to name it as a maintained index.
|
||||
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
|
||||
@@ -5102,14 +5102,14 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
// defaults. Lance writer-config keys are canonically snake_case.
|
||||
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
|
||||
const writerConfigDefaults = { durable_write: "false" };
|
||||
await table.enableMemWal({
|
||||
await table.enableRTE({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 4,
|
||||
maintainedIndexes: [indexName],
|
||||
writerConfigDefaults,
|
||||
});
|
||||
const spec = await table.getMemWalSpec();
|
||||
const spec = await table.getRTESpec();
|
||||
expect(spec).toBeDefined();
|
||||
expect(spec?.specType).toBe("bucket");
|
||||
expect(spec?.column).toBe("id");
|
||||
@@ -5118,19 +5118,19 @@ describe("enableMemWal / disableMemWal", () => {
|
||||
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
|
||||
|
||||
// After unset, undefined again.
|
||||
await table.disableMemWal();
|
||||
expect(await table.getMemWalSpec()).toBeUndefined();
|
||||
await table.disableRTE();
|
||||
expect(await table.getRTESpec()).toBeUndefined();
|
||||
|
||||
// Identity round-trips (column recovered from the schema).
|
||||
await table.enableMemWal({ specType: "identity", column: "id" });
|
||||
const identity = await table.getMemWalSpec();
|
||||
await table.enableRTE({ specType: "identity", column: "id" });
|
||||
const identity = await table.getRTESpec();
|
||||
expect(identity?.specType).toBe("identity");
|
||||
expect(identity?.column).toBe("id");
|
||||
await table.disableMemWal();
|
||||
await table.disableRTE();
|
||||
|
||||
// Unsharded round-trips (no routing column).
|
||||
await table.enableMemWal({ specType: "unsharded" });
|
||||
const unsharded = await table.getMemWalSpec();
|
||||
await table.enableRTE({ specType: "unsharded" });
|
||||
const unsharded = await table.getRTESpec();
|
||||
expect(unsharded?.specType).toBe("unsharded");
|
||||
expect(unsharded?.column).toBeFalsy();
|
||||
});
|
||||
@@ -5159,7 +5159,7 @@ describe("LSM merge insert", () => {
|
||||
]);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
// numBuckets = 1: every row routes to the single bucket.
|
||||
await table.enableMemWal({
|
||||
await table.enableRTE({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 1,
|
||||
@@ -5179,22 +5179,22 @@ describe("LSM merge insert", () => {
|
||||
{ id: "c", value: 3 },
|
||||
{ id: "d", value: 4 },
|
||||
]);
|
||||
// LSM path: rows go to the MemWAL, so only numRows is populated.
|
||||
// LSM path: rows go to the RTE, so only numRows is populated.
|
||||
expect(res.numRows).toBe(2);
|
||||
expect(res.version).toBe(0);
|
||||
expect(res.numInsertedRows).toBe(0);
|
||||
|
||||
await table.closeMemWalWriters();
|
||||
await table.closeRTEWriters();
|
||||
});
|
||||
|
||||
it("falls back to the standard path with useMemWal(false)", async () => {
|
||||
it("falls back to the standard path with useRTE(false)", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await bucketTable(conn);
|
||||
|
||||
const res = await table
|
||||
.mergeInsert("id")
|
||||
.whenNotMatchedInsertAll()
|
||||
.useMemWal(false)
|
||||
.useRTE(false)
|
||||
.execute([
|
||||
{ id: "b", value: 9 },
|
||||
{ id: "e", value: 5 },
|
||||
@@ -5217,7 +5217,7 @@ describe("LSM merge insert", () => {
|
||||
expect(res.numRows).toBe(1);
|
||||
});
|
||||
|
||||
it("rejects a non-upsert merge under a MemWAL spec", async () => {
|
||||
it("rejects a non-upsert merge under an RTE spec", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await bucketTable(conn);
|
||||
|
||||
@@ -5229,7 +5229,7 @@ describe("LSM merge insert", () => {
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("auto-routes reads through the MemWAL scanner", async () => {
|
||||
it("auto-routes reads through the RTE scanner", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await bucketTable(conn); // base ids "a", "b"
|
||||
|
||||
@@ -5243,24 +5243,22 @@ describe("LSM merge insert", () => {
|
||||
const lsm = await table.query().toArray();
|
||||
expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
|
||||
|
||||
// useMemWal(false) bypasses the MemWAL and reads the base table only.
|
||||
const baseOnly = await table.query().useMemWal(false).toArray();
|
||||
// useRTE(false) bypasses the RTE and reads the base table only.
|
||||
const baseOnly = await table.query().useRTE(false).toArray();
|
||||
expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("reads the base table when no MemWAL spec is installed", async () => {
|
||||
it("reads the base table when no RTE spec is installed", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await conn.createEmptyTable(
|
||||
"plain",
|
||||
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
|
||||
);
|
||||
// No spec: default read and useMemWal(false) both succeed against the base table.
|
||||
// No spec: default read and useRTE(false) both succeed against the base table.
|
||||
await expect(table.query().toArray()).resolves.toBeDefined();
|
||||
await expect(
|
||||
table.query().useMemWal(false).toArray(),
|
||||
).resolves.toBeDefined();
|
||||
// useMemWal(true) demands MemWAL routing; without a spec it errors.
|
||||
await expect(table.query().useMemWal(true).toArray()).rejects.toThrow();
|
||||
await expect(table.query().useRTE(false).toArray()).resolves.toBeDefined();
|
||||
// useRTE(true) demands RTE routing; without a spec it errors.
|
||||
await expect(table.query().useRTE(true).toArray()).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5278,42 +5276,42 @@ describe("LSM convergence and stats", () => {
|
||||
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
|
||||
);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
await table.enableMemWal({ specType: "unsharded" });
|
||||
await table.enableRTE({ specType: "unsharded" });
|
||||
return table;
|
||||
}
|
||||
|
||||
// These four route through the server that owns the MemWAL, so a local table
|
||||
// These four route through the server that owns the RTE, so a local table
|
||||
// rejects them rather than answering. What is asserted here is that the
|
||||
// bindings reach the core at all; the behavior against a real endpoint is
|
||||
// covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
|
||||
it("rejects flushMemWal on a local table", async () => {
|
||||
it("rejects flushRTE on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
await expect(table.flushMemWal()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.flushRTE()).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
|
||||
it("rejects compactMemWal on a local table", async () => {
|
||||
it("rejects compactRTE on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
await expect(table.compactMemWal()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.compactRTE()).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
|
||||
it("rejects getMemWalStats on a local table", async () => {
|
||||
it("rejects getRTEStats on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i);
|
||||
await expect(table.getRTEStats()).rejects.toThrow(/not supported/i);
|
||||
await expect(table.getRTEStats(true)).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
|
||||
it("rejects checkpointMemWal on a local table", async () => {
|
||||
it("rejects checkpointRTE on a local table", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await lsmTable(conn);
|
||||
|
||||
// checkpointMemWal freezes first, so it surfaces flushMemWal's rejection.
|
||||
await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i);
|
||||
// checkpointRTE freezes first, so it surfaces flushRTE's rejection.
|
||||
await expect(table.checkpointRTE()).rejects.toThrow(/not supported/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user