From ea1d2b172ae240c22a89d05b1c66ca6d3a5f90a4 Mon Sep 17 00:00:00 2001 From: XY Zhan Date: Fri, 16 Oct 2026 15:27:53 -0400 Subject: [PATCH 01/22] fix: require a primary key to enable MemWAL (#4561) ## Why Every MemWAL write and read resolves rows by the table's unenforced primary key, but `enable_memwal` accepted a table without one. The spec installed, and every MemWAL `merge_insert` then failed with "table has a MemWAL index but no unenforced primary key" until MemWAL was disabled. Lance's MemWAL initialization doesn't catch it: its key checks pass when there is no key. ## What changes `enable_memwal` refuses a spec on a table with no unenforced primary key, for every sharding mode, before anything is committed. The check runs against the dataset the spec is committed to; a key cannot be removed once set, and an overwrite or restore of the table conflicts with that commit. Servers that install specs through this method get the check too. The Rust doc on `Table::enable_memwal` now states the requirement; the Python and TypeScript docs already did. ## Tests - New `test_enable_memwal_requires_primary_key` (Rust and Python): unsharded, bucket and identity specs are all refused without a key, a refused spec leaves nothing installed, and the same table accepts a spec once a key is set. The Rust test fails with the check disabled. - Tests that installed a spec on a table without a key now declare one first (6 Rust, 3 Python). - `cargo nextest run -p lancedb --features remote --lib`: 1529 passed. `cargo clippy -p lancedb --features remote --tests --examples -- -D warnings` and `cargo fmt --check` are clean; no integration test or example installs a spec. - Python: `test_memwal_spec.py` and `test_merge_insert_lsm.py`, 43 passed; `ruff format` and `ruff check` are clean. --- python/python/tests/test_memwal_spec.py | 15 +++++++ rust/lancedb/src/table.rs | 59 +++++++++++++++++++++++++ rust/lancedb/src/table/merge/lsm.rs | 11 +++++ 3 files changed, 85 insertions(+) diff --git a/python/python/tests/test_memwal_spec.py b/python/python/tests/test_memwal_spec.py index a1ec6a90d..d292c9185 100644 --- a/python/python/tests/test_memwal_spec.py +++ b/python/python/tests/test_memwal_spec.py @@ -45,6 +45,7 @@ def _make_table(tmp_path): def test_enable_memwal_validates(tmp_path): _db, table = _make_table(tmp_path) + table.set_unenforced_primary_key("id") # Out-of-range num_buckets. with pytest.raises(Exception, match="num_buckets"): @@ -60,6 +61,7 @@ def test_enable_memwal_validates(tmp_path): def test_disable_memwal(tmp_path): _db, table = _make_table(tmp_path) + table.set_unenforced_primary_key("id") # unset errors when no spec is set. with pytest.raises(Exception, match="no MemWAL write spec"): @@ -74,6 +76,18 @@ def test_disable_memwal(tmp_path): table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) +def test_enable_memwal_requires_primary_key(tmp_path): + _db, table = _make_table(tmp_path) + for spec in ( + MemWalWriteSpec.unsharded(), + MemWalWriteSpec.bucket("id", 4), + MemWalWriteSpec.identity("id"), + ): + with pytest.raises(Exception, match="primary key"): + table.enable_memwal(spec) + assert table.get_memwal_spec() is None + + def test_set_unsharded_spec(tmp_path): _db, table = _make_table(tmp_path) # Lance MemWAL still requires a primary key on the dataset; Unsharded @@ -206,6 +220,7 @@ async def test_async_get_memwal_spec(tmp_path): ) assert await table.get_memwal_spec() is None + await table.set_unenforced_primary_key("id") # A real scalar index is needed to name it as a maintained index. await table.create_index("id", config=BTree()) diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index c0a4a719e..c6193d750 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -2015,6 +2015,9 @@ impl Table { /// - [`MemWalWriteSpec::identity`] — shard by the raw value of a scalar column. /// - [`MemWalWriteSpec::unsharded`] — route every write to a single shard. /// + /// Every spec requires the table to declare an unenforced primary key + /// ([`Table::set_unenforced_primary_key`]); otherwise this fails. + /// /// A table carries one spec: this fails while one is installed, since the /// SSTables already written were homed under it. /// [`Table::disable_memwal`] removes one. @@ -5541,6 +5544,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); // Reject num_buckets out of range. for bad in [0u32, 1025] { @@ -5585,6 +5589,56 @@ mod tests { assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); } + /// A spec is refused on a table without an unenforced primary key, for every + /// sharding mode, and a refused spec leaves nothing installed. + #[tokio::test] + async fn test_enable_memwal_requires_primary_key() { + use lance::dataset::mem_wal::DatasetMemWalExt; + + let tmp_dir = tempdir().unwrap(); + let conn = ConnectBuilder::new(tmp_dir.path().to_str().unwrap()) + .read_consistency_interval(Duration::from_secs(0)) + .execute() + .await + .unwrap(); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::Int64Array::from(vec![1, 2]))], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)); + let table = conn.create_table("t", reader).execute().await.unwrap(); + let installed = |table: &Table| { + let table = table.clone(); + async move { + let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); + dataset.mem_wal_index_details().await.unwrap().is_some() + } + }; + + for spec in [ + MemWalWriteSpec::unsharded(), + MemWalWriteSpec::bucket("id", 4), + MemWalWriteSpec::identity("id"), + ] { + let err = table.enable_memwal(spec).await.unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("primary key")), + "got {err:?}" + ); + assert!(!installed(&table).await); + } + + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .enable_memwal(MemWalWriteSpec::unsharded()) + .await + .unwrap(); + assert!(installed(&table).await); + } + #[tokio::test] async fn test_enable_memwal_unsharded() { use lance::dataset::mem_wal::DatasetMemWalExt; @@ -5606,6 +5660,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); table .enable_memwal(MemWalWriteSpec::unsharded()) @@ -5652,6 +5707,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); table .enable_memwal( @@ -5703,6 +5759,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); // unset errors when no spec is set. table.disable_memwal().await.unwrap_err(); @@ -5760,6 +5817,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); // No spec installed yet. assert_eq!(table.get_memwal_spec().await.unwrap(), None); @@ -5826,6 +5884,7 @@ mod tests { .await .unwrap(); let table = conn.create_table("t", reader).execute().await.unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); table .create_index(&["id"], Index::BTree(Default::default())) diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index d26be28cc..39e4fbf28 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -127,6 +127,17 @@ pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> .into(), }); } + // Every MemWAL write and read resolves rows by the unenforced primary key, + // so a spec on a table without one leaves the table unwritable through the + // MemWAL path. A key cannot be removed once set, and an overwrite or restore + // of the table conflicts with the commit below. + if dataset.schema().unenforced_primary_key().is_empty() { + return Err(Error::InvalidInput { + message: "a MemWAL write spec requires the table to declare an unenforced \ + primary key; set one with set_unenforced_primary_key first" + .into(), + }); + } let mut builder = dataset .initialize_mem_wal() .mem_index_registry(table.mem_index_registry()); From b301a47aa8b4ed0e1bad6ae7706029b376f9a107 Mon Sep 17 00:00:00 2001 From: Bilal Atique <74999164+BilalAtique@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:50:38 +0500 Subject: [PATCH 02/22] feat(node): add blobMode to return blob bytes from queries (#4353) --- docs/src/js/globals.md | 1 + .../js/interfaces/QueryExecutionOptions.md | 32 +++ docs/src/js/type-aliases/BlobMode.md | 13 + nodejs/__test__/table.test.ts | 183 +++++++++++++ nodejs/lancedb/index.ts | 1 + nodejs/lancedb/query.ts | 255 ++++++++++++++++-- nodejs/lancedb/table.ts | 10 +- nodejs/src/query.rs | 27 +- 8 files changed, 497 insertions(+), 25 deletions(-) create mode 100644 docs/src/js/type-aliases/BlobMode.md diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 739b9033a..7310f3f90 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -161,6 +161,7 @@ - [BaseTokenizer](type-aliases/BaseTokenizer.md) - [BlobData](type-aliases/BlobData.md) - [BlobInput](type-aliases/BlobInput.md) +- [BlobMode](type-aliases/BlobMode.md) - [BlobOptions](type-aliases/BlobOptions.md) - [BlobRange](type-aliases/BlobRange.md) - [BlobReadOptions](type-aliases/BlobReadOptions.md) diff --git a/docs/src/js/interfaces/QueryExecutionOptions.md b/docs/src/js/interfaces/QueryExecutionOptions.md index 46b06ce7e..8d3749f38 100644 --- a/docs/src/js/interfaces/QueryExecutionOptions.md +++ b/docs/src/js/interfaces/QueryExecutionOptions.md @@ -10,6 +10,38 @@ Options that control the behavior of a particular query execution ## Properties +### blobMode? + +```ts +optional blobMode: BlobMode; +``` + +How to return blob v2 columns. Applies to `toArrow()` and `toArray()`. + +- `"descriptions"` (default): each blob is a descriptor. Read the bytes + with [Table.fetchBlobs](../classes/Table.md#fetchblobs) or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles). +- `"bytes"`: each top-level blob column holds the blob bytes as + `LargeBinary`. The query also reads `_rowid` to fetch the bytes, and + leaves it out of the result unless [QueryBase.withRowId](../classes/QueryBase.md#withrowid) was + called. All the bytes are held in memory. + +Blobs nested in a struct or a list keep their descriptors. + +The rows and the bytes come from the same table version. If the table +changes while the query runs, the query runs again, up to three times. + +#### Example + +```ts +const rows = await table + .query() + .select(["id", "image"]) + .toArray({ blobMode: "bytes" }); +const image: Uint8Array | null = rows[0].image; +``` + +*** + ### maxBatchLength? ```ts diff --git a/docs/src/js/type-aliases/BlobMode.md b/docs/src/js/type-aliases/BlobMode.md new file mode 100644 index 000000000..67d7c47ad --- /dev/null +++ b/docs/src/js/type-aliases/BlobMode.md @@ -0,0 +1,13 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobMode + +# Type Alias: BlobMode + +```ts +type BlobMode: "descriptions" | "bytes"; +``` + +How a query returns blob v2 columns. See [QueryExecutionOptions.blobMode](../interfaces/QueryExecutionOptions.md#blobmode). diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index f53f859b8..b30121725 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -10,6 +10,7 @@ import * as tmp from "tmp"; import { AutoQuery, type BlobInput, + type BlobMode, Connection, MatchQuery, PhraseQuery, @@ -18,6 +19,7 @@ import { VectorQuery, blob, connect, + isBlobField, tokenize, } from "../lancedb"; import { @@ -3016,6 +3018,161 @@ describe("when dealing with blob columns", () => { expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4)); }); + it("toArrow returns blob bytes with blobMode bytes", async () => { + const { table, alpha, beta } = await openBlobTable(); + const result = await table.query().toArrow({ blobMode: "bytes" }); + const field = result.schema.fields.find((f) => f.name === "image")!; + expect(arrow.DataType.isLargeBinary(field.type)).toBe(true); + expect(isBlobField(field)).toBe(false); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]); + const byId = new Map( + result.toArray().map((row) => [Number(row.id), row.image]), + ); + expect(Buffer.from(byId.get(1)!)).toEqual(alpha); + expect(Buffer.from(byId.get(2)!)).toEqual(beta); + expect(byId.get(3)).toBeNull(); + }); + + it("toArray returns blob bytes and keeps _rowid only when asked", async () => { + const { table, rowIds, alpha } = await openBlobTable(); + const [plain] = await table + .query() + .where("id = 1") + .select(["id", "image"]) + .toArray({ blobMode: "bytes" }); + expect(Object.keys(plain)).toEqual(["id", "image"]); + expect(Buffer.from(plain.image)).toEqual(alpha); + + const [withRowId] = await table + .query() + .where("id = 1") + .withRowId() + .toArray({ blobMode: "bytes" }); + expect(withRowId._rowid).toBe(rowIds[0]); + expect(Buffer.from(withRowId.image)).toEqual(alpha); + }); + + it("fills a renamed blob column with bytes", async () => { + const { table, beta } = await openBlobTable(); + const [row] = await table + .query() + .where("id = 2") + .select( + new Map([ + ["key", "id"], + ["picture", "image"], + ]), + ) + .toArray({ blobMode: "bytes" }); + expect(Number(row.key)).toBe(2); + expect(Buffer.from(row.picture)).toEqual(beta); + }); + + it("returns blob bytes from a take query", async () => { + const { table, rowIds, alpha, beta } = await openBlobTable(); + const rows = await table + .takeRowIds([rowIds[1], rowIds[0]]) + .toArray({ blobMode: "bytes" }); + const byId = new Map(rows.map((row) => [Number(row.id), row])); + expect(Buffer.from(byId.get(1)!.image)).toEqual(alpha); + expect(Buffer.from(byId.get(2)!.image)).toEqual(beta); + expect(Object.keys(rows[0])).not.toContain("_rowid"); + }); + + it("does not change later executions of the same query", async () => { + const { table } = await openBlobTable(); + const query = table.query().where("id = 1"); + await query.toArray({ blobMode: "bytes" }); + const [row] = await query.toArray(); + expect(Object.keys(row)).not.toContain("_rowid"); + expect(Number(row.image.size)).toBe(5); + }); + + it("skips row ids when no blob column is selected", async () => { + const { table } = await openBlobTable(); + const result = await table + .query() + .select(["id"]) + .toArrow({ blobMode: "bytes" }); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id"]); + expect(result.numRows).toBe(3); + }); + + it("returns an empty bytes column for an empty result", async () => { + const { table } = await openBlobTable(); + const result = await table + .query() + .where("id > 100") + .toArrow({ blobMode: "bytes" }); + expect(result.numRows).toBe(0); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]); + const field = result.schema.fields.find((f) => f.name === "image")!; + expect(arrow.DataType.isLargeBinary(field.type)).toBe(true); + expect(isBlobField(field)).toBe(false); + }); + + it("keeps the descriptor schema for an empty result", async () => { + const { table } = await openBlobTable(); + const result = await table.query().where("id > 100").toArrow(); + expect(result.numRows).toBe(0); + expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]); + const field = result.schema.fields.find((f) => f.name === "image")!; + expect(arrow.DataType.isStruct(field.type)).toBe(true); + }); + + it("rejects an unknown blobMode", async () => { + const { table } = await openBlobTable(); + await expect( + table.query().toArray({ blobMode: "lazy" as BlobMode }), + ).rejects.toThrow(/blobMode must be/); + }); + + it("reruns the query when the table changes before the version check", async () => { + const { table, native, update } = await openVersionedBlobTable(); + const liveVersion = native.version.bind(native); + native.version = async () => { + native.version = liveVersion; + await update(2n, "new"); + return liveVersion(); + }; + const [row] = await table.query().toArray({ blobMode: "bytes" }); + expect(row.generation).toBe(2n); + expect(Buffer.from(row.image).toString()).toBe("new"); + }); + + it("reads bytes from the query's version when a write lands before the fetch", async () => { + const { table, native, update } = await openVersionedBlobTable(); + const liveVersion = native.version.bind(native); + native.version = async () => { + native.version = liveVersion; + const version = await liveVersion(); + await update(2n, "new"); + return version; + }; + const [row] = await table.query().toArray({ blobMode: "bytes" }); + expect(row.generation).toBe(1n); + expect(Buffer.from(row.image).toString()).toBe("old"); + }); + + it("fails when the table changes during every attempt", async () => { + const { table, native, update } = await openVersionedBlobTable(); + const liveVersion = native.version.bind(native); + let generation = 1n; + native.version = async () => { + generation += 1n; + await update(generation, `generation ${generation}`); + return liveVersion(); + }; + try { + await expect( + table.query().toArray({ blobMode: "bytes" }), + ).rejects.toThrow(/changed during each of 3 attempts/); + expect(generation).toBe(4n); + } finally { + native.version = liveVersion; + } + }); + it("rejects fetchBlobs on a non-blob column", async () => { const { table, rowIds } = await openBlobTable(); await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i); @@ -3198,6 +3355,32 @@ describe("when dealing with blob columns", () => { ).map((value) => (value == null ? null : Number(value.size))); } + async function openVersionedBlobTable() { + const db = await connect(tmpDir.name, { + storageOptions: { newTableEnableStableRowIds: "true" }, + }); + const schema = new Schema([ + new Field("id", new Int64(), false), + new Field("generation", new Int64(), true), + blob("image"), + ]); + const table = await db.createTable( + "versions", + [{ id: 1n, generation: 1n, image: Buffer.from("old") }], + { schema }, + ); + const native = ( + table as unknown as { inner: { version: () => Promise } } + ).inner; + const update = async (generation: bigint, image: string) => { + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .execute([{ id: 1n, generation, image: Buffer.from(image) }]); + }; + return { table, native, update }; + } + async function openBlobTable() { const db = await connect(tmpDir.name); const schema = new Schema([ diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 3254a0742..3096cbe6b 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -122,6 +122,7 @@ export { VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, + BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index 3583e5992..ef7443bbb 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -3,12 +3,18 @@ import { Table as ArrowTable, + Field, type IntoVector, + LargeBinary, RecordBatch, + Schema, + Struct, extractVectorBuffer, fromBufferToRecordBatch, fromRecordBatchToBuffer, + makeData, tableFromIPC, + vectorFromArray, } from "./arrow"; import { type IvfPqOptions } from "./indices"; import { @@ -41,6 +47,112 @@ export async function* RecordBatchIterator( } } +/** + * How a query returns blob v2 columns. See {@link QueryExecutionOptions.blobMode}. + */ +export type BlobMode = "descriptions" | "bytes"; + +/** + * The table a query reads, and the parts of the query that decide which blob + * columns `blobMode: "bytes"` fills in. + * + * @hidden + */ +export interface QueryBlobContext { + table: NativeTable; + withRowId: boolean; + projection?: [string, string][]; +} + +const BLOB_BYTES_ATTEMPTS = 3; + +function blobSources( + blobColumns: string[], + projection: [string, string][] | undefined, +): Map { + const columns = new Set(blobColumns); + const pairs = + projection ?? + blobColumns.map((column): [string, string] => [column, column]); + return new Map(pairs.filter(([, source]) => columns.has(source))); +} + +async function readBlobBytes( + table: ArrowTable, + sources: Map, + snapshot: NativeTable, + dropRowId: boolean, +): Promise { + const rowIdColumn = table.getChild("_rowid"); + if (table.numRows > 0 && rowIdColumn === null) { + throw new Error('blobMode "bytes" needs _rowid in the query result'); + } + const rowIds = + rowIdColumn === null + ? [] + : Array.from(rowIdColumn.toArray() as BigUint64Array); + const replacements = new Map(); + for (const [output, source] of sources) { + const index = table.schema.fields.findIndex( + (field) => field.name === output, + ); + if (index < 0) { + continue; + } + const values = + rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds); + replacements.set( + index, + values.map((value) => value ?? null), + ); + } + const result = withBinaryColumns(table, replacements); + if (!dropRowId) { + return result; + } + return result.select( + result.schema.fields + .map((field) => field.name) + .filter((name) => name !== "_rowid"), + ); +} + +// A fresh field, not a clone: the descriptor field's metadata must not follow +// the bytes. +function withBinaryColumns( + table: ArrowTable, + replacements: Map, +): ArrowTable { + if (replacements.size === 0) { + return table; + } + const fields = table.schema.fields.map((field, index) => + replacements.has(index) + ? new Field(field.name, new LargeBinary(), true) + : field, + ); + const schema = new Schema(fields, table.schema.metadata); + let offset = 0; + const batches = table.batches.map((batch) => { + const children = batch.data.children.map((child, index) => { + const values = replacements.get(index); + if (values === undefined) { + return child; + } + return vectorFromArray( + values.slice(offset, offset + batch.numRows), + new LargeBinary(), + ).data[0]; + }); + offset += batch.numRows; + return new RecordBatch( + schema, + makeData({ type: new Struct(fields), length: batch.numRows, children }), + ); + }); + return new ArrowTable(schema, batches); +} + /** * Options that control the behavior of a particular query execution */ @@ -57,6 +169,32 @@ export interface QueryExecutionOptions { * Timeout for query execution in milliseconds */ timeoutMs?: number; + + /** + * How to return blob v2 columns. Applies to `toArrow()` and `toArray()`. + * + * - `"descriptions"` (default): each blob is a descriptor. Read the bytes + * with {@link Table.fetchBlobs} or {@link Table.fetchBlobFiles}. + * - `"bytes"`: each top-level blob column holds the blob bytes as + * `LargeBinary`. The query also reads `_rowid` to fetch the bytes, and + * leaves it out of the result unless {@link QueryBase.withRowId} was + * called. All the bytes are held in memory. + * + * Blobs nested in a struct or a list keep their descriptors. + * + * The rows and the bytes come from the same table version. If the table + * changes while the query runs, the query runs again, up to three times. + * + * @example + * ```ts + * const rows = await table + * .query() + * .select(["id", "image"]) + * .toArray({ blobMode: "bytes" }); + * const image: Uint8Array | null = rows[0].image; + * ``` + */ + blobMode?: BlobMode; } export type AnalyzePlanDistributedMetrics = "aggregate" | "per_worker" | "full"; @@ -119,10 +257,19 @@ export class QueryBase< /** * @hidden */ - protected constructor(inner?: NativeQueryType | Promise) { + protected blobContext?: QueryBlobContext; + + /** + * @hidden + */ + protected constructor( + inner?: NativeQueryType | Promise, + blobContext?: QueryBlobContext, + ) { if (inner !== undefined) { this.inner = inner; } + this.blobContext = blobContext; } // call a function on the inner (either a promise or the actual object) @@ -193,14 +340,22 @@ export class QueryBase< }); }; + let projection: [string, string][]; if (typeof columns === "string") { selectColumns([columns]); + projection = [[columns, columns]]; } else if (Array.isArray(columns)) { selectColumns(columns); + projection = columns.map((column) => [column, column]); } else if (columns instanceof Map) { - selectMapping(Array.from(columns.entries())); + projection = Array.from(columns.entries()); + selectMapping(projection); } else { - selectMapping(Object.entries(columns)); + projection = Object.entries(columns); + selectMapping(projection); + } + if (this.blobContext !== undefined) { + this.blobContext = { ...this.blobContext, projection }; } return this; @@ -215,6 +370,9 @@ export class QueryBase< */ withRowId(): this { this.doCall((inner: NativeQueryType) => inner.withRowId()); + if (this.blobContext !== undefined) { + this.blobContext = { ...this.blobContext, withRowId: true }; + } return this; } @@ -223,9 +381,14 @@ export class QueryBase< */ protected async nativeExecute( options?: Partial, + withRowId = false, ): Promise { const inner = await this.getInner(); - return inner.execute(options?.maxBatchLength, options?.timeoutMs); + return inner.execute( + options?.maxBatchLength, + options?.timeoutMs, + withRowId, + ); } /** @@ -257,8 +420,41 @@ export class QueryBase< * Empty results retain the query's output schema. */ async toArrow(options?: Partial): Promise { + const context = this.blobBytesContext(options?.blobMode); + if (context === undefined) { + return this.collect(options, false); + } + for (let attempt = 1; ; attempt++) { + // The snapshot pins the bytes. The query runs on the live table, so an + // unchanged version afterwards proves it read the snapshot's version. + const snapshot = await context.table.querySnapshot(); + const version = await snapshot.version(); + const sources = blobSources( + await snapshot.blobColumns(), + context.projection, + ); + if (sources.size === 0) { + return this.collect(options, false); + } + const addRowId = !context.withRowId; + const table = await this.collect(options, addRowId); + if ((await context.table.version()) === version) { + return readBlobBytes(table, sources, snapshot, addRowId); + } + if (attempt === BLOB_BYTES_ATTEMPTS) { + throw new Error( + `blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`, + ); + } + } + } + + private async collect( + options: Partial | undefined, + withRowId: boolean, + ): Promise { const batches = []; - const iterator = this.nativeExecute(options); + const iterator = this.nativeExecute(options, withRowId); for await (const batch of RecordBatchIterator(iterator)) { batches.push(batch); } @@ -268,6 +464,23 @@ export class QueryBase< return new ArrowTable(batches); } + private blobBytesContext( + mode: BlobMode | undefined, + ): QueryBlobContext | undefined { + if (mode === undefined || mode === "descriptions") { + return undefined; + } + if (mode !== "bytes") { + throw new Error( + `blobMode must be "descriptions" or "bytes", got "${String(mode)}"`, + ); + } + if (this.blobContext === undefined) { + throw new Error('blobMode "bytes" is not supported for this query'); + } + return this.blobContext; + } + /** Collect the results as an array of objects. */ // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[] async toArray(options?: Partial): Promise { @@ -355,8 +568,11 @@ export class StandardQueryBase< extends QueryBase implements ExecutableQuery { - constructor(inner?: NativeQueryType | Promise) { - super(inner); + constructor( + inner?: NativeQueryType | Promise, + blobContext?: QueryBlobContext, + ) { + super(inner, blobContext); } /** @@ -505,8 +721,11 @@ export class VectorQuery extends StandardQueryBase { /** * @hidden */ - constructor(inner: NativeVectorQuery | Promise) { - super(inner); + constructor( + inner: NativeVectorQuery | Promise, + blobContext?: QueryBlobContext, + ) { + super(inner, blobContext); } /** @@ -760,7 +979,7 @@ export class VectorQuery extends StandardQueryBase { addQueryVectorToNative(inner, outcome.value); return inner; })(); - return new VectorQuery(res); + return new VectorQuery(res, this.blobContext); } else { this.doVectorCall((inner) => addQueryVectorToNative(inner, vector)); return this; @@ -849,7 +1068,7 @@ export function createAutoQuery( return nearestToNative(route.table.query(), vector); }; - return new AutoQuery(createInner); + return new AutoQuery(createInner, { table, withRowId: false }); } /** @@ -858,8 +1077,8 @@ export function createAutoQuery( * @hideconstructor */ export class TakeQuery extends QueryBase { - constructor(inner: NativeTakeQuery) { - super(inner); + constructor(inner: NativeTakeQuery, blobContext?: QueryBlobContext) { + super(inner, blobContext); } /** @@ -898,8 +1117,9 @@ export class AutoQuery extends StandardQueryBase< private readonly createInner: () => Promise< NativeQuery | NativeVectorQuery >, + blobContext?: QueryBlobContext, ) { - super(); + super(undefined, blobContext); } /** @hidden */ @@ -933,7 +1153,7 @@ export class Query extends StandardQueryBase { * @hidden */ constructor(tbl: NativeTable) { - super(tbl.query()); + super(tbl.query(), { table: tbl, withRowId: false }); } /** @@ -979,14 +1199,15 @@ export class Query extends StandardQueryBase { const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector), ); - return new VectorQuery(nativeQuery); + return new VectorQuery(nativeQuery, this.blobContext); } if (vector instanceof Promise) { return new VectorQuery( vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), + this.blobContext, ); } - return new VectorQuery(nearestToNative(inner, vector)); + return new VectorQuery(nearestToNative(inner, vector), this.blobContext); } nearestToText(query: string | FullTextQuery, columns?: string[]): Query { diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4d6500a5f..2b5319bec 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -1240,11 +1240,17 @@ export class LocalTable extends Table { } takeOffsets(offsets: number[]): TakeQuery { - return new TakeQuery(this.inner.takeOffsets(offsets)); + return new TakeQuery(this.inner.takeOffsets(offsets), { + table: this.inner, + withRowId: false, + }); } takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery { - return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds))); + return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds)), { + table: this.inner, + withRowId: false, + }); } blobColumns(): Promise { diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index 68db6f810..dba8f665a 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -196,6 +196,7 @@ impl Query { &self, max_batch_length: Option, timeout_ms: Option, + with_row_id: Option, ) -> napi::Result { let mut execution_opts = QueryExecutionOptions::default(); if let Some(max_batch_length) = max_batch_length { @@ -204,8 +205,12 @@ impl Query { if let Some(timeout_ms) = timeout_ms { execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64)) } - let inner_stream = self - .inner + let query = if with_row_id == Some(true) { + self.inner.clone().with_row_id() + } else { + self.inner.clone() + }; + let inner_stream = query .execute_with_options(execution_opts) .await .map_err(|e| { @@ -432,6 +437,7 @@ impl VectorQuery { &self, max_batch_length: Option, timeout_ms: Option, + with_row_id: Option, ) -> napi::Result { let mut execution_opts = QueryExecutionOptions::default(); if let Some(max_batch_length) = max_batch_length { @@ -440,8 +446,12 @@ impl VectorQuery { if let Some(timeout_ms) = timeout_ms { execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64)) } - let inner_stream = self - .inner + let query = if with_row_id == Some(true) { + self.inner.clone().with_row_id() + } else { + self.inner.clone() + }; + let inner_stream = query .execute_with_options(execution_opts) .await .map_err(|e| { @@ -520,6 +530,7 @@ impl TakeQuery { &self, max_batch_length: Option, timeout_ms: Option, + with_row_id: Option, ) -> napi::Result { let mut execution_opts = QueryExecutionOptions::default(); if let Some(max_batch_length) = max_batch_length { @@ -528,8 +539,12 @@ impl TakeQuery { if let Some(timeout_ms) = timeout_ms { execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64)) } - let inner_stream = self - .inner + let query = if with_row_id == Some(true) { + self.inner.clone().with_row_id() + } else { + self.inner.clone() + }; + let inner_stream = query .execute_with_options(execution_opts) .await .map_err(|e| { From 5f746d1e0111ae9f59932af5e81a274bff64a9df Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 16 Oct 2026 12:58:37 -0700 Subject: [PATCH 03/22] fix: refuse a materialized view whose sources lead back to itself (#4494) Creating a view now follows its source through each view's stored coordinate, up to 32 levels, and refuses when the walk reaches the name being created. One table open per level; no catalog scan. Part of ENT-2922. --- rust/lancedb/src/materialized_view.rs | 98 +++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index c04a6cbbe..95e3baac7 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -42,6 +42,9 @@ pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; /// Schema metadata key holding the view definition; see [`DEFINITION_FORMAT`]. pub const DEFINITION_META_KEY: &str = "mv.definition"; +/// How many views may stand between a view and a plain table. +const MAX_SOURCE_DEPTH: usize = 32; + /// Field metadata namespace for declarations about schema structure, such as /// an unenforced primary key. const SCHEMA_DECLARATION_META_PREFIX: &str = "lance-schema:"; @@ -1238,6 +1241,7 @@ impl PreparedDeclaration { namespace_path: &[String], name: &str, ) -> Result { + ensure_no_source_cycle(&self.database, namespace_path, name, &self.definition).await?; let empty: Vec> = vec![]; let reader: Box = @@ -1254,6 +1258,65 @@ impl PreparedDeclaration { } } +/// Refuse to create `name` when following its source through each view's +/// stored definition leads back to `name`: with no plain table beneath it, +/// such a view would only ever refresh from itself. The walk follows the +/// coordinates the views already record, so it opens one table per level. +async fn ensure_no_source_cycle( + database: &Arc, + namespace_path: &[String], + name: &str, + definition: &MaterializedViewDefinition, +) -> Result<()> { + let mut path = vec![name.to_string()]; + let mut next = ( + definition.source_namespace.clone(), + definition.source_table.clone(), + ); + for _ in 0..MAX_SOURCE_DEPTH { + let (namespace, source) = next; + path.push(source.clone()); + if namespace == namespace_path && source == name { + return Err(Error::InvalidInput { + message: format!( + "materialized view '{name}' would read itself: {}", + path.join(" -> ") + ), + }); + } + let table = match database + .open_table(OpenTableRequest { + name: source, + namespace_path: namespace, + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + { + Ok(table) => table, + // A missing source ends the chain, so it cannot lead back here. + Err(Error::TableNotFound { .. }) => return Ok(()), + Err(e) => return Err(e), + }; + let schema = table.schema().await?; + // Only a definition this version reads can be followed. + match read_definition(schema.metadata()) { + Ok(Some(StoredDefinition::Query(upstream))) => { + next = (upstream.source_namespace, upstream.source_table); + } + _ => return Ok(()), + } + } + Err(Error::InvalidInput { + message: format!( + "materialized view '{name}' would nest deeper than {MAX_SOURCE_DEPTH} views" + ), + }) +} + /// Column definitions are positional over the view schema: carry each /// field's entry to its place in `fields`, physical for a field that had none. fn rewrite_column_definitions( @@ -2017,6 +2080,41 @@ mod tests { } } + #[tokio::test] + async fn a_view_cannot_lead_back_to_itself() { + let conn = connect("memory://").execute().await.unwrap(); + let rows = || record_batch!(("x", Int32, [1, 2])).unwrap(); + conn.create_table("src", rows()).execute().await.unwrap(); + conn.create_materialized_view("m", "src") + .execute() + .await + .unwrap(); + conn.create_materialized_view("m2", "m") + .execute() + .await + .unwrap(); + conn.drop_table("src", &[]).await.unwrap(); + + let err = conn + .create_materialized_view("src", "m2") + .execute() + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("materialized view 'src' would read itself: src -> m2 -> m -> src"), + "{err}" + ); + assert!(conn.open_table("src").execute().await.is_err()); + + // Back under a plain table, the same chain is fine to extend. + conn.create_table("src", rows()).execute().await.unwrap(); + conn.create_materialized_view("m3", "m2") + .execute() + .await + .unwrap(); + } + #[tokio::test] async fn direct_writes_to_a_view_are_refused() { use arrow_schema::Schema; From 44ba9c6dee530516cd00cb4e86477715c6a57ca1 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:43:55 -0700 Subject: [PATCH 04/22] fix: support remote blob range reads (#4553) --- python/python/lancedb/table.py | 5 +- python/python/tests/test_remote_db.py | 46 +++++++- rust/lancedb/src/remote/table.rs | 20 ++-- rust/lancedb/src/remote/table/blobs.rs | 151 ++++++++++++++++++++++++- rust/lancedb/src/table.rs | 2 + 5 files changed, 210 insertions(+), 14 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index f0cb05427..a1a54ee8e 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2190,8 +2190,9 @@ class Table(ABC): One missing or deleted row ID rejects the entire batch; no partial results are returned. - Row IDs can be obtained from a query with ``with_row_id(True)``. This - API is currently supported only by local tables. + Row IDs can be obtained from a query with ``with_row_id(True)``. + Remote tables read the requested ranges through version-pinned blob + handles using HTTP byte-range requests. """ @abstractmethod diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 6ee41aa0c..e9768b0bf 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1623,7 +1623,6 @@ def remote_unsupported_operations(): return [ ("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"), ("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"), - ("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"), ("optimize", (), {}, "optimize"), ("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"), ] @@ -3362,13 +3361,20 @@ def blob_remote_table(*, server_version=Version("0.5.0")): else: assert body["with_row_id"] is True if columns == ["image"]: - # fetch_blob_files sizes its handles from a descriptor take. + # Blob handles are sized from a descriptor take. assert body["filter"].startswith("_rowid IN") response_table = blob_descriptor_take_table() + row_ids = [ + int(value) for value in re.findall(r"\d+", body["filter"]) + ] + response_table = response_table.filter( + pa.compute.is_in(response_table["_rowid"], pa.array(row_ids)) + ) else: response_table = blob_query_response_table() request.send_response(200) request.send_header("Content-Type", "application/vnd.apache.arrow.file") + request.send_header("x-lancedb-version", "1") request.end_headers() with pa.ipc.new_file(request.wfile, response_table.schema) as writer: writer.write_table(response_table) @@ -3400,6 +3406,42 @@ def test_remote_blob_columns_and_fetch(): assert blobs.to_pylist() == [b"alpha", None, b"gamma"] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_remote_fetch_blob_ranges(use_async): + with blob_remote_table() as remote_table: + table = remote_table._table if use_async else remote_table + requests = [ + (30, 2, 3), + (10, 1, 3), + (20, 0, 1), + (10, 0, 2), + (40, 0, 0), + (10, 5, 0), + (30, 2, 3), + ] + result = table.fetch_blob_ranges("image", requests) + if use_async: + result = await result + assert isinstance(result, pa.LargeBinaryArray) + assert result.to_pylist() == [b"mma", b"lph", None, b"al", b"", b"", b"mma"] + + for invalid, message in [ + ([(10, 0, 1), (99, 0, 1)], "only 1 exist"), + ([(10, 4, 2)], "exceeds blob size"), + ([(20, 2**64 - 1, 1)], "overflows u64"), + ]: + with pytest.raises(ValueError, match=message): + result = table.fetch_blob_ranges("image", invalid) + if use_async: + await result + + result = table.fetch_blob_ranges("image", []) + if use_async: + result = await result + assert result.to_pylist() == [] + + @pytest.mark.asyncio async def test_async_remote_to_lance_rejects_storage_access(monkeypatch): with blob_remote_table() as remote_table: diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 57843ca55..d21583c91 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2884,12 +2884,10 @@ impl BaseTable for RemoteTable { async fn fetch_blob_ranges( &self, - _column: &str, - _requests: &[crate::blob::BlobRangeRequest], + column: &str, + requests: &[crate::blob::BlobRangeRequest], ) -> Result { - Err(Error::NotSupported { - message: "fetch_blob_ranges is not supported for remote tables.".into(), - }) + self.fetch_blob_ranges_impl(column, requests).await } async fn fetch_blob_files( @@ -5394,6 +5392,14 @@ mod tests { err.to_string() .contains("blobs inside lists cannot be fetched") ); + let err = table + .fetch_blob_ranges(column, [crate::blob::BlobRangeRequest::new(10, 0, 1)]) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("blobs inside lists cannot be fetched") + ); } } @@ -5955,10 +5961,6 @@ mod tests { "optimize", table.optimize(OptimizeAction::All).await.map(|_| ()), ), - ( - "fetch_blob_ranges", - table.fetch_blob_ranges("image", []).await.map(|_| ()), - ), ( "Wand factor", table diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index fdcb8f440..e7219280c 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; +use arrow_array::builder::LargeBinaryBuilder; use arrow_array::{Array, LargeBinaryArray, StructArray, UInt8Array, UInt64Array}; use arrow_schema::DataType; use bytes::Bytes; @@ -20,7 +21,7 @@ use reqwest::{Response, StatusCode, header}; use tokio::sync::{Mutex, OwnedSemaphorePermit}; use crate::Error; -use crate::blob::BlobFile; +use crate::blob::{BlobFile, BlobRangeRequest}; use crate::error::Result; use crate::query::{QueryFilter, QueryRequest, Select}; use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient}; @@ -538,6 +539,37 @@ impl RemoteTable { Ok(blobs) } + /// Materialize ranges through version-pinned handles, preserving nulls and order. + pub(super) async fn fetch_blob_ranges_impl( + &self, + column: &str, + requests: &[BlobRangeRequest], + ) -> Result { + let row_ids = requests + .iter() + .map(|request| request.row_id) + .collect::>(); + let files = self.fetch_blob_files_impl(column, &row_ids).await?; + let mut builder = LargeBinaryBuilder::with_capacity(requests.len(), 0); + for (file, request) in files.into_iter().zip(requests) { + let end = + request + .offset + .checked_add(request.length) + .ok_or_else(|| Error::InvalidInput { + message: "blob range offset + length overflows u64".into(), + })?; + match file { + Some(file) => { + let bytes = file.read_range(request.offset..end).await?; + builder.append_value(bytes); + } + None => builder.append_null(), + } + } + Ok(builder.finish()) + } + /// Open seekable handles for `row_ids`. /// /// Descriptors for the requested rows come from one `_rowid` take query per @@ -1191,6 +1223,123 @@ mod tests { (table, requests) } + #[tokio::test] + async fn remote_blob_ranges_preserve_order_nulls_and_empty_ranges() { + let (table, requests) = mock_take_blob_table( + vec![ + (10, Some((BlobKind::Inline, PAYLOAD.len() as u64))), + (20, None), + (30, Some((BlobKind::Packed, PAYLOAD.len() as u64))), + (40, Some((BlobKind::Inline, 0))), + ], + Some("7"), + ); + + let ranges = table + .fetch_blob_ranges( + "image", + &[ + BlobRangeRequest::new(30, 6, 5), + BlobRangeRequest::new(10, 0, 3), + BlobRangeRequest::new(20, 0, 1), + BlobRangeRequest::new(10, 2, 2), + BlobRangeRequest::new(40, 0, 0), + BlobRangeRequest::new(10, PAYLOAD.len() as u64, 0), + BlobRangeRequest::new(30, 6, 5), + ], + ) + .await + .unwrap(); + + assert_eq!( + ranges, + LargeBinaryArray::from(vec![ + Some(&PAYLOAD[6..11]), + Some(&PAYLOAD[0..3]), + None, + Some(&PAYLOAD[2..4]), + Some(b"".as_slice()), + Some(b"".as_slice()), + Some(&PAYLOAD[6..11]), + ]) + ); + let requests = requests.lock().unwrap(); + assert_eq!( + requests[0], + TakeMockRequest::Query { + row_ids: vec![10, 20, 30, 40], + version: None, + } + ); + // Only nonempty, non-null ranges are read, at the descriptor take's version. + let reads = requests[1..] + .iter() + .map(|request| match request { + TakeMockRequest::Range { + row_id, + range, + version, + } => { + assert_eq!(version.as_deref(), Some("7")); + (*row_id, range.as_str()) + } + other => panic!("unexpected request: {other:?}"), + }) + .collect::>(); + assert_eq!( + reads, + [ + (30, "bytes=6-10"), + (10, "bytes=0-2"), + (10, "bytes=2-3"), + (30, "bytes=6-10") + ] + ); + } + + #[tokio::test] + async fn remote_blob_ranges_reject_missing_rows_and_invalid_ranges() { + for (ranges, message) in [ + ( + vec![ + BlobRangeRequest::new(10, 0, 1), + BlobRangeRequest::new(99, 0, 1), + ], + "only 1 exist", + ), + (vec![BlobRangeRequest::new(10, 35, 2)], "exceeds blob size"), + (vec![BlobRangeRequest::new(10, 37, 0)], "exceeds blob size"), + ( + vec![BlobRangeRequest::new(10, u64::MAX, 1)], + "overflows u64", + ), + ( + vec![BlobRangeRequest::new(20, u64::MAX, 1)], + "overflows u64", + ), + ] { + let (table, requests) = mock_take_blob_table( + vec![ + (10, Some((BlobKind::Inline, PAYLOAD.len() as u64))), + (20, None), + ], + Some("7"), + ); + let error = table.fetch_blob_ranges("image", &ranges).await.unwrap_err(); + assert!(error.to_string().contains(message), "{error}"); + // Missing rows and invalid ranges fail without reading any bytes. + assert_eq!(requests.lock().unwrap().len(), 1); + } + } + + #[tokio::test] + async fn remote_blob_ranges_accept_empty_requests_without_http() { + let (table, requests) = mock_take_blob_table(vec![], Some("7")); + let ranges = table.fetch_blob_ranges("image", &[]).await.unwrap(); + assert!(ranges.is_empty()); + assert!(requests.lock().unwrap().is_empty()); + } + #[tokio::test] async fn remote_blob_files_are_sized_from_one_descriptor_take() { let payload_size = PAYLOAD.len() as u64; diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index c6193d750..802c2f517 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1304,6 +1304,8 @@ impl Table { /// ranges for the same blob. The output has the same length and order as /// the requests. Null blobs produce null output slots; empty ranges on /// non-null blobs produce empty byte strings. + /// Remote tables read through version-pinned [`BlobFile`] handles using + /// HTTP byte-range requests. /// /// `_rowid` values stay valid after compaction when the table has stable /// row ids. One missing or deleted row id rejects the whole request; no From 91541408e75214885834bca5f8a7ddd43da51664 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Fri, 16 Oct 2026 13:48:19 -0700 Subject: [PATCH 05/22] chore: update lance dependency to v14.0.0-beta.19 (#4562) Updates the Rust workspace Lance dependencies and Java lance-core from 14.0.0-beta.18 to [v14.0.0-beta.19](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.19), refreshing Cargo.lock. No compatibility fixes were required. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. Co-authored-by: Daniel Rammer --- Cargo.lock | 92 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 ++++++++-------- java/pom.xml | 2 +- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88aabd4f5..ad9eb6d41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5510,8 +5510,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "byteorder", "ethnum", @@ -5528,8 +5528,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5543,8 +5543,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "async-trait", @@ -5558,8 +5558,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arc-swap", "arrow", @@ -5613,8 +5613,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5628,8 +5628,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow", "arrow-array", @@ -5669,8 +5669,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "arrow-array", "arrow-schema", @@ -5683,8 +5683,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" +version = "14.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index c6b9e84fe..a4aaeeaca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index d6d1fdf98..c85bf04ca 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.18 + 14.0.0-beta.19 2.18.11 false 2.30.0 From ca275b6d0ecc0b72cb13978ff227200727f6334c Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Fri, 16 Oct 2026 17:05:29 -0500 Subject: [PATCH 06/22] ci: ignore yanked zip, update rustls, remove unneeded deny rules (#4560) Old versions of the zip crate have been yanked due to some rather serious vulnerabilities. However, it is a transitive dependency via `candle` and they have not yet released a patch. For now I think the best course of action is to continue to ship, knowing that we will need to address this issue as soon as candle upgrades. I also updated rustls since we have passed the 48 hour mark and I removed all deny exclusions that were no longer needed. --- Cargo.lock | 12 ++++++------ deny.toml | 37 ------------------------------------- 2 files changed, 6 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad9eb6d41..66e777cb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3083,7 +3083,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -6573,7 +6573,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -9173,9 +9173,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -11919,9 +11919,9 @@ dependencies = [ [[package]] name = "zip" -version = "7.2.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +checksum = "cc12baa6db2b15a140161ce53d72209dacea594230798c24774139b54ecaa980" dependencies = [ "crc32fast", "indexmap 2.14.0", diff --git a/deny.toml b/deny.toml index d5f42dbff..1a6977d1c 100644 --- a/deny.toml +++ b/deny.toml @@ -68,43 +68,12 @@ ignore = [ # https://rustsec.org/advisories/RUSTSEC-2025-0119 { id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" }, - # rustls-webpki 0.101.7 (old major line): name-constraint checks for - # URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain - # from aws-smithy-http-client. The 0.103 line we actively use is patched. - # Clearing the 0.101 copy requires the aws-sdk chain to migrate off - # rustls 0.21. - # https://rustsec.org/advisories/RUSTSEC-2026-0098 - # https://rustsec.org/advisories/RUSTSEC-2026-0099 - { id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, - { id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, - - # rustls 0.23.45: four advisories fixed in 0.23.46 (Debug impls leaking - # TLS secrets, ECH rejection buffering, Brotli certificate decompression - # allocation, ServerHello key-exchange downgrade). 0.23.46 was published - # 2026-10-14 and the dependency age policy requires releases to be 48 - # hours old, so the upgrade cannot land yet. Remove these entries and run - # `cargo update -p rustls` once the window has passed. Mirrors lance #9985. - # https://rustsec.org/advisories/RUSTSEC-2026-0346 - # https://rustsec.org/advisories/RUSTSEC-2026-0347 - # https://rustsec.org/advisories/RUSTSEC-2026-0348 - # https://rustsec.org/advisories/RUSTSEC-2026-0349 - { id = "RUSTSEC-2026-0346", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, - { id = "RUSTSEC-2026-0347", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, - { id = "RUSTSEC-2026-0348", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, - { id = "RUSTSEC-2026-0349", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" }, - # unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera # → serde_yaml_ng. No maintained upgrade exists; clearing it requires # lindera to move off serde_yaml_ng. # https://rustsec.org/advisories/RUSTSEC-2024-0449 { id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" }, - # rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy - # rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line - # we actively use is upgraded to 0.103.13 which contains the fix. - # https://rustsec.org/advisories/RUSTSEC-2026-0104 - { id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" }, - # pyo3 advisories in the Python bindings; tracked pending a patched pyo3 release. # https://rustsec.org/advisories/RUSTSEC-2026-0176 # https://rustsec.org/advisories/RUSTSEC-2026-0177 @@ -129,12 +98,6 @@ ignore = [ # compact_str/smol_str, so clearing this requires polars to migrate. # https://rustsec.org/advisories/RUSTSEC-2026-0249 { id = "RUSTSEC-2026-0249", reason = "smartstring unmaintained via polars; no fixed upstream release" }, - - # h2 0.3: empty DATA frames can be queued without limit. The patched - # h2 0.4 line is locked to 0.4.16, but no patched 0.3 release exists. - # The old copy is pulled in by aws-smithy's legacy hyper 0.14 client. - # https://rustsec.org/advisories/RUSTSEC-2026-0258 - { id = "RUSTSEC-2026-0258", reason = "h2 0.3 via legacy aws-smithy/hyper 0.14; no patched 0.3 release" }, ] # --------------------------------------------------------------------------- From 0220f3af17325bb64ea2cc5a953159caa7680640 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Sun, 18 Oct 2026 11:13:18 -0700 Subject: [PATCH 07/22] feat(python): accept large_binary in Function signatures (#4577) udf() rejected a large_binary input or output, and large_binary nested in a list or struct, outside a Blob v2 payload. It is now a grammar primitive like large_utf8, and the shared type golden lists it. --- python/python/lancedb/functions.py | 7 ++---- .../tests/test_first_class_function_slice2.py | 23 ++++++++++++++++++- .../first_class_functions/v1/arrow_types.json | 21 +++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 445d31027..7f5776b32 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -697,6 +697,7 @@ _GRAMMAR_PRIMITIVES = ( (pa.string(), "utf8"), (pa.large_string(), "large_utf8"), (pa.binary(), "binary"), + (pa.large_binary(), "large_binary"), (pa.date32(), "date32"), (pa.date64(), "date64"), ) @@ -846,11 +847,7 @@ def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]: { "name": child.name, "nullable": child.nullable, - "type": ( - {"type": "large_binary"} - if pa.types.is_large_binary(child.type) - else _exact_arrow_type(child.type) - ), + "type": _exact_arrow_type(child.type), } for child in storage ], diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 975310ed4..9ee3e388a 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -622,7 +622,6 @@ def test_canonical_arrow_type_prefers_the_compact_grammar(): for outside in [ pa.timestamp("us"), pa.decimal128(10, 2), - pa.large_binary(), pa.binary(4), pa.duration("s"), pa.list_(pa.float32(), 0), @@ -1280,6 +1279,28 @@ def test_explicit_large_utf8_schemas_use_the_canonical_function_name(): } +def test_large_binary_schemas_use_the_canonical_function_name(): + payload = pa.field("data", pa.large_binary(), nullable=True) + + @udf( + input_schema=pa.schema([payload]), + output_schema=pa.field("result", pa.large_binary(), nullable=False), + ) + def preserve(data): + return data + + signature = preserve.registration_request.signature + assert signature.inputs[0].arrow_type == "large_binary" + assert signature.output.arrow_type == "large_binary" + cells = pa.large_list(pa.field("item", pa.struct([payload]), nullable=True)) + assert json.loads(_canonical_arrow_type(cells))["fields"][0]["type"] == { + "type": "struct", + "fields": [ + {"name": "data", "nullable": True, "type": {"type": "large_binary"}} + ], + } + + def test_nested_struct_output_uses_canonical_exact_json(): token = pa.struct( [ diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json index 38d17821b..26e8db430 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json @@ -90,6 +90,12 @@ "type": "binary" } }, + { + "arrow_type": "large_binary", + "json": { + "type": "large_binary" + } + }, { "arrow_type": "date32", "json": { @@ -192,6 +198,21 @@ ] } }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "large_binary" + } + } + ] + } + }, { "arrow_type": "large_list", "json": { From c54b4027ef4e83f6936e2ceb377a71580f0e1c51 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 18 Oct 2026 18:15:14 +0000 Subject: [PATCH 08/22] =?UTF-8?q?Bump=20version:=200.41.0-beta.7=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index e36bbe1f9..47ab20a04 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.7" +current_version = "0.41.0-beta.8" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 66e777cb1..915ffb7b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5697,7 +5697,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.7" +version = "0.41.0-beta.8" dependencies = [ "ahash", "anyhow", @@ -5795,7 +5795,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.7" +version = "0.41.0-beta.8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5820,7 +5820,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.7" +version = "0.41.0-beta.8" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index d9207be9d..f36007a20 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.7 + 0.41.0-beta.8 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 4bc763f6a..2f4729cb1 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.7 + 0.41.0-beta.8 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index c85bf04ca..43a68201a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.7 + 0.41.0-beta.8 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 0a76c20b7..c3e86ae92 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.7" +version = "0.41.0-beta.8" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 2b95f8f3e..678e1dbc5 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 567766ad1..2a2e7d6c6 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index ab4224bef..31cebd96a 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index bef819b77..2c785801d 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 7df785698..851062c13 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index dd155262e..8252c0301 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 85b72ee02..b80b8f2da 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 6aefb4ba5..3757b1b16 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.7", + "version": "0.41.0-beta.8", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 368b3b24e..22802633d 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.7" +version = "0.41.0-beta.8" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 591fb72de..2668aef8f 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.7" +version = "0.41.0-beta.8" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From a4f6e5fe3708b622e665bbacc63fb02ce00f5878 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:17:58 -0700 Subject: [PATCH 09/22] fix(python): reject blob ranges unsupported by storage schemas (#4566) --- python/python/lancedb/table.py | 17 +++++-- python/python/tests/test_blob.py | 84 ++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index a1a54ee8e..a9cf13b65 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -341,10 +341,19 @@ def _blob_input_to_arrow(data: Any, schema: Optional[pa.Schema]) -> Optional[pa. if isinstance(field.type, pa.ExtensionType) else field.type ) - storage = pa.array( - [_blob_value_to_storage(value) for value in values[field.name]], - type=storage_type, - ) + storage_values = [_blob_value_to_storage(value) for value in values[field.name]] + missing_range_fields = {"position", "size"} - set(storage_type.names) + # Arrow silently ignores dictionary keys absent from the storage type. + if missing_range_fields and any( + value is not None + and any(value.get(name) is not None for name in missing_range_fields) + for value in storage_values + ): + raise ValueError( + f"Blob column '{field.name}' storage schema cannot store byte ranges. " + "Create the column with lancedb.blob() to use position and size." + ) + storage = pa.array(storage_values, type=storage_type) column = ( pa.ExtensionArray.from_storage(field.type, storage) if isinstance(field.type, pa.ExtensionType) diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 8be90115c..130526eaf 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -2305,6 +2305,90 @@ def test_add_external_uri_string_round_trips_with_flag(tmp_path): assert blobs[0].as_py() == payload +@pytest.mark.parametrize("as_pandas", [False, True]) +@pytest.mark.parametrize("as_blob", [False, True]) +@pytest.mark.parametrize("minimal_schema", [False, True]) +@pytest.mark.parametrize("position", [0, 4]) +def test_add_external_blob_range_matches_storage_schema( + tmp_path, as_pandas, as_blob, minimal_schema, position +): + payload = b"0123456789abcdef" + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(payload) + field = ( + pa.field("image", pa.large_binary(), metadata={"lance-encoding:blob": "true"}) + if minimal_schema + else lancedb.blob("image") + ) + db = lancedb.connect(tmp_path / "db") + table = db.create_table( + "ranges", schema=pa.schema([pa.field("id", pa.int64()), field]) + ) + value = ( + Blob.from_uri(blob_path.as_uri(), position=position, size=6) + if as_blob + else {"uri": blob_path.as_uri(), "position": position, "size": 6} + ) + rows = [{"id": 1, "image": b"inline"}, {"id": 2, "image": value}] + data = pd.DataFrame(rows) if as_pandas else rows + + if minimal_schema: + with pytest.raises(ValueError, match="Blob column 'image'.*storage schema"): + table.add(data, allow_external_blob_outside_bases=True) + assert table.count_rows() == 0 + else: + table.add(data, allow_external_blob_outside_bases=True) + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [ + b"inline", + payload[position : position + 6], + ] + + +@pytest.mark.parametrize("as_pandas", [False, True]) +def test_add_minimal_blob_schema_preserves_whole_file_and_inline_values( + tmp_path, as_pandas +): + payload = b"whole-file" + blob_path = tmp_path / "payload.bin" + blob_path.write_bytes(payload) + uri = blob_path.as_uri() + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "image", pa.large_binary(), metadata={"lance-encoding:blob": "true"} + ), + ] + ) + table = lancedb.connect(tmp_path / "db").create_table("minimal", schema=schema) + values = [ + None, + b"inline", + Blob.from_bytes(b"from-blob"), + uri, + Blob.from_uri(uri), + {"uri": uri, "position": None, "size": None}, + ] + rows = [{"id": i, "image": value} for i, value in enumerate(values)] + table.add( + pd.DataFrame(rows) if as_pandas else rows, + allow_external_blob_outside_bases=True, + ) + + by_id = _row_ids_by_id(table) + assert table.fetch_blobs( + "image", [by_id[i] for i in range(len(values))] + ).to_pylist() == [ + None, + b"inline", + b"from-blob", + payload, + payload, + payload, + ] + + @pytest.mark.parametrize("as_pandas", [False, True]) def test_add_bytes_and_uri_in_one_batch_preserves_external_uri(tmp_path, as_pandas): payload = b"external-payload" From ec53c82ef8de9d6811605bcafa76ad33f7a04c28 Mon Sep 17 00:00:00 2001 From: Weston Pace Date: Mon, 19 Oct 2026 11:09:27 -0700 Subject: [PATCH 10/22] fix(python): bound StreamingDataset's raw_batches queue against io_queue_depth (#4530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - `StreamingDataset`'s Stage 1 (I/O) queue (`raw_batches`) had no cap: `_fill_io` only limited *in-flight* fetch futures (`io_pending`) to `io_queue_depth`, but every completed fetch was unconditionally drained into `raw_batches` and the freed slot immediately refilled, so I/O could race arbitrarily far ahead of a slow transform stage with no bound on buffered memory. - `transform_queue_depth`'s admission check (`_try_submit_tx`) couldn't compensate either, since it only counts `cooked` rows and in-flight transform futures — it never looks at `raw_batches`. - Fix: `_fill_io` now counts both `io_pending` and `raw_batches` against the `io_queue_depth` budget, matching the parameter's documented "batches to keep in flight per split" contract. ## Test plan - [x] Added `test_raw_queue_depth_bounded_when_transform_is_the_bottleneck` (deterministic, via synchronous I/O) asserting `raw_queue_depth <= io_queue_depth * read_batch_size` even when the transform stage is the bottleneck. - [x] Full `python/python/tests/test_elastic_dataloader.py` suite passes (190/190), confirming no change to sample ordering, counts, or checkpoint/resumability behavior. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 5 --- python/python/lancedb/streaming.py | 24 ++++-- .../python/tests/test_elastic_dataloader.py | 86 +++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index 76b3702a1..91ca6ce28 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -261,10 +261,15 @@ class StreamingDataset(IterableDataset): storage) at the cost of higher memory usage per split buffer. Defaults to ``DEFAULT_READ_BATCH_SIZE`` (64). io_queue_depth: - Number of I/O batches to keep in flight per split. Higher values - overlap storage latency with transform and training compute at the cost - of more memory and threads. Must be greater than zero. Defaults to - ``DEFAULT_PREFETCH_BATCHES`` (4). + Number of fetched batches to keep ahead of the transform stage per + split: this bounds the combined count of in-flight fetches and + already-fetched batches still waiting for a transform slot. Higher + values overlap storage latency with transform and training compute + at the cost of more memory and threads, but I/O stops running ahead + once this many batches are buffered, applying backpressure when the + transform stage (or ``transform_queue_depth``) is the bottleneck. + Must be greater than zero. Defaults to ``DEFAULT_PREFETCH_BATCHES`` + (4). columns: Optional list of column names to read. When set, only those columns are fetched from storage; all others are omitted. ``None`` (the @@ -836,8 +841,17 @@ class StreamingDataset(IterableDataset): io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices))) def _fill_io(i: int) -> None: + # Budget covers both in-flight fetches (io_pending) and batches + # that have already landed but are still waiting for a transform + # slot (raw_batches). Without raw_batches in the count, a slow + # transform stage never applies backpressure to Stage 1: each + # completed fetch is drained into raw_batches and immediately + # replaced by a new one, so I/O keeps racing arbitrarily far + # ahead of the transform stage instead of merely overlapping + # storage latency with it. while ( - len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i] + len(io_pending[i]) + len(raw_batches[i]) < io_queue_depth + and fetch_head[i] < split_sizes[i] ): _submit_io(i) diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index da27e5bfc..cc6874e6c 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -2036,6 +2036,92 @@ def test_transform_queue_depth_bounds_cooked_rows(lance_table): ) +def test_raw_queue_depth_bounded_when_transform_is_the_bottleneck(tmp_path): + """io_queue_depth bounds raw_batches even when transform is the bottleneck. + + Regression test for a bug where Stage 1 (I/O) applied no backpressure at + all: _fill_io only capped io_pending (in-flight fetches) at + io_queue_depth, but every completed fetch was drained into raw_batches + unconditionally (_drain_io) and immediately replaced by a new fetch + (_fill_io again), so I/O kept racing arbitrarily far ahead of a slow + transform stage -- raw_batches had no cap of its own. + transform_queue_depth did not help either: its admission check in + _try_submit_tx only counts cooked rows and in-flight transform futures + (`len(cooked[i]) + len(tx_pending[i]) * batch_size`); it never looks at + len(raw_batches[i]), so it cannot throttle Stage 1. + + The fix makes _fill_io count both io_pending and raw_batches against the + io_queue_depth budget, so raw_queue_depth should never exceed + io_queue_depth * read_batch_size per split, regardless of how slow the + transform stage is. + + I/O is forced to run synchronously (same technique as + test_transform_queue_depth_does_not_admit_at_capacity_minus_one) so the + assertion is deterministic regardless of machine load, rather than + relying on a real-time race between I/O and transform threads. Only the + transform runs on a real background thread (with a short sleep) so the + consumer genuinely blocks waiting for it, letting Stage 1 attempt to run + ahead between blocking waits. + """ + import concurrent.futures as cf + from concurrent.futures import ThreadPoolExecutor + from unittest.mock import patch + import pyarrow as pa + import time + + n_rows = 200 + read_batch_size = 10 + io_queue_depth = 2 + transform_queue_depth = 2 + max_allowed = io_queue_depth * read_batch_size + + db = lancedb.connect(tmp_path) + table = db.create_table("t", pa.table({"id": list(range(n_rows))})) + + def slow_transform(batch: pa.RecordBatch) -> list: + time.sleep(0.005) + return batch.column("id").to_pylist() + + original_submit = ThreadPoolExecutor.submit + + def synchronous_io_submit(self, fn, *args, **kwargs): + if getattr(fn, "__name__", "") == "_io_call": + fut = cf.Future() + try: + fut.set_result(fn(*args, **kwargs)) + except Exception as exc: + fut.set_exception(exc) + return fut + return original_submit(self, fn, *args, **kwargs) + + with patch.object(ThreadPoolExecutor, "submit", synchronous_io_submit): + ds = StreamingDataset( + table, + num_splits=1, + shuffle_seed=SHUFFLE_SEED, + read_batch_size=read_batch_size, + io_queue_depth=io_queue_depth, + transform_queue_depth=transform_queue_depth, + transform_parallelism=1, + transform=slow_transform, + ) + + peak_raw = 0 + ids = [] + for item in ds: + ids.append(item) # slow_transform yields bare ids, not dicts + if ds.raw_queue_depth > peak_raw: + peak_raw = ds.raw_queue_depth + + assert peak_raw <= max_allowed, ( + f"raw_queue_depth peaked at {peak_raw}, expected <= {max_allowed} " + "(io_queue_depth * read_batch_size)" + ) + # The backpressure fix must not come at the cost of dropping or + # duplicating rows. + assert sorted(ids) == list(range(n_rows)) + + def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path): """Admission requires a full read_batch_size of free space, not just one slot. From 7d52c7deb75c9533aff86c94dc46d9379eba3e8b Mon Sep 17 00:00:00 2001 From: LuQQiu Date: Mon, 19 Oct 2026 11:28:23 -0700 Subject: [PATCH 11/22] chore(deny): ignore zip advisories reachable only through candle-core (#4593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo deny` fails on `main` (e.g. [run 39153525802](https://github.com/lancedb/lancedb/actions/runs/39153525802)) and on #4590 with three new `zip 7.4.0` advisories: - RUSTSEC-2026-0367, RUSTSEC-2026-0368, RUSTSEC-2026-0369 (SUID/SGID bits, symlink targets and permissions applied through symlinks during archive extraction), all patched only in `zip >= 9.0.0`. `zip` is in the lock only through `candle-core` (optional local-embedding feature). Every published candle release pins `zip < 9` (0.9.x/0.10.x `^7.2`, 0.11.0 `^8.6`, git main 8.6), so neither `cargo update -p zip` nor a candle bump clears it. LanceDB never extracts archives; the crate is only linked. Same approach as #4551 (rustls): ignore with the reason recorded in `deny.toml`. `cargo deny check advisories` passes locally with this change. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5.1 --- deny.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deny.toml b/deny.toml index 1a6977d1c..7d60ae63f 100644 --- a/deny.toml +++ b/deny.toml @@ -80,6 +80,18 @@ ignore = [ { id = "RUSTSEC-2026-0176", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" }, { id = "RUSTSEC-2026-0177", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" }, + # zip < 9.0.0: SUID/SGID bits, symlink targets and permissions applied through + # symlinks during archive extraction. Transitive via candle-core (optional + # local-embedding feature); every published candle release pins zip < 9 + # (0.9.x/0.10.x ^7.2, 0.11.0 ^8.6), so no dependency bump clears it. + # LanceDB never extracts archives; the zip crate is only linked. + # https://rustsec.org/advisories/RUSTSEC-2026-0367 + # https://rustsec.org/advisories/RUSTSEC-2026-0368 + # https://rustsec.org/advisories/RUSTSEC-2026-0369 + { id = "RUSTSEC-2026-0367", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" }, + { id = "RUSTSEC-2026-0368", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" }, + { id = "RUSTSEC-2026-0369", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" }, + # quick-xml < 0.41.0: quadratic runtime on duplicate attribute names (DoS). # quick-xml < 0.41.0: unbounded namespace-declaration allocation in NsReader (DoS). # Pulled in transitively by inferno (dev-only flame-graph dep), lance-namespace-impls From 9020640538c885b8e112c5b860ef94d6d9709ddb Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 19 Oct 2026 12:11:25 -0700 Subject: [PATCH 12/22] fix(python)!: don't trust embedding settings from table metadata (#4550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: `SentenceTransformerEmbeddings.trust_remote_code` now defaults to `False` (was `True`). When an embedding function is restored from table metadata, `trust_remote_code=True` and custom endpoints (OpenAI `base_url`, Ollama `host`/proxy, watsonx `url`) are only honored if the process approved them; otherwise computing embeddings raises `UntrustedEmbeddingConfigError`. Functions created in the same process are approved automatically. To reopen such a table in a new process, call `get_registry().allow_remote_code(model, revision=...)` or `get_registry().allow_endpoint(url)` first. ## Summary Embedding functions are rebuilt from table metadata, which anyone who can write the table controls. Since sentence-transformers defaulted `trust_remote_code` to `True`, opening and querying a crafted table ran arbitrary code from a model repository (GHSA-x8v2-vj73-gfwm). Metadata could likewise point `base_url` at another server, sending it queries and the `OPENAI_API_KEY` from the environment. The table can no longer grant itself trust: ```python registry = get_registry() registry.allow_remote_code("nomic-ai/nomic-embed-text-v1.5", revision="") registry.allow_endpoint("http://my-ollama-host:11434") ``` Trust is keyed by model name (optionally pinned to a revision), so approving one model doesn't let a table swap in another. Restored functions that aren't approved are replaced by a placeholder that raises on use without constructing the real function (`huggingface` loads its model in `__init__`), so non-embedding operations on the table keep working. `sentence-transformers` and `huggingface` gain a `revision` parameter. Custom embedding functions can opt in by overriding the new `remote_code_model()` / `endpoints()` hooks. Not covered: TypeScript (its built-ins have neither remote code nor configurable endpoints), Bedrock `assumed_role`, and third-party functions that don't implement the hooks. ## Evidence `test_embeddings_trust.py` covers restore with and without approval, revision pinning, swapped model names, endpoint gating, and that the table stays usable. Opening a table with crafted metadata and searching it now gives: ``` UntrustedEmbeddingConfigError: Embedding function 'sentence-transformers' was restored from table metadata with settings this process has not approved: - runs code from model 'attacker/model' at revision None. If you trust it, call get_registry().allow_remote_code('attacker/model', revision=None) ``` ## Merge Risk **Door:** One-way: changes a default and makes previously working reopen-and-search flows require explicit approval. **Blast Radius:** public-api (Python embeddings); users of remote-code models or custom endpoints who reopen tables in a new process. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- docs/web/embedding/index.mdx | 23 ++ .../integrations/embedding/huggingface.mdx | 2 +- .../embedding/sentence-transformers.mdx | 5 +- python/python/lancedb/embeddings/__init__.py | 8 +- python/python/lancedb/embeddings/base.py | 98 +++++- python/python/lancedb/embeddings/ollama.py | 19 +- python/python/lancedb/embeddings/openai.py | 3 + python/python/lancedb/embeddings/registry.py | 137 ++++++++- .../embeddings/sentence_transformers.py | 26 +- .../python/lancedb/embeddings/transformers.py | 18 +- python/python/lancedb/embeddings/utils.py | 5 + python/python/lancedb/embeddings/watsonx.py | 3 + python/python/tests/test_embeddings_trust.py | 291 ++++++++++++++++++ 13 files changed, 616 insertions(+), 22 deletions(-) create mode 100644 python/python/tests/test_embeddings_trust.py diff --git a/docs/web/embedding/index.mdx b/docs/web/embedding/index.mdx index 9f1531c05..ee893f661 100644 --- a/docs/web/embedding/index.mdx +++ b/docs/web/embedding/index.mdx @@ -134,6 +134,29 @@ For non-sensitive settings such as inference device selection, you can also use Find the full list of arguments for each provider in the [integrations](/integrations/embedding) section. +### Trusting remote code and custom endpoints {#trusting-remote-code-and-custom-endpoints} + +In Python, the embedding definition is stored in the table, so anyone who can write to the table can change it. +To keep a table from running code or sending data where you didn't intend, two kinds of settings are only honored +when this process has approved them: + +- `trust_remote_code=True`, which runs code from the model's Hugging Face repository. +- Custom endpoints such as OpenAI `base_url`, Ollama `host`, or watsonx `url`, which receive your data and API keys. + +Embedding functions you create in the current process are approved automatically. When you open a table in a new +process, approve these settings explicitly before computing embeddings: + +```python +from lancedb.embeddings import get_registry + +registry = get_registry() +# Pinning a commit hash is recommended, since the code at a branch can change. +registry.allow_remote_code("nomic-ai/nomic-embed-text-v1.5", revision="") +registry.allow_endpoint("http://my-ollama-host:11434") +``` + +Until then, computing embeddings raises `UntrustedEmbeddingConfigError`. Other operations on the table still work. + ## Multiple embedding columns {#multiple-embedding-columns} A single table can include more than one embedding definition when you want to store multiple semantic views diff --git a/docs/web/integrations/embedding/huggingface.mdx b/docs/web/integrations/embedding/huggingface.mdx index 69cf85599..046b4dfb4 100644 --- a/docs/web/integrations/embedding/huggingface.mdx +++ b/docs/web/integrations/embedding/huggingface.mdx @@ -5,7 +5,7 @@ sidebarTitle: Hugging Face import { PyEmbeddingHuggingfaceUsage } from '/snippets/integrations.mdx'; -We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. +We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. Pin `revision` to a commit hash, and note that tables using such a model need it approved with `get_registry().allow_remote_code(...)` when opened in a new process. See [Trusting remote code and custom endpoints](/embedding#trusting-remote-code-and-custom-endpoints). Example usage: diff --git a/docs/web/integrations/embedding/sentence-transformers.mdx b/docs/web/integrations/embedding/sentence-transformers.mdx index aa0c9d1ca..0b425e50e 100644 --- a/docs/web/integrations/embedding/sentence-transformers.mdx +++ b/docs/web/integrations/embedding/sentence-transformers.mdx @@ -12,7 +12,7 @@ Sentence transformer embeddings are normalized by default. It is recommended to -The `trust_remote_code` parameter defaults to `True`, which allows models to execute arbitrary code from their Hugging Face repository during loading. If you are loading untrusted models, set `trust_remote_code=False` to prevent remote code execution. +Setting `trust_remote_code=True` allows the model to execute arbitrary code from its Hugging Face repository during loading. Only enable it for repositories you trust, and pin `revision` to a commit hash. When a table using such a model is opened in a new process, the model must be approved with `get_registry().allow_remote_code(...)`. See [Trusting remote code and custom endpoints](/embedding#trusting-remote-code-and-custom-endpoints). | Parameter | Type | Default Value | Description | @@ -20,7 +20,8 @@ The `trust_remote_code` parameter defaults to `True`, which allows models to exe | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model | | `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) | | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model | -| `trust_remote_code` | `bool` | `True` | Whether to trust and execute remote code from the model's Huggingface repository | +| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository | +| `revision` | `str` | `None` | The model revision to load, such as a commit hash |
diff --git a/python/python/lancedb/embeddings/__init__.py b/python/python/lancedb/embeddings/__init__.py index 4a39b7900..636502c9e 100644 --- a/python/python/lancedb/embeddings/__init__.py +++ b/python/python/lancedb/embeddings/__init__.py @@ -3,7 +3,12 @@ # ruff: noqa: F401 -from .base import EmbeddingFunction, EmbeddingFunctionConfig, TextEmbeddingFunction +from .base import ( + EmbeddingFunction, + EmbeddingFunctionConfig, + TextEmbeddingFunction, + UntrustedEmbeddingConfigError, +) from .bedrock import BedRockText from .cohere import CohereEmbeddingFunction from .gemini_text import GeminiText @@ -29,6 +34,7 @@ __all__ = [ "EmbeddingFunction", "EmbeddingFunctionConfig", "TextEmbeddingFunction", + "UntrustedEmbeddingConfigError", "EmbeddingFunctionRegistry", "get_registry", "register", diff --git a/python/python/lancedb/embeddings/base.py b/python/python/lancedb/embeddings/base.py index fbe43860d..f26a940df 100644 --- a/python/python/lancedb/embeddings/base.py +++ b/python/python/lancedb/embeddings/base.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod import copy -from typing import List, Union +from typing import List, Optional, Tuple, Union from lancedb.util import add_note import numpy as np @@ -13,6 +13,18 @@ from pydantic import BaseModel, Field, PrivateAttr from .utils import TEXT, retry_with_exponential_backoff +class UntrustedEmbeddingConfigError(PermissionError): + """ + Raised when using an embedding function restored from table metadata whose + settings this process has not approved. + + See + [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code] + and + [allow_endpoint][lancedb.embeddings.EmbeddingFunctionRegistry.allow_endpoint]. + """ + + class EmbeddingFunction(BaseModel, ABC): """ An ABC for embedding functions. @@ -37,11 +49,29 @@ class EmbeddingFunction(BaseModel, ABC): """ Create an instance of the embedding function """ + from .registry import EmbeddingFunctionRegistry + resolved_kwargs = cls.__resolveVariables(kwargs) instance = cls(**resolved_kwargs) - instance._original_args = kwargs + instance._original_args = dict(kwargs) + if ( + instance.remote_code_model() is not None + and "trust_remote_code" in cls.model_fields + ): + # Record the flag even when left at its default, so a missing flag + # in table metadata only ever means a table written before this. + instance._original_args.setdefault( + "trust_remote_code", instance.trust_remote_code + ) + # The caller wrote this config, so this process trusts it. Without this, + # a table created with it could not be reopened in the same process. + EmbeddingFunctionRegistry.get_instance()._trust(instance) return instance + @classmethod + def _resolve_variables(cls, args: dict) -> dict: + return cls.__resolveVariables(args) + @classmethod def __resolveVariables(cls, args: dict) -> dict: """ @@ -92,6 +122,38 @@ class EmbeddingFunction(BaseModel, ABC): """ return [] + def remote_code_model(self) -> Optional[Tuple[str, Optional[str]]]: + """ + Return the model whose repository code this function runs when its + `trust_remote_code` field is enabled, as a `(name, revision)` pair. + + Return None if the function never runs code from a model repository. + When the function is restored from table metadata, `trust_remote_code` + is only enabled for models allowed with + [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code]. + """ + return None + + @classmethod + def _legacy_trust_remote_code_default(cls) -> bool: + """ + The `trust_remote_code` to use for table metadata that omits it. Only + tables written before the flag was always recorded omit it, so this is + the default from that time. + """ + field = cls.model_fields.get("trust_remote_code") + return bool(field.default) if field is not None else False + + def endpoints(self) -> List[str]: + """ + Return the non-default URLs this function sends data or credentials to. + + When the function is restored from table metadata, each of these must be + allowed with + [allow_endpoint][lancedb.embeddings.EmbeddingFunctionRegistry.allow_endpoint]. + """ + return [] + @abstractmethod def compute_query_embeddings(self, *args, **kwargs) -> list[Union[np.array, None]]: """ @@ -233,3 +295,35 @@ class TextEmbeddingFunction(EmbeddingFunction): ) -> list[Union[np.array, None]]: """Generate the embeddings for the given texts""" pass + + +class UntrustedEmbeddingFunction(EmbeddingFunction): + """ + Stands in for an embedding function restored from table metadata whose + settings this process has not approved. + + The table stays usable for anything that doesn't compute embeddings, while + computing embeddings raises + [UntrustedEmbeddingConfigError][lancedb.embeddings.UntrustedEmbeddingConfigError]. + The real function is never constructed, since some load their model in + `__init__`. + """ + + function_name: str + reasons: Tuple[str, ...] + + def _error(self) -> UntrustedEmbeddingConfigError: + details = "\n".join(f" - {reason}" for reason in self.reasons) + return UntrustedEmbeddingConfigError( + f"Embedding function '{self.function_name}' was restored from table " + "metadata with settings this process has not approved:\n" + details + ) + + def compute_query_embeddings(self, *args, **kwargs): + raise self._error() + + def compute_source_embeddings(self, *args, **kwargs): + raise self._error() + + def ndims(self) -> int: + raise self._error() diff --git a/python/python/lancedb/embeddings/ollama.py b/python/python/lancedb/embeddings/ollama.py index 871a04247..aad8742de 100644 --- a/python/python/lancedb/embeddings/ollama.py +++ b/python/python/lancedb/embeddings/ollama.py @@ -14,6 +14,9 @@ if TYPE_CHECKING: import ollama +DEFAULT_OLLAMA_HOST = "http://localhost:11434" + + @register("ollama") class OllamaEmbeddings(TextEmbeddingFunction): """ @@ -24,11 +27,25 @@ class OllamaEmbeddings(TextEmbeddingFunction): """ name: str = "nomic-embed-text" - host: str = "http://localhost:11434" + host: str = DEFAULT_OLLAMA_HOST options: Optional[dict] = None # type = ollama.Options keep_alive: Optional[Union[float, str]] = None ollama_client_kwargs: Optional[dict] = {} + def endpoints(self) -> List[str]: + urls = [] + if self.host.rstrip("/") != DEFAULT_OLLAMA_HOST: + urls.append(self.host) + # These kwargs go to the underlying httpx client, where a proxy would + # see every request. + for key in ("proxy", "proxies"): + proxy = (self.ollama_client_kwargs or {}).get(key) + if isinstance(proxy, str): + urls.append(proxy) + elif isinstance(proxy, dict): + urls.extend(v for v in proxy.values() if isinstance(v, str)) + return urls + def ndims(self) -> int: return len(self.generate_embeddings(["foo"])[0]) diff --git a/python/python/lancedb/embeddings/openai.py b/python/python/lancedb/embeddings/openai.py index 8f3073d49..1edfaa2ca 100644 --- a/python/python/lancedb/embeddings/openai.py +++ b/python/python/lancedb/embeddings/openai.py @@ -58,6 +58,9 @@ class OpenAIEmbeddings(TextEmbeddingFunction): def sensitive_keys(): return ["api_key"] + def endpoints(self) -> List[str]: + return [self.base_url] if self.base_url else [] + @staticmethod def model_names(): return [ diff --git a/python/python/lancedb/embeddings/registry.py b/python/python/lancedb/embeddings/registry.py index 4c91445a5..1dc7ffcb8 100644 --- a/python/python/lancedb/embeddings/registry.py +++ b/python/python/lancedb/embeddings/registry.py @@ -2,9 +2,15 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors import json -from typing import Dict, Optional, Type +from typing import Dict, Optional, Set, Tuple, Type -from .base import EmbeddingFunction, EmbeddingFunctionConfig +from pydantic import TypeAdapter + +from .base import ( + EmbeddingFunction, + EmbeddingFunctionConfig, + UntrustedEmbeddingFunction, +) class EmbeddingFunctionRegistry: @@ -42,6 +48,9 @@ class EmbeddingFunctionRegistry: def __init__(self): self._functions = {} self._variables = {} + # Revision is a str, None for the default branch, or _ANY_REVISION. + self._allowed_remote_code: Set[Tuple[str, object]] = set() + self._allowed_endpoints: Set[str] = set() def register(self, alias: Optional[str] = None): """ @@ -122,20 +131,129 @@ class EmbeddingFunctionRegistry: obj["vector_column"]: EmbeddingFunctionConfig( vector_column=obj["vector_column"], source_column=obj["source_column"], - function=self.get(obj["name"]).create(**obj["model"]), + function=self._restore(obj["name"], obj["model"]), ) for obj in raw_list } + def _restore(self, name: str, args: dict) -> EmbeddingFunction: + """ + Create an embedding function from table metadata. Anyone who can write + the table controls `args`, so settings that run model code or send data + elsewhere must already be trusted by this process. + """ + cls = self.get(name) + resolved = cls._resolve_variables(args) + # Inspect the config without running __init__, which may load a model. + probe = cls.model_construct(**resolved) + + reasons = [] + model = probe.remote_code_model() + if model is not None: + allowed = self._remote_code_allowed(*model) + explicit = "trust_remote_code" in resolved + if explicit: + requested = TypeAdapter(bool).validate_python( + resolved["trust_remote_code"] + ) + else: + requested = cls._legacy_trust_remote_code_default() + # Always set the flag, since a subclass may default it to True. + resolved["trust_remote_code"] = requested and allowed + # A legacy table that omits the flag is downgraded rather than + # blocked: most sentence-transformers models don't need remote code, + # and those tables relied on the old True default without asking. + if requested and not allowed and explicit: + model_name, revision = model + reasons.append( + f"runs code from model '{model_name}' at revision " + f"{revision!r}. If you trust it, call " + f"get_registry().allow_remote_code({model_name!r}, " + f"revision={revision!r})" + ) + reasons.extend( + f"sends data to endpoint '{url}'. If you trust it, call " + f"get_registry().allow_endpoint({url!r})" + for url in probe.endpoints() + if _normalize_url(url) not in self._allowed_endpoints + ) + + if reasons: + func = UntrustedEmbeddingFunction( + function_name=name, reasons=tuple(reasons) + ) + else: + func = cls(**resolved) + func._original_args = args + return func + + def allow_remote_code(self, model: str, revision: Optional[str] = None) -> None: + """ + Trust the code in a model repository when an embedding function that + loads it is restored from table metadata. + + Table metadata cannot enable `trust_remote_code` by itself, since anyone + who can write to the table could then run code in this process. Embedding + functions created in this process with `trust_remote_code=True` are + allowed automatically. + + Parameters + ---------- + model : str + The model name, such as `"nomic-ai/nomic-embed-text-v1.5"`. + revision : Optional[str] + The revision the table's embedding function must pin. Pinning a + commit hash is recommended, since the code at a branch or tag can + change after you review it. If None, any revision is trusted, + including pull request refs that anyone can push to the repository. + """ + self._allowed_remote_code.add( + (model, _ANY_REVISION if revision is None else revision) + ) + + def allow_endpoint(self, url: str) -> None: + """ + Trust a custom endpoint, such as an OpenAI `base_url` or Ollama `host`, + when an embedding function that uses it is restored from table metadata. + + Table metadata cannot pick an endpoint by itself, since it could send + your data and API keys to a server you don't control. Embedding functions + created in this process are allowed to use their endpoints automatically. + + Parameters + ---------- + url : str + The endpoint URL. Trailing slashes are ignored. + """ + self._allowed_endpoints.add(_normalize_url(url)) + + def _remote_code_allowed(self, model: str, revision: Optional[str]) -> bool: + return (model, _ANY_REVISION) in self._allowed_remote_code or ( + model, + revision, + ) in self._allowed_remote_code + + def _trust(self, func: EmbeddingFunction) -> None: + model = func.remote_code_model() + if model is not None and getattr(func, "trust_remote_code", False): + # Only the revision this code selected, where None means the default + # branch. Other revisions, such as PR refs, may hold other code. + self._allowed_remote_code.add(model) + for url in func.endpoints(): + self.allow_endpoint(url) + def function_to_metadata(self, conf: "EmbeddingFunctionConfig"): """ Convert the given embedding function and source / vector column configs into a config dictionary that can be serialized into arrow metadata """ func = conf.function - name = getattr( - func, "__embedding_function_registry_alias__", func.__class__.__name__ - ) + if isinstance(func, UntrustedEmbeddingFunction): + name = func.function_name + else: + name = getattr( + func, "__embedding_function_registry_alias__", func.__class__.__name__ + ) json_data = func.safe_model_dump() return { "name": name, @@ -180,6 +298,13 @@ class EmbeddingFunctionRegistry: return self._variables[name] +_ANY_REVISION = object() + + +def _normalize_url(url: str) -> str: + return url.rstrip("/") + + # Global instance __REGISTRY__ = EmbeddingFunctionRegistry() diff --git a/python/python/lancedb/embeddings/sentence_transformers.py b/python/python/lancedb/embeddings/sentence_transformers.py index 8be02e9f7..e9f47a7b9 100644 --- a/python/python/lancedb/embeddings/sentence_transformers.py +++ b/python/python/lancedb/embeddings/sentence_transformers.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors -from typing import List, Union +from typing import List, Optional, Tuple, Union import numpy as np @@ -27,19 +27,32 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction): The device to use for the model normalize : bool, default True Whether to normalize the embeddings - trust_remote_code : bool, default True - Whether to trust the remote code + trust_remote_code : bool, default False + Whether to run code from the model's repository. Only enable this for + repositories you trust. When restored from table metadata, this is only + enabled for models allowed with + [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code]. + revision : Optional[str], default None + The model revision to load, such as a commit hash. """ name: str = "all-MiniLM-L6-v2" device: str = "cpu" normalize: bool = True - trust_remote_code: bool = True + trust_remote_code: bool = False + revision: Optional[str] = None def __init__(self, **kwargs): super().__init__(**kwargs) self._ndims = None + def remote_code_model(self) -> Optional[Tuple[str, Optional[str]]]: + return (self.name, self.revision) + + @classmethod + def _legacy_trust_remote_code_default(cls) -> bool: + return True + @property def embedding_model(self): """ @@ -84,5 +97,8 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction): "sentence_transformers", "sentence-transformers" ) return sentence_transformers.SentenceTransformer( - self.name, device=self.device, trust_remote_code=self.trust_remote_code + self.name, + device=self.device, + trust_remote_code=self.trust_remote_code, + revision=self.revision, ) diff --git a/python/python/lancedb/embeddings/transformers.py b/python/python/lancedb/embeddings/transformers.py index 3f42edfdd..92182c916 100644 --- a/python/python/lancedb/embeddings/transformers.py +++ b/python/python/lancedb/embeddings/transformers.py @@ -3,7 +3,7 @@ from functools import cached_property -from typing import List, Any +from typing import List, Any, Optional, Tuple import numpy as np @@ -34,7 +34,11 @@ class TransformersEmbeddingFunction(EmbeddingFunction): Whether or not to allow for custom models defined on the HuggingFace Hub in their own modeling files. This option should only be set to True for repositories you trust and in which you have read the code, as it - will execute code present on the Hub on your local machine. + will execute code present on the Hub on your local machine. When + restored from table metadata, this is only enabled for models allowed with + [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code]. + revision : Optional[str] + The model revision to load, such as a commit hash. Default is None. to download package, run : `pip install transformers` @@ -45,6 +49,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction): name: str = "colbert-ir/colbertv2.0" device: str = "cpu" trust_remote_code: bool = False + revision: Optional[str] = None _tokenizer: Any = PrivateAttr() _model: Any = PrivateAttr() @@ -52,12 +57,17 @@ class TransformersEmbeddingFunction(EmbeddingFunction): super().__init__(*args, **kwargs) self._ndims = None transformers = attempt_import_or_raise("transformers") - self._tokenizer = transformers.AutoTokenizer.from_pretrained(self.name) + self._tokenizer = transformers.AutoTokenizer.from_pretrained( + self.name, revision=self.revision + ) self._model = transformers.AutoModel.from_pretrained( - self.name, trust_remote_code=self.trust_remote_code + self.name, trust_remote_code=self.trust_remote_code, revision=self.revision ) self._model.to(self.device) + def remote_code_model(self) -> Optional[Tuple[str, Optional[str]]]: + return (self.name, self.revision) + model_config = ConfigDict(ignored_types=(cached_property,)) def ndims(self): diff --git a/python/python/lancedb/embeddings/utils.py b/python/python/lancedb/embeddings/utils.py index 98806fa52..f76c9b1a5 100644 --- a/python/python/lancedb/embeddings/utils.py +++ b/python/python/lancedb/embeddings/utils.py @@ -304,6 +304,11 @@ def _is_non_retryable_error(error: Exception) -> bool: Returns: True if the error should not be retried, False otherwise """ + from .base import UntrustedEmbeddingConfigError + + if isinstance(error, UntrustedEmbeddingConfigError): + return True + # Check for OpenAI authentication errors error_type = type(error).__name__ if error_type == "AuthenticationError": diff --git a/python/python/lancedb/embeddings/watsonx.py b/python/python/lancedb/embeddings/watsonx.py index 82a0ad070..90ab4ea0d 100644 --- a/python/python/lancedb/embeddings/watsonx.py +++ b/python/python/lancedb/embeddings/watsonx.py @@ -87,6 +87,9 @@ class WatsonxEmbeddings(TextEmbeddingFunction): url: Optional[str] = None params: Optional[Dict] = None + def endpoints(self) -> List[str]: + return [self.url] if self.url else [] + @staticmethod def sensitive_keys(): return ["api_key"] diff --git a/python/python/tests/test_embeddings_trust.py b/python/python/tests/test_embeddings_trust.py new file mode 100644 index 000000000..92221c898 --- /dev/null +++ b/python/python/tests/test_embeddings_trust.py @@ -0,0 +1,291 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +""" +Restoring an embedding function from table metadata must not let the table grant +itself trust: remote model code and custom endpoints need approval from the +process that opens the table. +""" + +import json +from unittest.mock import patch + +import lancedb +import pyarrow as pa +import pytest +from lancedb.embeddings import ( + EmbeddingFunctionConfig, + OllamaEmbeddings, + OpenAIEmbeddings, + SentenceTransformerEmbeddings, + UntrustedEmbeddingConfigError, + get_registry, +) +from lancedb.embeddings.base import UntrustedEmbeddingFunction + + +@pytest.fixture(autouse=True) +def fresh_trust(monkeypatch): + registry = get_registry() + monkeypatch.setattr(registry, "_allowed_remote_code", set()) + monkeypatch.setattr(registry, "_allowed_endpoints", set()) + + +def metadata_for(name: str, model: dict) -> dict: + entry = { + "name": name, + "model": model, + "source_column": "text", + "vector_column": "vector", + } + return {b"embedding_functions": json.dumps([entry]).encode()} + + +def restore(name: str, model: dict): + return get_registry().parse_functions(metadata_for(name, model))["vector"].function + + +def test_sentence_transformers_does_not_trust_remote_code_by_default(): + assert SentenceTransformerEmbeddings.create().trust_remote_code is False + + +@pytest.mark.parametrize("name", ["sentence-transformers", "huggingface", "colbert"]) +def test_restored_remote_code_requires_allowlist(name): + model = {"name": "attacker/model", "trust_remote_code": True} + func = restore(name, model) + + with pytest.raises(UntrustedEmbeddingConfigError, match="attacker/model"): + func.compute_query_embeddings("hello") + with pytest.raises(UntrustedEmbeddingConfigError): + func.ndims() + + +def test_blocked_function_does_not_construct_model(): + # huggingface loads its model in __init__, so the check has to happen first. + with patch( + "lancedb.embeddings.transformers.attempt_import_or_raise" + ) as attempt_import: + restore("huggingface", {"name": "attacker/model", "trust_remote_code": True}) + attempt_import.assert_not_called() + + +def test_allowlisted_model_restores_with_remote_code(): + get_registry().allow_remote_code("org/model") + func = restore( + "sentence-transformers", {"name": "org/model", "trust_remote_code": True} + ) + assert isinstance(func, SentenceTransformerEmbeddings) + assert func.trust_remote_code is True + + +def test_allowlist_does_not_cover_other_models(): + get_registry().allow_remote_code("org/model") + func = restore( + "sentence-transformers", {"name": "org/other", "trust_remote_code": True} + ) + with pytest.raises(UntrustedEmbeddingConfigError, match="org/other"): + func.compute_query_embeddings("hello") + + +def test_pinned_revision_must_match(): + get_registry().allow_remote_code("org/model", revision="abc123") + + pinned = restore( + "sentence-transformers", + {"name": "org/model", "revision": "abc123", "trust_remote_code": True}, + ) + assert pinned.trust_remote_code is True + + for model in [ + {"name": "org/model", "revision": "def456", "trust_remote_code": True}, + {"name": "org/model", "trust_remote_code": True}, + ]: + func = restore("sentence-transformers", model) + with pytest.raises(UntrustedEmbeddingConfigError, match="org/model"): + func.compute_query_embeddings("hello") + + +def test_restored_trust_flag_follows_allowlist_when_absent(): + # Tables written before the default changed don't record trust_remote_code. + assert restore( + "sentence-transformers", {"name": "org/model"} + ).trust_remote_code is (False) + get_registry().allow_remote_code("org/model") + assert restore("sentence-transformers", {"name": "org/model"}).trust_remote_code + + +def test_created_function_records_trust_flag(): + # Otherwise a new table using the default would look like a pre-fix table, + # where a missing flag meant True. + conf = EmbeddingFunctionConfig( + source_column="text", + vector_column="vector", + function=SentenceTransformerEmbeddings.create(name="org/model"), + ) + assert get_registry().function_to_metadata(conf)["model"] == { + "name": "org/model", + "trust_remote_code": False, + } + + get_registry().allow_remote_code("org/model") + metadata = get_registry().get_table_metadata([conf]) + restored = get_registry().parse_functions(metadata)["vector"].function + assert restored.trust_remote_code is False + + +def test_huggingface_missing_flag_keeps_false_default(): + # huggingface always defaulted to False, so approval must not flip it. + get_registry().allow_remote_code("org/model") + with patch("lancedb.embeddings.transformers.attempt_import_or_raise"): + func = restore("huggingface", {"name": "org/model"}) + assert func.trust_remote_code is False + + +def test_restored_explicit_false_stays_false(): + get_registry().allow_remote_code("org/model") + func = restore( + "sentence-transformers", {"name": "org/model", "trust_remote_code": False} + ) + assert func.trust_remote_code is False + + +def test_create_trusts_its_own_config(tmp_path): + func = SentenceTransformerEmbeddings.create( + name="org/model", trust_remote_code=True + ) + db = lancedb.connect(tmp_path) + schema = pa.schema( + [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 4))] + ) + db.create_table( + "t", + schema=schema, + embedding_functions=[ + EmbeddingFunctionConfig( + source_column="text", vector_column="vector", function=func + ) + ], + ) + + restored = db.open_table("t").embedding_functions["vector"].function + assert isinstance(restored, SentenceTransformerEmbeddings) + assert restored.trust_remote_code is True + + +def test_create_trusts_only_the_revision_it_selected(): + # Anyone can push a PR ref to a Hugging Face repo, so trusting a model's + # default branch must not extend to its other revisions. + SentenceTransformerEmbeddings.create(name="org/model", trust_remote_code=True) + func = restore( + "sentence-transformers", + {"name": "org/model", "revision": "refs/pr/123", "trust_remote_code": True}, + ) + assert isinstance(func, UntrustedEmbeddingFunction) + + same = restore( + "sentence-transformers", {"name": "org/model", "trust_remote_code": True} + ) + assert same.trust_remote_code is True + + +def test_explicit_unpinned_approval_covers_any_revision(): + get_registry().allow_remote_code("org/model") + func = restore( + "sentence-transformers", + {"name": "org/model", "revision": "abc123", "trust_remote_code": True}, + ) + assert func.trust_remote_code is True + + +def test_missing_flag_does_not_inherit_unapproved_trust_default(): + class LegacySentenceTransformer(SentenceTransformerEmbeddings): + trust_remote_code: bool = True + + registry = get_registry() + registry.register("legacy-trust-default")(LegacySentenceTransformer) + try: + func = restore("legacy-trust-default", {"name": "attacker/model"}) + assert not func.trust_remote_code + finally: + del registry._functions["legacy-trust-default"] + + +def test_trusting_one_model_does_not_trust_a_swapped_name(): + SentenceTransformerEmbeddings.create(name="org/model", trust_remote_code=True) + func = restore( + "sentence-transformers", {"name": "attacker/model", "trust_remote_code": True} + ) + with pytest.raises(UntrustedEmbeddingConfigError): + func.compute_query_embeddings("hello") + + +ATTACKER_URL = "https://attacker.example/v1" + + +@pytest.mark.parametrize( + "name,model", + [ + ("openai", {"base_url": ATTACKER_URL}), + ("ollama", {"host": ATTACKER_URL}), + ("ollama", {"ollama_client_kwargs": {"proxy": ATTACKER_URL}}), + ("watsonx", {"url": ATTACKER_URL}), + ], +) +def test_restored_custom_endpoint_requires_allowlist(name, model): + func = restore(name, model) + with pytest.raises(UntrustedEmbeddingConfigError, match="attacker.example"): + func.compute_query_embeddings("hello") + + get_registry().allow_endpoint(ATTACKER_URL) + assert isinstance(restore(name, model), get_registry().get(name)) + + +def test_default_endpoints_need_no_allowlist(): + assert isinstance(restore("openai", {}), OpenAIEmbeddings) + assert isinstance( + restore("ollama", {"host": "http://localhost:11434"}), OllamaEmbeddings + ) + + +def test_endpoint_allowlist_ignores_trailing_slash(): + get_registry().allow_endpoint("http://my-host:11434/") + assert isinstance( + restore("ollama", {"host": "http://my-host:11434"}), OllamaEmbeddings + ) + + +def test_create_trusts_its_own_endpoint(): + OpenAIEmbeddings.create(base_url="http://my-proxy/v1") + assert isinstance( + restore("openai", {"base_url": "http://my-proxy/v1"}), OpenAIEmbeddings + ) + + +def test_blocked_function_round_trips_metadata(): + model = {"name": "attacker/model", "trust_remote_code": True} + func = restore("sentence-transformers", model) + conf = EmbeddingFunctionConfig( + source_column="text", vector_column="vector", function=func + ) + entry = get_registry().function_to_metadata(conf) + assert entry["name"] == "sentence-transformers" + assert entry["model"] == model + + +def test_table_with_blocked_function_still_opens(tmp_path): + db = lancedb.connect(tmp_path) + schema = pa.schema( + [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 2))], + metadata=metadata_for( + "sentence-transformers", + {"name": "attacker/model", "trust_remote_code": True}, + ), + ) + db.create_table("t", schema=schema) + tbl = db.open_table("t") + + tbl.add([{"text": "hi", "vector": [1.0, 2.0]}]) + assert tbl.search([1.0, 2.0]).limit(1).to_list()[0]["text"] == "hi" + + with pytest.raises(UntrustedEmbeddingConfigError, match="allow_remote_code"): + tbl.search("hi").limit(1).to_list() From 52eed381bb43a2faf8d8d4135f1c9613465aafa9 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Mon, 19 Oct 2026 12:47:25 -0700 Subject: [PATCH 13/22] ci: add an advisory Vale check for documentation prose (#4581) Checks `docs/web` against the shared LanceDB style rules, so the same mechanical mistakes reach review. **Change.** - Add `.vale.ini`, pinning the shared LanceDB Vale package by release (`vale-lancedb-v0.1.0`), and ignore the synced `.vale/` directory. - Add the Docs prose style workflow. On pull requests that change `docs/web` pages or `.vale.ini`, it annotates only added or changed lines and never fails on a style finding; an install, sync or configuration error still fails it. - Link the style guide from `CONTRIBUTING.md` and show how to check a page locally. Validation: Tested in [temporary PR #4595](https://github.com/lancedb/lancedb/pull/4595). Vale flagged all five deliberate style issues on the changed prose lines, left inline code and unchanged lines alone, and kept the check green. [Passing workflow run](https://github.com/lancedb/lancedb/actions/runs/39299187786/job/118219554825). --- .github/workflows/docs-style.yml | 46 ++++++++++++++++++++++++++++++++ .gitignore | 3 +++ .vale.ini | 5 ++++ CONTRIBUTING.md | 9 +++++++ 4 files changed, 63 insertions(+) create mode 100644 .github/workflows/docs-style.yml create mode 100644 .vale.ini diff --git a/.github/workflows/docs-style.yml b/.github/workflows/docs-style.yml new file mode 100644 index 000000000..f0bac9cd0 --- /dev/null +++ b/.github/workflows/docs-style.yml @@ -0,0 +1,46 @@ +name: Docs prose style + +# Vale checks the mechanical rules in the LanceDB style guide +# (lancedb/docs, STYLE.md) with the package pinned in .vale.ini. Its findings +# are advisory annotations on the lines a pull request adds or changes and +# never fail the job; an install, sync or configuration error still does. +on: + pull_request: + paths: + - "docs/web/**/*.mdx" + - ".vale.ini" + +permissions: + contents: read + pull-requests: read + +env: + VALE_VERSION: 3.24.0 + VALE_SHA256: 867534ddb678abca7f214bc4706ead0922bd5252ffe8035fc227013f66c8b214 + +jobs: + prose: + name: Lint changed prose + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Install Vale + run: | + archive="vale_${VALE_VERSION}_Linux_64-bit.tar.gz" + curl --fail --location --silent --show-error --output "$RUNNER_TEMP/$archive" \ + "https://github.com/vale-cli/vale/releases/download/v${VALE_VERSION}/$archive" + echo "${VALE_SHA256} $RUNNER_TEMP/$archive" | sha256sum --check + tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" vale + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + + - name: Annotate changed lines + uses: vale-cli/vale-action@518a9136acc6e6668ce7c00d367051e0941e87ff # v3.0.0 + with: + version: none + files: docs/web + reporter: github-pr-annotations + filter_mode: added + fail_on_error: false diff --git a/.gitignore b/.gitignore index 0679f3c8a..61f9edb9b 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ dist target **/sccache.log + +# Vale styles installed by `vale sync` +/.vale/ diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 000000000..13b421863 --- /dev/null +++ b/.vale.ini @@ -0,0 +1,5 @@ +# Prose style for the pages in docs/web: the shared LanceDB package, pinned by +# release. The rules and guide live in lancedb/docs (STYLE.md and vale/). +# Run `vale sync` once, then `vale docs/web/.mdx`. +StylesPath = .vale/styles +Packages = https://github.com/lancedb/docs/releases/download/vale-lancedb-v0.1.0/LanceDB.zip diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0aabd8fec..f7aeac21a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,6 +88,15 @@ npm i -g mint # https://mintlify.com/docs/installation cd docs/web && mint dev ``` +Wording, names and punctuation follow the +[LanceDB style guide](https://github.com/lancedb/docs/blob/main/STYLE.md). +[Vale](https://vale.sh) checks part of it, and pull requests show its +suggestions on the lines they change. To check a page yourself: + +```bash +vale sync && vale docs/web/indexing/vector-index.mdx +``` + This repository holds a page at every path the site publishes, so the whole navigation resolves locally. Enterprise pages exist here as open-source pages -- what the capability is, and what the embedded form does instead. A separate From 1ad97b148be0dd08b889eec70ecb2c72a2ece9f2 Mon Sep 17 00:00:00 2001 From: Tobias <5562156+tobocop2@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:35:07 -0400 Subject: [PATCH 14/22] build(python): publish an sdist alongside the wheels (#4522) Closes #4246. lancedb publishes no source distribution, so a platform without a prebuilt wheel (#4104, #4239) has nothing to install. This adds an sdist to the Python release. ## The sdist job The release workflow builds an sdist with the maturin version of the wheel jobs. On a tag, the publish job uploads it with the wheels: to PyPI, and to Fury for beta tags. The job also runs on the pull request dry run, which now starts on a change to `python/pyproject.toml`. ## Checks - The job builds a wheel from the sdist with `--locked` and smoke-tests it in a fresh venv. - `ci/check_sdist.py` fails if a tracked file of the Python package is missing from the tarball. - `twine check --strict` checks that the metadata renders. `pydistcheck` enforces PyPI's 100 MiB size limit. - The unit tests of both check scripts run in the CI scripts workflow. ## Cargo.lock maturin ships the repository lockfile, which pins crates that only the dropped `nodejs` member uses, so `cargo --locked` fails (PyO3/maturin#2609). The job first runs `cargo metadata --locked` on the repository workspace, so a lockfile that has drifted from `Cargo.toml` fails there. It then removes the nodejs-only pins and repacks the sdist. `ci/check_cargo_lock.py` fails if the prune changes any package it keeps, or any dependency of one. The published file is the repacked one. ## Behavior change After the first release with an sdist, pip on a platform with no current wheel builds the newest version from source, which needs a Rust toolchain and `protoc`. Before, pip installed the last version with a wheel for that platform. --- .github/workflows/ci-scripts.yml | 4 ++ .github/workflows/pypi-publish.yml | 108 +++++++++++++++++++++++++++-- ci/check_cargo_lock.py | 56 +++++++++++++++ ci/check_sdist.py | 42 +++++++++++ ci/tests/test_check_cargo_lock.py | 97 ++++++++++++++++++++++++++ ci/tests/test_check_sdist.py | 71 +++++++++++++++++++ python/README.md | 2 + 7 files changed, 373 insertions(+), 7 deletions(-) create mode 100644 ci/check_cargo_lock.py create mode 100644 ci/check_sdist.py create mode 100644 ci/tests/test_check_cargo_lock.py create mode 100644 ci/tests/test_check_sdist.py diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml index 9b9124ab2..f7f0dbb07 100644 --- a/.github/workflows/ci-scripts.yml +++ b/.github/workflows/ci-scripts.yml @@ -5,11 +5,15 @@ on: branches: - main paths: + - ci/check_cargo_lock.py + - ci/check_sdist.py - ci/set_lance_version.py - ci/tests/** - .github/workflows/ci-scripts.yml pull_request: paths: + - ci/check_cargo_lock.py + - ci/check_sdist.py - ci/set_lance_version.py - ci/tests/** - .github/workflows/ci-scripts.yml diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index d16203979..e71564473 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -12,6 +12,9 @@ on: - .github/workflows/build_mac_wheel/action.yml - .github/workflows/build_windows_wheel/action.yml - .github/actions/setup-protoc/action.yml + - ci/check_cargo_lock.py + - ci/check_sdist.py + - python/pyproject.toml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock @@ -161,10 +164,91 @@ jobs: name: wheels-windows path: target/wheels/lancedb-*.whl if-no-files-found: error + sdist: + name: Python sdist + timeout-minutes: 60 + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + lfs: true + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + maturin-version: "1.12.4" + command: sdist + working-directory: python + args: --out dist + # maturin ships the nodejs-only pins in Cargo.lock. Remove this step when + # the pinned maturin fixes https://github.com/PyO3/maturin/issues/2609. + - name: Sync Cargo.lock to the packaged workspace + shell: bash + run: | + # A repo lock in sync with every manifest means the prune only drops pins. + cargo metadata --locked --format-version 1 > /dev/null + mkdir "$RUNNER_TEMP/sdist-src" + tar -xzf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src" + lock=$(echo "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.lock) + cp "$lock" "$RUNNER_TEMP/Cargo.lock.before" + cargo metadata --format-version 1 \ + --manifest-path "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.toml > /dev/null + python ci/check_cargo_lock.py "$RUNNER_TEMP/Cargo.lock.before" "$lock" + tar -czf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src" \ + "$(basename python/dist/lancedb-*.tar.gz .tar.gz)" + - name: Install sdist check tools + shell: bash + run: python -m pip install twine==7.0.0 pydistcheck==0.11.3 + - name: Check sdist size + shell: bash + run: | + pydistcheck --select distro-too-large-compressed \ + --max-allowed-size-compressed 100M python/dist/lancedb-*.tar.gz + - name: Check sdist contents + shell: bash + run: python ci/check_sdist.py python/dist/lancedb-*.tar.gz + - name: Check sdist with twine + shell: bash + run: python -m twine check --strict python/dist/lancedb-*.tar.gz + - uses: ./.github/actions/setup-protoc + # Build the way `pip install lancedb` does when no wheel matches. The dev + # profile keeps the build within the runner's disk. + - name: Build wheel from sdist + shell: bash + env: + MATURIN_PEP517_ARGS: "--profile dev --locked" + CARGO_PROFILE_DEV_DEBUG: "0" + run: | + python -m pip wheel --no-deps --wheel-dir "$RUNNER_TEMP/sdist-wheel" \ + python/dist/lancedb-*.tar.gz + - name: Smoke test wheel built from sdist + shell: bash + working-directory: ${{ runner.temp }} + run: | + python -m venv sdist-venv + sdist-venv/bin/pip install sdist-wheel/lancedb-*.whl + sdist-venv/bin/python -c ' + import tempfile + import lancedb + db = lancedb.connect(tempfile.mkdtemp()) + table = db.create_table("t", [{"vector": [1.0, 2.0], "id": 1}, {"vector": [3.0, 4.0], "id": 2}]) + assert table.search([1.0, 2.0]).limit(1).to_list()[0]["id"] == 1 + print("lancedb", lancedb.__version__, "built from sdist OK") + ' + - uses: actions/upload-artifact@v7 + if: startsWith(github.ref, 'refs/tags/v') + with: + name: sdist + path: python/dist/lancedb-*.tar.gz + if-no-files-found: error publish: - name: Publish wheels + name: Publish wheels and sdist if: startsWith(github.ref, 'refs/tags/v') - needs: [linux, mac, windows] + needs: [linux, mac, windows, sdist] runs-on: ubuntu-latest permissions: id-token: write @@ -177,7 +261,12 @@ jobs: pattern: wheels-* path: target/wheels merge-multiple: true - - name: List wheels + - name: Download sdist artifact + uses: actions/download-artifact@v8 + with: + name: sdist + path: target/wheels + - name: List packages run: ls -la target/wheels - name: Choose repo id: choose_repo @@ -194,13 +283,18 @@ jobs: run: | shopt -s nullglob WHEELS=(target/wheels/*.whl) + SDISTS=(target/wheels/*.tar.gz) if [[ ${#WHEELS[@]} -eq 0 ]]; then echo "No wheels found in target/wheels/" >&2 exit 1 fi - for WHEEL in "${WHEELS[@]}"; do - echo "Uploading $WHEEL to Fury" - curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/" + if [[ ${#SDISTS[@]} -ne 1 ]]; then + echo "Expected one sdist in target/wheels/, found ${#SDISTS[@]}" >&2 + exit 1 + fi + for PACKAGE in "${WHEELS[@]}" "${SDISTS[@]}"; do + echo "Uploading $PACKAGE to Fury" + curl -f -F package=@"$PACKAGE" "https://$FURY_TOKEN@push.fury.io/lancedb/" done # NOTE: pypa/gh-action-pypi-publish must be invoked directly from a # workflow file, not from inside a composite action. When called from a @@ -215,7 +309,7 @@ jobs: report-failure: name: Report Workflow Failure runs-on: ubuntu-latest - needs: [linux, mac, windows, publish] + needs: [linux, mac, windows, sdist, publish] permissions: contents: read issues: write diff --git a/ci/check_cargo_lock.py b/ci/check_cargo_lock.py new file mode 100644 index 000000000..551b76794 --- /dev/null +++ b/ci/check_cargo_lock.py @@ -0,0 +1,56 @@ +"""Fail if pruning Cargo.lock did more than remove packages and the edges to them. + +Usage: python ci/check_cargo_lock.py +""" + +import sys +import tomllib +from pathlib import Path + + +def key(package: dict) -> tuple: + return tuple(package.get(k) for k in ("name", "version", "source", "checksum")) + + +def graph(lock: dict) -> dict[tuple, set[tuple]]: + """Map each package to the packages its dependency strings resolve to.""" + by_name = {} + for p in lock["package"]: + by_name.setdefault(p["name"], []).append(p) + + def resolve(dep: str) -> tuple: + # Cargo writes "name", "name version" or "name version (source)". + name, *rest = dep.split(" ", 2) + (match,) = [ + p + for p in by_name[name] + if (not rest or p["version"] == rest[0]) + and (len(rest) < 2 or rest[1] == f"({p.get('source')})") + ] + return key(match) + + return { + key(p): {resolve(d) for d in p.get("dependencies", [])} for p in lock["package"] + } + + +def main() -> None: + before, after = (tomllib.loads(Path(p).read_text()) for p in sys.argv[1:]) + errors = [] + for field in (before.keys() | after.keys()) - {"package"}: + if before.get(field) != after.get(field): + errors.append(f"top-level {field!r} changed") + edges_before, edges_after = graph(before), graph(after) + removed = edges_before.keys() - edges_after.keys() + for package, deps in edges_after.items(): + if package not in edges_before: + errors.append(f"added or changed {package}") + elif deps != edges_before[package] - removed: + errors.append(f"dependencies of {package[:2]} changed") + if errors: + sys.exit("error: the prune changed Cargo.lock:\n" + "\n".join(errors)) + print(f"Cargo.lock prune only removed {len(removed)} packages") + + +if __name__ == "__main__": + main() diff --git a/ci/check_sdist.py b/ci/check_sdist.py new file mode 100644 index 000000000..6c0c88931 --- /dev/null +++ b/ci/check_sdist.py @@ -0,0 +1,42 @@ +"""Fail if the lancedb sdist lacks a git-tracked file of the Python package. + +Usage: python ci/check_sdist.py (run from the repository root) +""" + +import subprocess +import sys +import tarfile +from pathlib import Path + +# Where git tracks the package, and where maturin puts it under the sdist root. +TRACKED_PACKAGE = "python/python/lancedb" +SDIST_PACKAGE = "python/lancedb" + + +def main() -> None: + sdist = Path(sys.argv[1]) + root = sdist.name.removesuffix(".tar.gz") + listing = subprocess.run( + ["git", "ls-files", "-z", "--", TRACKED_PACKAGE], + check=True, + capture_output=True, + text=True, + ).stdout + tracked = [p for p in listing.split("\0") if p] + if not tracked: + sys.exit(f"error: git tracks no files under {TRACKED_PACKAGE}") + + with tarfile.open(sdist) as tar: + members = {m.name for m in tar if m.isfile()} + missing = [ + p + for p in tracked + if f"{root}/{SDIST_PACKAGE}{p.removeprefix(TRACKED_PACKAGE)}" not in members + ] + if missing: + sys.exit(f"error: {sdist.name} lacks tracked sources: {', '.join(missing)}") + print(f"{sdist.name} OK: all {len(tracked)} tracked sources present") + + +if __name__ == "__main__": + main() diff --git a/ci/tests/test_check_cargo_lock.py b/ci/tests/test_check_cargo_lock.py new file mode 100644 index 000000000..82570d087 --- /dev/null +++ b/ci/tests/test_check_cargo_lock.py @@ -0,0 +1,97 @@ +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "ci" / "check_cargo_lock.py" +CRATES_IO = "registry+https://github.com/rust-lang/crates.io-index" + + +def package(name, version, deps=(), source=CRATES_IO, checksum="aa"): + text = f'[[package]]\nname = "{name}"\nversion = "{version}"\n' + if source: + text += f'source = "{source}"\nchecksum = "{checksum}"\n' + if deps: + text += f"dependencies = {list(deps)!r}\n".replace("'", '"') + return text + + +LIBC = package("libc", "0.2.1") +UNTRUSTED = package("untrusted", "0.9.0") +# The prune drops napi and untrusted 0.7.1, the edge from app to it, and the +# version qualifier on app's remaining untrusted edge. +BEFORE = "version = 4\n\n" + "".join( + [ + package( + "app", "1.0.0", ["libc", "untrusted 0.7.1", "untrusted 0.9.0"], source=None + ), + package("napi", "3.0.0", ["untrusted 0.7.1"]), + LIBC, + package("untrusted", "0.7.1"), + UNTRUSTED, + ] +) +APP = package("app", "1.0.0", ["libc", "untrusted"], source=None) + + +class CheckCargoLockTest(unittest.TestCase): + def run_check(self, after, header="version = 4\n\n"): + with tempfile.TemporaryDirectory() as tmp: + before_path = Path(tmp, "before.lock") + after_path = Path(tmp, "after.lock") + before_path.write_text(BEFORE) + after_path.write_text(header + after) + return subprocess.run( + [sys.executable, SCRIPT, before_path, after_path], + capture_output=True, + text=True, + ) + + def test_prune_passes(self): + result = self.run_check(APP + LIBC + UNTRUSTED) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_added_entry_fails(self): + result = self.run_check(APP + LIBC + UNTRUSTED + package("ring", "0.17.0")) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ring", result.stderr) + + def test_changed_version_fails(self): + result = self.run_check(APP + package("libc", "0.2.2") + UNTRUSTED) + self.assertNotEqual(result.returncode, 0) + self.assertIn("0.2.2", result.stderr) + + def test_changed_source_fails(self): + libc = package("libc", "0.2.1", source="git+https://example.com/libc") + result = self.run_check(APP + libc + UNTRUSTED) + self.assertNotEqual(result.returncode, 0) + self.assertIn("example.com", result.stderr) + + def test_changed_checksum_fails(self): + libc = package("libc", "0.2.1", checksum="bb") + result = self.run_check(APP + libc + UNTRUSTED) + self.assertNotEqual(result.returncode, 0) + self.assertIn("'bb'", result.stderr) + + def test_added_dependency_fails(self): + libc = package("libc", "0.2.1", ["untrusted"]) + result = self.run_check(APP + libc + UNTRUSTED) + self.assertNotEqual(result.returncode, 0) + self.assertIn("dependencies of ('libc'", result.stderr) + + def test_removed_dependency_fails(self): + app = package("app", "1.0.0", ["untrusted"], source=None) + result = self.run_check(app + LIBC + UNTRUSTED) + self.assertNotEqual(result.returncode, 0) + self.assertIn("dependencies of ('app'", result.stderr) + + def test_changed_lock_version_fails(self): + result = self.run_check(APP + LIBC + UNTRUSTED, header="version = 3\n\n") + self.assertNotEqual(result.returncode, 0) + self.assertIn("top-level 'version'", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_check_sdist.py b/ci/tests/test_check_sdist.py new file mode 100644 index 000000000..1bb34306e --- /dev/null +++ b/ci/tests/test_check_sdist.py @@ -0,0 +1,71 @@ +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "ci" / "check_sdist.py" +ROOT = "lancedb-1.0.0" + + +class CheckSdistTest(unittest.TestCase): + def run_check(self, tracked, shipped, links=(), root=ROOT): + """Run the check in a git repo tracking `tracked`, on an sdist holding `shipped` + as files and `links` as symlinks, under the folder `root`.""" + with tempfile.TemporaryDirectory() as tmp: + subprocess.run(["git", "init", "-q", tmp], check=True) + for name in tracked: + path = Path(tmp, "python/python/lancedb", name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("x = 1\n") + if tracked: + subprocess.run(["git", "-C", tmp, "add", "."], check=True) + sdist = Path(tmp, f"{ROOT}.tar.gz") + with tarfile.open(sdist, "w:gz") as tar: + for name in shipped: + member = Path(tmp, "member") + member.write_text("x = 1\n") + tar.add(member, arcname=f"{root}/python/lancedb/{name}") + for name in links: + link = tarfile.TarInfo(f"{root}/python/lancedb/{name}") + link.type = tarfile.SYMTYPE + link.linkname = "__init__.py" + tar.addfile(link) + return subprocess.run( + [sys.executable, SCRIPT, sdist], + cwd=tmp, + capture_output=True, + text=True, + ) + + def test_complete_sdist_passes(self): + result = self.run_check(["__init__.py", "db.py"], ["__init__.py", "db.py"]) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_missing_tracked_file_fails(self): + result = self.run_check(["__init__.py", "db.py"], ["__init__.py"]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("python/python/lancedb/db.py", result.stderr) + + def test_tracked_file_shipped_as_symlink_fails(self): + result = self.run_check( + ["__init__.py", "db.py"], ["__init__.py"], links=["db.py"] + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("python/python/lancedb/db.py", result.stderr) + + def test_files_outside_the_sdist_root_fail(self): + result = self.run_check(["__init__.py"], ["__init__.py"], root="other-1.0.0") + self.assertNotEqual(result.returncode, 0) + self.assertIn("python/python/lancedb/__init__.py", result.stderr) + + def test_no_tracked_files_fails(self): + result = self.run_check([], ["__init__.py"]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("git tracks no files", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/README.md b/python/README.md index 3a81c486a..f941c9e20 100644 --- a/python/README.md +++ b/python/README.md @@ -8,6 +8,8 @@ A Python library for [LanceDB](https://github.com/lancedb/lancedb). pip install lancedb ``` +If no wheel is published for your platform, pip builds `lancedb` from the source distribution. That build needs a [Rust toolchain](https://rustup.rs/) and [protoc](https://grpc.io/docs/protoc-installation/) (the Protocol Buffers compiler) on your `PATH`, and can take a while. + ### Pre-Haswell x86_64 hosts: `lancedb-compat` The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C) for full performance on modern hardware. Pre-Haswell hosts — Intel Sandy Bridge / Ivy Bridge / Westmere; AMD Bulldozer / Piledriver / Steamroller — don't have AVX2 and crash with `Illegal instruction` at `import lancedb`. From 198812fd53e11cdc3035c1dc2ce008a7cb61feee Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Mon, 19 Oct 2026 13:45:55 -0700 Subject: [PATCH 15/22] docs: clean up prose flagged by the shared Vale rules (#4596) The existing docs have 254 findings from our shared Vale rules. Contributors would otherwise inherit these when editing the same lines. Clean up the prose across 80 pages: correct product names and a repeated word, simplify a few phrases, and replace dashes with punctuation that fits each sentence. The pinned Vale scan now reports zero findings, with no suppressions. --- docs/web/api-reference/rest/index.mdx | 2 +- docs/web/basics/agentic-skills.mdx | 6 +-- docs/web/deployment/monitoring.mdx | 2 +- docs/web/deployment/storage.mdx | 4 +- docs/web/embedding/index.mdx | 2 +- docs/web/enterprise/architecture.mdx | 2 +- docs/web/enterprise/authentication.mdx | 10 ++--- docs/web/enterprise/benchmarks.mdx | 6 +-- docs/web/enterprise/deployment/azure.mdx | 2 +- docs/web/enterprise/deployment/index.mdx | 4 +- docs/web/enterprise/index.mdx | 2 +- docs/web/enterprise/security.mdx | 6 +-- .../add-computed-column.mdx | 8 ++-- .../feature-engineering/data-evolution.mdx | 2 +- docs/web/feature-engineering/index.mdx | 8 ++-- .../feature-engineering/refresh-column.mdx | 10 ++--- docs/web/governance/authorization.mdx | 12 +++--- docs/web/indexing/reindexing.mdx | 2 +- docs/web/indexing/vector-index.mdx | 6 +-- docs/web/integrations/ai/genkit.mdx | 4 +- docs/web/integrations/ai/hermes-agent.mdx | 40 +++++++++---------- docs/web/integrations/ai/kiln.mdx | 2 +- docs/web/integrations/ai/langchain.mdx | 8 ++-- docs/web/integrations/data/dlt.mdx | 2 +- docs/web/integrations/data/voxel51.mdx | 2 +- .../integrations/embedding/huggingface.mdx | 2 +- .../embedding/sentence-transformers.mdx | 4 +- docs/web/integrations/embedding/voyageai.mdx | 8 ++-- .../integrations/reranking/answerdotai.mdx | 2 +- docs/web/integrations/reranking/cohere.mdx | 2 +- docs/web/integrations/reranking/colbert.mdx | 2 +- docs/web/integrations/reranking/jina.mdx | 2 +- docs/web/integrations/reranking/openai.mdx | 2 +- docs/web/integrations/reranking/voyageai.mdx | 10 ++--- docs/web/integrations/reranking/watsonx.mdx | 2 +- docs/web/maintenance/background-jobs.mdx | 10 ++--- docs/web/maintenance/cleanup.mdx | 8 ++-- docs/web/maintenance/compaction.mdx | 6 +-- docs/web/management/function.mdx | 10 ++--- docs/web/management/index.mdx | 2 +- docs/web/management/materialized-view.mdx | 4 +- docs/web/management/namespace.mdx | 2 +- docs/web/management/table.mdx | 28 ++++++------- docs/web/search/full-text-search.mdx | 4 +- docs/web/sql/data-types.mdx | 4 +- docs/web/sql/ddl/function.mdx | 8 ++-- docs/web/sql/ddl/index.mdx | 2 +- docs/web/sql/ddl/job.mdx | 10 ++--- docs/web/sql/ddl/materialized-view.mdx | 6 +-- docs/web/sql/ddl/namespace.mdx | 8 ++-- docs/web/sql/ddl/table.mdx | 16 ++++---- docs/web/sql/ddl/view.mdx | 14 +++---- docs/web/sql/dml/copy.mdx | 4 +- docs/web/sql/dml/delete.mdx | 4 +- docs/web/sql/dml/insert.mdx | 4 +- docs/web/sql/dml/refresh.mdx | 2 +- docs/web/sql/dml/update.mdx | 6 +-- docs/web/sql/dql/aggregation.mdx | 2 +- docs/web/sql/dql/explain.mdx | 6 +-- docs/web/sql/dql/fts.mdx | 14 +++---- docs/web/sql/dql/join.mdx | 6 +-- docs/web/sql/dql/sampling.mdx | 6 +-- docs/web/sql/dql/select.mdx | 4 +- docs/web/sql/dql/window.mdx | 2 +- docs/web/sql/functions.mdx | 10 ++--- docs/web/sql/index.mdx | 8 ++-- docs/web/sql/operators.mdx | 12 +++--- docs/web/sql/scalar-functions.mdx | 2 +- docs/web/tables/branching.mdx | 24 +++++------ docs/web/training/data-loading.mdx | 2 +- docs/web/training/object-detection.mdx | 30 +++++++------- docs/web/tutorials/agents/agentic-rag.mdx | 4 +- docs/web/tutorials/agents/contextual-rag.mdx | 2 +- docs/web/tutorials/agents/hyde.mdx | 4 +- .../agents/matryoshka-embeddings.mdx | 2 +- .../agents/nvidia-rag-blueprint/index.mdx | 2 +- .../agents/time-travel-rag/index.mdx | 2 +- .../multivector-needle-in-a-haystack.mdx | 2 +- docs/web/tutorials/search/ner-search.mdx | 4 +- docs/web/tutorials/search/video-search.mdx | 2 +- 80 files changed, 252 insertions(+), 252 deletions(-) diff --git a/docs/web/api-reference/rest/index.mdx b/docs/web/api-reference/rest/index.mdx index b2dd10016..d9bb26867 100644 --- a/docs/web/api-reference/rest/index.mdx +++ b/docs/web/api-reference/rest/index.mdx @@ -82,7 +82,7 @@ curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \ -d '{}' ``` -That's it -- you're connected! Now, you can start adding data and querying it. +That's it. You're connected! Now, you can start adding data and querying it. You can visit the tutorial section to build your own applications with LanceDB. Enterprise A deployment emits its own metrics from the server side, using Prometheus for collection and OpenTelemetry (OTel) to -export them with data enrichment -- no client-side instrumentation needed. +export them with data enrichment. No client-side instrumentation is needed. | Component | Metrics | | --- | --- | diff --git a/docs/web/deployment/storage.mdx b/docs/web/deployment/storage.mdx index 4994e8f0c..35b6dd80f 100644 --- a/docs/web/deployment/storage.mdx +++ b/docs/web/deployment/storage.mdx @@ -16,7 +16,7 @@ availability: This page is about the embedded library, where your connection chooses and configures the storage backend. A LanceDB Enterprise deployment sets up its own storage when it is deployed, so a `db://` connection passes none of these -options except an Azure storage account name -- see +options except an Azure storage account name. See [Deployment](/enterprise/deployment) for how a deployment is set up. LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs). @@ -49,7 +49,7 @@ LanceDB separates storage and compute and writes immutable fragments, making it **Concurrent writers on S3** -S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box — no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier. +S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box: no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier. ### 2\. File storage (EFS, GCS Filestore, Azure File) {#2-file-storage-efs-gcs-filestore-azure-file} diff --git a/docs/web/embedding/index.mdx b/docs/web/embedding/index.mdx index ee893f661..4402feddf 100644 --- a/docs/web/embedding/index.mdx +++ b/docs/web/embedding/index.mdx @@ -1,7 +1,7 @@ --- title: "Managing Embeddings" sidebarTitle: "Managing Embeddings" -description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support." +description: "Use the embedding API in LanceDB: registry, functions, schemas, and multi-language SDK support." icon: "bars" availability: oss: available diff --git a/docs/web/enterprise/architecture.mdx b/docs/web/enterprise/architecture.mdx index fc1c1c7db..c47dcbd2e 100644 --- a/docs/web/enterprise/architecture.mdx +++ b/docs/web/enterprise/architecture.mdx @@ -9,7 +9,7 @@ keywords: ["architecture", "control plane", "data plane", "indexer"] Enterprise This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see +of LanceDB has no server to provide it. See [Architecture](/architecture) for what differs between the two. A deployment separates compute from storage so each scales on its own: table diff --git a/docs/web/enterprise/authentication.mdx b/docs/web/enterprise/authentication.mdx index 4d04120fc..3a570d1d9 100644 --- a/docs/web/enterprise/authentication.mdx +++ b/docs/web/enterprise/authentication.mdx @@ -17,8 +17,8 @@ A deployment authenticates every request before [authorization](/governance/authorization) decides what it may do, against a `db://` remote table, in one of two ways: -- **API keys** -- a long-lived shared secret passed as `api_key` on connect. Works in every SDK. -- **OAuth 2.0** -- short-lived bearer tokens obtained from your identity provider, refreshed automatically by the client. +- **API keys**: a long-lived shared secret passed as `api_key` on connect. Works in every SDK. +- **OAuth 2.0**: short-lived bearer tokens obtained from your identity provider, refreshed automatically by the client. OAuth is the recommended option when you want to rotate credentials centrally, plug into an existing identity provider, or run on Azure with managed @@ -48,7 +48,7 @@ An API key belongs in the environment rather than in source. See The async Python client and the TypeScript client can obtain bearer tokens from an OIDC issuer and attach them to every request. Token acquisition, -caching, and refresh are handled inside the client -- your application code +caching, and refresh are handled inside the client. Your application code only provides the configuration. A deployment accepts these tokens only when its administrator has configured @@ -162,7 +162,7 @@ field names; TypeScript uses `camelCase`. | `flow` | No | Selects the OAuth flow. Defaults to Client Credentials. | | `client_secret` / `clientSecret` | Conditional | Required for Client Credentials. Redacted from the config's `repr` (Python) and `Debug` output (TypeScript native binding) so it does not leak into logs. | | `managed_identity_client_id` / `managedIdentityClientId` | No | Client ID of a user-assigned managed identity. Only used with Azure Managed Identity; omit for system-assigned identities. | -| `refresh_buffer_secs` / `refreshBufferSecs` | No | How many seconds before token expiry to proactively refresh. Defaults to 300. Keep this well below the token TTL -- setting it greater than or equal to the TTL forces a refresh on every request. | +| `refresh_buffer_secs` / `refreshBufferSecs` | No | How many seconds before token expiry to proactively refresh. Defaults to 300. Keep this well below the token TTL: setting it greater than or equal to the TTL forces a refresh on every request. | Treat the client secret like any other production credential. Load it from a @@ -171,5 +171,5 @@ control. Both SDKs deliberately omit the secret from their debug/repr output so accidental log lines do not expose the value. -Embedded LanceDB has nothing to authenticate to -- your process opens the files +Embedded LanceDB has nothing to authenticate to: your process opens the files directly, and the object store's own credentials are the only gate. diff --git a/docs/web/enterprise/benchmarks.mdx b/docs/web/enterprise/benchmarks.mdx index 3626f7b81..feec4dfc4 100644 --- a/docs/web/enterprise/benchmarks.mdx +++ b/docs/web/enterprise/benchmarks.mdx @@ -9,11 +9,11 @@ keywords: ["benchmarks", "performance", "throughput", "latency"] Enterprise This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see +of LanceDB has no server to provide it. See [Architecture](/architecture) for what differs between the two. -Benchmark results are specific to a deployment's shape -- cluster size, storage -class, dataset -- so they are published with LanceDB Enterprise. +Benchmark results are specific to a deployment's shape (cluster size, storage +class, dataset), so they are published with LanceDB Enterprise. For tuning your own workload rather than comparing published numbers, see [Query plans](/search/explain) and the performance notes on diff --git a/docs/web/enterprise/deployment/azure.mdx b/docs/web/enterprise/deployment/azure.mdx index 349393249..702c6a2eb 100644 --- a/docs/web/enterprise/deployment/azure.mdx +++ b/docs/web/enterprise/deployment/azure.mdx @@ -9,7 +9,7 @@ keywords: ["azure", "deployment", "aks"] Enterprise This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see +of LanceDB has no server to provide it. See [Architecture](/architecture) for what differs between the two. The Azure deployment procedure is published with LanceDB Enterprise. See diff --git a/docs/web/enterprise/deployment/index.mdx b/docs/web/enterprise/deployment/index.mdx index c718cc32b..efaa2c786 100644 --- a/docs/web/enterprise/deployment/index.mdx +++ b/docs/web/enterprise/deployment/index.mdx @@ -9,12 +9,12 @@ keywords: ["deployment", "install", "cluster", "kubernetes"] Enterprise This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see +of LanceDB has no server to provide it. See [Architecture](/architecture) for what differs between the two. Deployment is specific to your cloud and account, so the procedure is published with LanceDB Enterprise. [Contact us](https://www.lancedb.com/contact) to get started. -Embedded LanceDB has nothing to deploy -- it is a library your process links +Embedded LanceDB has nothing to deploy: it is a library your process links against. See [Architecture](/architecture#embedded). diff --git a/docs/web/enterprise/index.mdx b/docs/web/enterprise/index.mdx index a601e850d..acf41f6d1 100644 --- a/docs/web/enterprise/index.mdx +++ b/docs/web/enterprise/index.mdx @@ -9,7 +9,7 @@ keywords: ["enterprise", "deployment", "cluster"] Enterprise This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see +of LanceDB has no server to provide it. See [Architecture](/architecture) for what differs between the two. LanceDB Enterprise runs the lakehouse as a service: a control plane for diff --git a/docs/web/enterprise/security.mdx b/docs/web/enterprise/security.mdx index a2b60a242..f462203d4 100644 --- a/docs/web/enterprise/security.mdx +++ b/docs/web/enterprise/security.mdx @@ -9,11 +9,11 @@ keywords: ["compliance", "security", "soc2", "encryption", "audit"] Enterprise This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see +of LanceDB has no server to provide it. See [Architecture](/architecture) for what differs between the two. -Compliance posture -- certifications, data residency, encryption at rest and in -transit, audit logging -- is a property of a deployment, so it is documented +Compliance posture (certifications, data residency, encryption at rest and in +transit, audit logging) is a property of a deployment, so it is documented with LanceDB Enterprise. For embedded LanceDB, these are properties of the object store you point it at diff --git a/docs/web/feature-engineering/add-computed-column.mdx b/docs/web/feature-engineering/add-computed-column.mdx index 331918715..c287d42f5 100644 --- a/docs/web/feature-engineering/add-computed-column.mdx +++ b/docs/web/feature-engineering/add-computed-column.mdx @@ -28,7 +28,7 @@ table.add_columns(computed={"doubled": "x * 2"}) ``` That commits a new table version with the column present and empty. It is an -ordinary table version, so the usual [versioning](/management/table#versioning) applies -- +ordinary table version, so the usual [versioning](/management/table#versioning) applies, including that you can travel back to before the column existed. ## Compute now or on refresh {#computed-vs-transforms} @@ -61,7 +61,7 @@ table.add_columns(computed={ }) ``` -Declaration order is dependency order -- a computed column may reference one +Declaration order is dependency order: a computed column may reference one declared earlier in the same mapping, but not one declared after it. The functions available are the ones the @@ -92,8 +92,8 @@ reshape later. Dropping the computed column releases the constraint. Enterprise -When the value cannot be expressed in SQL -- an embedding, a model call, an -image decode -- [Geneva](/geneva), the feature engineering package for LanceDB +When the value cannot be expressed in SQL (an embedding, a model call, an +image decode), [Geneva](/geneva), the feature engineering package for LanceDB Enterprise, fills the column with a Python function instead. {/* diff --git a/docs/web/feature-engineering/data-evolution.mdx b/docs/web/feature-engineering/data-evolution.mdx index f70612cf9..37e396b8b 100644 --- a/docs/web/feature-engineering/data-evolution.mdx +++ b/docs/web/feature-engineering/data-evolution.mdx @@ -111,7 +111,7 @@ The expressions above are evaluated once, as part of the call. For other workflo expression and leaves the column empty, so you choose when the work happens. Use it when you want to schedule the computation separately. - On LanceDB Enterprise, [Geneva](/geneva) fills a column with Python, for - values SQL cannot express -- an embedding, a model call, an image decode. + values SQL cannot express: an embedding, a model call, an image decode. {/* Draft for the Functions release. Replaces the Geneva item in the list above: diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 7fe699648..8f0253444 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -24,7 +24,7 @@ then refresh the derived data when its inputs change. */} -In LanceDB Enterprise, every operation here also has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. +In LanceDB Enterprise, every operation here also has a SQL form. See [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. ## Computed columns and materialized views {#the-shape} @@ -91,8 +91,8 @@ Draft for the Functions release. Replace the Geneva row of the table above with: **Computed columns** are the common case: a column declared with a SQL expression, filled by [a refresh](/feature-engineering/refresh-column). -**Materialized views** derive a whole table -- a filtered, projected, or -aggregated view of a source -- and bring it up to date on each refresh. The +**Materialized views** derive a whole table (a filtered, projected, or +aggregated view of a source) and bring it up to date on each refresh. The result is an ordinary table: queryable, indexable, searchable. **Geneva** is LanceDB Enterprise's Python package for work SQL cannot express: @@ -128,7 +128,7 @@ result.rows_assigned ``` A backfill over a large table can therefore be started, left, and checked on -from a different process -- the connection does not have to stay open. +from a different process: the connection does not have to stay open. Enterprise manages the compute for the refresh. You still control when to submit it; declaring a computed column does not schedule recurring refreshes. diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index ae430e5f8..f4550ed24 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -31,8 +31,8 @@ REFRESH COLUMN doubled ON my_table ``` -The refresh commits ordinary table versions -- the values, then a -metadata-only record of the inputs they were computed from -- so +The refresh commits ordinary table versions (the values, then a +metadata-only record of the inputs they were computed from), so [versioning](/management/table#versioning) applies to it like any other write. @@ -53,7 +53,7 @@ column and then ingesting more data does not mean recomputing the backfill. derives from `x` and a row's `x` is later updated, `doubled` keeps its old value until the next refresh recomputes it. -That is what makes the call idempotent and safe to retry -- but unlike a +That is what makes the call idempotent and safe to retry. However, unlike a spreadsheet formula, nothing is recomputed until you refresh, so a derived column is stale between an update and the next refresh. @@ -122,8 +122,8 @@ values can remain after a failure. Either way, recover by fixing the cause and refreshing again. The new refresh fills rows that are still empty or whose inputs changed, and normally skips values that were kept. That is not guaranteed: a refresh can recompute values -that were already written -- for example when the record of the inputs they -were computed from was not saved -- so do not count on a retry costing only the +that were already written (for example when the record of the inputs they +were computed from was not saved), so do not count on a retry costing only the rows that did not complete. {/* diff --git a/docs/web/governance/authorization.mdx b/docs/web/governance/authorization.mdx index 9ef978c38..dc5b6fd26 100644 --- a/docs/web/governance/authorization.mdx +++ b/docs/web/governance/authorization.mdx @@ -23,8 +23,8 @@ A grant is held by one of four kinds of subject, distinguished by a prefix: | Prefix | Subject | | --- | --- | -| `p:` | a principal, by ID -- `P:` by name | -| `g:` | a group, by ID -- `G:` by name | +| `p:` | a principal, by ID (`P:` by name) | +| `g:` | a group, by ID (`G:` by name) | | `r:` | a role | | `a:` | the principal holding an API key, named by the key | @@ -67,7 +67,7 @@ A nested namespace joins its path with `$`, as in `namespace:db:parent$child` or `table:db:parent$child$table`. The hierarchy is not decorative. Reaching a table requires `USAGE` on the -namespace containing it and on the database containing that -- a grant on the +namespace containing it and on the database containing that. A grant on the table alone is not enough to traverse to it. ## Privileges {#privileges} @@ -118,14 +118,14 @@ A subtlety worth knowing, because it explains grants that appear to work on objects that do not exist yet. When you create an object, the deployment checks the database and each -namespace that will contain it. One that **has no owner** -- because it does -not exist yet, or predates per-object authorization -- is treated as one you are +namespace that will contain it. One that **has no owner** (because it does +not exist yet, or predates per-object authorization) is treated as one you are creating: the check falls back to `CREATE_DATABASE` on `system` for the database, or to `CREATE_NAMESPACE` on its parent for a namespace, and you become its owner. One that has an owner needs `USAGE` as usual. So a principal holding `CREATE_DATABASE` can create a table in a database that -has not been created -- which is what makes creation possible at all, and why +has not been created, which is what makes creation possible at all, and why an empty ACL set is not the same as a deny. The fallback applies only when creating; other operations on a database or namespace need `USAGE` on it. diff --git a/docs/web/indexing/reindexing.mdx b/docs/web/indexing/reindexing.mdx index fdbfe3b5f..9df3235db 100644 --- a/docs/web/indexing/reindexing.mdx +++ b/docs/web/indexing/reindexing.mdx @@ -21,7 +21,7 @@ import { } from '/snippets/indexing.mdx'; As you add new data to your LanceDB tables, your indexes may become outdated. -Reindexing is the process of updating the index to account for new data -- this applies to either a full-text search (FTS) index or a vector index. Reindexing is an important operation to run periodically as your data grows, as it has performance implications. +Reindexing is the process of updating the index to account for new data. This applies to either a full-text search (FTS) index or a vector index. Reindexing is an important operation to run periodically as your data grows, as it has performance implications. As data is being added and a reindex operation is running, LanceDB will combine results from the existing index with exhaustive/flat search on the new data. This is done to ensure that you're still retrieving results over all your data, but it does come at a performance cost. The more data that you add without reindexing, the impact on latency (due to exhaustive search) can be noticeable. diff --git a/docs/web/indexing/vector-index.mdx b/docs/web/indexing/vector-index.mdx index e9768efa6..2742a992f 100644 --- a/docs/web/indexing/vector-index.mdx +++ b/docs/web/indexing/vector-index.mdx @@ -40,7 +40,7 @@ LanceDB implements **ANN (Approximate Nearest-Neighbor)** queries with several t ## Do you need one? {#do-you-need-one} Not always. LanceDB's disk-based brute-force kNN search is fast even without an -index -- computing 100K pairs of 1000-dimension vectors takes under 20ms in +index: computing 100K pairs of 1000-dimension vectors takes under 20ms in benchmarks. For datasets around 100K rows or fewer, or applications that can accept ~100ms latency, a vector index is usually unnecessary overhead. @@ -276,7 +276,7 @@ The snippet below builds and queries an `IVF_HNSW_SQ` index. In LanceDB Enterprise, vector indexes are managed **automatically**. The system asynchronously updates and optimizes indexes as a background process: - Automatically manages indexing parameters and storage - Can build an `IVF_PQ` index automatically on a vector column the deployment is configured to index -- GPU-based `IVF_PQ` builds are available on request -- [contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources. +- GPU-based `IVF_PQ` builds are available on request. [Contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources. Open-Source LanceDB OSS users can manually create vector indexes by calling `table.create_index()`. @@ -350,7 +350,7 @@ discussion of LanceDB vector indexes and quantized vector indexes can be found [ ### Quantization Techniques LanceDB provides $3$ distinct quantization techniques: Product Quantization (PQ), RaBitQ Quantization (RQ), and Scalar Quantization (SQ). -Recall that all quantizations perform **lossy** compression, in that they irreversibly lose some degree of precision in order to +Recall that all quantizations perform **lossy** compression, in that they irreversibly lose some degree of precision to compactly store an index. #### Product Quantization (PQ) diff --git a/docs/web/integrations/ai/genkit.mdx b/docs/web/integrations/ai/genkit.mdx index 6314842b8..1ccbf834d 100644 --- a/docs/web/integrations/ai/genkit.mdx +++ b/docs/web/integrations/ai/genkit.mdx @@ -51,7 +51,7 @@ On running this query, you'll get 5 results fetched from the lancedb table, wher Now that we've seen how you can use LanceDB in a Genkit pipeline, let's refine the flow and create a RAG. A RAG flow will consist of an index and a retriever with its outputs postprocessed and fed into an LLM for final response ### Creating custom indexer flows {#creating-custom-indexer-flows} -You can also create custom indexer flows, utilizing more options and features provided by LanceDB. +You can also create custom indexer flows, using more options and features provided by LanceDB. {TsFrameworksGenkitCustomIndexer} @@ -64,7 +64,7 @@ In your console, you can see the logs Screenshot 2025-05-11 at 7 19 14 PM ### Creating custom retriever flows {#creating-custom-retriever-flows} -You can also create custom retriever flows, utilizing more options and features provided by LanceDB. +You can also create custom retriever flows, using more options and features provided by LanceDB. {TsFrameworksGenkitCustomRetriever} diff --git a/docs/web/integrations/ai/hermes-agent.mdx b/docs/web/integrations/ai/hermes-agent.mdx index 621b53129..12fc39da3 100644 --- a/docs/web/integrations/ai/hermes-agent.mdx +++ b/docs/web/integrations/ai/hermes-agent.mdx @@ -11,7 +11,7 @@ dedicated slot for external *memory providers* that run alongside its built-in n The [LanceDB memory plugin](https://github.com/lancedb/hermes-agent-memory) fills that slot. It gives Hermes durable, semantic recall across sessions: state a preference or a project -convention once, and the agent can retrieve it weeks later in a brand-new session — even when +convention once, and the agent can retrieve it weeks later in a brand-new session, even when you ask for it in completely different words. Everything runs inside Hermes' own Python process, storing a single LanceDB table on local disk. There's no memory server to operate. @@ -26,20 +26,20 @@ process, storing a single LanceDB table on local disk. There's no memory server Out of the box, Hermes remembers with a small curated notes file frozen into the system prompt, plus lexical (keyword) search over past sessions. Both are useful, but keyword search -misses paraphrases of what you originally typed — the exact thing you need when recalling a +misses paraphrases of what you originally typed, the exact thing you need when recalling a fact you phrased differently months ago. LanceDB OSS is an embedded retrieval library, which makes it a natural fit here: -- **No server to stand up** — it reads and writes a table on local disk, so the plugin ships +- **No server to stand up**: it reads and writes a table on local disk, so the plugin ships as a dependency rather than a service to operate. -- **One table holds everything** — content, metadata, and embeddings live together. A memory +- **One table holds everything**: content, metadata, and embeddings live together. A memory becomes a structured row with a category, tags, timestamps, and provenance, not just a text blob. -- **Query it any way you need** — vector similarity for meaning, BM25 full-text for exact +- **Query it any way you need**: vector similarity for meaning, BM25 full-text for exact names and jargon, a hybrid of the two, or plain metadata filters to keep recall scoped to the right workspace. -- **It scales up** — the same table abstraction carries over to larger LanceDB deployments +- **It scales up**: the same table abstraction carries over to larger LanceDB deployments later, so the local setup is never a dead end. ## Install and activate {#install-and-activate} @@ -68,7 +68,7 @@ hermes plugins install lancedb/hermes-agent-memory -Hermes loads plugins inside its own Python interpreter, so the dependencies go *there* — not +Hermes loads plugins inside its own Python interpreter, so the dependencies go *there*, not into a separate virtualenv. (This interpreter is shared across profiles, so you only install once.) @@ -83,7 +83,7 @@ is OpenAI, so set `OPENAI_API_KEY` in your environment or in `~/.hermes/.env`. Prefer a local or non-OpenAI model? The plugin uses an OpenAI-compatible client, so you can -point it at any compatible endpoint (OpenRouter, Ollama, vLLM, …) in your config — no code +point it at any compatible endpoint (OpenRouter, Ollama, vLLM, …) in your config, with no code change needed. See [Configuration](#configuration) below. @@ -95,7 +95,7 @@ Switch memory on and pick this plugin: hermes memory setup # choose "lancedb" ``` -Then confirm it's actually active before you start chatting — this is the one step worth not +Then confirm it's actually active before you start chatting. This is the one step worth not skipping, because Hermes quietly falls back to its built-in notes if the provider isn't set: ```bash @@ -128,7 +128,7 @@ Once activated, the agent has four tools for working with long-term memory: | `lancedb_forget` | Deletes safely: previews candidates first, then deletes by exact ID, so nothing disappears by accident. | Beyond these tools, the plugin also captures durable facts from your conversations -automatically — an auxiliary model distills them before context is compressed and again when a +automatically: an auxiliary model distills them before context is compressed and again when a session ends, so insights survive even when the raw messages are summarized away. ## Walkthrough {#walkthrough} @@ -157,8 +157,8 @@ line below: ### Recall {#recall} -First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB — -the two layers run side by side otherwise, and either could answer: +First, take Hermes' built-in notes out of the picture so recall can *only* come from LanceDB +(the two layers run side by side otherwise, and either could answer): ```bash hermes config set memory.memory_enabled false @@ -179,13 +179,13 @@ differently from how you saved it: ``` This is the whole point in one exchange. You asked about your *preferences* using none of the -words you stored the fact with — no *uv*, no *pip*, no *type hints* — yet a fresh session, with +words you stored the fact with (no *uv*, no *pip*, no *type hints*), yet a fresh session, with none of the earlier conversation in context, pulls it straight back. Semantic search matches on meaning rather than exact wording, which is precisely where keyword search falls short. -Turn the built-in layer back on for everyday use with `hermes config set memory.memory_enabled true` -— the two memory layers happily coexist. We only disabled it here to prove the recall came from +Turn the built-in layer back on for everyday use with `hermes config set memory.memory_enabled true`. +The two memory layers happily coexist. We only disabled it here to prove the recall came from LanceDB. @@ -231,7 +231,7 @@ lifecycle of a durable memory. ## Retrieval modes {#retrieval-modes} -Recall ships in `vector` mode by default — pure semantic search, which is what survives the +Recall ships in `vector` mode by default: pure semantic search, which is what survives the paraphrasing you saw above. If you also need exact name or jargon matching, switch to `hybrid` (vector + BM25) and choose how the two legs are fused: RRF, a vector-biased linear blend, or a cross-encoder reranker. Mode is set per call; fusion is a config setting. @@ -256,8 +256,8 @@ defaults to the compact 17M-parameter [ettin reranker](https://huggingface.co/cr ## Inspect the store {#inspect-the-store} Everything lives in one table named `memories` at `~/.hermes/lancedb/memories.lance`. Because -it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored -— a `kind` column separates extracted `fact` rows from the raw `turn` rows they were drawn +it's a plain LanceDB table, you can open it directly and see exactly what the agent has stored. +A `kind` column separates extracted `fact` rows from the raw `turn` rows they were drawn from: ```python @@ -270,7 +270,7 @@ print(tbl.to_pandas()[["kind", "category", "content"]].head()) ## Configuration {#configuration} -The plugin runs on sensible defaults once activated — you don't have to configure anything. +The plugin runs on sensible defaults once activated: you don't have to configure anything. `~/.hermes/config.yaml` is purely for overrides. Two common ones: Use a cheaper model for the auxiliary fact-extraction calls: @@ -298,7 +298,7 @@ plugins: Changing the embedding model (or its dimension) against an existing store requires recreating -the table — the plugin fails loudly on a dimension mismatch rather than silently returning +the table. The plugin fails loudly on a dimension mismatch rather than silently returning nothing. Every option is documented in the plugin's [`default_config.yaml`](https://github.com/lancedb/hermes-agent-memory/blob/main/src/default_config.yaml). diff --git a/docs/web/integrations/ai/kiln.mdx b/docs/web/integrations/ai/kiln.mdx index c93f31e17..91da617c4 100644 --- a/docs/web/integrations/ai/kiln.mdx +++ b/docs/web/integrations/ai/kiln.mdx @@ -20,7 +20,7 @@ Kiln's [app](https://kiln.tech/download) makes it easy to: ## Find the Best RAG Pipeline for Your Use Case {#find-the-best-rag-pipeline-for-your-use-case} -There is no universal best RAG solution—only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you. +There is no universal best RAG solution, only the best solution for your specific use case. Kiln makes it easy to compare state-of-the-art configurations and find which works best for you. Start with pre-configured templates for state-of-the-art RAG at various performance/quality/cost levels, or experiment with any combination of options: diff --git a/docs/web/integrations/ai/langchain.mdx b/docs/web/integrations/ai/langchain.mdx index 14ed628a8..8d012e01e 100644 --- a/docs/web/integrations/ai/langchain.mdx +++ b/docs/web/integrations/ai/langchain.mdx @@ -23,7 +23,7 @@ import { LangChain streamlines these stages (in figure above) by providing pre-built components and tools for integration, memory management, and deployment, allowing developers to focus on application logic rather than underlying complexities. -Integration of **Langchain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs. +Integration of **LangChain** with **LanceDB** enables applications to retrieve the most relevant data by comparing query vectors against stored vectors, facilitating effective information retrieval. It results in better and context aware replies and actions by the LLMs. ## Quick Start {#quick-start} You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. @@ -42,7 +42,7 @@ The exhaustive list of parameters for `LanceDB` vector store are : |Name|type|Purpose|default| |:----|:----|:----|:----| |`connection`| (Optional) `Any` |`lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.|`None`| -|`embedding`| (Optional) `Embeddings` | Langchain embedding model.|Provided by user.| +|`embedding`| (Optional) `Embeddings` | LangChain embedding model.|Provided by user.| |`uri`| (Optional) `str` |It specifies the directory location of **LanceDB database** and establishes a connection that can be used to interact with the database. |`/tmp/lancedb`| |`vector_key` |(Optional) `str`| Column name to use for vector's in the table.|`'vector'`| |`id_key` |(Optional) `str`| Column name to use for id's in the table.|`'id'`| @@ -51,10 +51,10 @@ The exhaustive list of parameters for `LanceDB` vector store are : |`api_key` |(Optional `str`) |API key to use for LanceDB Enterprise deployment.|`None`| |`region` |(Optional) `str`| Region to use for LanceDB Enterprise deployment.| Only for LanceDB Enterprise : `None`.| |`mode` |(Optional) `str` | Mode to use for adding data to the table. Valid values are "append" and "overwrite".|`'overwrite'`| -|`table`| (Optional) `Any`| You can connect to an existing table of LanceDB, created outside of langchain, and utilize it.|`None`| +|`table`| (Optional) `Any`| You can connect to an existing table of LanceDB, created outside of LangChain, and use it.|`None`| |`distance`|(Optional) `str`| The choice of distance metric used to calculate the similarity between vectors.|`'l2'`| |`reranker` |(Optional) `Any`| The reranker to use for LanceDB.|`None`| -|`relevance_score_fn` |(Optional) `Callable[[float], float]` | Langchain relevance score function to be used.|`None`| +|`relevance_score_fn` |(Optional) `Callable[[float], float]` | LangChain relevance score function to be used.|`None`| |`limit`|`int`|Set the maximum number of results to return.| `DEFAULT_K` (it is 4)| diff --git a/docs/web/integrations/data/dlt.mdx b/docs/web/integrations/data/dlt.mdx index f15a4b776..04d688fb9 100644 --- a/docs/web/integrations/data/dlt.mdx +++ b/docs/web/integrations/data/dlt.mdx @@ -40,7 +40,7 @@ In this example, we will be fetching movie information from the [Open Movie Data 3. **Specify necessary credentials and/or embedding model details:** - In order to fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB Enterprise, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/secrets.toml`. + To fetch data from the OMDb API, you will need to pass a valid API key into your pipeline. Depending on whether you're using LanceDB OSS or LanceDB Enterprise, you also may need to provide the necessary credentials to connect to the LanceDB instance. These can be pasted inside `.dlt/secrets.toml`. dlt's LanceDB integration also allows you to automatically embed the data during ingestion. Depending on the embedding model chosen, you may need to paste the necessary credentials inside `.dlt/secrets.toml`: ```toml diff --git a/docs/web/integrations/data/voxel51.mdx b/docs/web/integrations/data/voxel51.mdx index 1ee782353..a04cd0c3e 100644 --- a/docs/web/integrations/data/voxel51.mdx +++ b/docs/web/integrations/data/voxel51.mdx @@ -37,7 +37,7 @@ You can then sort your samples by similarity or use this information to find pot We'll be doing the following : -1. **Create Index** - In order to run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function. +1. **Create Index** - To run similarity queries against our media, we need to **index** the data. We can do this via the `compute_similarity()` function. - In the function, specify the **model** you want to use to generate the embedding vectors, and what **vector search engine** you want to use on the **backend** (here LanceDB). You can also give the similarity index a name(`brain_key`), which is useful if you want to run vector searches against multiple indexes. diff --git a/docs/web/integrations/embedding/huggingface.mdx b/docs/web/integrations/embedding/huggingface.mdx index 046b4dfb4..c62ba03ac 100644 --- a/docs/web/integrations/embedding/huggingface.mdx +++ b/docs/web/integrations/embedding/huggingface.mdx @@ -5,7 +5,7 @@ sidebarTitle: Hugging Face import { PyEmbeddingHuggingfaceUsage } from '/snippets/integrations.mdx'; -We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the HuggingFace Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. Pin `revision` to a commit hash, and note that tables using such a model need it approved with `get_registry().allow_remote_code(...)` when opened in a new process. See [Trusting remote code and custom endpoints](/embedding#trusting-remote-code-and-custom-endpoints). +We offer support for all Hugging Face models (which can be loaded via [transformers](https://huggingface.co/docs/transformers/en/index) library). The default model is `colbert-ir/colbertv2.0` which also has its own special callout - `registry.get("colbert")`. Some Hugging Face models might require custom models defined on the Hugging Face Hub in their own modeling files. You may enable this by setting `trust_remote_code=True`. This option should only be set to True for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. Pin `revision` to a commit hash, and note that tables using such a model need it approved with `get_registry().allow_remote_code(...)` when opened in a new process. See [Trusting remote code and custom endpoints](/embedding#trusting-remote-code-and-custom-endpoints). Example usage: diff --git a/docs/web/integrations/embedding/sentence-transformers.mdx b/docs/web/integrations/embedding/sentence-transformers.mdx index 0b425e50e..0096d11ab 100644 --- a/docs/web/integrations/embedding/sentence-transformers.mdx +++ b/docs/web/integrations/embedding/sentence-transformers.mdx @@ -20,7 +20,7 @@ Setting `trust_remote_code=True` allows the model to execute arbitrary code from | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model | | `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) | | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model | -| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository | +| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Hugging Face repository | | `revision` | `str` | `None` | The model revision to load, such as a commit hash | @@ -166,5 +166,5 @@ Here is an example that uses the BAAI embedding model from the Hugging Face Hub -Visit sentence-transformers [HuggingFace HUB](https://huggingface.co/sentence-transformers) page for more information on the available models. +Visit the sentence-transformers [Hugging Face Hub](https://huggingface.co/sentence-transformers) page for more information on the available models. diff --git a/docs/web/integrations/embedding/voyageai.mdx b/docs/web/integrations/embedding/voyageai.mdx index 8babfc214..cfa795841 100644 --- a/docs/web/integrations/embedding/voyageai.mdx +++ b/docs/web/integrations/embedding/voyageai.mdx @@ -1,6 +1,6 @@ --- -title: VoyageAI -sidebarTitle: VoyageAI +title: Voyage AI +sidebarTitle: Voyage AI --- import { PyEmbeddingVoyageaiUsage, PyEmbeddingVoyageaiMultimodal } from '/snippets/integrations.mdx'; @@ -9,7 +9,7 @@ Voyage AI provides cutting-edge embedding and rerankers. Using voyageai API requires voyageai package, which can be installed using `pip install voyageai`. Voyage AI embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification. -You also need to set the `VOYAGE_API_KEY` environment variable to use the VoyageAI API. +You also need to set the `VOYAGE_API_KEY` environment variable to use the Voyage AI API. ```shell export VOYAGE_API_KEY="your-api-key" @@ -32,7 +32,7 @@ Supported models are: - voyage-multimodal-3.5 (multimodal - supports text, images, and video) -**Multimodal Model:** `voyage-multimodal-3.5` supports text, images, and video inputs. It outputs 1024-dimensional embeddings by default, configurable via the `output_dimension` parameter (256, 512, 1024, 2048). See the [VoyageAI multimodal embeddings documentation](https://docs.voyageai.com/docs/multimodal-embeddings) for more details. +**Multimodal Model:** `voyage-multimodal-3.5` supports text, images, and video inputs. It outputs 1024-dimensional embeddings by default, configurable via the `output_dimension` parameter (256, 512, 1024, 2048). See the [Voyage AI multimodal embeddings documentation](https://docs.voyageai.com/docs/multimodal-embeddings) for more details. Supported parameters (to be passed in `create` method) are: diff --git a/docs/web/integrations/reranking/answerdotai.mdx b/docs/web/integrations/reranking/answerdotai.mdx index e3f500d49..22814e822 100644 --- a/docs/web/integrations/reranking/answerdotai.mdx +++ b/docs/web/integrations/reranking/answerdotai.mdx @@ -11,7 +11,7 @@ import { PyRerankingAnswerdotaiUsage } from '/snippets/integrations.mdx'; This integration uses [AnswersDotAI's rerankers](https://github.com/AnswerDotAI/rerankers) to rerank the search results, providing a lightweight, low-dependency, unified API to use all common reranking and cross-encoder models. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. diff --git a/docs/web/integrations/reranking/cohere.mdx b/docs/web/integrations/reranking/cohere.mdx index 26b98a7c8..bc731b39e 100644 --- a/docs/web/integrations/reranking/cohere.mdx +++ b/docs/web/integrations/reranking/cohere.mdx @@ -12,7 +12,7 @@ import { PyRerankingCohereUsage } from '/snippets/provider_examples.mdx'; This reranker uses the [Cohere](https://cohere.ai/) API to rerank the search results. You can use this reranker by passing `CohereReranker()` to the `rerank()` method. Note that you'll either need to set the `COHERE_API_KEY` environment variable or pass the `api_key` argument to use this reranker. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. ```shell pip install cohere diff --git a/docs/web/integrations/reranking/colbert.mdx b/docs/web/integrations/reranking/colbert.mdx index 463b3d78d..ffb3f8d2c 100644 --- a/docs/web/integrations/reranking/colbert.mdx +++ b/docs/web/integrations/reranking/colbert.mdx @@ -10,7 +10,7 @@ import { PyRerankingColbertUsage } from '/snippets/integrations.mdx'; # ColBERT Reranker This reranker uses ColBERT model to rerank the search results. You can use this reranker by passing `ColbertReranker()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. diff --git a/docs/web/integrations/reranking/jina.mdx b/docs/web/integrations/reranking/jina.mdx index 450a18ecf..c2b1db6db 100644 --- a/docs/web/integrations/reranking/jina.mdx +++ b/docs/web/integrations/reranking/jina.mdx @@ -12,7 +12,7 @@ import { PyRerankingJinaUsage } from '/snippets/provider_examples.mdx'; This reranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this reranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this reranker. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. Replace the credential placeholders with your provider credentials. In application code, load credentials from environment variables or a secrets manager. diff --git a/docs/web/integrations/reranking/openai.mdx b/docs/web/integrations/reranking/openai.mdx index 22d2af2cc..6ceb8a030 100644 --- a/docs/web/integrations/reranking/openai.mdx +++ b/docs/web/integrations/reranking/openai.mdx @@ -10,7 +10,7 @@ import { PyRerankingOpenaiUsage } from '/snippets/provider_examples.mdx'; # OpenAI Reranker (Experimental) This reranker uses OpenAI chat model to rerank the search results. You can use this reranker by passing `OpenAI()` to the `rerank()` method. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. > **Warning:** This reranker is experimental. OpenAI does not have a dedicated reranking model, so it uses a chat model under the hood. diff --git a/docs/web/integrations/reranking/voyageai.mdx b/docs/web/integrations/reranking/voyageai.mdx index 80a666f56..eaf2d83f8 100644 --- a/docs/web/integrations/reranking/voyageai.mdx +++ b/docs/web/integrations/reranking/voyageai.mdx @@ -1,20 +1,20 @@ --- -title: VoyageAI Reranker +title: Voyage AI Reranker sidebarTitle: "Voyage AI" -description: Integrate VoyageAI's cutting-edge reranking models with LanceDB. Features model selection, API key management, and comprehensive scoring options for all search types. +description: Integrate Voyage AI's cutting-edge reranking models with LanceDB. Features model selection, API key management, and comprehensive scoring options for all search types. --- import { PyRerankingVoyageaiUsage } from '/snippets/provider_examples.mdx'; -# VoyageAI Reranker +# Voyage AI Reranker {#voyageai-reranker} Voyage AI provides cutting-edge embedding and rerankers. -This reranker uses the [VoyageAI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker. +This reranker uses the [Voyage AI](https://docs.voyageai.com/docs/) API to rerank the search results. You can use this reranker by passing `VoyageAIReranker()` to the `rerank()` method. Note that you'll either need to set the `VOYAGE_API_KEY` environment variable or pass the `api_key` argument to use this reranker. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. Replace `your-api-key` with your provider API key. In application code, load diff --git a/docs/web/integrations/reranking/watsonx.mdx b/docs/web/integrations/reranking/watsonx.mdx index 5e4c6322c..13420ebe6 100644 --- a/docs/web/integrations/reranking/watsonx.mdx +++ b/docs/web/integrations/reranking/watsonx.mdx @@ -11,7 +11,7 @@ import { PyRerankingWatsonxUsage } from '/snippets/provider_examples.mdx'; This reranker uses the [IBM watsonx.ai](https://cloud.ibm.com/docs/apis/watsonx-ai#text-rerank) text rerank API to reorder search results. Pass `WatsonxReranker()` to the `rerank()` method on a query. Credentials come from the `WATSONX_API_KEY` and `WATSONX_PROJECT_ID` (or `WATSONX_SPACE_ID`) environment variables, or can be passed explicitly as arguments. -> **Note:** Supported query types – Hybrid, Vector, and FTS. +> **Note:** Supported query types: Hybrid, Vector, and FTS. ```shell pip install ibm-watsonx-ai diff --git a/docs/web/maintenance/background-jobs.mdx b/docs/web/maintenance/background-jobs.mdx index 2fe5d549c..dcfa31321 100644 --- a/docs/web/maintenance/background-jobs.mdx +++ b/docs/web/maintenance/background-jobs.mdx @@ -12,9 +12,9 @@ availability: cluster configuration. --- -The jobs documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), +The jobs documented elsewhere in these docs ([Build & Repair](/indexing/reindexing), [Compaction](/maintenance/compaction), [Cleanup](/maintenance/cleanup), -[Column Refresh](/feature-engineering/refresh-column) -- start from a call you +[Column Refresh](/feature-engineering/refresh-column)) start from a call you make or, for compaction, cleanup and index updates, from the deployment's own maintenance. This page is the rest: jobs the deployment runs to keep itself correct and fast, which you may see named in [`SHOW JOBS`](/sql/ddl/job#show) @@ -32,14 +32,14 @@ log from growing without bound: | Job type | What it does | | --- | --- | -| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments -- the two are unrelated. | +| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments. The two are unrelated. | | `wal_index_catchup` | Extends a base-table index over the rows `sstable_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. | ## Index maintenance {#index} | Job type | What it does | | --- | --- | -| `index_remap` | Reassigns index entries after row ids shift underneath an index -- the kind of remap a delta compaction can require. | +| `index_remap` | Reassigns index entries after row ids shift underneath an index (the kind of remap a delta compaction can require). | | `spfresh_merge` | Merges SPFresh-managed segments for indexes that use it. | ## Cache warming {#cache} @@ -58,7 +58,7 @@ log from growing without bound: | `job_events_export` | Copies terminal jobs' event histories into a curated system table for metering. Registry-scoped, like the archive sweep. | These two are why an old job you completed weeks ago no longer shows in -`SHOW JOBS` -- it was archived, not deleted. Where the deployment enables the +`SHOW JOBS`: it was archived, not deleted. Where the deployment enables the export, its events also survive in the exported table. ## Object teardown {#teardown} diff --git a/docs/web/maintenance/cleanup.mdx b/docs/web/maintenance/cleanup.mdx index 60a2f6bb0..397def690 100644 --- a/docs/web/maintenance/cleanup.mdx +++ b/docs/web/maintenance/cleanup.mdx @@ -13,11 +13,11 @@ availability: Every write keeps the version before it, which is what makes [time travel](/management/table#versioning) possible. Cleanup is what ends -that -- pruning versions older than a retention window, and the files that only +that: pruning versions older than a retention window, and the files that only those versions referenced. -**Enterprise job type:** `cleanup`. Distinct from `compact` -- compaction +**Enterprise job type:** `cleanup`. Distinct from `compact`: compaction merges files, cleanup removes them. `optimize()` in OSS runs both in one call; Enterprise schedules them as separate jobs, and runs cleanup only when the deployment's configuration enables it. @@ -100,7 +100,7 @@ cleanup jobs never set it. `delete_unverified=True` removes them regardless of age. Only set it when you -can guarantee **no other process is writing to the table** -- otherwise it can +can guarantee **no other process is writing to the table**. Otherwise it can delete files belonging to a write that is genuinely in progress, and corrupt the table. @@ -109,7 +109,7 @@ the table. - Merging small files -- the other half of `optimize()`. + Merging small files, the other half of `optimize()`. The Enterprise jobs that run without a call from you. diff --git a/docs/web/maintenance/compaction.mdx b/docs/web/maintenance/compaction.mdx index 757a344fc..5bee9c65e 100644 --- a/docs/web/maintenance/compaction.mdx +++ b/docs/web/maintenance/compaction.mdx @@ -34,8 +34,8 @@ Three operations, in one call: | | | | --- | --- | | **Compaction** | merges small files into larger ones | -| **Prune** | removes versions older than the retention window -- see [Cleanup](/maintenance/cleanup) | -| **Index** | folds newly added rows into existing indexes -- see [Build & Repair](/indexing/reindexing) | +| **Prune** | removes versions older than the retention window (see [Cleanup](/maintenance/cleanup)) | +| **Index** | folds newly added rows into existing indexes (see [Build & Repair](/indexing/reindexing)) | Compaction itself is only the first of the three: merging small fragments back into larger ones, which is what keeps a scan from opening more files than it @@ -43,7 +43,7 @@ needs to. **Enterprise job type:** `compact`. Runs automatically as data accumulates, -separately from [`cleanup`](/maintenance/cleanup) -- the two are distinct jobs +separately from [`cleanup`](/maintenance/cleanup). The two are distinct jobs even though `optimize()` bundles both in one OSS call. diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 5734d2545..4467c6abb 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -24,7 +24,7 @@ import { PyConnectEnterprisePlaceholders as FunctionConnect } from '/snippets/co Some columns cannot be written as a SQL expression: an embedding, a model call, an image decode. A function is a Python callable registered with the database, -versioned, and then applied as a column — the code runs where the data is +versioned, and then applied as a column. The code runs where the data is rather than pulling rows to your process. Registering and running a function needs a LanceDB Enterprise deployment with @@ -61,7 +61,7 @@ your process, without the database, so unit tests stay ordinary tests: Where annotations cannot express the Arrow type precisely, pass `input_schema` -and `output_schema` together — one without the other is rejected. A scalar +and `output_schema` together: one without the other is rejected. A scalar output must be non-nullable, because a physical NULL is what marks a computed-column row as unfilled. A named-struct output may declare nullable fields: whether a row has a result at all is the struct's own validity, which @@ -181,7 +181,7 @@ pydantic**. The client version defaults `python_version`, so a function registered from a 3.11 environment runs on 3.11 unless you say otherwise. That default is usually -right and occasionally surprising -- pin it when the function is sensitive to it. +right and occasionally surprising. Pin it when the function is sensitive to it. ## Failure handling {#failures} @@ -200,8 +200,8 @@ count on the retry costing only the rows that did not complete. The artifact is a **snapshot**: the function source, plus exactly the -module-level names it references — modules as imports, importable classes and -functions as imports, literals inlined. +module-level names it references (modules as imports, importable classes and +functions as imports, literals inlined). Code that reaches the module namespace another way is rejected where it can be seen and unsupported where it cannot: `globals()`, `eval`, `sys.modules`, diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx index b6bc07e88..21222f418 100644 --- a/docs/web/management/index.mdx +++ b/docs/web/management/index.mdx @@ -118,7 +118,7 @@ See [Jobs](/basics/jobs). | drop | `drop_index` | `dropIndex` | `DROP INDEX` | -SQL `CREATE INDEX` covers one column and takes no user-chosen name -- the server +SQL `CREATE INDEX` covers one column and takes no user-chosen name: the server names it. For a multi-column or named index, use the SDK. diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index 1475bd93a..221e02dcb 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -15,7 +15,7 @@ A materialized view is a table whose contents are defined by a query over another table. The query is recorded in the view's schema, and a refresh brings its stored rows up to date with the source. -The result is an ordinary table -- queryable, indexable, searchable, and present +The result is an ordinary table: queryable, indexable, searchable, and present in `table_names()`. Nothing about reading it is special. This page covers creating and managing one. Keeping it current is @@ -67,7 +67,7 @@ views. On Enterprise, it asks the server for the view listing, as A view is dropped with `drop_table`, because a materialized view **is** a table. -That also means dropping one removes stored rows -- unlike dropping a +That also means dropping one removes stored rows, unlike dropping a [view](/management/view), which removes only a saved query. ## Read the view and its definition {#working} diff --git a/docs/web/management/namespace.mdx b/docs/web/management/namespace.mdx index a3485f46d..dda6b918f 100644 --- a/docs/web/management/namespace.mdx +++ b/docs/web/management/namespace.mdx @@ -253,7 +253,7 @@ Below, we list some best practices for working with namespaces: - For simple use cases and single, stand-alone applications, the directory-based root namespace is sufficient and requires no special configuration. - For remote storage locations, introduce explicit namespaces when multiple teams, environments, or domains share the same catalog. - Treat namespace paths as stable identifiers (for example `"prod/search"`, `"staging/recs"`). -- For maintainability reasons, avoid hard-coding object-store table paths in application code -- instead, prefer catalog identifiers + namespaces. +- For maintainability reasons, avoid hard-coding object-store table paths in application code. Instead, prefer catalog identifiers + namespaces. ## Table operations with namespace paths {#table-operations-with-namespace-paths} diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 1b8ede070..df44a8fd3 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -215,7 +215,7 @@ Renaming is not supported on a local directory connection. In LanceDB Enterprise whether a table can be renamed or cloned at all depends on how the deployment stores its catalog. -A **shallow** clone -- the default -- references the source's data files rather +A **shallow** clone (the default) references the source's data files rather than copying them, so it is fast and cheap regardless of table size. It is the practical way to branch an experiment off a known version, or to pin a training set to the exact version a model saw. @@ -286,7 +286,7 @@ returned job has already finished. ## Data types {#data-types} -A table's schema is an **Arrow schema** -- there is no separate LanceDB type +A table's schema is an **Arrow schema**: there is no separate LanceDB type system mapped onto storage, so a column is the same object to the query engine as it is to the format. @@ -298,7 +298,7 @@ as it is to the format. | Nested | `struct`, `list`, `large_list`, `fixed_size_list`, `map` | | Lance extensions | vectors, [JSON](/sql/data-types#json), [blobs](#multimodal), bfloat16 | -A vector column is a `fixed_size_list` -- an ordinary Arrow type, +A vector column is a `fixed_size_list`, an ordinary Arrow type, which is why it is selectable and joinable but why arithmetic over it is list arithmetic rather than scalar. @@ -307,7 +307,7 @@ extension types and pseudo-columns. ## Multimodal data {#multimodal} -Binary data -- images, audio, video, model artifacts -- lives in the table +Binary data (images, audio, video, model artifacts) lives in the table beside the structured columns and embeddings that describe it, and is governed by the same versioning and search. @@ -520,7 +520,7 @@ precision does. ## Schema evolution {#schema} -Schema evolution enables non-breaking modifications to a database table's structure — such as adding columns, altering data types, or dropping fields — to adapt to evolving data requirements without service interruptions. +Schema evolution enables non-breaking modifications to a database table's structure (such as adding columns, altering data types, or dropping fields) to adapt to evolving data requirements without service interruptions. LanceDB supports ACID-compliant schema evolution through granular operations (add/alter/drop columns), allowing you to: * Iterate Safely: Modify schemas in production with versioned datasets and backward compatibility @@ -575,7 +575,7 @@ First, let's create a sample table with product data to demonstrate schema evolu -Columns derived from existing data -- SQL expressions over other columns -- are +Columns derived from existing data (SQL expressions over other columns) are covered in [Data evolution](/feature-engineering/data-evolution), because deriving values is a feature-engineering job rather than a schema one. @@ -776,7 +776,7 @@ Changing data types requires rewriting the column data and may be resource-inten ### Update field metadata {#schema-update-field-metadata} -Each column in a LanceDB table can carry a small key/value map of Arrow field metadata — useful +Each column in a LanceDB table can carry a small key/value map of Arrow field metadata, which is useful for annotating columns with units, provenance, PII flags, embedding model versions, or any other schema-level context your application needs. @@ -788,7 +788,7 @@ the new `version`. Each update targets one field by **dot-path**: top-level columns are addressed by name (for example `"embedding"`), and nested fields by their full path (for example `"address.zip"`). By -default, the keys you pass are **merged** into the field's existing metadata — keys you do not +default, the keys you pass are **merged** into the field's existing metadata: keys you do not mention are preserved, and passing `None` (Python) or `null` (TypeScript) deletes a key. Set `replace: true` to swap the field's entire metadata map instead of merging. @@ -823,8 +823,8 @@ To overwrite a field's metadata entirely instead of merging, set `replace` to `t -You can pass multiple updates in a single call to change metadata on several fields at once — -each call commits a single new table version. +You can pass multiple updates in a single call to change metadata on several fields at once. +Each call commits a single new table version. #### Canonical metadata keys {#schema-canonical-metadata-keys} @@ -908,7 +908,7 @@ Remove several columns at once for efficiency: Dropping a column changes the latest version only. Until [cleanup](/maintenance/cleanup) prunes the versions from before the drop, -[restoring](#versioning) one of them brings the column and its values back -- +[restoring](#versioning) one of them brings the column and its values back, along with the rest of the table as it was then. Once those versions are pruned, the drop cannot be undone, so check before dropping a column you may need. @@ -1071,11 +1071,11 @@ Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags let you attach human-readable labels (e.g., `"prod"`, `"baseline"`, `"q3-evaluation"`) to specific versions and check those out by name. They are conceptually similar to git tags, and unlike numeric versions, **tagged versions -are never pruned** -- a tagged version that is older than the retention window and +are never pruned**. A tagged version that is older than the retention window and is not the latest stops cleanup instead (see the cleanup note at the bottom of this page). -The tags API supports the standard CRUD operations — create, list, update, delete — +The tags API supports the standard CRUD operations (create, list, update, delete), plus checking out by tag name. @@ -1099,7 +1099,7 @@ deletion, the underlying table version becomes eligible for cleanup again. ### Branches {#versioning-branches} -Beyond linear history, LanceDB also supports **branches** — isolated, writable +Beyond linear history, LanceDB also supports **branches**: isolated, writable lines of history forked from `main` (or a specific version). Whereas tags and `checkout` give you read-only views of existing versions, a branch has its own writable history, making it ideal for experiments, backfills, and migrations diff --git a/docs/web/search/full-text-search.mdx b/docs/web/search/full-text-search.mdx index 0dc0096f7..bc174d5bd 100644 --- a/docs/web/search/full-text-search.mdx +++ b/docs/web/search/full-text-search.mdx @@ -823,8 +823,8 @@ const shouldResults = await table **How to use booleans?** -- Use `&` (Python), `Occur.Must`(Typescript) for intersection (documents must match all queries). -- Use `|` (Python), `Occur.Should`(Typescript) for union (documents must match at least one query). +- Use `&` (Python), `Occur.Must` (TypeScript) for intersection (documents must match all queries). +- Use `|` (Python), `Occur.Should` (TypeScript) for union (documents must match at least one query). diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index 8c7fa1781..863212385 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -96,7 +96,7 @@ JSON scalar index. See [Functions](/sql/scalar-functions#json). ### Blobs {#blobs} -A blob column is read either as its binary contents or as a **description** — a +A blob column is read either as its binary contents or as a **description**: a pointer plus metadata, without fetching the bytes. That choice changes the column's Arrow type, and therefore the schema your query sees. @@ -121,7 +121,7 @@ error naming the missing feature rather than an unknown-function error, and the spatial index is gated the same way. -Geometry can still be stored -- it is binary, and binary columns work -- but +Geometry can still be stored (it is binary, and binary columns work), but nothing interprets it. Filter on a bounding box held in ordinary numeric columns if you need spatial predicates today. diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index f0782008d..caf4c4cd5 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -23,7 +23,7 @@ Parameter and result types are Arrow type names, not SQL ones: `int64`, unless it is declared `NOT NULL`. The body is Python source. Its dependencies are declared with it and are part -of the **immutable function version** -- changing one produces a new version +of the **immutable function version**. Changing one produces a new version rather than altering the registered one, so a column filled by an earlier version is unaffected. @@ -40,7 +40,7 @@ version is unaffected. | | | | --- | --- | | `LANGUAGE` other than `python` | refused | -| `pip` and `conda` together | refused -- pick one | +| `pip` and `conda` together | refused (pick one) | | `conda_channels` without `conda` | refused | | `gpu` set to anything but `'true'` | refused | | An empty body | refused | @@ -69,13 +69,13 @@ DROP FUNCTION IF EXISTS embed WITH (version = '1') DROP FUNCTION embed WITH (version = '1', cascade = 'true') ``` -`version` is required -- a function is a set of immutable versions, and +`version` is required: a function is a set of immutable versions, and dropping is always of one specific version, never "the function" as a whole. Without `cascade = 'true'`, dropping a version that a column still pins fails rather than breaking that column silently. Passing it drops the version and -lets columns pinned to it stop refreshing -- they keep their last-filled +lets columns pinned to it stop refreshing. They keep their last-filled values, per [Guarantees](/basics/guarantees#jobs). diff --git a/docs/web/sql/ddl/index.mdx b/docs/web/sql/ddl/index.mdx index 3181210c0..fb84e881c 100644 --- a/docs/web/sql/ddl/index.mdx +++ b/docs/web/sql/ddl/index.mdx @@ -1,7 +1,7 @@ --- title: "Index" sidebarTitle: "Index" -description: "CREATE INDEX and DROP INDEX — one column, with the algorithm chosen by USING." +description: "CREATE INDEX and DROP INDEX: one column, with the algorithm chosen by USING." icon: bolt keywords: ["create index", "drop index", "sql"] --- diff --git a/docs/web/sql/ddl/job.mdx b/docs/web/sql/ddl/job.mdx index bc985d34d..323dc1b1e 100644 --- a/docs/web/sql/ddl/job.mdx +++ b/docs/web/sql/ddl/job.mdx @@ -8,8 +8,8 @@ keywords: ["show jobs", "show errors", "pause job", "resume job", "cancel job", Enterprise [SQL runs in LanceDB Enterprise.](/sql) -A [job](/basics/jobs) is not a catalog object -- it has no schema and is not -namespaced -- but it has its own SQL statements, grouped here the same way the +A [job](/basics/jobs) is not a catalog object (it has no schema and is not +namespaced), but it has its own SQL statements, grouped here the same way the catalog objects are. ## SHOW JOBS {#show} @@ -18,7 +18,7 @@ catalog objects are. SHOW JOBS ``` -Every job the server knows about -- id, type, state, creation time. The SQL +Every job the server knows about: id, type, state, creation time. The SQL counterpart of `list_jobs`. ## SHOW ERRORS {#errors} @@ -44,7 +44,7 @@ fewer filled rows than you expected. | Tables the server does not open directly | not supported | Without `FOR JOB` this surveys recent failures and stops at 32 jobs. When the -listing stops at your `LIMIT`, the message says so -- narrow with `FOR JOB` +listing stops at your `LIMIT`, the message says so. Narrow with `FOR JOB` rather than raising the limit blindly. ## PAUSE JOB {#pause} @@ -66,7 +66,7 @@ CANCEL JOB 'j-ci-abc-def' ``` The job id is always a quoted string literal. All three take exactly one and -nothing after it -- a second token on the line is a syntax error rather than +nothing after it: a second token on the line is a syntax error rather than being ignored. diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index 19dba5aa4..a8dfb8fbe 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -8,8 +8,8 @@ keywords: ["materialized view", "create materialized view", "show materialized v Enterprise [SQL runs in LanceDB Enterprise.](/sql) -A materialized view stores a query's result as rows, kept current by a refresh --- the counterpart to a [view](/sql/ddl/view), which stores only the query. +A materialized view stores a query's result as rows, kept current by a refresh. +It is the counterpart to a [view](/sql/ddl/view), which stores only the query. Materialized views are off by default in LanceDB Enterprise and have to be enabled @@ -51,7 +51,7 @@ and compatibility notes for older server versions. ## Dropping one {#drop} A materialized view **is** a table, so it is dropped with -[`DROP TABLE`](/sql/ddl/table#drop) -- which is also why dropping one removes +[`DROP TABLE`](/sql/ddl/table#drop). For the same reason, dropping one removes stored rows, unlike dropping a [view](/sql/ddl/view#drop). diff --git a/docs/web/sql/ddl/namespace.mdx b/docs/web/sql/ddl/namespace.mdx index efd209b02..472f3c1ad 100644 --- a/docs/web/sql/ddl/namespace.mdx +++ b/docs/web/sql/ddl/namespace.mdx @@ -1,14 +1,14 @@ --- title: "Namespace" sidebarTitle: "Namespace" -description: "CREATE SCHEMA and DROP SCHEMA — the SQL spelling of namespace DDL." +description: "CREATE SCHEMA and DROP SCHEMA: the SQL spelling of namespace DDL." icon: folder-tree keywords: ["namespace", "schema", "create schema", "drop schema", "sql"] --- Enterprise [SQL runs in LanceDB Enterprise.](/sql) -A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA` -- the +A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA`, the standard's name for the same concept. ## CREATE SCHEMA {#create} @@ -39,8 +39,8 @@ DROP SCHEMA `a$b` RESTRICT `CASCADE` is rejected: **"DROP SCHEMA CASCADE is not supported; drop the objects inside first."** A namespace's tables, views and other objects must be -dropped explicitly before the namespace itself can go. `RESTRICT` -- the -default -- states that requirement rather than changing it. +dropped explicitly before the namespace itself can go. `RESTRICT` (the +default) states that requirement rather than changing it. ## What to read next {#next} diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index 05406e677..ecb522fe6 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -15,7 +15,7 @@ CREATE TABLE documents (id BIGINT, title VARCHAR, created_at TIMESTAMP) CREATE TABLE recent AS SELECT * FROM documents WHERE created_at > now() - INTERVAL '7 days' ``` -Column types are Arrow types under SQL names -- see [Data types](/sql/data-types). +Column types are Arrow types under SQL names. See [Data types](/sql/data-types). A table inside a [namespace](/sql/ddl/namespace) is written as a dotted path: `prod.events.clicks`. @@ -25,7 +25,7 @@ A table inside a [namespace](/sql/ddl/namespace) is written as a dotted path: DESCRIBE documents ``` -The table's columns and types, without a plan -- the SQL counterpart of +The table's columns and types, without a plan. The SQL counterpart of `table.schema`. ## ALTER TABLE {#alter} @@ -37,7 +37,7 @@ name. SQL `ADD COLUMN` requires an `AS` binding, so a plain column such as `ALTER TABLE t ADD COLUMN c INT` is rejected. Add an empty, nullable column -with the SDK instead -- see +with the SDK instead. See [Add nullable columns](/management/table#schema-add-nullable-columns). ### A computed column {#computed-column} @@ -47,7 +47,7 @@ ALTER TABLE t ADD COLUMN doubled AS value * 2 ``` `AS ` declares a [computed column](/feature-engineering/add-computed-column). -The column is committed empty and the expression is stored, not evaluated -- +The column is committed empty and the expression is stored, not evaluated: declaring one on a billion-row table costs what declaring one on an empty table costs. [`REFRESH COLUMN`](/sql/dml/refresh) fills it. @@ -129,9 +129,9 @@ rows a sample would draw from are not all in the fragments yet. | --- | --- | | `ALTER TABLE ... DROP COLUMN` | not supported | -Dropping, renaming, or retyping a column is an SDK operation -- -[`drop_columns`](/management/table#schema) and -[`alter_columns`](/management/table#schema) -- not a SQL statement. +Dropping, renaming, or retyping a column is an SDK operation +([`drop_columns`](/management/table#schema) and +[`alter_columns`](/management/table#schema)), not a SQL statement. ## CLONE {#clone} @@ -147,7 +147,7 @@ copying them, so it costs one metadata commit regardless of the source's size. `AT` pins the point in the source's history to clone. A bare integer is a version and a bare quoted string is a timestamp; `VERSION`, `TAG` and -`TIMESTAMP` say so explicitly, and a tag always needs its keyword -- a bare +`TIMESTAMP` say so explicitly, and a tag always needs its keyword: a bare quoted string cannot be both a tag name and a timestamp. diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index 8b4337fac..ec923063b 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -1,7 +1,7 @@ --- title: "View" sidebarTitle: "View" -description: "CREATE VIEW and DROP VIEW — a saved query, evaluated when read." +description: "CREATE VIEW and DROP VIEW: a saved query, evaluated when read." icon: eye keywords: ["create view", "drop view", "sql", "catalog"] --- @@ -9,7 +9,7 @@ keywords: ["create view", "drop view", "sql", "catalog"] Enterprise [SQL runs in LanceDB Enterprise.](/sql) A view stores a **query**, not rows. Reading it runs the query against current -data, so a view is never stale -- and never faster than the query it wraps. It +data, so a view is never stale, and never faster than the query it wraps. It is the counterpart to a [materialized view](/sql/ddl/materialized-view), which stores rows and is brought up to date by a refresh; both are [catalog objects](/basics/catalog) defined by a query, differing only in when @@ -27,8 +27,8 @@ FROM "documents" AS t1 INNER JOIN "categories" AS t2 ON t1.ints = t2.ints ``` -The statement returns the view's schema -- column name, data type, nullability --- resolved from the query at creation time. Anything the +The statement returns the view's schema (column name, data type, nullability), +resolved from the query at creation time. Anything the [dialect](/sql/functions) supports can define one: joins, predicates, aggregates, window functions. @@ -50,7 +50,7 @@ the view would have produced and then discarded. DROP VIEW "recent_matches" ``` -Removes the saved query. Unlike dropping a table, no data is removed -- there +Removes the saved query. Unlike dropping a table, no data is removed: there was none to begin with. ## Listing {#listing} @@ -62,10 +62,10 @@ catalog distinguishes them: | --- | --- | | view names | the views in the database | | table names | the tables | -| relation names | everything queryable -- both | +| relation names | everything queryable (both) | Use the relation listing when the question is "what can I select from," and the -separate listings when the distinction matters -- before dropping, for +separate listings when the distinction matters: before dropping, for instance, since dropping a view removes only the saved query while dropping a table removes data. diff --git a/docs/web/sql/dml/copy.mdx b/docs/web/sql/dml/copy.mdx index 658b20b59..90b655d5b 100644 --- a/docs/web/sql/dml/copy.mdx +++ b/docs/web/sql/dml/copy.mdx @@ -1,7 +1,7 @@ --- title: "Copy" sidebarTitle: "Copy" -description: "COPY ... TO — writing a query result to object storage as files." +description: "COPY ... TO: writing a query result to object storage as files." icon: file-export keywords: ["copy", "export", "parquet", "sql"] --- @@ -17,7 +17,7 @@ COPY (SELECT id, title FROM documents) TO 's3://bucket/export/' STORED AS PARQUE ``` Writes a query result to object storage. This is the export path when the -consumer is another engine rather than an application -- results land as files +consumer is another engine rather than an application: results land as files rather than streaming to a client. ## What to read next {#next} diff --git a/docs/web/sql/dml/delete.mdx b/docs/web/sql/dml/delete.mdx index 90965062b..429271de3 100644 --- a/docs/web/sql/dml/delete.mdx +++ b/docs/web/sql/dml/delete.mdx @@ -1,7 +1,7 @@ --- title: "Delete" sidebarTitle: "Delete" -description: "DELETE FROM ... WHERE — rows are marked, not erased." +description: "DELETE FROM ... WHERE: rows are marked, not erased." icon: trash keywords: ["delete", "sql"] --- @@ -21,7 +21,7 @@ for as long as the version holding it survives [retention](/maintenance/cleanup). Commits a new table version, so [versioning](/management/table#versioning) and -the [write guarantees](/basics/guarantees#writes) apply -- including that the +the [write guarantees](/basics/guarantees#writes) apply, including that the version advances even when the statement affects zero rows. ## What to read next {#next} diff --git a/docs/web/sql/dml/insert.mdx b/docs/web/sql/dml/insert.mdx index 61fcb9994..44007c6a3 100644 --- a/docs/web/sql/dml/insert.mdx +++ b/docs/web/sql/dml/insert.mdx @@ -18,7 +18,7 @@ INSERT INTO archive SELECT * FROM documents WHERE created_at < '2026-01-01' server-side, without a round trip through your process. Commits a new table version, so [versioning](/management/table#versioning) and -the [write guarantees](/basics/guarantees#writes) apply -- including that the +the [write guarantees](/basics/guarantees#writes) apply, including that the version advances even when the statement affects zero rows. ## What to read next {#next} @@ -28,6 +28,6 @@ version advances even when the statement affects zero rows. The SDK equivalent. - Upsert -- matching incoming rows against existing keys -- has no SQL form. + Upsert (matching incoming rows against existing keys) has no SQL form. diff --git a/docs/web/sql/dml/refresh.mdx b/docs/web/sql/dml/refresh.mdx index 1a14adef8..d454069a9 100644 --- a/docs/web/sql/dml/refresh.mdx +++ b/docs/web/sql/dml/refresh.mdx @@ -27,7 +27,7 @@ REFRESH COLUMN embedding ON documents WHERE created_at > '2026-01-01' A refresh gives values to rows that lack them, recomputes rows whose inputs changed, and **leaves other filled rows alone**. That makes it idempotent, cheap -to repeat as data arrives, and safe to re-run after a failure -- though a re-run +to repeat as data arrives, and safe to re-run after a failure, though a re-run can recompute values the failed run had already written. `WHERE` scopes the refresh, which is how you fill a slice and check it before diff --git a/docs/web/sql/dml/update.mdx b/docs/web/sql/dml/update.mdx index c4332d185..b527d1ec0 100644 --- a/docs/web/sql/dml/update.mdx +++ b/docs/web/sql/dml/update.mdx @@ -1,7 +1,7 @@ --- title: "Update" sidebarTitle: "Update" -description: "UPDATE ... SET ... WHERE — and why it does not recompute derived columns." +description: "UPDATE ... SET ... WHERE, and why it does not recompute derived columns." icon: pen keywords: ["update", "sql"] --- @@ -19,12 +19,12 @@ clause every row is updated. An update does not recompute [computed columns](/feature-engineering/add-computed-column) that derive from the column you changed. Those rows keep the value computed from -the old input until the next refresh recomputes them -- see +the old input until the next refresh recomputes them. See [Guarantees](/basics/guarantees#jobs). Commits a new table version, so [versioning](/management/table#versioning) and -the [write guarantees](/basics/guarantees#writes) apply -- including that the +the [write guarantees](/basics/guarantees#writes) apply, including that the version advances even when the statement affects zero rows. ## What to read next {#next} diff --git a/docs/web/sql/dql/aggregation.mdx b/docs/web/sql/dql/aggregation.mdx index 48ae8453f..fe02dc17a 100644 --- a/docs/web/sql/dql/aggregation.mdx +++ b/docs/web/sql/dql/aggregation.mdx @@ -32,7 +32,7 @@ SELECT category, source, count(*) FROM documents GROUP BY category, source ## Aggregate without grouping {#whole-table} -Omitting `GROUP BY` aggregates the whole table into one row -- the usual way to +Omitting `GROUP BY` aggregates the whole table into one row, which is the usual way to ask how big something is: ```sql diff --git a/docs/web/sql/dql/explain.mdx b/docs/web/sql/dql/explain.mdx index 8c296c2d5..1fc4dc191 100644 --- a/docs/web/sql/dql/explain.mdx +++ b/docs/web/sql/dql/explain.mdx @@ -1,7 +1,7 @@ --- title: "Explain" sidebarTitle: "Explain" -description: "EXPLAIN, EXPLAIN ANALYZE and EXPLAIN VERBOSE — reading and measuring a plan from SQL." +description: "EXPLAIN, EXPLAIN ANALYZE and EXPLAIN VERBOSE: reading and measuring a plan from SQL." icon: map keywords: ["explain", "explain analyze", "explain verbose", "query plan"] --- @@ -20,7 +20,7 @@ EXPLAIN ANALYZE SELECT * FROM documents WHERE identifier < 1000 | `EXPLAIN VERBOSE ` | Adds intermediate optimizer plans on top of `EXPLAIN`. | The result is a small relation with `plan_type` and `plan` columns, read -through the same client you use for regular queries -- see +through the same client you use for regular queries. See [Connecting](/sql#connecting). SQL plans and the SDK's @@ -46,7 +46,7 @@ SQL can also add joins, aggregates and other operators: | `FilterExec` above a search | the filter ran after search rather than pushing into it | | `ANNSubIndex` / `ANNIvfPartition` | the vector index was used | -A `LanceScan` feeding a join is the common one worth acting on -- see +A `LanceScan` feeding a join is the common one worth acting on. See [Joins](/sql/dql/join#performance). For a table's columns and types without a plan, see diff --git a/docs/web/sql/dql/fts.mdx b/docs/web/sql/dql/fts.mdx index 71673645f..f6d5bc696 100644 --- a/docs/web/sql/dql/fts.mdx +++ b/docs/web/sql/dql/fts.mdx @@ -1,7 +1,7 @@ --- title: "Full-Text Search" sidebarTitle: "Full-Text Search" -description: "The fts() table function — match, fuzzy, phrase, boolean, boost and multi-match queries from SQL." +description: "The fts() table function: match, fuzzy, phrase, boolean, boost and multi-match queries from SQL." icon: magnifying-glass keywords: ["fts", "sql", "full text search", "fuzzy", "phrase", "boolean", "boost", "multi-match"] --- @@ -9,7 +9,7 @@ keywords: ["fts", "sql", "full text search", "fuzzy", "phrase", "boolean", "boos Enterprise [SQL runs in LanceDB Enterprise.](/sql) Full-text search is a **table function**, `fts(table, query)`, rather than a -predicate -- it produces a relevance score alongside the matching rows, and a +predicate: it produces a relevance score alongside the matching rows, and a `WHERE` clause has nowhere to put one. The two string-literal arguments are the table name and a JSON-encoded query, built with the same query-builder classes used from the SDK and serialized with `.to_json()`. @@ -32,8 +32,8 @@ CREATE INDEX ON my_docs USING fts (text); ``` -Phrase queries need position information, which costs extra index size -- -add `WITH (with_position = true)` to `CREATE INDEX`. Without it, +Phrase queries need position information, which costs extra index size. +Add `WITH (with_position = true)` to `CREATE INDEX`. Without it, [phrase queries](#phrase) find nothing. See [FTS parameters](/indexing/fts-index#fts-parameters). @@ -57,7 +57,7 @@ FROM fts('my_docs', '{"match":{"column":"text","terms":"puppy"}}') LIMIT 5 ``` -By default the top 5 matches come back in **arbitrary order** -- select +By default the top 5 matches come back in **arbitrary order**. Select `_score` and order by it for relevance ranking: ```sql @@ -126,8 +126,8 @@ MultiMatchQuery("puppy", ["text", "category"], boosts=[2.0, 1.0]) # text weight ## Combining with SQL {#combining} -`fts()` is a relation, so it composes with the rest of the SQL surface -- -filter, join, or aggregate over it same as any table: +`fts()` is a relation, so it composes with the rest of the SQL surface. +Filter, join, or aggregate over it same as any table: ```sql SELECT id, text, category diff --git a/docs/web/sql/dql/join.mdx b/docs/web/sql/dql/join.mdx index 7d4f84a8b..5ccf49f50 100644 --- a/docs/web/sql/dql/join.mdx +++ b/docs/web/sql/dql/join.mdx @@ -24,14 +24,14 @@ JOIN categories AS c ON c.id = d.category_id | `LEFT [OUTER] JOIN` | every left row, nulls where the right has no match | | `RIGHT [OUTER] JOIN` | every right row | | `FULL [OUTER] JOIN` | every row from both | -| `CROSS JOIN` | every combination -- the product of the two | +| `CROSS JOIN` | every combination (the product of the two) | `USING (col)` is shorthand for an equality join on a shared column name, and avoids repeating it in the select list. ## Across namespaces {#namespaces} -Tables in different [namespaces](/management/namespace) join by their dotted paths -- the +Tables in different [namespaces](/management/namespace) join by their dotted paths. The catalog is one catalog, so there is nothing to federate: ```sql @@ -56,7 +56,7 @@ ORDER BY hits DESC ``` That composition is the reason full-text search is a relation rather than a -predicate -- a `WHERE` clause has nowhere to put the relevance score. +predicate: a `WHERE` clause has nowhere to put the relevance score. ## Performance {#performance} diff --git a/docs/web/sql/dql/sampling.mdx b/docs/web/sql/dql/sampling.mdx index 8b36fa275..03e518f1b 100644 --- a/docs/web/sql/dql/sampling.mdx +++ b/docs/web/sql/dql/sampling.mdx @@ -35,7 +35,7 @@ SELECT * FROM documents TABLESAMPLE SYSTEM (10) | `SYSTEM` | `BLOCK` | **Lance fragments**, and takes whatever rows they hold | `BERNOULLI` gives a genuinely uniform sample and reads across the whole table to -get it. `SYSTEM` reads far less -- it skips entire fragments -- but the sample is +get it. `SYSTEM` reads far less (it skips entire fragments), but the sample is only as uniform as the data's layout. If rows were written in an order that correlates with what you are measuring, a fragment sample inherits that correlation. @@ -46,7 +46,7 @@ independent row sampling. Omitting the method gives ## Repeatability {#repeatable} -Without a seed, every execution resamples -- running the same query twice gives +Without a seed, every execution resamples: running the same query twice gives different rows. `REPEATABLE` fixes the draw: ```sql @@ -98,5 +98,5 @@ would draw from are not all in the fragments yet. ## Sampling outside SQL {#sdk} For a training loop rather than a query, the SDKs read a table in a shuffled -order without going through SQL at all -- see [Training](/training/data-loading), where a +order without going through SQL at all. See [Training](/training/data-loading), where a permutation drives the data loader. diff --git a/docs/web/sql/dql/select.mdx b/docs/web/sql/dql/select.mdx index 202a3d60a..cfc02c9c4 100644 --- a/docs/web/sql/dql/select.mdx +++ b/docs/web/sql/dql/select.mdx @@ -107,8 +107,8 @@ A CTE is scoped to the statement. One you want to keep is a ## Subqueries {#subqueries} -Subqueries are allowed anywhere an expression or a relation is -- in `FROM`, in -`WHERE` with `IN` or `EXISTS`, and as a scalar in the select list: +Subqueries are allowed anywhere an expression or a relation is (in `FROM`, in +`WHERE` with `IN` or `EXISTS`, and as a scalar in the select list): ```sql SELECT title diff --git a/docs/web/sql/dql/window.mdx b/docs/web/sql/dql/window.mdx index ed02994c3..14b052861 100644 --- a/docs/web/sql/dql/window.mdx +++ b/docs/web/sql/dql/window.mdx @@ -89,7 +89,7 @@ WITH ranked AS ( SELECT * FROM ranked WHERE rn = 1 ``` -The outer query is needed because a window function cannot appear in `WHERE` -- +The outer query is needed because a window function cannot appear in `WHERE`: windows are computed after filtering, so the predicate would have nothing to read. diff --git a/docs/web/sql/functions.mdx b/docs/web/sql/functions.mdx index 79c4258b7..9f7b8985b 100644 --- a/docs/web/sql/functions.mdx +++ b/docs/web/sql/functions.mdx @@ -14,9 +14,9 @@ plus what Lance adds. | | | | --- | --- | -| [Scalar functions](/sql/scalar-functions) | per row — math, string, regex, date/time, array, struct, JSON | -| [Aggregate functions](/sql/dql/aggregation#functions) | per group — `count`, `sum`, statistical, approximate | -| [Window functions](/sql/dql/window#functions) | per window — ranking, offset, running aggregates | +| [Scalar functions](/sql/scalar-functions) | per row: math, string, regex, date/time, array, struct, JSON | +| [Aggregate functions](/sql/dql/aggregation#functions) | per group: `count`, `sum`, statistical, approximate | +| [Window functions](/sql/dql/window#functions) | per window: ranking, offset, running aggregates | | [Table functions](#table-functions) | produce a relation rather than a value | @@ -51,7 +51,7 @@ and multi-match queries use this JSON encoding. Its schema contains the dataset's columns plus `_score`, so every stored column is available in the result. Projections push into the scan. -Being a relation is also what makes it compose — join it, filter it, aggregate +Being a relation is also what makes it compose: join it, filter it, aggregate over it. This join additionally requires `my_docs.category_id` and a `categories` table with `id` and `name` columns: @@ -73,7 +73,7 @@ multi-match query forms. ### `_score` {#score} A pseudo-column produced by `fts()`, carrying relevance for the row. It behaves -like any other column — order by it, filter on it, project it — and exists only +like any other column (order by it, filter on it, project it) and exists only in a relation that came from `fts()`. See [Data types](/sql/data-types#pseudo-columns). diff --git a/docs/web/sql/index.mdx b/docs/web/sql/index.mdx index a3f411c81..5793b5235 100644 --- a/docs/web/sql/index.mdx +++ b/docs/web/sql/index.mdx @@ -38,7 +38,7 @@ collecting the entire result in application memory. See The endpoint executes one statement per request and is meant for the queries and DDL/DML pages below. Reach for the SDKs instead for table-management -operations that have no SQL form, such as a merge insert -- see +operations that have no SQL form, such as a merge insert. See [What SQL cannot do](#cannot). ## Reference {#reference} @@ -66,7 +66,7 @@ Draft for the Functions release. Add this row to the table above, after Index: | [Insert](/sql/dml/insert) | `INSERT INTO`, including `INSERT ... SELECT` | | [Update](/sql/dml/update) | `UPDATE ... SET` | | [Delete](/sql/dml/delete) | `DELETE FROM` | -| [Copy](/sql/dml/copy) | `COPY` -- refused by recent deployment versions | +| [Copy](/sql/dml/copy) | `COPY` (refused by recent deployment versions) | | [Refresh](/sql/dml/refresh) | filling computed columns and materialized views | ### DQL {#dql} @@ -97,9 +97,9 @@ Draft for the Functions release. Add this row to the table above, after Index: | | | | --- | --- | | Vector search | See [SQL vector search](/basics/sql#no-vector-sql) for supported query forms and server-version requirements. The [SDK query builder](/search/vector-search) also supports vector search. | -| Merge insert | no SQL form -- needs a source relation and a per-match policy | +| Merge insert | no SQL form: it needs a source relation and a per-match policy | | `information_schema` | not enabled, so `SHOW TABLES` and `SHOW COLUMNS` fail | -| `COPY` | refused by recent deployment versions -- see [Copy](/sql/dml/copy) | +| `COPY` | refused by recent deployment versions (see [Copy](/sql/dml/copy)) | | `PREPARE` / `EXECUTE` | preparation is a [protocol feature](/sql/prepared-statements), not a statement | | Run on a local database | no engine to plan the statement | diff --git a/docs/web/sql/operators.mdx b/docs/web/sql/operators.mdx index 05ccbc3b4..3e27e6bb2 100644 --- a/docs/web/sql/operators.mdx +++ b/docs/web/sql/operators.mdx @@ -17,7 +17,7 @@ operator precedence. | | | | --- | --- | | `=` `!=` `<>` `<` `<=` `>` `>=` | ordinary comparison | -| `IS NULL` / `IS NOT NULL` | null test — `= NULL` is never true | +| `IS NULL` / `IS NOT NULL` | null test (`= NULL` is never true) | | `IS DISTINCT FROM` | comparison that treats null as a value | | `BETWEEN a AND b` | inclusive range | | `IN (...)` | set membership | @@ -34,7 +34,7 @@ its wildcard semantics. `AND`, `OR`, `NOT`. Three-valued: a comparison against null is **unknown**, and `NOT unknown` is still unknown. A `WHERE` clause keeps only rows where the -predicate is true, so unknown behaves like false there — but not inside `NOT +predicate is true, so unknown behaves like false there, but not inside `NOT IN`, which is the usual place this bites. See [Subqueries](/sql/dql/subqueries#in-exists). @@ -43,7 +43,7 @@ IN`, which is the usual place this bites. See `+` `-` `*` `/` `%`, with the usual precedence. String concatenation is `||`. -Arithmetic over a vector column is **list** arithmetic, not scalar arithmetic — +Arithmetic over a vector column is **list** arithmetic, not scalar arithmetic: a vector is a `FixedSizeList`. When the operation you want is a similarity comparison, use [vector search](/search/vector-search); whether SQL can express it [depends on the Enterprise server version](/basics/sql#no-vector-sql). @@ -53,7 +53,7 @@ can express it [depends on the Enterprise server version](/basics/sql#no-vector- | | | | --- | --- | -| String | `'text'` — doubled quote to escape: `'it''s'` | +| String | `'text'`, with a doubled quote to escape: `'it''s'` | | Number | `42`, `3.14` | | Boolean | `TRUE`, `FALSE` | | Null | `NULL` | @@ -61,8 +61,8 @@ can express it [depends on the Enterprise server version](/basics/sql#no-vector- | Interval | `INTERVAL '7 days'` | | Array | `[1, 2, 3]` | -Identifiers are quoted with double quotes when they need to be — a reserved word -or a name with mixed case: `SELECT "order" FROM "MyTable"`. +Identifiers are quoted with double quotes when they need to be (a reserved word +or a name with mixed case): `SELECT "order" FROM "MyTable"`. ## Casting {#casting} diff --git a/docs/web/sql/scalar-functions.mdx b/docs/web/sql/scalar-functions.mdx index d13fac7c3..f44c935c4 100644 --- a/docs/web/sql/scalar-functions.mdx +++ b/docs/web/sql/scalar-functions.mdx @@ -143,7 +143,7 @@ punctuation. Equivalent to an `fts()` match query with fuzziness off, `AND` semantics and the simple tokenizer. **Index-accelerated.** When the column has a full-text index that suits the -query, the planner uses it instead of scanning — which is the reason to prefer +query, the planner uses it instead of scanning, which is the reason to prefer it over `LIKE` for keyword tests. ### JSON accessors {#json} diff --git a/docs/web/tables/branching.mdx b/docs/web/tables/branching.mdx index 79eca8546..1e088c791 100644 --- a/docs/web/tables/branching.mdx +++ b/docs/web/tables/branching.mdx @@ -69,7 +69,7 @@ well as on LanceDB Enterprise (remote) tables. ## Connect to a table {#connect-to-a-table} -The branch API is identical no matter how you connect — only the connection +The branch API is identical no matter how you connect: only the connection itself differs between OSS and Enterprise. Establish a connection (`db`) and open a `table` (see [Ingesting data](/ingestion)), then use the same branch calls in every example that follows. @@ -150,7 +150,7 @@ somewhere else, pass a branch name, a specific version, or both. ### Write to a branch {#write-to-a-branch} -Writes go through the branch handle and stay there — the `main` handle keeps +Writes go through the branch handle and stay there: the `main` handle keeps reporting its original row count. Listing branches returns a mapping of each branch name to its metadata, including the version it was forked from. @@ -170,7 +170,7 @@ branch name to its metadata, including the version it was forked from. ### Reopen a branch {#reopen-a-branch} -A branch outlives the handle that created it. Reopen it later by name — either +A branch outlives the handle that created it. Reopen it later by name, either from an existing table handle or straight from the connection when you open the table. Both routes give you a writable handle tracking the branch's latest state. @@ -193,7 +193,7 @@ table. Both routes give you a writable handle tracking the branch's latest state Deleting a branch removes it and its branch-local history; `main` is untouched. A deployment refuses to delete a branch whose columns have been promoted onto `main`, because `main` still depends on it. -Before deleting a branch, make sure you've retained any results you need — see +Before deleting a branch, make sure you've retained any results you need. See [Apply branch-tested changes to `main`](#apply-branch-tested-changes-to-main) below. @@ -215,8 +215,8 @@ below. A branch has its own writable history. Outside of the [diff and cherry-pick -APIs](#compare-and-merge-a-branch-into-main) — which are available on -LanceDB Enterprise and promote added columns only — LanceDB does not reconcile +APIs](#compare-and-merge-a-branch-into-main) (which are available on +LanceDB Enterprise and promote added columns only), LanceDB does not reconcile one branch's history with another or detect conflicts between them. To carry other accepted work forward, rerun the validated operation against `main` or explicitly write selected results to it. @@ -242,7 +242,7 @@ If the result you want to retain is a set of inserted or updated rows, use matches incoming rows by key and does not merge branch histories. Read the rows you want from the reviewed branch, then upsert them into `main` on -your key column — `id` in this example. Here, `candidate` is the branch handle: +your key column (`id` in this example). Here, `candidate` is the branch handle: @@ -259,7 +259,7 @@ your key column — `id` in this example. Here, `candidate` is the branch handle With LanceDB Enterprise, the synchronous Python table raises for `to_arrow()`; read -the branch's rows with a query instead -- see +the branch's rows with a query instead. See [Avoid materializing the whole table](/search#avoid-materializing-the-whole-table). This operation transfers only the inserted or updated rows in its input. It does @@ -293,8 +293,8 @@ row and index changes. `diff` reads the branch and `main` without modifying either, and returns what has changed: columns added, removed or changed, indexes added or removed, row -counts, and a list of `errors` -- the reasons a cherry-pick of the branch could -not land now. An empty list means it could. +counts, and a list of `errors` (the reasons a cherry-pick of the branch could +not land now). An empty list means it could. ```python Python icon="python" @@ -436,6 +436,6 @@ give you to work with table history. Choose these approaches based on whether yo | **[`checkout(version)`](/management/table#versioning-rollback-to-previous-versions)** | ❌ No | Read a historical version of `main` without forking. Read-only until you `restore`. | -For linear version history — creating versions, listing them, rolling back, and -tagging — see the [Versioning and Reproducibility](/management/table#versioning) guide. +For linear version history (creating versions, listing them, rolling back, and +tagging), see the [Versioning and Reproducibility](/management/table#versioning) guide. diff --git a/docs/web/training/data-loading.mdx b/docs/web/training/data-loading.mdx index fd9df51b2..89e546257 100644 --- a/docs/web/training/data-loading.mdx +++ b/docs/web/training/data-loading.mdx @@ -155,7 +155,7 @@ for sample in eval_ds: ### Parallelism & Elasticity -When loading data, `StreamingDataset` partitions the the source dataset among disjoint parallel threads called **splits**. Each rank is allocated a contiguous group of splits, and each DataLoader worker associated to a rank gets a contiguous subgroup of its rank's splits. +When loading data, `StreamingDataset` partitions the source dataset among disjoint parallel threads called **splits**. Each rank is allocated a contiguous group of splits, and each DataLoader worker associated to a rank gets a contiguous subgroup of its rank's splits. If using `shuffle=False`, data is allocated to splits naively in ordered contiguous blocks of equal size. With `shuffle=True` (the default), the rows of the source dataset are shuffled and distributed among splits such that each split receives the same diff --git a/docs/web/training/object-detection.mdx b/docs/web/training/object-detection.mdx index 735b0af45..620409728 100644 --- a/docs/web/training/object-detection.mdx +++ b/docs/web/training/object-detection.mdx @@ -5,7 +5,7 @@ description: End-to-end fine-tuning of a Faster R-CNN object detector on curated icon: car --- -This example walks through fine-tuning an autonomous vehicle (AV) perception model on targeted failure-mode slices of [BDD100K](https://www.bdd100k.com/) — riders, nighttime pedestrians, and distant pedestrians — using LanceDB as a single multimodal table from raw JPEG bytes through to the PyTorch training loop. +This example walks through fine-tuning an autonomous vehicle (AV) perception model on targeted failure-mode slices of [BDD100K](https://www.bdd100k.com/) (riders, nighttime pedestrians, and distant pedestrians) using LanceDB as a single multimodal table from raw JPEG bytes through to the PyTorch training loop. The full pipeline lives in the [lancedb/training](https://github.com/lancedb/training/tree/main/object-detection) repository. This page focuses on the parts most relevant to training: defining curated splits as materialized views, loading them through the [`Permutation`](/training/data-loading) API, and pinning checkpoints to an exact data version. @@ -22,13 +22,13 @@ Fine-tuning Faster R-CNN ResNet50 FPN v2 for 10 epochs on each curated slice (ba | **Distant pedestrian** | mAP@0.5 | 0.4746 | **0.5788** | **+22.0%** | | | Recall | 0.6794 | **0.8024** | **+18.1%** | -No external data added — only training-distribution correction via SQL filters over a single Lance table. Each panel below shows the same frame with three overlaid predictions: **green** = ground truth · **red** = pretrained COCO baseline · **blue** = fine-tuned model. +No external data added, only training-distribution correction via SQL filters over a single Lance table. Each panel below shows the same frame with three overlaid predictions: **green** = ground truth · **red** = pretrained COCO baseline · **blue** = fine-tuned model. -![Rider detection — ground truth vs baseline vs fine-tuned](/static/assets/images/training/rider_04.jpg) +![Rider detection: ground truth vs baseline vs fine-tuned](/static/assets/images/training/rider_04.jpg) -![Nighttime pedestrian detection — ground truth vs baseline vs fine-tuned](/static/assets/images/training/nighttime_person_01.jpg) +![Nighttime pedestrian detection: ground truth vs baseline vs fine-tuned](/static/assets/images/training/nighttime_person_01.jpg) -![Distant pedestrian detection — ground truth vs baseline vs fine-tuned](/static/assets/images/training/distant_person_00.jpg) +![Distant pedestrian detection: ground truth vs baseline vs fine-tuned](/static/assets/images/training/distant_person_00.jpg) The rest of the page walks through the pipeline that produced these checkpoints. @@ -42,7 +42,7 @@ A perception model fine-tuned on a generic dataset typically misses the long-tai | **Nighttime pedestrians** | `timeofday = 'night' AND has_person = true` | | **Distant pedestrians** | `has_person = true AND person_bbox_area_pct < 30.0` | -Each curated slice becomes a [Geneva materialized view](/geneva/jobs/materialized-views) — a named, refreshable SQL filter over the source table — and the training script loads it by name. New footage flows in through `add()` → `backfill()` → `refresh()`; no manifests, no exports, no reshuffling on disk. +Each curated slice becomes a [Geneva materialized view](/geneva/jobs/materialized-views) (a named, refreshable SQL filter over the source table), and the training script loads it by name. New footage flows in through `add()` → `backfill()` → `refresh()`; no manifests, no exports, no reshuffling on disk. ## 1\. Schema {#1-schema} @@ -70,7 +70,7 @@ BDD_SCHEMA = pa.schema([ ]) ``` -Ingestion streams `pa.RecordBatch`es of raw frames + annotations directly into a Lance table — no intermediate preprocessing job. The table can live on local disk, S3, GCS, or Azure; everything downstream (backfills, views, the training loader) opens it in place via `lancedb.connect("s3://...")` with no local copy step. +Ingestion streams `pa.RecordBatch`es of raw frames + annotations directly into a Lance table, with no intermediate preprocessing job. The table can live on local disk, S3, GCS, or Azure; everything downstream (backfills, views, the training loader) opens it in place via `lancedb.connect("s3://...")` with no local copy step. ## 2\. Backfill curation features with Geneva {#2-backfill-curation-features-with-geneva} @@ -117,12 +117,12 @@ with gconn.local_ray_context(): ``` -Because the curation features are flat scalar columns on the same table, all four retrieval modes — SQL, full-text search, vector search, and SQL-filtered vector search — work directly without joins or exports. See [Geneva backfilling](/geneva/jobs/backfilling) for how a backfill fills only the rows that lack values. +Because the curation features are flat scalar columns on the same table, all four retrieval modes (SQL, full-text search, vector search, and SQL-filtered vector search) work directly without joins or exports. See [Geneva backfilling](/geneva/jobs/backfilling) for how a backfill fills only the rows that lack values. ## 3\. Define training splits as materialized views {#3-define-training-splits-as-materialized-views} -A training split is a named SQL filter, not a CSV manifest. Each view stays in sync with the source table and bumps its `version` on every refresh — the link between a checkpoint and the exact data that produced it. +A training split is a named SQL filter, not a CSV manifest. Each view stays in sync with the source table and bumps its `version` on every refresh. That version is the link between a checkpoint and the exact data that produced it. ```py Python icon=Python import geneva @@ -155,7 +155,7 @@ with gconn.local_ray_context(): ## 4\. PyTorch DataLoader via the Permutation API {#4-pytorch-dataloader-via-the-permutation-api} -The training script doesn't know about the filter — it opens a view by name and reads through the [`Permutation`](/training/data-loading) API. Each DataLoader worker reopens its own connection lazily, reads Arrow batches directly from Lance (zero-copy, no intermediate file format), and the collate function decodes the whole batch in one pass. `Permutation` provides random-access indexing over the table, so shuffling is a cheap pointer rewrite rather than a full-dataset shuffle on disk. +The training script doesn't know about the filter: it opens a view by name and reads through the [`Permutation`](/training/data-loading) API. Each DataLoader worker reopens its own connection lazily, reads Arrow batches directly from Lance (zero-copy, no intermediate file format), and the collate function decodes the whole batch in one pass. `Permutation` provides random-access indexing over the table, so shuffling is a cheap pointer rewrite rather than a full-dataset shuffle on disk. ```py Python icon=Python import lancedb @@ -246,12 +246,12 @@ def make_loader(uri, table_name, batch_size=64, num_workers=8, shuffle=False): ``` -`with_format("arrow")` keeps batches as zero-copy `pa.RecordBatch`es — no per-row Python boxing, no pickling between worker and main. Each DataLoader worker reopens its own `Permutation` after fork (the Rust async handle is cleared in `__getstate__`), so reads scale with `num_workers` and stream straight from the underlying object store. JPEG decode overlaps with GPU compute via `pin_memory` + `prefetch_factor`, which is what keeps the loader from becoming the bottleneck on a fast GPU. +`with_format("arrow")` keeps batches as zero-copy `pa.RecordBatch`es: no per-row Python boxing, no pickling between worker and main. Each DataLoader worker reopens its own `Permutation` after fork (the Rust async handle is cleared in `__getstate__`), so reads scale with `num_workers` and stream straight from the underlying object store. JPEG decode overlaps with GPU compute via `pin_memory` + `prefetch_factor`, which is what keeps the loader from becoming the bottleneck on a fast GPU. ## 5\. Fine-tune Faster R-CNN {#5-fine-tune-faster-r-cnn} -The training loop is plain PyTorch — the Lance integration ends at the loader. Mixed precision is enabled on CUDA for ~2× speedup on Ampere GPUs. +The training loop is plain PyTorch: the Lance integration ends at the loader. Mixed precision is enabled on CUDA for ~2× speedup on Ampere GPUs. ```py Python icon=Python import time @@ -312,7 +312,7 @@ for epoch in range(1, 11): ## 6\. Pin the checkpoint to a data version {#6-pin-the-checkpoint-to-a-data-version} -Every Lance table — including a materialized view — exposes a monotonically increasing `version`. Logging it next to the weights gives a permanent, deterministic link between a checkpoint and the exact data snapshot that produced it. +Every Lance table, including a materialized view, exposes a monotonically increasing `version`. Logging it next to the weights gives a permanent, deterministic link between a checkpoint and the exact data snapshot that produced it. ```py Python icon=Python import json @@ -340,7 +340,7 @@ tbl.checkout(version=7) # exact snapshot the checkpoint was trained on ## 7\. Continuous updates {#7-continuous-updates} -When new footage arrives, the same three calls update every downstream view — no view definitions change, no training-script edits required: +When new footage arrives, the same three calls update every downstream view (no view definitions change, no training-script edits required): ```py Python icon=Python # 1. ingest the new footage into the source table @@ -361,7 +361,7 @@ for view_name in gconn.table_names(): print(f"[{view_name}] {before} → {mv.count_rows()} rows (version {mv.version})") ``` -The next training run picks up the new data automatically — and pins itself to the new `version`. +The next training run picks up the new data automatically and pins itself to the new `version`. ## Full source {#full-source} diff --git a/docs/web/tutorials/agents/agentic-rag.mdx b/docs/web/tutorials/agents/agentic-rag.mdx index 505ae0791..c6cd21c76 100644 --- a/docs/web/tutorials/agents/agentic-rag.mdx +++ b/docs/web/tutorials/agents/agentic-rag.mdx @@ -4,8 +4,8 @@ sidebarTitle: "Agentic RAG" description: "Several agents collaborating to retrieve and answer." --- -Builds a RAG system where multiple agents collaborate -- retrieving -information, checking it, and generating answers -- rather than a single +Builds a RAG system where multiple agents collaborate (retrieving +information, checking it, and generating answers) rather than a single retrieve-then-generate pass. diff --git a/docs/web/tutorials/agents/contextual-rag.mdx b/docs/web/tutorials/agents/contextual-rag.mdx index 5f332d819..eeb8834e2 100644 --- a/docs/web/tutorials/agents/contextual-rag.mdx +++ b/docs/web/tutorials/agents/contextual-rag.mdx @@ -6,7 +6,7 @@ description: "Prepend LLM-generated context to each chunk before embedding it." Addresses the "lost in the middle" problem in retrieval. An LLM generates a succinct piece of context for each document chunk, which is prepended to the -chunk before embedding -- so a chunk carries enough surrounding meaning to be +chunk before embedding, so a chunk carries enough surrounding meaning to be retrieved for the right queries. diff --git a/docs/web/tutorials/agents/hyde.mdx b/docs/web/tutorials/agents/hyde.mdx index 76d674fae..e034b0b70 100644 --- a/docs/web/tutorials/agents/hyde.mdx +++ b/docs/web/tutorials/agents/hyde.mdx @@ -4,8 +4,8 @@ sidebarTitle: "HyDE" description: "Generate a hypothetical answer, then retrieve documents similar to it." --- -An LLM writes a hypothetical document answering the query, and that document -- -not the query -- is embedded and used for retrieval. It works because a +An LLM writes a hypothetical document answering the query, and that document, +not the query, is embedded and used for retrieval. It works because a plausible answer sits nearer the real answers in embedding space than a question does. diff --git a/docs/web/tutorials/agents/matryoshka-embeddings.mdx b/docs/web/tutorials/agents/matryoshka-embeddings.mdx index 6a6bc7f44..3ab1b181f 100644 --- a/docs/web/tutorials/agents/matryoshka-embeddings.mdx +++ b/docs/web/tutorials/agents/matryoshka-embeddings.mdx @@ -5,7 +5,7 @@ description: "Nested, variable-size embeddings with LanceDB and LlamaIndex." --- A RAG pipeline built on Matryoshka embeddings with LlamaIndex. The embeddings -nest, so one model serves several dimensionalities -- store the full vector and +nest, so one model serves several dimensionalities: store the full vector and retrieve with a truncated one when the cheaper comparison is good enough. diff --git a/docs/web/tutorials/agents/nvidia-rag-blueprint/index.mdx b/docs/web/tutorials/agents/nvidia-rag-blueprint/index.mdx index 667c876c1..d7d033f1e 100644 --- a/docs/web/tutorials/agents/nvidia-rag-blueprint/index.mdx +++ b/docs/web/tutorials/agents/nvidia-rag-blueprint/index.mdx @@ -150,7 +150,7 @@ This example is meant to prove more than a trivial vector lookup. - the adapter supports `RRFReranker`, `MRRReranker`, and `CrossEncoderReranker` - the default example uses `MRRReranker`, not a plain weighted linear combination -That matters for NVIDIA partner workloads because product names, storage platforms, and technical +Hybrid retrieval is important for NVIDIA partner workloads because product names, storage platforms, and technical jargon often need exact lexical matching as well as semantic retrieval. ## How this can be extended {#how-this-can-be-extended} diff --git a/docs/web/tutorials/agents/time-travel-rag/index.mdx b/docs/web/tutorials/agents/time-travel-rag/index.mdx index fd653bf50..c1cef0ca9 100644 --- a/docs/web/tutorials/agents/time-travel-rag/index.mdx +++ b/docs/web/tutorials/agents/time-travel-rag/index.mdx @@ -11,7 +11,7 @@ All the scripts and code for this tutorial are available in the ## Use case: Financial services regulatory knowledge base {#use-case-financial-services-regulatory-knowledge-base} -Imagine you're a major investment bank. Your team is tasked with building a critical Retrieval-Augmented Generation (RAG) system. This system must provide instant, accurate answers to compliance officers about ever-changing financial regulations. A wrong or out-of-date answer isn't just an inconvenience—it could lead to multi-million dollar fines, reputational damage, and regulatory audits. +Imagine you're a major investment bank. Your team is tasked with building a critical Retrieval-Augmented Generation (RAG) system. This system must provide instant, accurate answers to compliance officers about ever-changing financial regulations. A wrong or out-of-date answer isn't just an inconvenience: it could lead to multi-million dollar fines, reputational damage, and regulatory audits. Your knowledge base is a living entity, constantly evolving with: diff --git a/docs/web/tutorials/search/multivector-needle-in-a-haystack.mdx b/docs/web/tutorials/search/multivector-needle-in-a-haystack.mdx index ba5cefe35..7b6e1318a 100644 --- a/docs/web/tutorials/search/multivector-needle-in-a-haystack.mdx +++ b/docs/web/tutorials/search/multivector-needle-in-a-haystack.mdx @@ -53,7 +53,7 @@ A bi-encoder maps an entire piece of content (a query, a document page) to a *si ### Multi-vector (Late-Interaction) Models {#multi-vector-late-interaction-models} -Multi-vector models, pioneered by ColBERT, take a different approach. Instead of one vector per page, they generate a *set of vectors* for each page—one for every token (or image patch). +Multi-vector models, pioneered by ColBERT, take a different approach. Instead of one vector per page, they generate a *set of vectors* for each page: one for every token (or image patch). * **Mechanism (MaxSim):** The search process is more sophisticated. For each token in the query, the system finds the most similar token on the page. These maximum similarity scores are then summed up to get the final relevance score. This "late-interaction" preserves fine-grained, token-level details. * **The Models:** We used several vision-language models adapted for this architecture, including `ColPali`, `ColQwen2`, and `ColSmol`. While their underlying transformer backbones differ, they all share the ColBERT philosophy of representing documents as a bag of contextualized token embeddings. diff --git a/docs/web/tutorials/search/ner-search.mdx b/docs/web/tutorials/search/ner-search.mdx index 462475246..faae27770 100644 --- a/docs/web/tutorials/search/ner-search.mdx +++ b/docs/web/tutorials/search/ner-search.mdx @@ -4,8 +4,8 @@ sidebarTitle: "NER Search" description: "Extract named entities and embed them to sharpen retrieval." --- -Uses Named Entity Recognition to power vector search. Extracting entities -- -people, places, organizations -- and embedding them alongside the text improves +Uses Named Entity Recognition to power vector search. Extracting entities +(people, places, organizations) and embedding them alongside the text improves accuracy when queries are about specific things rather than general topics. diff --git a/docs/web/tutorials/search/video-search.mdx b/docs/web/tutorials/search/video-search.mdx index 0a8323597..8e51ec312 100644 --- a/docs/web/tutorials/search/video-search.mdx +++ b/docs/web/tutorials/search/video-search.mdx @@ -6,7 +6,7 @@ description: "Embed video with V-JEPA and search it by similarity." Builds a video search application on V-JEPA (Video Joint Embedding Predictive Architecture). Generates vector embeddings for video content and uses -[vector search](/search/vector-search) to find visually similar clips -- a +[vector search](/search/vector-search) to find visually similar clips: a multimodal case where the searchable column is not text. From 1befe331d20af2144f33feb98eb3b6e39d2c979b Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Mon, 19 Oct 2026 14:21:42 -0700 Subject: [PATCH 16/22] chore: update lance dependency to v14.0.0-beta.21 (#4590) Updates the Rust workspace Lance dependencies and Java lance-core from v14.0.0-beta.19 to [v14.0.0-beta.21](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.21), refreshing Cargo.lock. No compatibility changes were required. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. Co-authored-by: LuQQiu --- Cargo.lock | 93 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 ++++++++-------- java/pom.xml | 2 +- 3 files changed, 62 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 915ffb7b8..82975301a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arc-swap", "arrow", @@ -5092,6 +5092,7 @@ dependencies = [ "either", "fst", "futures", + "hex", "humantime", "itertools 0.14.0", "lance-arrow", @@ -5133,8 +5134,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5157,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5171,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5180,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5191,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5229,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5261,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5279,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5289,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5323,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5356,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5371,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5443,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5467,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow", "arrow-array", @@ -5510,8 +5511,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "byteorder", "ethnum", @@ -5528,8 +5529,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-schema", @@ -5543,8 +5544,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow", "async-trait", @@ -5558,8 +5559,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arc-swap", "arrow", @@ -5613,8 +5614,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -5628,8 +5629,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow", "arrow-array", @@ -5669,8 +5670,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "arrow-array", "arrow-schema", @@ -5683,8 +5684,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.19" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.19#42e593fb4ad0e1ccc22c1fa5e3c30bfe92a5384d" +version = "14.0.0-beta.21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index a4aaeeaca..e302d2891 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.19", default-features = false, "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.19", "tag" = "v14.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 43a68201a..85fcc2777 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.19 + 14.0.0-beta.21 2.18.11 false 2.30.0 From 6507615a6f8450157abfadc3b9a411f424387601 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 19 Oct 2026 21:22:43 +0000 Subject: [PATCH 17/22] =?UTF-8?q?Bump=20version:=200.41.0-beta.8=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 47ab20a04..f5be5a9ab 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.8" +current_version = "0.41.0-beta.9" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 82975301a..b339cf454 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5698,7 +5698,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.8" +version = "0.41.0-beta.9" dependencies = [ "ahash", "anyhow", @@ -5796,7 +5796,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.8" +version = "0.41.0-beta.9" dependencies = [ "arrow-array", "arrow-buffer", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.8" +version = "0.41.0-beta.9" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index f36007a20..7a9a89641 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.8 + 0.41.0-beta.9 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 2f4729cb1..83b27fefd 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.8 + 0.41.0-beta.9 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 85fcc2777..4dd145b31 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.8 + 0.41.0-beta.9 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index c3e86ae92..3ef5438de 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.8" +version = "0.41.0-beta.9" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 678e1dbc5..da0158b7d 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 2a2e7d6c6..451766110 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 31cebd96a..d818cb60f 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 2c785801d..4fe1c12cc 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 851062c13..7dc63ccaa 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 8252c0301..dc97bbed8 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index b80b8f2da..235bb0e55 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 3757b1b16..22f51ce61 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.8", + "version": "0.41.0-beta.9", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 22802633d..ce4230f9d 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.8" +version = "0.41.0-beta.9" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 2668aef8f..1daeef984 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.8" +version = "0.41.0-beta.9" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 4a03859b2a95f38dbef8fab962570de93d70eded Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:44:00 -0700 Subject: [PATCH 18/22] fix!: normalize blob layouts and support external byte ranges (#4589) --- docs/src/js/functions/blob.md | 9 + docs/src/js/type-aliases/BlobInput.md | 4 + nodejs/__test__/blob.test.ts | 188 ++- nodejs/__test__/remote.test.ts | 135 ++ nodejs/__test__/table.test.ts | 187 +++ nodejs/lancedb/arrow.ts | 34 +- nodejs/lancedb/blob.ts | 99 +- python/python/lancedb/table.py | 36 +- python/python/tests/test_blob.py | 146 +++ python/python/tests/test_remote_db.py | 83 ++ rust/lancedb/src/blob.rs | 63 +- rust/lancedb/src/remote/table.rs | 302 +++++ rust/lancedb/src/table/add_data.rs | 112 +- .../src/table/datafusion/blob_coerce.rs | 1151 ++++++++++++++++- rust/lancedb/src/table/datafusion/cast.rs | 61 +- rust/lancedb/src/table/merge.rs | 71 +- rust/lancedb/src/table/merge/lsm.rs | 9 +- rust/lancedb/src/table/refresh.rs | 6 +- rust/lancedb/tests/blob_integration.rs | 270 +++- rust/lancedb/tests/duplicate_pairs.rs | 2 + 20 files changed, 2837 insertions(+), 131 deletions(-) diff --git a/docs/src/js/functions/blob.md b/docs/src/js/functions/blob.md index 1a65b5133..d56735aca 100644 --- a/docs/src/js/functions/blob.md +++ b/docs/src/js/functions/blob.md @@ -11,6 +11,15 @@ function blob(name, options): Field ``` Declares a `lance.blob.v2` column. +Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes, +whole external files, and external byte ranges. Set both `position` and +positive `size` for a range, or omit both for a whole file. + +Existing two-field columns remain writable. Appends and merge inserts fill +missing range fields with nulls for four-field targets and drop all-null +range fields for two-field targets. Non-null ranges require a four-field +target. Overwrite with Arrow input replaces the table's schema; row objects +are converted using the current table schema. Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs) or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes. diff --git a/docs/src/js/type-aliases/BlobInput.md b/docs/src/js/type-aliases/BlobInput.md index d1738a760..53a47e5fd 100644 --- a/docs/src/js/type-aliases/BlobInput.md +++ b/docs/src/js/type-aliases/BlobInput.md @@ -12,6 +12,7 @@ type BlobInput: | BlobUri | object | object + | object | null | undefined; ``` @@ -20,6 +21,9 @@ A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)) Either inline bytes, a URI pointing at external bytes, or a struct that sets exactly one of `data` or `uri`. `null` and `undefined` write a null blob. +External byte ranges require both +`position` (non-negative) and `size` (positive). Use safe integer numbers or +`bigint` values for these unsigned 64-bit byte counts. ## Example diff --git a/nodejs/__test__/blob.test.ts b/nodejs/__test__/blob.test.ts index b41d5716f..12ff56b83 100644 --- a/nodejs/__test__/blob.test.ts +++ b/nodejs/__test__/blob.test.ts @@ -21,7 +21,41 @@ import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob"; import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding"; import { getRegistry } from "../lancedb/embedding/registry"; +function legacyBlob(name: string): Field { + const field = blob(name); + return new Field( + name, + new Struct((field.type as Struct).children.slice(0, 2)), + field.nullable, + field.metadata, + ); +} + describe("blob()", () => { + it("declares the four-field range layout", () => { + const field = blob("image", { + nullable: false, + inlineSizeThreshold: 12, + }); + expect(field.nullable).toBe(false); + expect(isBlobField(field)).toBe(true); + expect( + field.metadata.get("lance-encoding:blob-inline-size-threshold"), + ).toBe("12"); + expect( + (field.type as Struct).children.map((f) => [ + f.name, + f.type.toString(), + f.nullable, + ]), + ).toEqual([ + ["data", "LargeBinary", true], + ["uri", "Utf8", true], + ["position", "Uint64", true], + ["size", "Uint64", true], + ]); + }); + it("marks the field as lance.blob.v2", () => { const field = blob("image", { nullable: false }); expect(field.nullable).toBe(false); @@ -156,7 +190,7 @@ describe("coerceBlobValue", () => { [ "unknown struct field", { data: Buffer.from("y"), mime: "image/png" }, - /Blob struct values only support 'data' and 'uri', got 'mime'/, + /Blob struct values only support .*got 'mime'/, ], [ "misspelled struct field", @@ -174,6 +208,158 @@ describe("coerceBlobValue", () => { }); }); +describe("external blob ranges", () => { + const uri = "s3://bucket/key"; + + it.each([0, 4, 0n, 4n, 9007199254740993n])( + "preserves position %s", + (position) => { + expect(coerceBlobValue({ uri, position, size: 6 })).toEqual({ + data: null, + uri, + position: BigInt(position), + size: 6n, + }); + }, + ); + + it.each([ + [{ uri, position: 0 }, /both position and size/], + [{ uri, size: 6 }, /both position and size/], + [{ uri, position: null, size: 6 }, /both position and size/], + [{ uri, position: 0, size: 0 }, /size must be positive/], + [{ uri, position: -1, size: 6 }, /position must be non-negative/], + [{ uri, position: 0, size: -1n }, /size must be positive/], + [{ uri, position: 0.5, size: 6 }, /safe integer or bigint/], + [ + { uri, position: Number.MAX_SAFE_INTEGER + 1, size: 6 }, + /safe integer or bigint/, + ], + [{ uri, position: 0, size: Number.NaN }, /safe integer or bigint/], + [{ uri, position: "4", size: 6 }, /safe integer or bigint/], + [{ uri, position: false, size: 6 }, /safe integer or bigint/], + [{ uri, position: 1n << 64n, size: 6 }, /fit in uint64/], + [{ uri, position: 0xffffffffffffffffn, size: 1 }, /overflows uint64/], + [{ data: Buffer.from("x"), position: 0, size: 6 }, /require a uri/], + ])("rejects malformed input %p", (value, message) => { + expect(() => coerceBlobValue(value)).toThrow(message); + }); + + it("preserves ranges while normalizing a URL held by getters", () => { + class Range { + get uri() { + return new URL(uri); + } + get position() { + return 4; + } + get size() { + return 6; + } + } + expect(coerceBlobValue(new Range())).toEqual({ + data: null, + uri, + position: 4n, + size: 6n, + }); + }); + + it("writes ranges and mixed whole-file, inline, and null values to Arrow", () => { + const schema = new Schema([blob("image")]); + const rows = [ + null, + Buffer.from("x"), + uri, + { uri, position: 4, size: 6 }, + ].map((image) => ({ image })); + const column = makeArrowTable(rows, { schema }).getChild("image")!; + expect(column.get(0)).toBeNull(); + expect(Buffer.from(column.getChild("data")!.get(1))).toEqual( + Buffer.from("x"), + ); + expect(column.getChild("uri")!.get(2)).toBe(uri); + expect(column.getChild("position")!.toArray()).toEqual( + new BigUint64Array([0n, 0n, 0n, 4n]), + ); + expect(column.getChild("position")!.get(2)).toBeNull(); + expect(column.getChild("size")!.get(3)).toBe(6n); + }); + + it.each([ + Buffer.from("inline"), + uri, + { uri, position: null, size: null }, + null, + ])("retains the full declared range layout when seeded with %p", (image) => { + const schema = new Schema([blob("image")]); + const table = makeArrowTable([{ image }], { schema }); + expect(table.schema).toEqual(schema); + expect(table.getChild("image")!.getChild("position")!.get(0)).toBeNull(); + }); + + it.each([0, 4])( + "rejects position %s on a minimal schema before dropping keys", + (position) => { + expect(() => + makeArrowTable( + [ + { image: Buffer.from("inline") }, + { image: { uri, position, size: 6 } }, + ], + { schema: new Schema([legacyBlob("image")]) }, + ), + ).toThrow( + /field image at row 1: Blob storage schema cannot store byte ranges/, + ); + }, + ); + + it("accepts omitted or null range fields on the minimal schema", () => { + const column = makeArrowTable( + [{ image: { uri, position: null, size: null } }], + { + schema: new Schema([legacyBlob("image")]), + }, + ).getChild("image")!; + expect(column.getChild("uri")!.get(0)).toBe(uri); + }); + + it("validates ranges nested in structs and lists against their storage fields", () => { + const value = { uri, position: 4, size: 6 }; + for (const field of [legacyBlob("image"), blob("image")]) { + const schema = new Schema([ + new Field("info", new Struct([field]), true), + new Field("images", new List(field), true), + ]); + if ((field.type as Struct).children.length === 2) { + expect(() => + makeArrowTable([{ info: { image: value } }], { schema }), + ).toThrow(/field info.image.*storage schema/); + expect(() => makeArrowTable([{ images: [value] }], { schema })).toThrow( + /field images\[0\].*storage schema/, + ); + } else { + const table = makeArrowTable( + [{ info: { image: value }, images: [value] }], + { schema }, + ); + const info = table.getChild("info")!.get(0) as { + image: { position: bigint; size: bigint }; + }; + expect(info.image.position).toBe(4n); + const images = Array.from( + table.getChild("images")!.get(0) as Iterable<{ + position: bigint; + size: bigint; + }>, + ); + expect(images[0].size).toBe(6n); + } + } + }); +}); + describe("resolveBlobInputs", () => { it("reads Blob and File values in blob columns only", async () => { const schema = new Schema([ diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index c06cb737e..7e9479f94 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -12,6 +12,7 @@ import { OAuthConfig, OAuthFlowType, TlsConfig, + blob, connect, } from "../lancedb"; import { @@ -19,7 +20,10 @@ import { Field, Int64, RecordBatch, + RecordBatchReader, Schema, + Struct, + makeArrowTable, tableToIPC, } from "../lancedb/arrow"; import { @@ -94,6 +98,137 @@ async function withMockDatabase( } describe("remote connection", () => { + it.each([false, true])( + "normalizes remote blob writes (range target: %s)", + async (wide) => { + const field = (full: boolean) => { + const f = blob("image"); + return full + ? f + : new Field( + f.name, + new Struct((f.type as Struct).children.slice(0, 2)), + f.nullable, + f.metadata, + ); + }; + const schema = (full: boolean) => + new Schema([new Field("id", new Int64(), true), field(full)]); + const writes: { path: string; schema: Schema }[] = []; + let storedWide = wide; + const describe = () => ({ + version: writes.length + 1, + schema: { + fields: [ + { name: "id", type: { type: "int64" }, nullable: true }, + { + name: "image", + nullable: true, + metadata: { "ARROW:extension:name": "lance.blob.v2" }, + type: { + type: "struct", + fields: [ + { + name: "data", + nullable: true, + type: { type: "large_binary" }, + }, + { name: "uri", nullable: true, type: { type: "string" } }, + ...(storedWide + ? ["position", "size"].map((name) => ({ + name, + nullable: true, + type: { type: "uint64" }, + })) + : []), + ], + }, + }, + ], + }, + }); + await withMockDatabase( + (req, res) => { + if (req.url?.includes("/describe/")) { + res + .writeHead(200, { "content-type": "application/json" }) + .end(JSON.stringify(describe())); + return; + } + const chunks: Buffer[] = []; + // Failed streaming input aborts before end; only complete bodies commit. + req.on("error", () => { + /* Aborted inputs must not commit. */ + }); + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const reader = RecordBatchReader.from(Buffer.concat(chunks)).open(); + // Only read the schema: Rust separately checks compressed wire values. + writes.push({ path: req.url ?? "", schema: reader.schema }); + storedWide = + (reader.schema.fields[1].type as Struct).children.length === 4; + res.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + version: writes.length + 1, + // biome-ignore lint/style/useNamingConvention: server wire format + num_inserted_rows: 1, + }), + ); + }); + }, + async (db) => { + const table = await db.openTable("test"); + const input = makeArrowTable( + [{ id: 1n, image: Buffer.from("payload") }], + { schema: schema(!wide) }, + ); + await table.add(input); + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute(input); + expect(writes).toHaveLength(2); + for (const write of writes) + expect(write.schema).toEqual(schema(wide)); + expect(writes[0].path).toContain("/insert/"); + expect(writes[1].path).toContain("/merge_insert/"); + if (!wide) { + const invalid = makeArrowTable( + [ + { + id: 1n, + image: { + uri: "s3://registered/payload", + position: 4, + size: 6, + }, + }, + ], + { schema: schema(true) }, + ); + await expect(table.add(invalid)).rejects.toThrow( + /cannot store byte ranges/, + ); + await expect( + table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute(invalid), + ).rejects.toThrow(/cannot store byte ranges/); + expect(writes).toHaveLength(2); + } + await table.add(input, { mode: "overwrite" }); + expect(writes).toHaveLength(3); + expect(writes[2].path).toContain("mode=overwrite"); + expect(writes[2].schema).toEqual(schema(!wide)); + expect(await table.schema()).toEqual(schema(!wide)); + }, + ); + }, + ); + it("reports unsupported operations with backend-neutral errors", async () => { await withMockDatabase( (req, res) => { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index b30121725..1c4592a14 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2583,6 +2583,16 @@ describe("when dealing with versioning", () => { }); }); +function legacyBlob(name: string): Field { + const field = blob(name); + return new Field( + name, + new Struct((field.type as Struct).children.slice(0, 2)), + field.nullable, + field.metadata, + ); +} + describe("when dealing with blob columns", () => { let tmpDir: tmp.DirResult; beforeEach(() => { @@ -2592,6 +2602,89 @@ describe("when dealing with blob columns", () => { tmpDir.removeCallback(); }); + it.each([false, true])( + "normalizes Arrow blob layouts for append and merge (range target: %s)", + async (wide) => { + const db = await connect(tmpDir.name); + const schema = (full: boolean) => + new Schema([ + new Field("id", new Int64(), true), + full ? blob("image") : legacyBlob("image"), + ]); + const table = await db.createTable( + "layouts", + [{ id: 1n, image: Buffer.from("old") }], + { schema: schema(wide) }, + ); + await table.add( + makeArrowTable([{ id: 2n, image: Buffer.from("appended") }], { + schema: schema(!wide), + }), + ); + await table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute( + makeArrowTable( + [ + { id: 1n, image: Buffer.from("updated") }, + { id: 3n, image: Buffer.from("inserted") }, + ], + { schema: schema(!wide) }, + ), + ); + expect(await table.schema()).toEqual(schema(wide)); + const rows = await table.query().withRowId().toArray(); + rows.sort((a, b) => Number(a.id - b.id)); + expect( + await table.fetchBlobs( + "image", + rows.map((row) => row._rowid as bigint), + ), + ).toEqual([ + Buffer.from("updated"), + Buffer.from("appended"), + Buffer.from("inserted"), + ]); + if (!wide) { + const bad = makeArrowTable( + [ + { + id: 1n, + image: { uri: "s3://registered/file", position: 0, size: 6 }, + }, + ], + { schema: schema(true) }, + ); + const version = await table.version(); + await expect(table.add(bad)).rejects.toThrow( + /cannot store byte ranges/, + ); + await expect( + table + .mergeInsert("id") + .whenMatchedUpdateAll() + .whenNotMatchedInsertAll() + .execute(bad), + ).rejects.toThrow(/cannot store byte ranges/); + expect(await table.version()).toBe(version); + expect(await table.countRows()).toBe(3); + } + const replacement = makeArrowTable( + [{ id: 4n, image: Buffer.from("replacement") }], + { schema: schema(!wide) }, + ); + await table.add(replacement, { mode: "overwrite" }); + expect(await table.schema()).toEqual(schema(!wide)); + expect(await table.countRows()).toBe(1); + const [row] = await table.query().withRowId().toArray(); + expect(await table.fetchBlobs("image", [row._rowid as bigint])).toEqual([ + Buffer.from("replacement"), + ]); + }, + ); + it("rejects unsupported blob values in createTable and add", async () => { const db = await connect(tmpDir.name); const schema = new Schema([ @@ -2668,6 +2761,100 @@ describe("when dealing with blob columns", () => { expect(bytes[0]).toEqual(payload); }); + it.each([0, 4])( + "fetches the exact external range at position %s after reopening", + async (position) => { + const { db, uri, payload } = await openExternalBlobTable(); + const schema = new Schema([ + new Field("id", new Int64(), true), + blob("payload"), + ]); + const rows: { id: bigint; payload: BlobInput }[] = [ + { id: 1n, payload: Buffer.from("inline") }, + { id: 2n, payload: { uri: new URL(uri), position, size: 6 } }, + { id: 3n, payload: uri }, + { id: 4n, payload: null }, + ]; + const table = await db.createTable("ranges", rows.slice(0, 1), { + schema, + }); + await table.add(rows.slice(1), { allowExternalBlobOutsideBases: true }); + table.close(); + const reopened = await db.openTable("ranges"); + const results = await reopened.query().withRowId().toArray(); + results.sort((a, b) => Number(a.id - b.id)); + expect(results[1].payload.position).toBe(BigInt(position)); + expect(results[1].payload.size).toBe(6n); + expect( + await reopened.fetchBlobs( + "payload", + results.map((r) => r._rowid as bigint), + ), + ).toEqual([ + Buffer.from("inline"), + payload.subarray(position, position + 6), + payload, + null, + ]); + }, + ); + + it.each([0, 4])( + "rejects ranges at position %s on reopened minimal tables without writing", + async (position) => { + const { db, uri, payload } = await openExternalBlobTable(); + const table = await db.createEmptyTable( + "legacy_blobs", + new Schema([new Field("id", new Int64(), true), legacyBlob("payload")]), + ); + table.close(); + const reopened = await db.openTable("legacy_blobs"); + await expect( + reopened.add( + [ + { id: 1n, payload: Buffer.from("inline") }, + { id: 2n, payload: { uri, position, size: 6 } }, + ], + { allowExternalBlobOutsideBases: true }, + ), + ).rejects.toThrow(/storage schema cannot store byte ranges/); + expect(await reopened.countRows()).toBe(0); + await reopened.add( + [ + { id: 3n, payload: uri }, + { id: 4n, payload: Buffer.from("inline") }, + ], + { allowExternalBlobOutsideBases: true }, + ); + const rows = await reopened.query().withRowId().toArray(); + rows.sort((a, b) => Number(a.id - b.id)); + expect( + await reopened.fetchBlobs( + "payload", + rows.map((r) => r._rowid as bigint), + ), + ).toEqual([payload, Buffer.from("inline")]); + }, + ); + + it.each([{ position: 0 }, { size: 6 }, { position: 0, size: 0 }])( + "rejects partial or empty ranges without writing: %p", + async (range) => { + const { db, uri } = await openExternalBlobTable(); + const table = await db.createEmptyTable( + "invalid_ranges", + new Schema([blob("payload")]), + ); + await expect( + table.add( + [{ payload: Buffer.from("inline") }, { payload: { uri, ...range } }], + { allowExternalBlobOutsideBases: true }, + ), + ).rejects.toThrow(/both position and size|size must be positive/); + expect(await table.countRows()).toBe(0); + }, + ); + it("applies the external blob opt-in only to the current add", async () => { const { table, uri } = await openExternalBlobTable(); diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 3130da8eb..68b35e256 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -456,14 +456,29 @@ export function makeArrowTable( if (schema !== undefined) { // Validate blob values up front and give every one the full - // `{ data, uri }` shape, so inference sees the same struct whether a row + // storage shape, so inference sees the same struct whether a row // passed bytes, a URI, or a partial struct. - data = mapBlobInputs(data, schema, (value, field, row) => { + data = mapBlobInputs(data, schema, (value, field, row, storageField) => { if (value === undefined) { return value; } try { - return coerceBlobValue(value); + const normalized = coerceBlobValue(value, storageField); + if ( + normalized === null || + !(storageField.type instanceof Struct) || + storageField.type.children.length === 2 + ) { + return normalized; + } + // Keep even unused range children during schema inference. Otherwise + // creating a range-capable table with inline data would lose them. + return Object.fromEntries( + storageField.type.children.map((child) => [ + child.name, + normalized[child.name as keyof typeof normalized] ?? null, + ]), + ); } catch (e) { throw new Error( `Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`, @@ -533,7 +548,12 @@ function containsBlobField(field: Field): boolean { ); } -type BlobInputVisitor = (value: unknown, field: string, row: number) => unknown; +type BlobInputVisitor = ( + value: unknown, + field: string, + row: number, + storageField: Field, +) => unknown; /** * Calls `visit` on every value that lands in a blob column of `schema`, @@ -592,7 +612,7 @@ function mapBlobField( visit: BlobInputVisitor, ): unknown { if (isBlobField(field)) { - return visit(value, label, row); + return visit(value, label, row, field); } if (field.type instanceof Struct && isObject(value)) { return mapBlobFields(value, field.type.children, `${label}.`, row, visit); @@ -697,11 +717,11 @@ function transposeData( const valuesPath = [...path, field.name]; if (isBlobField(field) && field.type instanceof Struct) { const blobRows = data.map((datum) => - coerceBlobValue(valueAtPath(datum, valuesPath)), + coerceBlobValue(valueAtPath(datum, valuesPath), field), ); const childVectors = field.type.children.map((child) => { const values = blobRows.map((row) => - row == null ? null : (row[child.name as "data" | "uri"] ?? null), + row == null ? null : (row[child.name as keyof typeof row] ?? null), ); return makeVector(values, child.type, undefined, child.nullable); }); diff --git a/nodejs/lancedb/blob.ts b/nodejs/lancedb/blob.ts index 7b0276f14..82729644c 100644 --- a/nodejs/lancedb/blob.ts +++ b/nodejs/lancedb/blob.ts @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -import { Field, LargeBinary, Struct, Utf8 } from "apache-arrow"; +import { Field, LargeBinary, Struct, Uint64, Utf8 } from "apache-arrow"; import { BlobFile as NativeBlobFile } from "./native"; const BLOB_V2_EXTENSION_NAME = "lance.blob.v2"; @@ -28,6 +28,9 @@ export type BlobUri = string | URL; * * Either inline bytes, a URI pointing at external bytes, or a struct that sets * exactly one of `data` or `uri`. `null` and `undefined` write a null blob. + * External byte ranges require both + * `position` (non-negative) and `size` (positive). Use safe integer numbers or + * `bigint` values for these unsigned 64-bit byte counts. * * @example * ```ts @@ -45,8 +48,14 @@ export type BlobUri = string | URL; export type BlobInput = | BlobData | BlobUri - | { data: BlobData; uri?: null } - | { data?: null; uri: BlobUri } + | { data: BlobData; uri?: null; position?: null; size?: null } + | { data?: null; uri: BlobUri; position?: null; size?: null } + | { + data?: null; + uri: BlobUri; + position: number | bigint; + size: number | bigint; + } | null | undefined; @@ -54,6 +63,8 @@ export type BlobInput = export type BlobValue = { data: Buffer | Uint8Array | null; uri: string | null; + position?: bigint; + size?: bigint; }; export type BlobOptions = { @@ -78,6 +89,15 @@ export type BlobOptions = { /** * Declares a `lance.blob.v2` column. + * Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes, + * whole external files, and external byte ranges. Set both `position` and + * positive `size` for a range, or omit both for a whole file. + * + * Existing two-field columns remain writable. Appends and merge inserts fill + * missing range fields with nulls for four-field targets and drop all-null + * range fields for two-field targets. Non-null ranges require a four-field + * target. Overwrite with Arrow input replaces the table's schema; row objects + * are converted using the current table schema. * * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs} * or {@link Table.fetchBlobFiles} to read bytes. @@ -142,6 +162,8 @@ export function blob(name: string, options: BlobOptions = {}): Field { new Struct([ new Field("data", new LargeBinary(), true), new Field("uri", new Utf8(), true), + new Field("position", new Uint64(), true), + new Field("size", new Uint64(), true), ]), options.nullable ?? true, metadata, @@ -317,6 +339,8 @@ export function normalizeBlobInput(value: unknown): unknown { ...value, data: value.data, uri: value.uri, + position: value.position, + size: value.size, }; let changed = false; if (value.data instanceof ArrayBuffer) { @@ -355,10 +379,19 @@ export function withBlobBytes(value: unknown, bytes: Uint8Array): unknown { return bytes; } const record = value as Record; - return { ...record, uri: record.uri, data: bytes }; + return { + ...record, + uri: record.uri, + position: record.position, + size: record.size, + data: bytes, + }; } -export function coerceBlobValue(input: unknown): BlobValue | null { +export function coerceBlobValue( + input: unknown, + field?: Field, +): BlobValue | null { const value = normalizeBlobInput(input); if (value == null) { return null; @@ -385,11 +418,11 @@ export function coerceBlobValue(input: unknown): BlobValue | null { ); } const unknownKeys = Object.keys(record).filter( - (key) => key !== "data" && key !== "uri", + (key) => !["data", "uri", "position", "size"].includes(key), ); if (unknownKeys.length > 0) { throw new Error( - `Blob struct values only support 'data' and 'uri', got ${unknownKeys + `Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys .map((key) => `'${key}'`) .join(", ")}`, ); @@ -416,13 +449,63 @@ export function coerceBlobValue(input: unknown): BlobValue | null { "Blob struct values must set exactly one of 'data' or 'uri'", ); } + const hasPosition = record.position != null; + const hasSize = record.size != null; + if (hasPosition !== hasSize) { + throw new Error( + "External blob must set both position and size, or neither", + ); + } + if (hasPosition) { + if (uriValue == null) { + throw new Error("External blob byte ranges require a uri"); + } + const position = blobRangeInteger(record.position, "position", 0n); + const size = blobRangeInteger(record.size, "size", 1n); + if (position + size > 0xffffffffffffffffn) { + throw new Error("External blob range overflows uint64"); + } + if ( + field !== undefined && + (!(field.type instanceof Struct) || + ["position", "size"].some( + (name) => + !field.type.children.some((child: Field) => child.name === name), + )) + ) { + throw new Error( + "Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size", + ); + } + return { data: null, uri: uriValue, position, size }; + } return { data: bytes, uri: uriValue }; } throw new Error( - "Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }", + "Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }", ); } +function blobRangeInteger( + value: unknown, + name: string, + minimum: bigint, +): bigint { + if ( + typeof value !== "bigint" && + (typeof value !== "number" || !Number.isSafeInteger(value)) + ) { + throw new Error(`Blob ${name} must be a safe integer or bigint`); + } + const integer = BigInt(value); + if (integer < minimum || integer > 0xffffffffffffffffn) { + throw new Error( + `Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`, + ); + } + return integer; +} + function isBlobLike(value: unknown): value is Blob { return typeof Blob !== "undefined" && value instanceof Blob; } diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index a9cf13b65..e0a93f33e 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -588,7 +588,7 @@ def _cast_to_target_schema( def _preserve_blob_v2_fields( inferred_schema: pa.Schema, existing_schema: pa.Schema ) -> pa.Schema: - """Keep existing blob fields when an overwrite infers a replacement schema.""" + """Keep inferred blob semantics without overriding explicit replacement types.""" existing_fields = {field.name: field for field in existing_schema} fields = [ _preserve_blob_v2_field(field, existing_fields[field.name]) @@ -600,6 +600,9 @@ def _preserve_blob_v2_fields( def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field: + if is_blob_v2_field(inferred): + # An explicitly declared blob layout is part of the replacement schema. + return inferred if is_blob_v2_field(existing): return existing @@ -637,12 +640,10 @@ def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field: def _coerce_blob_write_columns( batch: pa.RecordBatch, target_schema: pa.Schema ) -> pa.RecordBatch: - """Materialize blob storage structs before the stream leaves Python. + """Materialize Python bytes and URIs as blob structs before Arrow casting. - merge_insert requires its source reader to already match the table's - physical schema. Unlike add and insert, it does not pass through - LanceDB's Rust blob coercion, so preserving binary input here would - reach Lance as binary and fail the schema check. + Existing blob structs retain their input layout. Rust normalizes them to + the stored schema and rejects populated ranges on two-field targets. """ columns = [] fields = [] @@ -973,6 +974,17 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: "(Utf8, LargeUtf8, Utf8View), or a Struct with a " "'data' or 'uri' child" ) + if pa.types.is_struct(source_type) or ( + isinstance(source_type, pa.ExtensionType) + and ( + not isinstance(target_field.type, pa.ExtensionType) + or type(source_type) is type(target_field.type) + ) + ): + # Keep the source layout until Rust can fill missing range fields or + # validate every batch before dropping all-null range fields. Arrow + # casting here would reject differing layouts before that validation. + return field.with_metadata(target_field.metadata) # LanceDB exposes stored JSON columns as lance.json (JSONB-backed LargeBinary), but # casting the input to that storage type here merely relabels the raw JSON bytes as # JSONB. Lance must see arrow.json so it can perform the JSONB encoding. @@ -1832,8 +1844,8 @@ class Table(ABC): - pyarrow.Table or pyarrow.RecordBatch mode: str The mode to use when writing the data. Valid values are - "append" and "overwrite". Overwrite infers a replacement schema - but preserves existing blob v2 fields present in the new data. + "append" and "overwrite". Overwrite adopts explicitly declared blob + layouts, while inferred bytes retain the existing blob v2 fields. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill". @@ -4111,8 +4123,8 @@ class LanceTable(Table): The data to insert into the table. mode: str The mode to use when writing the data. Valid values are - "append" and "overwrite". Overwrite infers a replacement schema - but preserves existing blob v2 fields present in the new data. + "append" and "overwrite". Overwrite adopts explicitly declared blob + layouts, while inferred bytes retain the existing blob v2 fields. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". @@ -6145,8 +6157,8 @@ class AsyncTable: - pyarrow.Table or pyarrow.RecordBatch mode: str The mode to use when writing the data. Valid values are - "append" and "overwrite". Overwrite infers a replacement schema - but preserves existing blob v2 fields present in the new data. + "append" and "overwrite". Overwrite adopts explicitly declared blob + layouts, while inferred bytes retain the existing blob v2 fields. on_bad_vectors: str, default "error" What to do if any of the vectors are not the same size or contains NaNs. One of "error", "drop", "fill", "null". diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 130526eaf..ec1b8b27e 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -63,6 +63,25 @@ def _blob_array(name, values): return pa.ExtensionArray.from_storage(blob_type, storage) +def _blob_layout_data(rows, with_ranges, *, as_extension=True): + field = lancedb.blob("image") + storage_type = field.type.storage_type + if not with_ranges or not as_extension: + storage_type = pa.struct(list(storage_type)[: (4 if with_ranges else 2)]) + field = pa.field( + "image", + storage_type, + metadata={"ARROW:extension:name": "lance.blob.v2"}, + ) + images = pa.array([row["image"] for row in rows], type=storage_type) + if isinstance(field.type, pa.ExtensionType): + images = pa.ExtensionArray.from_storage(field.type, images) + return pa.Table.from_arrays( + [pa.array([row["id"] for row in rows], type=pa.int64()), images], + schema=pa.schema([pa.field("id", pa.int64()), field]), + ) + + def _row_ids_by_id(table): hits = table.search().with_row_id(True).limit(1000).to_arrow() assert "_rowid" in hits.column_names @@ -998,6 +1017,133 @@ def test_merge_insert_writes_python_bytes(): assert blobs.to_pylist() == [b"updated", b"inserted"] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("input_type", ["extension", "struct", "reader"]) +@pytest.mark.parametrize("target_with_ranges", [False, True]) +async def test_merge_insert_normalizes_blob_layouts( + use_async, input_type, target_with_ranges +): + data = _blob_layout_data( + [{"id": 1, "image": {"data": b"before"}}], target_with_ranges + ) + table = lancedb.connect("memory:///").create_table("merge_layouts", data=data) + original_schema = table.schema + updates = _blob_layout_data( + [ + {"id": 1, "image": {"data": b"updated"}}, + {"id": 2, "image": {"data": b"inserted"}}, + {"id": 3, "image": None}, + ], + not target_with_ranges, + as_extension=input_type != "struct", + ) + if input_type == "reader": + updates = updates.to_reader(max_chunksize=1) + writer = table._table if use_async else table + result = ( + writer.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(updates) + ) + if use_async: + result = await result + + assert result.num_updated_rows == 1 + assert result.num_inserted_rows == 2 + assert table.schema == original_schema + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[i] for i in (1, 2, 3)]).to_pylist() == [ + b"updated", + b"inserted", + None, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "byte_range", [{"position": 0}, {"size": 6}, {"position": 4, "size": 6}] +) +async def test_merge_insert_legacy_blob_rejects_ranges_atomically( + use_async, byte_range +): + data = _blob_layout_data([{"id": 1, "image": {"data": b"before"}}], False) + table = lancedb.connect("memory:///").create_table("reject_ranges", data=data) + original_schema, original_version = table.schema, table.version + updates = _blob_layout_data( + [ + {"id": 1, "image": {"data": b"updated"}}, + {"id": 2, "image": {"uri": "file:///payload", **byte_range}}, + ], + True, + ).to_reader(max_chunksize=1) + writer = table._table if use_async else table + with pytest.raises(RuntimeError, match="cannot store byte ranges"): + result = ( + writer.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(updates) + ) + if use_async: + await result + + assert table.version == original_version + assert table.schema == original_schema + assert table.count_rows() == 1 + assert table.fetch_blobs("image", [_row_ids_by_id(table)[1]]).to_pylist() == [ + b"before" + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("input_type", ["extension", "struct", "reader"]) +@pytest.mark.parametrize("source_with_ranges", [False, True]) +async def test_add_overwrite_adopts_explicit_blob_layout( + tmp_path, use_async, input_type, source_with_ranges +): + data = _blob_layout_data( + [{"id": 1, "image": {"data": b"before"}}], not source_with_ranges + ) + table = lancedb.connect("memory:///").create_table("overwrite_layouts", data=data) + payload = b"0123456789abcdef" + path = tmp_path / "payload.bin" + path.write_bytes(payload) + value = ( + {"uri": path.as_uri(), "position": 4, "size": 6} + if source_with_ranges + else {"data": b"updated"} + ) + updates = _blob_layout_data( + [{"id": 2, "image": value}, {"id": 3, "image": None}], + source_with_ranges, + as_extension=input_type != "struct", + ) + if input_type == "reader": + updates = updates.to_reader(max_chunksize=1) + writer = table._table if use_async else table + result = writer.add( + updates, mode="overwrite", allow_external_blob_outside_bases=True + ) + if use_async: + await result + + blob_type = table.schema.field("image").type + if isinstance(blob_type, pa.ExtensionType): + blob_type = blob_type.storage_type + assert blob_type.num_fields == (4 if source_with_ranges else 2) + assert table.blob_columns() == ["image"] + assert table.count_rows() == 2 + by_id = _row_ids_by_id(table) + assert table.fetch_blobs("image", [by_id[2], by_id[3]]).to_pylist() == [ + payload[4:10] if source_with_ranges else b"updated", + None, + ] + + @pytest.mark.parametrize("write", ["merge_insert", "create_table"]) def test_uri_string_blob_write_reaches_external_base_validation(tmp_path, write): blob_path = tmp_path / "payload.bin" diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index e9768b0bf..73fb6ab12 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -3254,6 +3254,89 @@ BLOB_DESCRIBE_RESPONSE = { } +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("mode", ["merge", "overwrite"]) +@pytest.mark.parametrize("target_with_ranges", [False, True]) +async def test_remote_writes_normalize_blob_layouts( + use_async, mode, target_with_ranges +): + description = json.loads(json.dumps(BLOB_DESCRIBE_RESPONSE)) + if target_with_ranges: + description["schema"]["fields"][1]["type"]["fields"].extend( + {"name": name, "type": {"type": "uint64"}, "nullable": True} + for name in ("position", "size") + ) + received = [] + + def handler(request): + if request.path == "/v1/table/test/describe/": + response = description + else: + expected = "merge_insert" if mode == "merge" else "insert" + assert urlparse(request.path).path == f"/v1/table/test/{expected}/" + if request.headers.get("Transfer-Encoding") == "chunked": + body = bytearray() + while size := int(request.rfile.readline().split(b";")[0], 16): + body.extend(request.rfile.read(size)) + assert request.rfile.read(2) == b"\r\n" + assert request.rfile.read(2) == b"\r\n" + else: + body = request.rfile.read(int(request.headers["Content-Length"])) + received.append(pa.ipc.open_stream(body).read_all()) + response = { + "version": 2, + "num_updated_rows": 0, + "num_inserted_rows": 2, + "num_deleted_rows": 0, + } + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.send_header("phalanx-version", "0.5.0") + request.end_headers() + request.wfile.write(json.dumps(response).encode()) + + field = lancedb.blob("image") + storage_type = field.type.storage_type + if target_with_ranges: + storage_type = pa.struct(list(storage_type)[:2]) + field = pa.field( + "image", + storage_type, + metadata={"ARROW:extension:name": "lance.blob.v2"}, + ) + images = pa.array([{"data": b"payload"}, None], type=storage_type) + if isinstance(field.type, pa.ExtensionType): + images = pa.ExtensionArray.from_storage(field.type, images) + data = pa.Table.from_arrays( + [pa.array([1, 2], type=pa.int64()), images], + schema=pa.schema([pa.field("id", pa.int64()), field]), + ) + + with mock_lancedb_connection(handler) as db: + remote_table = db.open_table("test") + table = remote_table._table if use_async else remote_table + if mode == "merge": + result = ( + table.merge_insert("id").when_not_matched_insert_all().execute(data) + ) + else: + result = table.add(data, mode="overwrite") + if use_async: + await result + + assert len(received) == 1 + images = received[0]["image"].combine_chunks() + if isinstance(images.type, pa.ExtensionType): + images = images.storage + with_ranges = target_with_ranges if mode == "merge" else not target_with_ranges + assert images.type.num_fields == (4 if with_ranges else 2) + expected = {"data": b"payload", "uri": None} + if with_ranges: + expected.update(position=None, size=None) + assert images.to_pylist() == [expected, None] + + def blob_query_response_table(): image_field = pa.field( "image", diff --git a/rust/lancedb/src/blob.rs b/rust/lancedb/src/blob.rs index ca68164e4..05650d7c2 100644 --- a/rust/lancedb/src/blob.rs +++ b/rust/lancedb/src/blob.rs @@ -244,8 +244,34 @@ impl BlobRangeRequest { /// Creates an Arrow field for a Lance blob v2 column. /// -/// `Struct` with the `lance.blob.v2` marker. Same layout Lance -/// expects on write. +/// `Struct` with nullable children and the +/// `lance.blob.v2` marker. Set both range fields to read a byte range from a +/// URI, or leave both null for a whole file. Inline bytes are also supported. +/// +/// # Migration +/// +/// This helper previously returned only `data` and `uri`. Rust callers building +/// a `StructArray` from this field must now supply four children: append typed +/// null UInt64 arrays for `position` and `size` when no range is needed: +/// +/// ``` +/// use std::sync::Arc; +/// use arrow_array::{ArrayRef, LargeBinaryArray, StringArray, StructArray, UInt64Array}; +/// use arrow_schema::DataType; +/// let field = lancedb::blob("image", true); +/// let DataType::Struct(children) = field.data_type() else { unreachable!() }; +/// let image = StructArray::new(children.clone(), vec![ +/// Arc::new(LargeBinaryArray::from(vec![Some(b"image".as_slice())])) as ArrayRef, +/// Arc::new(StringArray::from(vec![None::<&str>])), +/// Arc::new(UInt64Array::new_null(1)), +/// Arc::new(UInt64Array::new_null(1)), +/// ], None); +/// ``` +/// +/// Existing two-field tables remain writable: append and merge-insert fill +/// missing range fields or drop all-null range fields to match the table. +/// Non-null ranges cannot be written to a two-field table. Overwrite replaces +/// the table schema with the input schema rather than normalizing its layout. /// /// A blob column may be top-level or nested inside a struct or list. Blobs /// nested inside structs use dotted paths (e.g. `info.blob`) in the read APIs. @@ -261,9 +287,21 @@ impl BlobRangeRequest { /// ]); /// ``` pub fn blob(name: impl AsRef, nullable: bool) -> Field { - lance::blob::blob_field(name.as_ref(), nullable) + blob_with_options(name, nullable, BlobOptions::default()) } +static BLOB_STORAGE_TYPE: std::sync::LazyLock = std::sync::LazyLock::new(|| { + DataType::Struct( + vec![ + Field::new("data", DataType::LargeBinary, true), + Field::new("uri", DataType::Utf8, true), + Field::new("position", DataType::UInt64, true), + Field::new("size", DataType::UInt64, true), + ] + .into(), + ) +}); + /// Options for creating a blob v2 column field with tier thresholds. /// /// Re-exported from lance so the thresholds are built exactly the way @@ -293,6 +331,7 @@ pub use lance::blob::BlobFieldOptions as BlobOptions; /// ``` pub fn blob_with_options(name: impl AsRef, nullable: bool, options: BlobOptions) -> Field { lance::blob::blob_field_with_options(name.as_ref(), nullable, options) + .with_data_type(BLOB_STORAGE_TYPE.clone()) } /// Returns true if `field` is a blob v2 column. @@ -306,7 +345,7 @@ pub fn is_blob(field: &Field) -> bool { } /// Returns true if `field`, or any field nested under it, is a blob v2 column. -fn field_tree_has_blob_v2(field: &Field) -> bool { +pub(crate) fn field_tree_has_blob_v2(field: &Field) -> bool { if field.is_blob_v2() { return true; } @@ -673,7 +712,21 @@ mod tests { field.metadata().get(ARROW_EXT_NAME_KEY).map(String::as_str), Some("lance.blob.v2") ); - assert!(matches!(field.data_type(), DataType::Struct(_))); + let DataType::Struct(children) = field.data_type() else { + panic!("expected struct") + }; + assert_eq!( + children + .iter() + .map(|f| (f.name().as_str(), f.data_type(), f.is_nullable())) + .collect::>(), + vec![ + ("data", &DataType::LargeBinary, true), + ("uri", &DataType::Utf8, true), + ("position", &DataType::UInt64, true), + ("size", &DataType::UInt64, true), + ] + ); } #[test] diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index d21583c91..272ac83f7 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -3154,6 +3154,17 @@ impl BaseTable for RemoteTable { let rescannable = source.rescannable(); let input: Arc = Arc::new(crate::table::datafusion::scannable_exec::ScannableExec::new(source, None)); + let table_schema = self.schema().await?; + let input = if crate::blob::has_blob_columns(&table_schema) { + crate::table::merge::validate_merge_source_fields( + input.schema().fields(), + table_schema.fields(), + "", + )?; + crate::table::datafusion::cast::cast_to_table_schema(input, &table_schema)? + } else { + input + }; let freshness_request = self.snapshot_freshness_headers(); let mut merge: Arc = Arc::new( @@ -4499,6 +4510,253 @@ mod tests { assert!(image.column_by_name("data").unwrap().is_null(0)); } + #[tokio::test] + async fn add_external_blob_ranges_preserves_remote_wire_values() { + let field = crate::blob("image", true); + let DataType::Struct(children) = field.data_type().clone() else { + unreachable!(); + }; + let schema = Arc::new(Schema::new(vec![field])); + // Registered external bases are enforced by the server, so remote + // clients send the URI and range without the local-only opt-in. + let uri = "s3://registered/payload.bin"; + let image = StructArray::new( + children, + vec![ + arrow_array::new_null_array(&DataType::LargeBinary, 2), + Arc::new(StringArray::from(vec![uri, uri])), + Arc::new(arrow_array::UInt64Array::from(vec![0, 4])), + Arc::new(arrow_array::UInt64Array::from(vec![6, 6])), + ], + None, + ); + let input = RecordBatch::try_new(schema.clone(), vec![Arc::new(image)]).unwrap(); + let describe_body = describe_response(&schema).into_bytes(); + let payload = b"0123456789abcdef"; + let fetched = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "image", + DataType::LargeBinary, + true, + )])), + vec![Arc::new(LargeBinaryArray::from(vec![ + &payload[..6], + &payload[4..10], + ]))], + ) + .unwrap(); + let fetched_body = write_ipc_stream_uncompressed(&fetched); + let (sender, receiver) = std::sync::mpsc::channel(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |mut request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_body.clone()) + .unwrap(), + "/v1/table/my_table/insert/" => { + sender.send(request.body_mut().take().unwrap()).unwrap(); + http::Response::builder() + .status(200) + .body(br#"{"version": 2}"#.to_vec()) + .unwrap() + } + "/v1/table/my_table/fetch_blobs/" => { + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()) + .unwrap(); + assert_eq!(body["column"], "image"); + assert_eq!(body["row_ids"], json!([0, 1])); + http::Response::builder() + .status(200) + .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE) + .body(fetched_body.clone()) + .unwrap() + } + path => panic!("unexpected path: {path}"), + }, + ); + table.add(input).execute().await.unwrap(); + let body = collect_body(receiver.recv().unwrap()).await; + let batch = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(body), None) + .unwrap() + .next() + .unwrap() + .unwrap(); + assert_eq!(batch.schema(), schema); + let image = batch.column(0).as_struct(); + let positions = image + .column(2) + .as_primitive::(); + let sizes = image + .column(3) + .as_primitive::(); + assert_eq!(positions.values().as_ref(), &[0, 4]); + assert_eq!(sizes.values().as_ref(), &[6, 6]); + assert_eq!(image.column(1).as_string::().value(0), uri); + assert_eq!(image.column(1).as_string::().value(1), uri); + let bytes = table.fetch_blobs("image", &[0, 1]).await.unwrap(); + assert_eq!(bytes.value(0), &payload[..6]); + assert_eq!(bytes.value(1), &payload[4..10]); + } + + #[tokio::test] + async fn remote_blob_layout_normalization_and_overwrite() { + use arrow_array::{ArrayRef, UInt64Array}; + for wide_target in [false, true] { + let field = |wide| { + if wide { + crate::blob("image", true) + } else { + lance::blob::blob_field("image", true) + } + }; + let target = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + field(wide_target), + ]); + for operation in ["add", "merge", "overwrite"] { + for range in [ + None, + Some((Some(4), None)), + Some((None, Some(6))), + Some((Some(4), Some(6))), + ] { + if range.is_some() && (wide_target || operation == "overwrite") { + continue; + } + // Range input always uses the full layout. Otherwise use the + // opposite layout to exercise both normalization directions. + let wide_input = range.is_some() || !wide_target; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + field(wide_input), + ])); + let DataType::Struct(children) = schema.field(1).data_type().clone() else { + unreachable!() + }; + let mut arrays: Vec = vec![ + Arc::new(LargeBinaryArray::from(vec![Some(b"inline".as_slice())])), + Arc::new(StringArray::from(vec![None::<&str>])), + ]; + if wide_input { + arrays.push(Arc::new(UInt64Array::from(vec![range.and_then(|r| r.0)]))); + arrays.push(Arc::new(UInt64Array::from(vec![range.and_then(|r| r.1)]))); + } + let image = Arc::new(StructArray::new(children, arrays, None)); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from(vec![1])), image], + ) + .unwrap(); + let describe = describe_response(&target); + let (sender, receiver) = std::sync::mpsc::channel(); + let table = Table::new_with_handler("my_table", move |mut request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(describe.clone()) + .unwrap(); + } + assert!( + request.url().path().ends_with("/insert/") + || request.url().path().ends_with("/merge_insert/") + ); + if operation == "overwrite" { + assert!( + request + .url() + .query_pairs() + .any(|(k, v)| k == "mode" && v == "overwrite") + ); + } + sender.send(request.body_mut().take().unwrap()).unwrap(); + http::Response::builder() + .status(200) + .body(r#"{"version": 2, "num_inserted_rows": 1}"#.to_string()) + .unwrap() + }); + // A bad later batch must abort the request, including after + // an earlier valid batch was encoded. No write handler runs. + let mut batches = Vec::new(); + if range.is_some() { + let empty_range_arrays: Vec = vec![ + Arc::new(LargeBinaryArray::from(vec![Some(b"first".as_slice())])), + Arc::new(StringArray::from(vec![None::<&str>])), + Arc::new(UInt64Array::new_null(1)), + Arc::new(UInt64Array::new_null(1)), + ]; + let DataType::Struct(children) = schema.field(1).data_type().clone() else { + unreachable!() + }; + batches.push(Ok(RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![0])), + Arc::new(StructArray::new(children, empty_range_arrays, None)), + ], + ) + .unwrap())); + } + batches.push(Ok(batch)); + let reader: Box = + Box::new(RecordBatchIterator::new(batches, schema.clone())); + let result = match operation { + "merge" => { + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(reader).await.map(|_| ()) + } + "overwrite" => table + .add(reader) + .mode(AddDataMode::Overwrite) + .execute() + .await + .map(|_| ()), + _ => table.add(reader).execute().await.map(|_| ()), + }; + if !wide_target && range.is_some() && operation != "overwrite" { + assert!( + result + .unwrap_err() + .to_string() + .contains("cannot store byte ranges") + ); + assert!( + receiver.try_recv().is_err(), + "invalid stream must not reach a committed write" + ); + continue; + } + result.unwrap(); + let body = collect_body(receiver.recv().unwrap()).await; + let reader = StreamReader::try_new(Cursor::new(body), None).unwrap(); + let expected = if operation == "overwrite" { + schema.as_ref() + } else { + &target + }; + assert_eq!(reader.schema().as_ref(), expected); + for batch in reader { + let batch = batch.unwrap(); + let image = batch.column(1).as_struct(); + assert!( + image.column(0).as_binary::().value(0) == b"inline" + || image.column(0).as_binary::().value(0) == b"first" + ); + if image.num_columns() == 4 && range.is_none() { + assert!(image.column(2).is_null(0)); + assert!(image.column(3).is_null(0)); + } + } + } + } + } + } + #[rstest] #[case(true)] #[case(false)] @@ -4852,6 +5110,12 @@ mod tests { )); let table = Table::new_with_handler("my_table", move |request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(r#"{"version": 1, "schema": {"fields": []}}"#) + .unwrap(); + } if request.url().path() == "/v1/table/my_table/merge_insert/" { assert_eq!(request.method(), "POST"); @@ -4908,6 +5172,12 @@ mod tests { )); let table = Table::new_with_handler("my_table", move |request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(r#"{"version": 1, "schema": {"fields": []}}"#) + .unwrap(); + } assert_eq!(request.method(), "POST"); assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/"); let params = request.url().query_pairs().collect::>(); @@ -4993,6 +5263,12 @@ mod tests { let num_deleted_rows = if delete_unmatched { 3 } else { 0 }; let table = Table::new_with_handler("my_table", move |request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(describe_response(&Schema::empty())) + .unwrap(); + } assert_eq!(request.method(), "POST"); assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/"); assert_eq!(request.headers()[CONTENT_TYPE], ARROW_STREAM_CONTENT_TYPE); @@ -5063,6 +5339,12 @@ mod tests { )); let table = Table::new_with_handler("my_table", move |request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(r#"{"version": 1, "schema": {"fields": []}}"#) + .unwrap(); + } assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/"); // One repeated `on` per column, in the order the caller gave them. @@ -5134,6 +5416,12 @@ mod tests { // Default parameters let table = Table::new_with_handler("my_table", |request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(r#"{"version": 1, "schema": {"fields": []}}"#) + .unwrap(); + } assert_eq!(request.method(), "POST"); assert_eq!(request.url().path(), "/v1/table/my_table/merge_insert/"); @@ -14405,6 +14693,20 @@ mod tests { )); let table = Table::new_with_handler("my_table", move |request| { match request.url().path() { + "/v1/table/my_table/describe/" => { + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()) + .unwrap(); + assert_eq!(body["branch"], "exp"); + http::Response::builder() + .status(200) + .body(describe_response(&Schema::new(vec![Field::new( + "id", + DataType::Int32, + false, + )]))) + .unwrap() + } "/v1/table/my_table/branches/create/" => http::Response::builder() .status(200) .body("{}".to_string()) diff --git a/rust/lancedb/src/table/add_data.rs b/rust/lancedb/src/table/add_data.rs index c82b85971..12404ec73 100644 --- a/rust/lancedb/src/table/add_data.rs +++ b/rust/lancedb/src/table/add_data.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +use std::collections::HashSet; use std::sync::Arc; -use arrow_schema::{DataType, Fields, Schema}; +use arrow_schema::{DataType, Field, Fields, Schema}; use lance::dataset::WriteMode; +use lance_arrow::FieldExt; use serde::{Deserialize, Serialize}; use crate::data::scannable::Scannable; @@ -233,6 +235,7 @@ pub struct PreprocessingOutput { /// Fields can be in different orders, so match by name. /// /// If a column exists in input but not in table, error (no extra columns allowed). +/// Input field names must be unique within each struct. /// /// If a column exists in table but not in input, that is okay - it may be filled with nulls. /// @@ -244,7 +247,13 @@ fn validate_schema(input: &Schema, table: &Schema) -> Result<()> { } fn validate_fields(input: &Fields, table: &Fields) -> Result<()> { + let mut names = HashSet::with_capacity(input.len()); for field in input { + if !names.insert(field.name()) { + return Err(Error::InvalidInput { + message: format!("field '{}' is specified more than once", field.name()), + }); + } match table.iter().find(|f| f.name() == field.name()) { None => { return Err(Error::InvalidInput { @@ -252,17 +261,33 @@ fn validate_fields(input: &Fields, table: &Fields) -> Result<()> { }); } Some(table_field) => { - if let (DataType::Struct(in_children), DataType::Struct(tbl_children)) = - (field.data_type(), table_field.data_type()) - { - validate_fields(in_children, tbl_children)?; - } + validate_field(field, table_field)?; } } } Ok(()) } +fn validate_field(input: &Field, table: &Field) -> Result<()> { + // Blob coercion validates its children and permits dropping only + // all-null range fields when targeting a legacy two-field layout. + if table.is_blob_v2() { + return Ok(()); + } + match (input.data_type(), table.data_type()) { + (DataType::Struct(input), DataType::Struct(table)) => validate_fields(input, table), + ( + DataType::List(input) | DataType::LargeList(input) | DataType::FixedSizeList(input, _), + DataType::List(table) | DataType::LargeList(table) | DataType::FixedSizeList(table, _), + ) => { + // Preserve the item's extension metadata, and match nested struct + // children by name rather than comparing collection item names. + validate_field(input, table) + } + _ => Ok(()), + } +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -299,6 +324,81 @@ mod tests { conn.create_table("test", batch).execute().await.unwrap() } + #[rstest::rstest] + fn test_validate_append_collection_fields( + #[values("list", "large_list", "fixed_size_list")] source_kind: &str, + #[values("list", "large_list", "fixed_size_list")] target_kind: &str, + ) { + fn schema(kind: &str, item: Field) -> Schema { + let item = Arc::new(item); + let data_type = match kind { + "list" => DataType::List(item), + "large_list" => DataType::LargeList(item), + "fixed_size_list" => DataType::FixedSizeList(item, 1), + _ => unreachable!(), + }; + Schema::new(vec![Field::new("items", data_type, true)]) + } + let image = crate::blob("image", true); + let legacy_image = lance::blob::blob_field("image", true); + let caption = Field::new("caption", DataType::Utf8, true); + let input = schema( + source_kind, + Field::new( + "source_item", + DataType::Struct(vec![image.clone(), caption.clone()].into()), + true, + ), + ); + let target = schema( + target_kind, + Field::new( + "target_item", + DataType::Struct(vec![legacy_image.clone()].into()), + true, + ), + ); + let error = super::validate_schema(&input, &target) + .unwrap_err() + .to_string(); + assert!( + error.contains("caption") && error.contains("does not exist"), + "{error}" + ); + + let target = schema( + target_kind, + Field::new( + "target_item", + DataType::Struct(vec![caption.clone(), legacy_image.clone()].into()), + true, + ), + ); + // Declared siblings can be reordered, but must not occur more than once. + super::validate_schema(&input, &target).unwrap(); + let duplicate = schema( + source_kind, + Field::new( + "source_item", + DataType::Struct(vec![image.clone(), caption.clone(), caption].into()), + true, + ), + ); + let error = super::validate_schema(&duplicate, &target) + .unwrap_err() + .to_string(); + assert!( + error.contains("caption") && error.contains("specified more than once"), + "{error}" + ); + // Item names need not match, and blob children still reach range coercion. + super::validate_schema( + &schema(source_kind, image.with_name("source_item")), + &schema(target_kind, legacy_image.with_name("target_item")), + ) + .unwrap(); + } + async fn test_add_with_data(data: T) where T: Scannable + 'static, diff --git a/rust/lancedb/src/table/datafusion/blob_coerce.rs b/rust/lancedb/src/table/datafusion/blob_coerce.rs index 80b6cd6fc..07a200d16 100644 --- a/rust/lancedb/src/table/datafusion/blob_coerce.rs +++ b/rust/lancedb/src/table/datafusion/blob_coerce.rs @@ -5,6 +5,7 @@ //! //! [`super::cast::cast_to_table_schema`] calls [`coerce_blob_expr`]. +use std::collections::HashSet; use std::fmt; use std::hash::{Hash, Hasher}; use std::sync::Arc; @@ -12,9 +13,9 @@ use std::sync::Arc; use arrow_array::{ Array, ArrayRef, BooleanArray, FixedSizeListArray, LargeListArray, ListArray, RecordBatch, }; -use arrow_buffer::OffsetBuffer; +use arrow_buffer::{BooleanBuffer, NullBuffer, NullBufferBuilder, OffsetBuffer}; use arrow_cast::cast; -use arrow_schema::{DataType, Field, FieldRef, Fields, Schema}; +use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use arrow_select::nullif::nullif; use datafusion::functions::core::{get_field, named_struct}; use datafusion_common::config::ConfigOptions; @@ -24,7 +25,6 @@ use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions::{CastExpr, Literal}; use datafusion_physical_plan::PhysicalExpr; use datafusion_physical_plan::expressions::Column; -use lance_arrow::FieldExt; use crate::error::{Error, Result}; @@ -35,6 +35,7 @@ pub(super) fn coerce_blob_expr( input_field: &Field, table_field: &FieldRef, config: &Arc, + ancestors: &[Arc], ) -> Result<(Arc, FieldRef)> { let DataType::Struct(declared_fields) = table_field.data_type() else { return Err(Error::InvalidInput { @@ -69,6 +70,33 @@ pub(super) fn coerce_blob_expr( ), }); } + let mut names = HashSet::with_capacity(children.len()); + for child in children { + // Name-based extraction must not discard an ambiguous payload + // or silently ignore a duplicate range child during narrowing. + if !names.insert(child.name()) { + return Err(Error::InvalidInput { + message: format!( + "blob child '{}' in column '{}' is specified more than once", + child.name(), + table_field.name() + ), + }); + } + if !declared_fields + .iter() + .any(|field| field.name() == child.name()) + && !matches!(child.name().as_str(), "position" | "size") + { + return Err(Error::InvalidInput { + message: format!( + "unknown blob child '{}' in column '{}'", + child.name(), + table_field.name() + ), + }); + } + } BlobInputShape::Struct(children) } other => { @@ -155,22 +183,38 @@ pub(super) fn coerce_blob_expr( // `named_struct` always yields a valid struct, so a null input would land // as a row that set neither `data` nor `uri` -- not an absent blob but a // malformed one, which Lance rejects on write. + let dropped_ranges = match &input_shape { + BlobInputShape::Struct(children) => children + .iter() + .enumerate() + .filter(|(_, child)| { + matches!(child.name().as_str(), "position" | "size") + && !declared_fields + .iter() + .any(|field| field.name() == child.name()) + }) + .map(|(index, _)| index) + .collect(), + _ => Vec::new(), + }; let expr: Arc = Arc::new(AbsentBlobIsNull { source: input_expr, built, field: table_field.clone(), + dropped_ranges, + ancestors: ancestors.to_vec(), }); Ok((expr, table_field.clone())) } -/// Apply the normal blob coercion to list values while retaining the list's -/// offsets and validity. A child batch lets the existing blob expression handle -/// binary, null, and descriptor inputs exactly as it does for top-level fields. +/// Recurse through list items while retaining the list's offsets and validity. +/// A child batch reuses the field projection, including nested structs and lists, +/// so every blob leaf receives the same coercion as a top-level blob column. pub(super) fn coerce_blob_list_expr( input_expr: Arc, input_field: &Field, table_field: &FieldRef, - config: &Arc, + ancestors: &[Arc], ) -> Result, FieldRef)>> { let (DataType::List(input_item) | DataType::LargeList(input_item) @@ -184,23 +228,45 @@ pub(super) fn coerce_blob_list_expr( else { return Ok(None); }; - if !table_item.is_blob_v2() || input_item == table_item { + if !crate::blob::field_tree_has_blob_v2(table_item) || input_item == table_item { return Ok(None); } - let (item_expr, _) = coerce_blob_expr( - Arc::new(Column::new(input_item.name(), 0)), - input_item, - table_item, - config, - )?; + // List item names need not match across SDKs. Their position identifies the + // item, while named children inside a struct still match by name. + let input_fields = vec![Arc::new( + input_item.as_ref().clone().with_name(table_item.name()), + )] + .into(); + let validity_field = Field::new("__blob_parent_validity", DataType::Boolean, true); + let (item_expr, output_item) = super::cast::build_field_exprs( + &input_fields, + &vec![table_item.clone()].into(), + &|_| Arc::new(Column::new(input_item.name(), 0)), + &[Arc::new(Column::new(validity_field.name(), 1))], + )? + .pop() + .expect("one list item projection"); + let item_schema = Arc::new(Schema::new(vec![ + input_item.clone(), + Arc::new(validity_field), + ])); + let output_type = match table_field.data_type() { + DataType::List(_) => DataType::List(output_item), + DataType::LargeList(_) => DataType::LargeList(output_item), + DataType::FixedSizeList(_, size) => DataType::FixedSizeList(output_item, *size), + _ => unreachable!(), + }; + let output_field = Arc::new(table_field.as_ref().clone().with_data_type(output_type)); let expr: Arc = Arc::new(CoerceBlobList { source: input_expr, input_item: input_item.clone(), item_expr, - field: table_field.clone(), + field: output_field.clone(), + ancestors: ancestors.to_vec(), + item_schema, }); - Ok(Some((expr, table_field.clone()))) + Ok(Some((expr, output_field))) } #[derive(Debug, Clone)] @@ -209,6 +275,8 @@ struct CoerceBlobList { input_item: FieldRef, item_expr: Arc, field: FieldRef, + ancestors: Vec>, + item_schema: SchemaRef, } impl fmt::Display for CoerceBlobList { @@ -223,6 +291,7 @@ impl PartialEq for CoerceBlobList { && self.input_item == other.input_item && self.item_expr.eq(&other.item_expr) && self.field == other.field + && self.ancestors == other.ancestors } } @@ -234,6 +303,7 @@ impl Hash for CoerceBlobList { self.input_item.hash(state); self.item_expr.hash(state); self.field.hash(state); + self.ancestors.hash(state); } } @@ -263,34 +333,56 @@ impl PhysicalExpr for CoerceBlobList { } else { cast(source.as_ref(), &intermediate_type)? }; - let values = match source.data_type() { + let row_nulls = NullBuffer::union( + source.logical_nulls().as_ref(), + ancestor_nulls(&self.ancestors, batch)?.as_ref(), + ); + let (values, item_nulls) = match source.data_type() { DataType::List(_) => { let list = source.as_any().downcast_ref::().unwrap(); let first = list.offsets()[0] as usize; let last = *list.offsets().last().unwrap() as usize; - list.values().slice(first, last - first) + ( + list.values().slice(first, last - first), + expand_list_nulls(row_nulls.as_ref(), list.offsets().lengths(), last - first), + ) } DataType::LargeList(_) => { let list = source.as_any().downcast_ref::().unwrap(); let first = list.offsets()[0] as usize; let last = *list.offsets().last().unwrap() as usize; - list.values().slice(first, last - first) + ( + list.values().slice(first, last - first), + expand_list_nulls(row_nulls.as_ref(), list.offsets().lengths(), last - first), + ) } - DataType::FixedSizeList(_, _) => source - .as_any() - .downcast_ref::() - .unwrap() - .values() - .clone(), + DataType::FixedSizeList(_, size) => ( + source + .as_any() + .downcast_ref::() + .unwrap() + .values() + .clone(), + row_nulls.as_ref().map(|nulls| nulls.expand(*size as usize)), + ), other => { return Err(DataFusionError::Internal(format!( "coerce_blob_list expected a list, got {other}" ))); } }; + // Carry list/ancestor validity in a separate column. Applying it to the + // items themselves would introduce nulls into non-nullable child fields. + let validity = BooleanArray::new( + item_nulls + .as_ref() + .map(|nulls| nulls.inner().clone()) + .unwrap_or_else(|| BooleanBuffer::new_set(values.len())), + item_nulls, + ); let item_batch = RecordBatch::try_new( - Arc::new(Schema::new(vec![self.input_item.clone()])), - vec![values.clone()], + self.item_schema.clone(), + vec![values.clone(), Arc::new(validity)], )?; let coerced = self .item_expr @@ -335,7 +427,9 @@ impl PhysicalExpr for CoerceBlobList { } fn children(&self) -> Vec<&Arc> { - vec![&self.source] + std::iter::once(&self.source) + .chain(self.ancestors.iter()) + .collect() } fn with_new_children( @@ -347,6 +441,8 @@ impl PhysicalExpr for CoerceBlobList { input_item: self.input_item.clone(), item_expr: self.item_expr.clone(), field: self.field.clone(), + ancestors: children[1..].to_vec(), + item_schema: self.item_schema.clone(), })) } @@ -355,6 +451,35 @@ impl PhysicalExpr for CoerceBlobList { } } +fn ancestor_nulls( + ancestors: &[Arc], + batch: &RecordBatch, +) -> datafusion_common::Result> { + let mut nulls = None; + for ancestor in ancestors { + let array = ancestor.evaluate(batch)?.into_array(batch.num_rows())?; + nulls = NullBuffer::union(nulls.as_ref(), array.logical_nulls().as_ref()); + } + Ok(nulls) +} + +fn expand_list_nulls( + nulls: Option<&NullBuffer>, + lengths: impl Iterator, + item_count: usize, +) -> Option { + let nulls = nulls?; + let mut builder = NullBufferBuilder::new(item_count); + for (length, valid) in lengths.zip(nulls.iter()) { + if valid { + builder.append_n_non_nulls(length); + } else { + builder.append_n_nulls(length); + } + } + builder.finish() +} + /// Carries the source column's nullity onto the struct built for it. /// /// This is its own expression rather than a `CASE` because the projection @@ -366,6 +491,8 @@ struct AbsentBlobIsNull { source: Arc, built: Arc, field: FieldRef, + dropped_ranges: Vec, + ancestors: Vec>, } impl fmt::Display for AbsentBlobIsNull { @@ -376,7 +503,11 @@ impl fmt::Display for AbsentBlobIsNull { impl PartialEq for AbsentBlobIsNull { fn eq(&self, other: &Self) -> bool { - self.source.eq(&other.source) && self.built.eq(&other.built) && self.field == other.field + self.source.eq(&other.source) + && self.built.eq(&other.built) + && self.field == other.field + && self.dropped_ranges == other.dropped_ranges + && self.ancestors == other.ancestors } } @@ -387,6 +518,8 @@ impl Hash for AbsentBlobIsNull { self.source.hash(state); self.built.hash(state); self.field.hash(state); + self.dropped_ranges.hash(state); + self.ancestors.hash(state); } } @@ -401,8 +534,30 @@ impl PhysicalExpr for AbsentBlobIsNull { fn evaluate(&self, batch: &RecordBatch) -> datafusion_common::Result { let rows = batch.num_rows(); - let built = self.built.evaluate(batch)?.into_array(rows)?; let source = self.source.evaluate(batch)?.into_array(rows)?; + if !self.dropped_ranges.is_empty() { + let ancestor_nulls = ancestor_nulls(&self.ancestors, batch)?; + let values = source + .as_any() + .downcast_ref::() + .unwrap(); + let effective_nulls = NullBuffer::union(values.nulls(), ancestor_nulls.as_ref()); + for index in &self.dropped_ranges { + let nulls = NullBuffer::union( + effective_nulls.as_ref(), + values.column(*index).logical_nulls().as_ref(), + ); + if nulls.as_ref().map_or(values.len() != 0, |nulls| { + nulls.null_count() != values.len() + }) { + return Err(DataFusionError::Execution(format!( + "blob storage schema cannot store byte ranges in column '{}': position and size must both be null", + self.field.name() + ))); + } + } + } + let built = self.built.evaluate(batch)?.into_array(rows)?; let Some(nulls) = source.logical_nulls() else { return Ok(ColumnarValue::Array(built)); }; @@ -413,7 +568,10 @@ impl PhysicalExpr for AbsentBlobIsNull { } fn children(&self) -> Vec<&Arc> { - vec![&self.source, &self.built] + [&self.source, &self.built] + .into_iter() + .chain(self.ancestors.iter()) + .collect() } fn with_new_children( @@ -424,6 +582,8 @@ impl PhysicalExpr for AbsentBlobIsNull { source: children[0].clone(), built: children[1].clone(), field: self.field.clone(), + dropped_ranges: self.dropped_ranges.clone(), + ancestors: children[2..].to_vec(), })) } @@ -449,7 +609,6 @@ fn typed_null(data_type: &DataType) -> Result> { mod tests { use super::super::cast::cast_to_table_schema; use super::*; - use crate::blob::blob; use arrow_array::{ Array, ArrayRef, BinaryArray, BinaryViewArray, FixedSizeListArray, Int32Array, Int64Array, LargeBinaryArray, LargeListArray, ListArray, NullArray, RecordBatch, StringArray, @@ -461,6 +620,7 @@ mod tests { use datafusion_catalog::MemTable; use datafusion_physical_plan::ExecutionPlan; use futures::TryStreamExt; + use lance::blob::blob_field as blob; use lance_arrow::FieldExt; use std::collections::HashMap; @@ -946,6 +1106,47 @@ mod tests { assert!(image.column_by_name("data").unwrap().is_null(0)); } + #[tokio::test] + async fn dropping_ranges_respects_parent_validity_and_slices() { + for parent_valid in [false, true] { + let DataType::Struct(children) = wide_blob_field("image").data_type().clone() else { + unreachable!() + }; + let input = StructArray::new( + children, + vec![ + Arc::new(LargeBinaryArray::from(vec![ + Some(b"ignored".as_slice()), + Some(b"payload".as_slice()), + ])), + Arc::new(StringArray::from(vec![None::<&str>, None])), + Arc::new(UInt64Array::from(vec![None, Some(4)])), + Arc::new(UInt64Array::new_null(2)), + ], + Some(NullBuffer::from(vec![true, parent_valid])), + ) + .slice(1, 1); + let batch = batch_with_image( + Field::new("image", input.data_type().clone(), true), + Arc::new(input), + ); + if parent_valid { + let plan = cast_to_table_schema(plan_from_batch(batch).await, &blob_table_schema()) + .unwrap(); + let error = plan + .execute(0, SessionContext::new().task_ctx()) + .unwrap() + .try_collect::>() + .await + .unwrap_err(); + assert!(error.to_string().contains("cannot store byte ranges")); + } else { + let output = coerce(batch, &blob_table_schema()).await; + assert!(image_struct(&output).is_null(0)); + } + } + } + #[tokio::test] async fn descriptor_struct_without_value_child_is_rejected() { let descriptor = StructArray::new( @@ -1094,4 +1295,890 @@ mod tests { let plan = cast_to_table_schema(input, &table_schema).unwrap(); assert_eq!(Arc::as_ptr(&plan), input_ptr, "no projection inserted"); } + + fn nested_layout_batch( + wide: bool, + id: i64, + range: Option, + nesting: &[&str], + ) -> RecordBatch { + nested_layout_batch_with_null(wide, id, range, nesting, None) + } + + fn nested_layout_batch_with_null( + wide: bool, + id: i64, + range: Option, + nesting: &[&str], + null_depth: Option, + ) -> RecordBatch { + nested_layout_batch_with_options(wide, id, range, nesting, null_depth, &[]) + } + + fn nested_layout_batch_with_options( + wide: bool, + id: i64, + range: Option, + nesting: &[&str], + null_depth: Option, + captions: &[&str], + ) -> RecordBatch { + // Ancestor validity must not introduce nulls into a required blob child. + let field = if wide { + crate::blob("image", false) + } else { + blob("image", false) + }; + let DataType::Struct(children) = field.data_type().clone() else { + unreachable!() + }; + let mut columns: Vec = vec![ + Arc::new(LargeBinaryArray::from(vec![Some(b"payload".as_slice())])), + Arc::new(StringArray::from(vec![None::<&str>])), + ]; + if wide { + columns.push(Arc::new(UInt64Array::from(vec![range]))); + columns.push(Arc::new(UInt64Array::from(vec![range]))); + } + let array: ArrayRef = Arc::new(StructArray::new(children, columns, None)); + nest_blob_batch(field, array, id, nesting, null_depth, captions) + } + + fn nest_blob_batch( + mut field: Field, + mut array: ArrayRef, + id: i64, + nesting: &[&str], + null_depth: Option, + captions: &[&str], + ) -> RecordBatch { + for (depth, kind) in nesting.iter().enumerate().rev() { + let nulls = (null_depth == Some(depth)).then(|| NullBuffer::from(vec![false])); + let data_type = match *kind { + "struct" => { + let mut fields = vec![Arc::new(field)]; + let mut columns = vec![array]; + if depth + 1 == nesting.len() { + for caption in captions { + fields.push(Arc::new(Field::new("caption", DataType::Utf8, true))); + columns.push(Arc::new(StringArray::from(vec![*caption]))); + } + } + let fields: Fields = fields.into(); + array = Arc::new(StructArray::new(fields.clone(), columns, nulls)); + DataType::Struct(fields) + } + "list" => { + let item = Arc::new(field.with_name("item")); + array = Arc::new(ListArray::new( + item.clone(), + OffsetBuffer::new(vec![0, 1].into()), + array, + nulls, + )); + DataType::List(item) + } + "large_list" => { + let item = Arc::new(field.with_name("item")); + array = Arc::new(LargeListArray::new( + item.clone(), + OffsetBuffer::new(vec![0_i64, 1].into()), + array, + nulls, + )); + DataType::LargeList(item) + } + "fixed_size_list" => { + let item = Arc::new(field.with_name("item")); + array = Arc::new(FixedSizeListArray::new(item.clone(), 1, array, nulls)); + DataType::FixedSizeList(item, 1) + } + _ => unreachable!(), + }; + field = Field::new("value", data_type, true); + } + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + field, + ])), + vec![Arc::new(Int64Array::from(vec![id])), array], + ) + .unwrap() + } + + fn duplicate_blob_child_batch(wide: bool, nesting: &[&str], child_index: usize) -> RecordBatch { + let field = if wide { + crate::blob("image", false) + } else { + blob("image", false) + }; + let DataType::Struct(children) = field.data_type() else { + unreachable!() + }; + let duplicate = children[child_index].clone(); + let mut fields = children.to_vec(); + fields.push(duplicate.clone()); + let mut columns = children + .iter() + .map(|child| arrow_array::new_null_array(child.data_type(), 1)) + .collect::>(); + columns[0] = Arc::new(LargeBinaryArray::from(vec![b"first".as_slice()])); + // Distinct duplicate values expose silent name-based selection. Range + // duplicates also must not bypass null-only narrowing to old layouts. + columns.push(match duplicate.name().as_str() { + "data" => Arc::new(LargeBinaryArray::from(vec![b"second".as_slice()])) as ArrayRef, + "uri" => Arc::new(StringArray::from(vec!["file:///discarded"])), + "position" | "size" => Arc::new(UInt64Array::from(vec![4])), + _ => unreachable!(), + }); + let fields: Fields = fields.into(); + let array = Arc::new(StructArray::new(fields.clone(), columns, None)); + let field = field.with_data_type(DataType::Struct(fields)); + nest_blob_batch(field, array, 1, nesting, None, &[]) + } + + #[rstest::rstest] + #[case(&[])] + #[case(&["struct"])] + #[case(&["list"])] + #[case(&["list", "struct"])] + #[case(&["list", "list", "struct"])] + #[case(&["struct", "list", "struct"])] + #[case(&["large_list", "struct"])] + #[case(&["fixed_size_list", "struct", "large_list"])] + #[tokio::test] + async fn projection_rejects_duplicate_blob_children( + #[case] nesting: &[&str], + #[values(false, true)] wide_source: bool, + #[values(false, true)] wide_target: bool, + ) { + let target = nested_layout_batch(wide_target, 0, None, nesting); + for child_index in 0..if wide_source { 4 } else { 2 } { + let input = duplicate_blob_child_batch(wide_source, nesting, child_index); + let error = coerce_err(input, target.schema().as_ref()) + .await + .to_string(); + assert!(error.contains("specified more than once"), "{error}"); + } + } + + #[cfg(feature = "remote")] + #[rstest::rstest] + #[case(&[])] + #[case(&["struct"])] + #[case(&["list"])] + #[case(&["list", "struct"])] + #[case(&["list", "list", "struct"])] + #[case(&["struct", "list", "struct"])] + #[case(&["large_list", "struct"])] + #[tokio::test] + async fn writes_reject_duplicate_blob_children( + #[case] nesting: &[&str], + #[values(false, true)] wide_source: bool, + #[values(false, true)] wide_target: bool, + #[values(false, true)] remote: bool, + #[values(false, true)] merge: bool, + ) { + use crate::query::ExecutableQuery; + + let initial = nested_layout_batch(wide_target, 0, None, nesting); + let schema = initial.schema(); + let db = crate::connect("memory://").execute().await.unwrap(); + let (table, receiver) = if remote { + let (table, receiver) = remote_layout_table(schema.as_ref()); + (table, Some(receiver)) + } else { + ( + db.create_table("descriptors", initial) + .execute() + .await + .unwrap(), + None, + ) + }; + let snapshot = if remote { + None + } else { + Some(( + table.version().await.unwrap(), + table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(), + )) + }; + for child_index in 0..if wide_source { 4 } else { 2 } { + let input = duplicate_blob_child_batch(wide_source, nesting, child_index); + let result = if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let schema = input.schema(); + builder + .execute(Box::new(arrow_array::RecordBatchIterator::new( + vec![Ok(input)], + schema, + ))) + .await + .map(|_| ()) + } else { + table.add(input).execute().await.map(|_| ()) + }; + let error = result.unwrap_err().to_string(); + assert!(error.contains("specified more than once"), "{error}"); + if let Some(receiver) = &receiver { + assert!( + receiver.try_recv().is_err(), + "ambiguous write reached server" + ); + } else { + let (version, before) = snapshot.as_ref().unwrap(); + assert_eq!(table.version().await.unwrap(), *version); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + assert_eq!(table.schema().await.unwrap(), schema); + let batches = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(&batches, before); + } + } + } + + #[rstest::rstest] + #[case(&["struct"])] + #[case(&["struct", "struct"])] + #[case(&["list", "struct"])] + #[case(&["struct", "list", "struct"])] + #[case(&["large_list", "struct", "list"])] + #[case(&["fixed_size_list", "struct", "large_list"])] + #[tokio::test] + async fn nullable_nested_blob_layout_projection( + #[case] nesting: &[&str], + #[values(false, true)] wide_target: bool, + #[values(false, true)] sliced: bool, + ) { + for depth in 0..nesting.len() { + // Physical range slots beneath a null ancestor are not logical values. + let input = + nested_layout_batch_with_null(!wide_target, 1, Some(0), nesting, Some(depth)); + let input = if sliced { + let outside = nested_layout_batch(!wide_target, 0, Some(4), nesting); + arrow_select::concat::concat_batches( + &input.schema(), + &[outside.clone(), input, outside], + ) + .unwrap() + .slice(1, 1) + } else { + input + }; + let expected = + nested_layout_batch_with_null(wide_target, 1, None, nesting, Some(depth)); + let output = coerce(input, expected.schema().as_ref()).await; + assert_eq!(output, expected, "null ancestor at {depth} in {nesting:?}"); + } + } + + #[rstest::rstest] + #[case(&["list"])] + #[case(&["list", "struct"])] + #[case(&["list", "list"])] + #[case(&["large_list", "struct", "list"])] + #[case(&["fixed_size_list", "struct", "large_list"])] + #[case(&["struct", "list", "struct"])] + #[tokio::test] + async fn nested_blob_layout_projection( + #[case] nesting: &[&str], + #[values(false, true)] wide_target: bool, + #[values(false, true)] sliced: bool, + ) { + let input = nested_layout_batch(!wide_target, 1, None, nesting); + let input = if sliced { + // Populated ranges outside a sliced collection must not be examined. + let before = nested_layout_batch(!wide_target, 0, Some(4), nesting); + let after = nested_layout_batch(!wide_target, 2, Some(4), nesting); + arrow_select::concat::concat_batches(&input.schema(), &[before, input, after]) + .unwrap() + .slice(1, 1) + } else { + input + }; + let expected = nested_layout_batch(wide_target, 1, None, nesting); + let output = coerce(input, expected.schema().as_ref()).await; + assert_eq!(output, expected); + } + + #[cfg(feature = "remote")] + fn remote_layout_table( + schema: &Schema, + ) -> (crate::Table, std::sync::mpsc::Receiver) { + let describe = serde_json::json!({ + "version": 1, + "schema": lance::arrow::json::JsonSchema::try_from(schema).unwrap(), + }) + .to_string(); + let (sender, receiver) = std::sync::mpsc::channel(); + let table = crate::Table::new_with_handler("layouts", move |mut request| { + if request.url().path().ends_with("/describe/") { + return http::Response::builder() + .status(200) + .body(describe.clone()) + .unwrap(); + } + assert!( + request.url().path().ends_with("/insert/") + || request.url().path().ends_with("/merge_insert/") + ); + sender.send(request.body_mut().take().unwrap()).unwrap(); + http::Response::builder() + .status(200) + .body(r#"{"version":2,"num_inserted_rows":1}"#.to_string()) + .unwrap() + }); + (table, receiver) + } + + #[cfg(feature = "remote")] + #[rstest::rstest] + #[case(&["list", "struct"])] + #[case(&["list", "list", "struct"])] + #[case(&["struct", "list", "struct"])] + #[case(&["large_list", "struct"])] + #[case(&["large_list", "struct", "list", "struct"])] + #[tokio::test] + async fn nested_blob_append_validates_siblings( + #[case] nesting: &[&str], + #[values(false, true)] wide_source: bool, + #[values(false, true)] wide_target: bool, + #[values(false, true)] remote: bool, + ) { + use crate::query::ExecutableQuery; + use futures::StreamExt; + use http_body::Body; + + let db = crate::connect("memory://").execute().await.unwrap(); + let initial = nested_layout_batch(wide_target, 0, None, nesting); + let schema = initial.schema(); + let (table, receiver) = if remote { + let (table, receiver) = remote_layout_table(schema.as_ref()); + (table, Some(receiver)) + } else { + ( + db.create_table("no_caption", initial) + .execute() + .await + .unwrap(), + None, + ) + }; + let input = + nested_layout_batch_with_options(wide_source, 1, None, nesting, None, &["keep me"]); + let version = if remote { + None + } else { + Some(table.version().await.unwrap()) + }; + let error = table + .add(input.clone()) + .execute() + .await + .unwrap_err() + .to_string(); + assert!( + error.contains("caption") && error.contains("does not exist"), + "{error}" + ); + if let Some(receiver) = receiver { + assert!( + receiver.try_recv().is_err(), + "invalid append reached server" + ); + } else { + assert_eq!(table.version().await.unwrap(), version.unwrap()); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + assert_eq!(table.schema().await.unwrap(), schema); + } + + // A declared sibling must survive normalization in every layout direction. + let initial = + nested_layout_batch_with_options(wide_target, 0, None, nesting, None, &["keep me"]); + let schema = initial.schema(); + let (table, receiver) = if remote { + let (table, receiver) = remote_layout_table(schema.as_ref()); + (table, Some(receiver)) + } else { + ( + db.create_table("with_caption", initial) + .execute() + .await + .unwrap(), + None, + ) + }; + table.add(input).execute().await.unwrap(); + if let Some(receiver) = receiver { + let mut body = receiver.recv().unwrap(); + let mut bytes = Vec::new(); + while let Some(frame) = + futures::stream::poll_fn(|cx| std::pin::Pin::new(&mut body).poll_frame(cx)) + .next() + .await + { + bytes.extend_from_slice(frame.unwrap().data_ref().unwrap()); + } + let batches = + arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + batches, + vec![nested_layout_batch_with_options( + wide_target, + 1, + None, + nesting, + None, + &["keep me"] + )] + ); + } else { + assert_eq!(table.count_rows(None).await.unwrap(), 2); + assert_eq!(table.schema().await.unwrap(), schema); + let batches = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut found = false; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let Some(row) = (0..ids.len()).find(|row| ids.value(*row) == 1) else { + continue; + }; + let mut array = batch.column(1).slice(row, 1); + for kind in nesting { + array = match *kind { + "list" => array.as_any().downcast_ref::().unwrap().value(0), + "large_list" => array + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "struct" => { + let values = array.as_any().downcast_ref::().unwrap(); + if let Some(caption) = values.column_by_name("caption") { + assert_eq!( + caption + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "keep me" + ); + found = true; + break; + } + values.column(0).clone() + } + _ => unreachable!(), + }; + } + } + assert!(found, "declared caption was lost"); + } + } + + #[cfg(feature = "remote")] + #[rstest::rstest] + #[case(&["list", "struct"])] + #[case(&["list", "list", "struct"])] + #[case(&["struct", "list", "struct"])] + #[case(&["large_list", "struct"])] + #[case(&["large_list", "struct", "list", "struct"])] + #[tokio::test] + async fn nested_blob_append_rejects_duplicate_siblings( + #[case] nesting: &[&str], + #[values(false, true)] wide_source: bool, + #[values(false, true)] wide_target: bool, + #[values(false, true)] remote: bool, + ) { + use crate::query::ExecutableQuery; + + let initial = + nested_layout_batch_with_options(wide_target, 0, None, nesting, None, &["keep me"]); + let schema = initial.schema(); + let db = crate::connect("memory://").execute().await.unwrap(); + let (table, receiver) = if remote { + let (table, receiver) = remote_layout_table(schema.as_ref()); + (table, Some(receiver)) + } else { + ( + db.create_table("captions", initial) + .execute() + .await + .unwrap(), + None, + ) + }; + let snapshot = if remote { + None + } else { + Some(( + table.version().await.unwrap(), + table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(), + )) + }; + // Construct distinct arrays for the duplicate names: selecting only the + // first matching field must not silently discard the second value. + let input = nested_layout_batch_with_options( + wide_source, + 1, + None, + nesting, + None, + &["first", "second"], + ); + let error = table.add(input).execute().await.unwrap_err().to_string(); + assert!( + error.contains("caption") && error.contains("specified more than once"), + "{error}" + ); + if let Some(receiver) = receiver { + assert!( + receiver.try_recv().is_err(), + "ambiguous append reached server" + ); + } else { + let (version, before) = snapshot.unwrap(); + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + assert_eq!(table.schema().await.unwrap(), schema); + let batches = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches, before); + } + } + + #[cfg(feature = "remote")] + #[rstest::rstest] + #[case(&["struct"])] + #[case(&["list", "struct"])] + #[case(&["struct", "list", "struct"])] + #[case(&["large_list", "list"])] + #[tokio::test] + async fn nullable_nested_blob_layout_writes( + #[case] nesting: &[&str], + #[values(false, true)] merge: bool, + #[values(false, true)] remote: bool, + ) { + use crate::query::ExecutableQuery; + use futures::StreamExt; + use http_body::Body; + + for depth in 0..nesting.len() { + let initial = nested_layout_batch(false, 0, None, nesting); + let schema = initial.schema(); + let db = crate::connect("memory://").execute().await.unwrap(); + let (table, receiver) = if remote { + let (table, receiver) = remote_layout_table(schema.as_ref()); + (table, Some(receiver)) + } else { + ( + db.create_table("layouts", initial).execute().await.unwrap(), + None, + ) + }; + let visible_id = if merge { 0 } else { 1 }; + let visible = nested_layout_batch(true, visible_id, None, nesting); + let hidden = nested_layout_batch_with_null(true, 2, Some(0), nesting, Some(depth)); + let outside = nested_layout_batch(true, 99, Some(4), nesting); + let input = arrow_select::concat::concat_batches( + &visible.schema(), + &[outside.clone(), visible, hidden, outside], + ) + .unwrap() + .slice(1, 2); + let expected = arrow_select::concat::concat_batches( + &schema, + &[ + nested_layout_batch(false, visible_id, None, nesting), + nested_layout_batch_with_null(false, 2, None, nesting, Some(depth)), + ], + ) + .unwrap(); + if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let input_schema = input.schema(); + builder + .execute(Box::new(arrow_array::RecordBatchIterator::new( + vec![Ok(input)], + input_schema, + ))) + .await + .unwrap(); + } else { + table.add(input).execute().await.unwrap(); + } + let count = if merge { 2 } else { 3 }; + if let Some(receiver) = &receiver { + let mut body = receiver.recv().unwrap(); + let mut bytes = Vec::new(); + while let Some(frame) = + futures::stream::poll_fn(|cx| std::pin::Pin::new(&mut body).poll_frame(cx)) + .next() + .await + { + bytes.extend_from_slice(frame.unwrap().data_ref().unwrap()); + } + let batches = + arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(batches, vec![expected]); + } else { + assert_eq!(table.schema().await.unwrap(), schema); + assert_eq!(table.count_rows(None).await.unwrap(), count); + let batches = table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut found = false; + for batch in batches { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + if let Some(row) = (0..ids.len()).find(|row| ids.value(*row) == 2) { + let mut array = batch.column(1).slice(row, 1); + for kind in &nesting[..depth] { + array = match *kind { + "struct" => array + .as_any() + .downcast_ref::() + .unwrap() + .column(0) + .clone(), + "list" => { + array.as_any().downcast_ref::().unwrap().value(0) + } + "large_list" => array + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + _ => unreachable!(), + }; + } + assert!(array.is_null(0), "lost null ancestor at {depth}"); + found = true; + } + } + assert!(found); + } + + // A visible populated range in a later batch must still abort everything. + let version = if remote { + None + } else { + Some(table.version().await.unwrap()) + }; + let hidden = nested_layout_batch_with_null(true, 3, Some(0), nesting, Some(depth)); + let bad = nested_layout_batch(true, 4, Some(4), nesting); + let reader: Box = + Box::new(arrow_array::RecordBatchIterator::new( + vec![Ok(hidden), Ok(bad.clone())], + bad.schema(), + )); + let result = if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(reader).await.map(|_| ()) + } else { + table.add(reader).execute().await.map(|_| ()) + }; + let error = result.unwrap_err().to_string(); + assert!(error.contains("cannot store byte ranges"), "{error}"); + if let Some(receiver) = receiver { + assert!( + receiver.try_recv().is_err(), + "rejected write reached server" + ); + } else { + assert_eq!(table.version().await.unwrap(), version.unwrap()); + assert_eq!(table.count_rows(None).await.unwrap(), count); + } + } + } + + #[cfg(feature = "remote")] + #[rstest::rstest] + #[case(&["list"])] + #[case(&["list", "struct"])] + #[case(&["list", "list"])] + #[case(&["large_list", "struct", "list"])] + #[case(&["struct", "list", "struct"])] + #[tokio::test] + async fn nested_blob_layout_writes( + #[case] nesting: &[&str], + #[values(false, true)] wide_target: bool, + #[values(false, true)] merge: bool, + #[values(false, true)] remote: bool, + ) { + use crate::query::ExecutableQuery; + use futures::StreamExt; + use http_body::Body; + + let initial = nested_layout_batch(wide_target, 0, None, nesting); + let schema = initial.schema(); + let db = crate::connect("memory://").execute().await.unwrap(); + let (table, receiver) = if remote { + let (table, receiver) = remote_layout_table(schema.as_ref()); + (table, Some(receiver)) + } else { + ( + db.create_table("layouts", initial).execute().await.unwrap(), + None, + ) + }; + let input = nested_layout_batch(!wide_target, 1, None, nesting); + let expected = nested_layout_batch(wide_target, 1, None, nesting); + let (input, expected) = if merge { + ( + arrow_select::concat::concat_batches( + &input.schema(), + &[nested_layout_batch(!wide_target, 0, None, nesting), input], + ) + .unwrap(), + arrow_select::concat::concat_batches( + &schema, + &[nested_layout_batch(wide_target, 0, None, nesting), expected], + ) + .unwrap(), + ) + } else { + (input, expected) + }; + if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let schema = input.schema(); + builder + .execute(Box::new(arrow_array::RecordBatchIterator::new( + vec![Ok(input)], + schema, + ))) + .await + .unwrap(); + } else { + table.add(input).execute().await.unwrap(); + } + if let Some(receiver) = &receiver { + let mut body = receiver.recv().unwrap(); + let mut bytes = Vec::new(); + while let Some(frame) = + futures::stream::poll_fn(|cx| std::pin::Pin::new(&mut body).poll_frame(cx)) + .next() + .await + { + bytes.extend_from_slice(frame.unwrap().data_ref().unwrap()); + } + let batches = + arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(batches, vec![expected]); + } else { + assert_eq!(table.schema().await.unwrap(), schema); + assert_eq!(table.count_rows(None).await.unwrap(), 2); + // Read the stored nested descriptors as well as the row count. + table + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + } + if !wide_target { + let version = if remote { + None + } else { + Some(table.version().await.unwrap()) + }; + let first = nested_layout_batch(true, 2, None, nesting); + let bad = nested_layout_batch(true, 3, Some(4), nesting); + let reader: Box = + Box::new(arrow_array::RecordBatchIterator::new( + vec![Ok(first), Ok(bad.clone())], + bad.schema(), + )); + let result = if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(reader).await.map(|_| ()) + } else { + table.add(reader).execute().await.map(|_| ()) + }; + let error = result.unwrap_err().to_string(); + assert!(error.contains("cannot store byte ranges"), "{error}"); + if let Some(receiver) = receiver { + assert!( + receiver.try_recv().is_err(), + "a rejected range must not reach the server" + ); + } else { + assert_eq!(table.version().await.unwrap(), version.unwrap()); + assert_eq!(table.count_rows(None).await.unwrap(), 2); + } + } + } } diff --git a/rust/lancedb/src/table/datafusion/cast.rs b/rust/lancedb/src/table/datafusion/cast.rs index 063572cbe..b6e815d5e 100644 --- a/rust/lancedb/src/table/datafusion/cast.rs +++ b/rust/lancedb/src/table/datafusion/cast.rs @@ -33,9 +33,12 @@ pub fn cast_to_table_schema( return Ok(input); } - let exprs = build_field_exprs(input_schema.fields(), table_schema.fields(), &|idx| { - Arc::new(Column::new(input_schema.field(idx).name(), idx)) as Arc - })?; + let exprs = build_field_exprs( + input_schema.fields(), + table_schema.fields(), + &|idx| Arc::new(Column::new(input_schema.field(idx).name(), idx)) as Arc, + &[], + )?; let exprs: Vec<(Arc, String)> = exprs .into_iter() @@ -52,10 +55,13 @@ pub fn cast_to_table_schema( /// For each table field that exists in the input, produce an expression that /// reads from the input and casts if needed. Fields in the table but not in the /// input are omitted (the storage layer handles missing columns). -fn build_field_exprs( +/// `ancestors` carries validity for blob range validation without nullifying +/// non-nullable children hidden beneath a null parent. +pub(super) fn build_field_exprs( input_fields: &Fields, table_fields: &Fields, get_input_expr: &dyn Fn(usize) -> Arc, + ancestors: &[Arc], ) -> Result, FieldRef)>> { let config = Arc::new(ConfigOptions::default()); let mut result = Vec::new(); @@ -109,13 +115,14 @@ fn build_field_exprs( input_field, table_field, &config, + ancestors, )?); continue; } // Blob values in lists need the same coercion as top-level blob columns. if let Some(coerced) = - coerce_blob_list_expr(input_expr.clone(), input_field, table_field, &config)? + coerce_blob_list_expr(input_expr.clone(), input_field, table_field, ancestors)? { result.push(coerced); continue; @@ -138,30 +145,38 @@ fn build_field_exprs( (DataType::Struct(in_children), DataType::Struct(tbl_children)) if in_children != tbl_children => { - let sub_exprs = build_field_exprs(in_children, tbl_children, &|child_idx| { - let child_name = in_children[child_idx].name(); - Arc::new(ScalarFunctionExpr::new( - &format!("get_field({child_name})"), - get_field(), - vec![ - input_expr.clone(), - Arc::new(Literal::new(ScalarValue::from(child_name.as_str()))), - ], - Arc::new(in_children[child_idx].as_ref().clone()), - config.clone(), - )) as Arc - })?; + let mut child_ancestors = ancestors.to_vec(); + child_ancestors.push(input_expr.clone()); + let sub_exprs = build_field_exprs( + in_children, + tbl_children, + &|child_idx| { + let child_name = in_children[child_idx].name(); + Arc::new(ScalarFunctionExpr::new( + &format!("get_field({child_name})"), + get_field(), + vec![ + input_expr.clone(), + Arc::new(Literal::new(ScalarValue::from(child_name.as_str()))), + ], + Arc::new(in_children[child_idx].as_ref().clone()), + config.clone(), + )) as Arc + }, + &child_ancestors, + )?; let output_struct_fields: Fields = sub_exprs .iter() .map(|(_, f)| f.clone()) .collect::>() .into(); - let output_field: FieldRef = Arc::new(Field::new( - table_field.name(), - DataType::Struct(output_struct_fields), - table_field.is_nullable(), - )); + let output_field: FieldRef = Arc::new( + table_field + .as_ref() + .clone() + .with_data_type(DataType::Struct(output_struct_fields)), + ); // Build named_struct(lit("a"), expr_a, lit("b"), expr_b, ...) let mut ns_args: Vec> = Vec::new(); diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 66ec48baf..589353efa 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -254,7 +254,11 @@ fn canonicalize_merge_filter(filter: Option) -> Result Result<()> { +pub(crate) fn validate_merge_source_fields( + input: &Fields, + target: &Fields, + parent: &str, +) -> Result<()> { let mut names = HashSet::with_capacity(input.len()); for field in input { let path = if parent.is_empty() { @@ -273,7 +277,9 @@ fn validate_merge_source_fields(input: &Fields, target: &Fields, parent: &str) - .ok_or_else(|| Error::InvalidInput { message: format!("merge source field '{path}' is not present in the table schema"), })?; - validate_merge_source_type(field.data_type(), target_field.data_type(), &path)?; + if !crate::blob::is_blob(target_field) { + validate_merge_source_type(field.data_type(), target_field.data_type(), &path)?; + } } Ok(()) } @@ -296,7 +302,13 @@ fn validate_merge_source_type(input: &DataType, target: &DataType, path: &str) - | DataType::LargeListView(target), ) | (DataType::Map(input, _), DataType::Map(target, _)) => { - validate_merge_source_type(input.data_type(), target.data_type(), path) + // Collection items carry extension metadata just like named fields. + // Blob coercion, rather than struct validation, checks range narrowing. + if crate::blob::is_blob(target) { + Ok(()) + } else { + validate_merge_source_type(input.data_type(), target.data_type(), path) + } } (DataType::Dictionary(_, input), _) => validate_merge_source_type(input, target, path), (_, DataType::Dictionary(_, target)) => validate_merge_source_type(input, target, path), @@ -313,35 +325,18 @@ pub(crate) async fn execute_merge_insert( new_data: Box, ) -> Result { params.canonicalize_filters()?; + let schema = table.schema().await?; super::computed_columns::ensure_no_function_bindings_for_mutation( - table.schema().await?.as_ref(), + schema.as_ref(), "merge_insert", )?; - match lsm::lsm_dispatch_decision(table, ¶ms).await? { - lsm::LsmDispatch::Lsm(plan) => { - let future = - lsm::execute_lsm_merge_insert(table, plan, params.validate_single_shard, new_data); - return match params.timeout { - Some(timeout) => match tokio::time::timeout(timeout, future).await { - Ok(result) => result, - Err(_) => Err(Error::Runtime { - message: "merge insert timed out".to_string(), - }), - }, - None => future.await, - }; - } - lsm::LsmDispatch::Standard => {} - } - - let dataset = table.dataset.get().await?; - let schema = arrow_schema::Schema::from(dataset.schema()); - // JSON source fields must carry the stored extension metadata just as on - // append. Keep arrow.json text labelled until Lance encodes it as JSONB. - let source = if schema - .fields() - .iter() - .any(|field| lance_arrow::json::has_json_fields(field)) + // Normalize blob layouts and JSON extension metadata just as on append. + // Keep arrow.json text labelled until Lance encodes it as JSONB. + let source = if crate::blob::has_blob_columns(&schema) + || schema + .fields() + .iter() + .any(|field| lance_arrow::json::has_json_fields(field)) { validate_merge_source_fields(new_data.schema().fields(), schema.fields(), "")?; let plan = Arc::new(super::datafusion::scannable_exec::ScannableExec::new( @@ -356,6 +351,24 @@ pub(crate) async fn execute_merge_insert( } else { new_data.into_stream() }; + match lsm::lsm_dispatch_decision(table, ¶ms).await? { + lsm::LsmDispatch::Lsm(plan) => { + let future = + lsm::execute_lsm_merge_insert(table, plan, params.validate_single_shard, source); + return match params.timeout { + Some(timeout) => match tokio::time::timeout(timeout, future).await { + Ok(result) => result, + Err(_) => Err(Error::Runtime { + message: "merge insert timed out".to_string(), + }), + }, + None => future.await, + }; + } + lsm::LsmDispatch::Standard => {} + } + + let dataset = table.dataset.get().await?; let mut builder = LanceMergeInsertBuilder::try_new(dataset.clone(), params.on)?; match ( params.when_matched_update_all, diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 39e4fbf28..d8877428d 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -24,8 +24,10 @@ use arrow_array::cast::AsArray; use arrow_array::types::{ Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type, }; -use arrow_array::{Array, ArrayRef, Int32Array, RecordBatch, RecordBatchReader}; +use arrow_array::{Array, ArrayRef, Int32Array, RecordBatch}; use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef}; +use datafusion_physical_plan::SendableRecordBatchStream; +use futures::TryStreamExt; use lance::Dataset; use lance::dataset::mem_wal::{ DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec, @@ -707,7 +709,7 @@ pub(crate) async fn execute_lsm_merge_insert( table: &NativeTable, plan: LsmPlan, validate_single_shard: bool, - new_data: Box, + mut new_data: SendableRecordBatchStream, ) -> Result { let dataset = table.dataset.get().await?; let target_schema: SchemaRef = Arc::new(ArrowSchema::from(dataset.schema())); @@ -718,8 +720,7 @@ pub(crate) async fn execute_lsm_merge_insert( let mut batches: Vec = Vec::new(); let mut total_rows: u64 = 0; - for batch in new_data { - let batch = batch.map_err(|e| Error::Arrow { source: e })?; + while let Some(batch) = new_data.try_next().await? { if batch.num_rows() == 0 { continue; } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 4274aaa6f..0aebcaf2d 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -1747,7 +1747,7 @@ mod tests { RecordBatch::try_new( Arc::new(Schema::new(vec![ Field::new("id", arrow_schema::DataType::Int32, false), - crate::blob("image", true), + lance::blob::blob_field("image", true), ])), vec![Arc::new(Int32Array::from(ids)), builder.finish().unwrap()], ) @@ -1901,7 +1901,7 @@ mod tests { let mut blob_builder = lance::blob::BlobArrayBuilder::new(2); blob_builder.push_bytes(b"nested").unwrap(); blob_builder.push_null().unwrap(); - let blob_field = crate::blob("image", true); + let blob_field = lance::blob::blob_field("image", true); let metadata_fields = Fields::from(vec![blob_field.clone()]); let metadata = StructArray::new( metadata_fields.clone(), @@ -1968,7 +1968,7 @@ mod tests { blob_builder.push_bytes(b"a").unwrap(); blob_builder.push_bytes(b"bb").unwrap(); blob_builder.push_null().unwrap(); - let item = Arc::new(crate::blob("item", true)); + let item = Arc::new(lance::blob::blob_field("item", true)); let images = ListArray::new( item.clone(), OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 3])), diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index 063690129..91f3a978c 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -10,12 +10,13 @@ use arrow_array::{ use arrow_schema::{DataType, Field, Fields, Schema}; use futures::TryStreamExt; use lance::Dataset; +use lance::blob::blob_field as blob; use lance::dataset::WriteParams; use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_table::format::BasePath; use lancedb::{ Connection, Error, Result, Table, - blob::{BlobRangeRequest, blob}, + blob::BlobRangeRequest, connect, connect_namespace, database::listing::{ ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, @@ -1610,6 +1611,88 @@ async fn external_uri_struct_round_trips_with_flag() -> Result<()> { Ok(()) } +#[tokio::test] +async fn external_ranges_round_trip_with_registered_base_and_opt_in() -> Result<()> { + let tmp = tempdir().unwrap(); + let payload = b"0123456789abcdef"; + let uri = write_payload_file_uri(tmp.path(), "payload.bin", payload); + let base_uri = url::Url::from_directory_path(tmp.path()) + .unwrap() + .to_string(); + let db = connect(tmp.path().join("db").to_str().unwrap()) + .execute() + .await?; + + for registered_base in [false, true] { + let field = lancedb::blob("image", true); + let DataType::Struct(children) = field.data_type().clone() else { + unreachable!("blob field is a struct"); + }; + let schema = Arc::new(Schema::new(vec![field])); + let name = if registered_base { + "registered" + } else { + "opt_in" + }; + let table = db + .create_empty_table(name, schema.clone()) + .write_options(WriteOptions { + lance_write_params: Some(WriteParams { + initial_bases: registered_base.then(|| { + vec![BasePath { + id: 1, + name: Some("external".to_string()), + path: base_uri.clone(), + is_dataset_root: false, + }] + }), + ..Default::default() + }), + }) + .execute() + .await?; + let values = StructArray::new( + children, + vec![ + Arc::new(LargeBinaryArray::from(vec![ + None, + None, + None, + Some(b"inline".as_slice()), + ])), + Arc::new(StringArray::from(vec![ + Some(uri.as_str()), + Some(uri.as_str()), + Some(uri.as_str()), + None, + ])), + Arc::new(UInt64Array::from(vec![Some(0), Some(4), None, None])), + Arc::new(UInt64Array::from(vec![Some(6), Some(6), None, None])), + ], + None, + ); + table + .add(RecordBatch::try_new(schema, vec![Arc::new(values)])?) + .allow_external_blob_outside_bases(!registered_base) + .execute() + .await?; + let reopened = db.open_table(name).execute().await?; + let ids = collect_row_ids(&reopened).await?; + let bytes = reopened.fetch_blobs("image", &ids).await?; + assert_eq!(bytes.value(0), &payload[..6]); + assert_eq!(bytes.value(1), &payload[4..10]); + assert_eq!(bytes.value(2), payload); + assert_eq!(bytes.value(3), b"inline"); + let files = reopened.fetch_blob_files("image", &ids).await?; + assert_eq!(files[1].as_ref().unwrap().size(), 6); + assert_eq!( + files[1].as_ref().unwrap().read_range(1..3).await?, + &payload[5..7] + ); + } + Ok(()) +} + #[tokio::test] async fn external_uri_add_requires_opt_in() -> Result<()> { let tmp = tempdir().unwrap(); @@ -1761,3 +1844,188 @@ async fn malformed_string_uri_is_rejected_at_write() -> Result<()> { assert_eq!(table.count_rows(None).await?, 0); Ok(()) } + +fn layout_batch(wide: bool, id: i64, position: Option, size: Option) -> RecordBatch { + let field = if wide { + lancedb::blob("image", true) + } else { + blob("image", true) + }; + let DataType::Struct(children) = field.data_type().clone() else { + unreachable!() + }; + let mut columns: Vec = vec![ + Arc::new(LargeBinaryArray::from(vec![Some(b"payload".as_slice())])), + new_null_array(&DataType::Utf8, 1), + ]; + if wide { + columns.push(Arc::new(UInt64Array::from(vec![position]))); + columns.push(Arc::new(UInt64Array::from(vec![size]))); + } + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + field, + ])), + vec![ + Arc::new(Int64Array::from(vec![id])), + Arc::new(StructArray::new(children, columns, None)), + ], + ) + .unwrap() +} + +fn layout_reader(batches: Vec) -> Box { + let schema = batches[0].schema(); + Box::new(arrow_array::RecordBatchIterator::new( + batches.into_iter().map(Ok), + schema, + )) +} + +#[tokio::test] +async fn blob_layout_append_and_merge_normalize_both_directions() -> Result<()> { + for wide in [false, true] { + for merge in [false, true] { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let initial = layout_batch(wide, 1, None, None); + let schema = initial.schema(); + let table = db.create_table("layouts", initial).execute().await?; + let input = layout_batch(!wide, if merge { 1 } else { 2 }, None, None); + if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + let result = builder + .execute(layout_reader(vec![ + input, + layout_batch(!wide, 2, None, None), + ])) + .await?; + assert_eq!(result.num_updated_rows, 1); + assert_eq!(result.num_inserted_rows, 1); + } else { + table.add(input).execute().await?; + } + assert_eq!(table.schema().await?, schema); + assert_eq!(table.count_rows(None).await?, 2); + let ids = collect_row_ids(&table).await?; + let values = table.fetch_blobs("image", &ids).await?; + assert_eq!(values.value(0), b"payload"); + assert_eq!(values.value(1), b"payload"); + } + } + Ok(()) +} + +#[tokio::test] +async fn blob_layout_rejects_non_null_ranges_atomically_in_later_batches() -> Result<()> { + for merge in [false, true] { + for (position, size) in [(Some(0), None), (None, Some(6)), (Some(0), Some(6))] { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let table = db + .create_table("layouts", layout_batch(false, 1, None, None)) + .execute() + .await?; + let version = table.version().await?; + // A successful first batch must not commit when a later batch has a range. + let input = layout_reader(vec![ + layout_batch(true, 2, None, None), + layout_batch(true, 3, position, size), + ]); + let result = if merge { + let mut builder = table.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder.execute(input).await.map(|_| ()) + } else { + table.add(input).execute().await.map(|_| ()) + }; + let error = result.unwrap_err().to_string(); + assert!(error.contains("cannot store byte ranges"), "{error}"); + assert_eq!(table.version().await?, version); + assert_eq!(table.count_rows(None).await?, 1); + let ids = collect_row_ids(&table).await?; + assert_eq!(table.fetch_blobs("image", &ids).await?.value(0), b"payload"); + } + } + Ok(()) +} + +#[tokio::test] +async fn blob_layout_overwrite_replaces_schema_in_both_directions() -> Result<()> { + for wide in [false, true] { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let table = db + .create_table("layouts", layout_batch(wide, 1, None, None)) + .execute() + .await?; + let input = layout_batch(!wide, 2, None, None); + let expected_schema = input.schema(); + table + .add(input) + .mode(AddDataMode::Overwrite) + .execute() + .await?; + assert_eq!(table.schema().await?, expected_schema); + assert_eq!(table.count_rows(None).await?, 1); + let ids = collect_row_ids(&table).await?; + assert_eq!(table.fetch_blobs("image", &ids).await?.value(0), b"payload"); + } + Ok(()) +} + +#[tokio::test] +async fn blob_layout_memwal_merge_normalizes_before_writing() -> Result<()> { + use lancedb::{query::Select, table::MemWalWriteSpec}; + for wide in [false, true] { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let table = db + .create_table("layouts", layout_batch(wide, 1, None, None)) + .execute() + .await?; + table.set_unenforced_primary_key(["id"]).await?; + table.enable_memwal(MemWalWriteSpec::unsharded()).await?; + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge + .clone() + .execute(layout_reader(vec![layout_batch(!wide, 2, None, None)])) + .await?; + let batches = table + .query() + .select(Select::columns(&["id"])) + .execute() + .await? + .try_collect::>() + .await?; + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 2); + if !wide { + let error = merge + .execute(layout_reader(vec![ + layout_batch(true, 3, None, None), + layout_batch(true, 4, Some(0), Some(6)), + ])) + .await + .unwrap_err(); + assert!(error.to_string().contains("cannot store byte ranges")); + let batches = table + .query() + .select(Select::columns(&["id"])) + .execute() + .await? + .try_collect::>() + .await?; + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 2); + } + } + Ok(()) +} diff --git a/rust/lancedb/tests/duplicate_pairs.rs b/rust/lancedb/tests/duplicate_pairs.rs index 63f1c9f5a..6b602ac4f 100644 --- a/rust/lancedb/tests/duplicate_pairs.rs +++ b/rust/lancedb/tests/duplicate_pairs.rs @@ -410,6 +410,8 @@ async fn dedup_query_materializes_blob_payloads_and_null_vectors() -> anyhow::Re (0..6).map(|id| vec![id; 4096]), )), Arc::new(StringArray::from(vec![None::<&str>; 6])), + Arc::new(UInt64Array::new_null(6)), + Arc::new(UInt64Array::new_null(6)), ], None, ); From 9f616e6218873333a95208950350cca75a526aa8 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 20 Oct 2026 22:51:42 +0800 Subject: [PATCH 19/22] docs: explain managed blob ingestion, functions, and cleanup (#4604) Document explicit Managed Blob v2 opt-in, descriptor queries and payload fetching, Function inputs and outputs, and whole-object cleanup. Explain null versus empty values, snapshot ownership, packed-object retention, and the distinction from the legacy large-binary Blob API. The local ingestion and Enterprise Function examples have been executed end to end. The Function lifecycle description depends on lance-format/lance#10176 and the corresponding Enterprise migration. The existing Functions page remains outside the published navigation until launch. Validation: the 347-file assembled site has no broken links. Mintlify 4.2.888 build validation reports four existing `RsExampleBody`/`RsExampleTail` import warnings in the reranking and vector-search pages; the same diagnostics reproduce with the unchanged base versions of all three pages edited here. --- docs/web/maintenance/cleanup.mdx | 29 +++++++++-- docs/web/management/function.mdx | 62 +++++++++++++++++++++++ docs/web/management/table.mdx | 86 +++++++++++++++++++++++++++++++- 3 files changed, 172 insertions(+), 5 deletions(-) diff --git a/docs/web/maintenance/cleanup.mdx b/docs/web/maintenance/cleanup.mdx index 397def690..7833fa8a0 100644 --- a/docs/web/maintenance/cleanup.mdx +++ b/docs/web/maintenance/cleanup.mdx @@ -65,13 +65,34 @@ files, so a tag on what was the latest version can become an older one by the time cleanup runs. `optimize()` then raises after compacting, and its index update does not run. -A deployment's background cleanup follows the same rules. Its own background -jobs, such as compaction and index updates, also commit new versions, so a tag -on what is now the latest version can become an older one without a call from -you. If you rely on reading old versions, set the window to cover the range you +Enterprise background cleanup keeps tagged snapshots while pruning unrelated +expired versions. If you rely on reading untagged old versions, set the window +to cover the range you need before running this, not after. +## Managed blob objects {#managed-blobs} + +[Managed blobs](/management/table#managed-blobs) store out-of-line payloads +under the table's `_blobs/` directory. Cleanup follows the references from +retained snapshots, including protected tags and branches, before selecting +objects for deletion. Replacing a blob or deleting a row makes its old payload +eligible only after no retained snapshot references it. + +Cleanup removes whole objects. If a packed object contains several blobs, it +remains while any of them is live. Compaction preserves Managed payload objects +rather than copying their bytes, so compaction alone does not reclaim the +unused portion of a packed object. + +Use table maintenance to remove Managed objects; deleting `_blobs/` files +directly can corrupt current or historical snapshots. Upgrade all clients and +maintenance workers to a version that understands Managed blobs before using +them, and reopen handles created before that upgrade. + +For local tables, `optimize()` performs compaction and cleanup. Enterprise runs +these as background jobs; the remote Python table does not support `optimize()`. +See [Background jobs](/maintenance/background-jobs) for the Enterprise path. + ## Disk usage does not drop immediately {#disk} Compaction writes new, merged files before the old ones are removed, so disk diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 4467c6abb..8e5b53e4a 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -143,6 +143,68 @@ rows added since the last one and recomputes rows whose inputs changed, and leaves filled rows with unchanged inputs alone. See [Refresh a column](/feature-engineering/refresh-column). +## Read and produce blobs {#blobs} + +A Function can bind a [Blob v2 column](/management/table#managed-blobs) and +return another blob. In row mode, a top-level blob parameter arrives as Python +`bytes`, or `None` for a nullable input. The worker reads the payload before +calling your function. Return bytes and let the destination table store the +Managed payload and descriptor. + +Declare blob inputs and outputs explicitly. This example reverses the bytes +to demonstrate the round trip without requiring an image library. Here `db` +is an Enterprise database and `table` contains an `image` Blob v2 column +with table schema metadata `lance.blob.managed=true`, as in the +[table creation example](/management/table#managed-blobs-read-write): + +```python +from datetime import timedelta + +import lancedb +from lancedb import col +from lancedb.functions import udf +import pyarrow as pa + +@udf( + name="reverse_image", + python_version="3.12", + pip=["pyarrow==26.0.0"], + input_schema=pa.schema([lancedb.blob("image")]), + output_schema=lancedb.blob("reversed", nullable=False), +) +def reverse_image(image): + return b"missing" if image is None else image[::-1] + +function = db.create_function_async(reverse_image).wait( + timeout=timedelta(minutes=15) +) +table.add_columns({"reversed": function(image=col("image"))}) +result = table.refresh_column_async("reversed").wait( + timeout=timedelta(minutes=15) +) +assert result.rows_failed == 0 + +hits = table.search().select(["id", "reversed"]).with_row_id(True).to_arrow() +values = table.fetch_blobs("reversed", hits).to_pylist() +``` + +An empty byte string is a filled result. A scalar output cannot be nullable, +because a physical NULL marks an unfilled computed row. To return a nullable +blob as a value, use a named-struct output schema with a nullable blob field; +the struct itself records whether the result has been filled. Read a nested +blob with `table.fetch_blobs("payload.image", hits)`. + +On deployments that support computed-column inputs, another Function can bind +to `col("reversed")`. Refresh the producer first, then the consumer. Once +outputs are current, a later refresh fills appended rows. A filtered refresh +preserves payload references for unselected Managed outputs. An unfiltered +refresh can recompute a whole fragment when it cannot establish that the +fragment's outputs are current. + +Function output blobs follow the same compaction and cleanup lifecycle as +ingested Managed blobs. Retained versions continue to own their payloads; +cleanup reclaims an object only after its retained references are gone. + ## Declare the environment {#environment} The function runs remotely, so its dependencies are part of its definition: diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index df44a8fd3..ed9883277 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -427,7 +427,91 @@ Once you have the bytes back from the search result, you can decode them into th -### Large Blobs (Blob API) {#multimodal-large-blobs-blob-api} +### Managed blobs (Blob v2) {#managed-blobs} + +Use `lancedb.blob()` for images, audio, videos, and other binary values whose +payloads should load separately from the rest of a row. Write Python `bytes` +normally. Set the table schema metadata `lance.blob.managed` to `true` to +store small values inline and larger values in managed objects under the table's `_blobs/` directory. Medium values can share a packed +object; large values use a dedicated object. + +The table owns these objects. You do not upload them separately, keep an object +catalog, or delete their files yourself. A descriptor records where to read the +bytes. Compaction preserves the descriptor and its payload object, so merging +table fragments does not rewrite the blob payloads. + + +Use an SDK and Enterprise deployment that support Blob v2 Managed storage. +Upgrade every reader, writer, and maintenance worker before activating Managed +blobs on a table, and reopen existing table handles after the upgrade. Older +clients cannot safely maintain a table containing Managed blobs. + + +#### Write and read blobs {#managed-blobs-read-write} + +This example uses a local connection. The same schema, `add`, query, and +`fetch_blobs` calls work with an Enterprise database connection: + +```python +import lancedb +import pyarrow as pa + +db = lancedb.connect("./media") +table = db.create_table( + "images", + schema=pa.schema([ + pa.field("id", pa.int64(), nullable=False), + lancedb.blob("image"), + ], metadata={"lance.blob.managed": "true"}), +) +table.add([ + {"id": 1, "image": b""}, + {"id": 2, "image": bytes(range(256)) * 400}, + {"id": 3, "image": None}, +]) + +# Query descriptors first, then fetch only the selected rows' payloads. +hits = ( + table.search() + .where("id <= 2") + .select(["id", "image"]) + .with_row_id(True) + .to_arrow() +) +images = table.fetch_blobs("image", hits).to_pylist() +by_id = dict(zip(hits["id"].to_pylist(), images, strict=True)) +assert by_id[1] == b"" +assert len(by_id[2]) == 102400 +``` + +`None` is a null value; `b""` is a present, empty blob. `fetch_blobs` returns +bytes in the order of the supplied row IDs. It materializes the selected +payloads, so fetch bounded batches when blobs are large. Row IDs belong to +their table; fetch them from the same table and snapshot that produced the +query result. + +Keep descriptors opaque in application code. Their representation can differ +between inline values, Managed objects, and externally referenced blobs. For +streaming and range reads, see the +[Lance blob guide](https://lance.org/guide/blob/). + +#### Derived blobs and maintenance {#managed-blobs-lifecycle} + +Enterprise Functions can consume blob columns and +return new blobs. Their output payloads are managed by the destination table. +Local tables can also derive blob columns with +[SQL computed columns](/feature-engineering/add-computed-column). + +Deleting a row or replacing its blob does not immediately remove the old +payload: retained table versions may still reference it. [Cleanup](/maintenance/cleanup#managed-blobs) +reclaims objects after those references expire. A packed object remains while +any of its contained blobs is still referenced. + +### Legacy large-binary blobs {#multimodal-large-blobs-blob-api} + +The schema below describes the earlier large-binary Blob API. It is distinct +from the `lancedb.blob()` schema above; changing the field's metadata alone does +not migrate an existing column to Blob v2. For larger files like high-resolution images or videos, Lance provides a specialized **Blob API**. By using a large-binary Arrow type (`pa.large_binary()` in Python, `LargeBinary` in TypeScript, and `DataType::LargeBinary` in Rust) and specific metadata, you enable **lazy loading** and optimized encoding. This allows you to work with massive datasets without loading all binary data into memory upfront. From 1772f4ef8311b2f4cb31bca7a1a87faf94fb9a12 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Tue, 20 Oct 2026 09:16:18 -0700 Subject: [PATCH 20/22] fix: don't panic building permutations over tables with schema metadata (#4602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem `permutation_builder(table).execute()` and `StreamingDataset(table, ...)` (which builds a permutation at construction) panic with `RustPanic` on any table whose Arrow schema has schema-level metadata: ```python schema = pa.schema([pa.field("x", pa.int64())], metadata={b"huggingface": b"{}"}) t = lancedb.connect(tmp).create_table("t", pa.table({"x": list(range(100))}, schema=schema)) permutation_builder(t).split_random(fixed=2, seed=1).execute() # RustPanic ``` Every Hugging Face Lance dataset triggers this, e.g. `hf://datasets/lance-format/lerobot-xvla-soft-fold/data` table `frames` carries `source-column-name-map`. ## Root cause `PermutationBuilder::add_config_metadata` replaced the stream schema's metadata with the permutation config and then called `batch.with_schema(schema).unwrap()`. The base-table scan's batches carry the base schema metadata, and `RecordBatch::with_schema` requires the target to be a superset of the current schema, metadata included. It returned `SchemaError("target schema is not superset of current schema ...")`, and the `unwrap` turned that into a panic on a tokio worker. ## Fix - The permutation table deliberately does **not** inherit base-table schema metadata. That metadata describes columns the permutation table (`row_id`, `split_id`) does not have, and its keys can collide with config keys such as `split_names`. A merge would let a base table's `split_names` leak into, or override, the permutation's split names. Batches are rebuilt with `RecordBatch::try_new(schema, columns)`. - Errors are propagated (`map_ok` → `and_then`) instead of `unwrap`. - The same panic-on-worker pattern in the splitter's streaming closures (`split.rs`: `try_new`/`filter(...).unwrap()`) now propagates errors too. Nothing else in `dataloader/` used `with_schema(...).unwrap()`. ## Tests - Rust: `test_permutation_over_table_with_schema_metadata` in `builder.rs` builds a random split + shuffled permutation over a table with `huggingface` and a colliding `split_names` key. It panics with the original `SchemaError` on the old code and passes now. - Python: `test_permutation_over_table_with_schema_metadata` in `test_permutation.py` covers the repro and reads both splits through `Permutations`/`Permutation.from_tables`. `test_streaming_dataset_over_table_with_schema_metadata` in `test_elastic_dataloader.py` covers `StreamingDataset` construction and iteration. - `cargo fmt`, `cargo clippy --features remote --tests --examples`, the dataloader Rust tests (42), `test_permutation.py` (72), and `test_elastic_dataloader.py` all pass. `test_torch.py::test_remote_permutation_dataloader_multiprocessing` fails locally with `take query result did not include ordering column '__lancedb_take_row_offset'`. It uses `Permutation.identity`, which does not go through `PermutationBuilder`, so this change does not cause it. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../python/tests/test_elastic_dataloader.py | 16 ++++ python/python/tests/test_permutation.py | 29 +++++++ .../src/dataloader/permutation/builder.rs | 80 ++++++++++++++++++- .../src/dataloader/permutation/split.rs | 26 +++--- 4 files changed, 137 insertions(+), 14 deletions(-) diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index cc6874e6c..9d18752f1 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -3259,6 +3259,22 @@ def test_doc_example_checkpoint(lance_table): ) +def test_streaming_dataset_over_table_with_schema_metadata(tmp_path): + """StreamingDataset builds its permutation at construction; base table schema + metadata (e.g. Hugging Face's `source-column-name-map`) must not break it.""" + schema = pa.schema( + [pa.field("id", pa.int64())], + metadata={b"source-column-name-map": b"{}", b"huggingface": b"{}"}, + ) + table = lancedb.connect(tmp_path).create_table( + "data", pa.table({"id": list(range(NUM_ROWS))}, schema=schema) + ) + + ds = StreamingDataset(table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED) + + assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS)) + + # --------------------------------------------------------------------------- # Remote tables (LanceDB Cloud / Enterprise) # --------------------------------------------------------------------------- diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py index 142fc84f1..5e8bb1d76 100644 --- a/python/python/tests/test_permutation.py +++ b/python/python/tests/test_permutation.py @@ -115,6 +115,35 @@ def test_split_random_fixed(mem_db): assert split_ids.count(split_id) == 25 +def test_permutation_over_table_with_schema_metadata(tmp_path): + """Schema-level metadata on the base table must not break the build. + + Hugging Face datasets carry keys like `huggingface` or `source-column-name-map`. + """ + schema = pa.schema( + [pa.field("x", pa.int64())], + # `split_names` collides with a permutation config key; the config must win. + metadata={b"huggingface": b"{}", b"split_names": b'["base"]'}, + ) + tbl = connect(tmp_path).create_table( + "t", pa.table({"x": list(range(100))}, schema=schema) + ) + permutation_tbl = ( + permutation_builder(tbl) + .split_random(fixed=2, seed=1, split_names=["train", "test"]) + .execute() + ) + + permutations = Permutations(tbl, permutation_tbl) + assert permutations.split_names == ["train", "test"] + train = Permutation.from_tables(tbl, permutation_tbl, "train") + test = Permutation.from_tables(tbl, permutation_tbl, "test") + assert train.num_rows + test.num_rows == 100 + train_x = [row["x"] for batch in train.iter(16) for row in batch] + test_x = [row["x"] for batch in test.iter(16) for row in batch] + assert sorted(train_x + test_x) == list(range(100)) + + def test_split_random_with_seed(mem_db): """Test that seeded random splits are reproducible.""" tbl = mem_db.create_table("test_table", pa.table({"x": range(50), "y": range(50)})) diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index e08f3eddc..3837d9084 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, sync::Arc}; +use arrow_array::RecordBatch; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_execution::{disk_manager::DiskManagerBuilder, runtime_env::RuntimeEnvBuilder}; use datafusion_expr::col; @@ -220,6 +221,12 @@ impl PermutationBuilder { Ok(Box::pin(SimpleRecordBatchStream { schema, stream })) } + /// Replaces the stream's schema metadata with the permutation config. + /// + /// The scan of the base table can carry the base table's schema metadata (e.g. Hugging + /// Face's `huggingface` key). That metadata describes columns the permutation table does + /// not have, and its keys could collide with the config keys, so it is dropped rather + /// than merged. fn add_config_metadata( data: SendableRecordBatchStream, metadata: HashMap, @@ -227,7 +234,13 @@ impl PermutationBuilder { let schema = data.schema().as_ref().clone().with_metadata(metadata); let schema = Arc::new(schema); let schema_clone = schema.clone(); - let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap()); + // `RecordBatch::with_schema` only accepts a superset of the batch's schema, which a + // schema with replaced metadata is not, so rebuild the batch around the new schema. + let stream = data.and_then(move |batch| { + std::future::ready( + RecordBatch::try_new(schema.clone(), batch.columns().to_vec()).map_err(Error::from), + ) + }); Ok(Box::pin(SimpleRecordBatchStream { schema: schema_clone, stream, @@ -776,4 +789,69 @@ mod tests { "expected the pre-check to refuse the table, got: {err}" ); } + + /// Base table schema metadata (e.g. Hugging Face's `huggingface` key) describes the base + /// table's columns, so the permutation table drops it and keeps only its own config. + #[tokio::test] + async fn test_permutation_over_table_with_schema_metadata() { + use arrow_array::{Int64Array, RecordBatchIterator}; + use arrow_schema::{DataType, Field, Schema}; + + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let schema = Arc::new( + Schema::new(vec![Field::new("x", DataType::Int64, false)]).with_metadata( + HashMap::from([ + ("huggingface".to_string(), "{}".to_string()), + // Collides with a config key; the config value must win. + (SPLIT_NAMES_CONFIG_KEY.to_string(), "not json".to_string()), + ]), + ), + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(0..100))], + ) + .unwrap(); + let reader: Box = + Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())); + let table = db.create_table("tbl", reader).execute().await.unwrap(); + assert!( + table + .schema() + .await + .unwrap() + .metadata + .contains_key("huggingface") + ); + + let permutation_table = PermutationBuilder::new(table) + .with_split_strategy( + SplitStrategy::Random { + seed: Some(1), + sizes: SplitSizes::Fixed(2), + clump_size: None, + }, + Some(vec!["train".to_string(), "test".to_string()]), + ) + .with_shuffle_strategy(ShuffleStrategy::Random { + seed: Some(1), + clump_size: None, + }) + .build() + .await + .unwrap(); + + assert_eq!(permutation_table.count_rows(None).await.unwrap(), 100); + let metadata = permutation_table.schema().await.unwrap().metadata.clone(); + assert!(!metadata.contains_key("huggingface")); + assert_eq!( + metadata.get(SPLIT_NAMES_CONFIG_KEY).map(String::as_str), + Some(r#"["train","test"]"#) + ); + assert!(metadata.contains_key(BASE_VERSION_CONFIG_KEY)); + } } diff --git a/rust/lancedb/src/dataloader/permutation/split.rs b/rust/lancedb/src/dataloader/permutation/split.rs index b33544100..cfa8e5235 100644 --- a/rust/lancedb/src/dataloader/permutation/split.rs +++ b/rust/lancedb/src/dataloader/permutation/split.rs @@ -214,9 +214,9 @@ impl Splitter { } arrays.push(Arc::new(split_ids)); - std::future::ready(Some(Ok( - RecordBatch::try_new(new_schema.clone(), arrays).unwrap() - ))) + std::future::ready(Some( + RecordBatch::try_new(new_schema.clone(), arrays).map_err(Error::from), + )) }); Ok(Box::pin(SimpleRecordBatchStream::new( @@ -289,23 +289,23 @@ impl Splitter { .collect::>(); let new_schema_clone = new_schema.clone(); - let stream = source.map_ok(move |batch| { + let stream = source.and_then(move |batch| { let split_ids = Self::hash_split_id(&batch, &thresholds, total_weight); - if split_ids.null_count() > 0 { - let is_valid = split_ids.nulls().unwrap().inner(); - let is_valid_mask = BooleanArray::new(is_valid.clone(), None); - let split_ids = arrow::compute::filter(&split_ids, &is_valid_mask).unwrap(); - let row_ids = batch.column(row_id_index); - let row_ids = arrow::compute::filter(row_ids.as_ref(), &is_valid_mask).unwrap(); - RecordBatch::try_new(new_schema.clone(), vec![row_ids, split_ids]).unwrap() + let batch = if let Some(nulls) = split_ids.nulls().filter(|n| n.null_count() > 0) { + let is_valid_mask = BooleanArray::new(nulls.inner().clone(), None); + arrow::compute::filter(&split_ids, &is_valid_mask).and_then(|split_ids| { + let row_ids = batch.column(row_id_index); + let row_ids = arrow::compute::filter(row_ids.as_ref(), &is_valid_mask)?; + RecordBatch::try_new(new_schema.clone(), vec![row_ids, split_ids]) + }) } else { RecordBatch::try_new( new_schema.clone(), vec![batch.column(row_id_index).clone(), Arc::new(split_ids)], ) - .unwrap() - } + }; + std::future::ready(batch.map_err(Error::from)) }); Ok(Box::pin(SimpleRecordBatchStream::new( From 92825f6f13d35f04c804aa15647fe636735fb46d Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Tue, 20 Oct 2026 11:58:57 -0500 Subject: [PATCH 21/22] refactor!: rename MemWAL APIs to RTE (realtime engine) (#4598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Renames the user-facing MemWAL surface to **RTE** (realtime engine) across Rust, Python, TypeScript, and Java, including the REST wire format. This follows the LSM → MemWAL rename in #4075. | Before | After | |---|---| | `MemWalWriteSpec` / `MemWalStats` | `RTEWriteSpec` / `RTEStats` | | `enable_memwal` / `disable_memwal` / `get_memwal_spec` | `enable_rte` / `disable_rte` / `get_rte_spec` | | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | `flush_rte` / `compact_rte` / `checkpoint_rte` | | `get_memwal_stats` / `close_memwal_writers` | `get_rte_stats` / `close_rte_writers` | | `use_memwal` (query, merge insert) | `use_rte` | | TS/Java `enableMemWal`, `useMemWal`, ... | `enableRTE`, `useRTE`, ... (napi `js_name` set explicitly) | | Java `LanceDbTableMemWal` | `LanceDbTableRTE` | | `table/memwal_stats.rs` | `table/rte_stats.rs` | Docs and comments now say "realtime engine (RTE)". Text about Lance's underlying mechanism keeps "MemWAL" ("Lance's MemWAL", "MemWAL index"). **Left alone:** Lance-owned names (`MemWalIndexDetails`, `DatasetMemWalExt`, `InitializeMemWal*`, `mem_wal` paths), internal `Lsm*` names and modules, and the `does not support with_row_id` refusal text that the hybrid fallback matches on. ## Compatibility Breaking, for an experimental API. The wire format changes, so this must roll out together with the matching sophon change (lancedb/sophon#8870): - Routes: `/enable_memwal/` → `/enable_rte/`, `/disable_memwal/` → `/disable_rte/`, `/get_memwal_spec/` → `/get_rte_spec/`, `/get_memwal_stats/` → `/get_rte_stats/`, `/flush_memwal/` → `/flush_rte/`, `/compact_memwal/` → `/compact_rte/` - Response keys: `memwal_spec` → `rte_spec`, `memwal_stats` → `rte_stats` - Request field: `use_memwal` → `use_rte` on query and merge-insert bodies The SQL keyword becomes `ALTER TABLE ... SET RTE` server-side. The guides under `docs/web` will be updated separately. ## Validation - Rust: `cargo clippy --features remote --tests --examples` clean; `cargo test --features remote -p lancedb --lib`: 1533 passed - Java: `mvn test`: 37 passed (Spotless applied with JDK 11) - Python: `ruff format` / `ruff check` clean; targeted tests (`test_rte_spec`, `test_merge_insert_lsm`, `test_remote_db`, `test_wal_hybrid`, `test_query`, `test_permutation`, `test_table`): 708 passed - TypeScript: `pnpm build`; `table`, `query` tests: 606 passed; `biome check` clean; `docs/src/js` regenerated with `pnpm run docs` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- docs/src/js/classes/AutoQuery.md | 20 +- docs/src/js/classes/MergeInsertBuilder.md | 16 +- docs/src/js/classes/Query.md | 20 +- docs/src/js/classes/Table.md | 74 ++-- docs/src/js/classes/TakeQuery.md | 10 +- docs/src/js/classes/VectorQuery.md | 22 +- docs/src/js/globals.md | 4 +- .../{MemWalStats.md => RTEStats.md} | 6 +- .../{MemWalWriteSpec.md => RTEWriteSpec.md} | 10 +- docs/src/python/python.md | 2 +- java/.project | 28 ++ .../org.eclipse.core.resources.prefs | 2 + java/.settings/org.eclipse.m2e.core.prefs | 4 + java/README.md | 18 +- java/lancedb-core/.classpath | 57 +++ java/lancedb-core/.project | 34 ++ .../org.eclipse.core.resources.prefs | 5 + .../.settings/org.eclipse.jdt.apt.core.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 9 + .../.settings/org.eclipse.m2e.core.prefs | 4 + .../src/main/java/com/lancedb/JsonFields.java | 4 +- .../LanceDbNamespaceClientBuilder.java | 4 +- .../java/com/lancedb/LanceDbRestClient.java | 15 +- ...bTableMemWal.java => LanceDbTableRTE.java} | 116 +++---- .../{MemWalStats.java => RTEStats.java} | 14 +- ...MemWalWriteSpec.java => RTEWriteSpec.java} | 52 +-- ...mWalTest.java => LanceDbTableRTETest.java} | 262 +++++++------- nodejs/__test__/table.test.ts | 104 +++--- nodejs/lancedb/index.ts | 4 +- nodejs/lancedb/merge.ts | 16 +- nodejs/lancedb/query.ts | 30 +- nodejs/lancedb/table.ts | 108 +++--- nodejs/src/merge.rs | 6 +- nodejs/src/query.rs | 18 +- nodejs/src/table.rs | 97 +++--- python/python/lancedb/__init__.py | 4 +- python/python/lancedb/_lancedb.pyi | 52 ++- python/python/lancedb/_wal_hybrid.py | 38 +- python/python/lancedb/merge.py | 16 +- python/python/lancedb/permutation.py | 2 +- python/python/lancedb/query.py | 78 ++--- python/python/lancedb/remote/table.py | 48 +-- python/python/lancedb/table.py | 144 ++++---- python/python/tests/test_merge_insert_lsm.py | 106 +++--- python/python/tests/test_remote_db.py | 40 +-- .../{test_memwal_spec.py => test_rte_spec.py} | 120 +++---- python/python/tests/test_wal_hybrid.py | 14 +- python/python/tests/utils.py | 6 +- python/src/lib.rs | 6 +- python/src/query.rs | 28 +- python/src/table.rs | 117 +++---- rust/lancedb/src/database/listing.rs | 6 +- .../src/dataloader/permutation/builder.rs | 23 +- rust/lancedb/src/lib.rs | 2 +- rust/lancedb/src/materialized_view/refresh.rs | 6 +- rust/lancedb/src/query.rs | 38 +- rust/lancedb/src/query/wal_fusion.rs | 34 +- rust/lancedb/src/remote/table.rs | 249 +++++++------ rust/lancedb/src/remote/table/blobs.rs | 6 +- rust/lancedb/src/remote/table/insert.rs | 2 +- rust/lancedb/src/table.rs | 327 +++++++++--------- rust/lancedb/src/table/checkpoint.rs | 24 +- rust/lancedb/src/table/computed_columns.rs | 30 +- rust/lancedb/src/table/datafusion.rs | 14 +- rust/lancedb/src/table/dataset.rs | 10 +- rust/lancedb/src/table/merge.rs | 191 ++++------ rust/lancedb/src/table/merge/lsm.rs | 136 ++++---- rust/lancedb/src/table/query.rs | 38 +- rust/lancedb/src/table/query/lsm.rs | 50 +-- rust/lancedb/src/table/refresh.rs | 30 +- .../table/{memwal_stats.rs => rte_stats.rs} | 14 +- rust/lancedb/src/table/schema_evolution.rs | 4 +- rust/lancedb/tests/blob_integration.rs | 6 +- 73 files changed, 1656 insertions(+), 1600 deletions(-) rename docs/src/js/interfaces/{MemWalStats.md => RTEStats.md} (66%) rename docs/src/js/interfaces/{MemWalWriteSpec.md => RTEWriteSpec.md} (80%) create mode 100644 java/.project create mode 100644 java/.settings/org.eclipse.core.resources.prefs create mode 100644 java/.settings/org.eclipse.m2e.core.prefs create mode 100644 java/lancedb-core/.classpath create mode 100644 java/lancedb-core/.project create mode 100644 java/lancedb-core/.settings/org.eclipse.core.resources.prefs create mode 100644 java/lancedb-core/.settings/org.eclipse.jdt.apt.core.prefs create mode 100644 java/lancedb-core/.settings/org.eclipse.jdt.core.prefs create mode 100644 java/lancedb-core/.settings/org.eclipse.m2e.core.prefs rename java/lancedb-core/src/main/java/com/lancedb/{LanceDbTableMemWal.java => LanceDbTableRTE.java} (73%) rename java/lancedb-core/src/main/java/com/lancedb/{MemWalStats.java => RTEStats.java} (80%) rename java/lancedb-core/src/main/java/com/lancedb/{MemWalWriteSpec.java => RTEWriteSpec.java} (83%) rename java/lancedb-core/src/test/java/com/lancedb/{LanceDbTableMemWalTest.java => LanceDbTableRTETest.java} (67%) rename python/python/tests/{test_memwal_spec.py => test_rte_spec.py} (65%) rename rust/lancedb/src/table/{memwal_stats.rs => rte_stats.rs} (93%) diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md index 1e9859215..f12668b90 100644 --- a/docs/src/js/classes/AutoQuery.md +++ b/docs/src/js/classes/AutoQuery.md @@ -425,16 +425,16 @@ Empty results retain the query's output schema. *** -### useMemWal() +### useRTE() ```ts -useMemWal(enable): this +useRTE(enable): this ``` -Control MemWAL read routing for this query. +Control RTE read routing for this query. -By default (unset), when the table carries a MemWAL write spec (see -[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so +By default (unset), when the table carries an RTE write spec (see +[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and the SSTables), deduplicated by primary key; a table without a spec @@ -444,12 +444,12 @@ reads the base table. * **enable**: `boolean` `true` forces the LSM scanner and errors if the table has no - MemWAL write spec. `false` bypasses the MemWAL and reads the base table only, + RTE write spec. `false` bypasses the RTE and reads the base table only, even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, - hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useMemWal(false)` is set, because a base-only read would silently exclude - un-compacted MemWAL data. + hybrid search, `orderBy`). On an RTE table those shapes error unless + `useRTE(false)` is set, because a base-only read would silently exclude + un-compacted RTE data. #### Returns @@ -457,7 +457,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useMemWal` +`StandardQueryBase.useRTE` *** diff --git a/docs/src/js/classes/MergeInsertBuilder.md b/docs/src/js/classes/MergeInsertBuilder.md index cf40837d3..d4ee34b13 100644 --- a/docs/src/js/classes/MergeInsertBuilder.md +++ b/docs/src/js/classes/MergeInsertBuilder.md @@ -76,23 +76,23 @@ the query optimizer chooses a suboptimal path. *** -### useMemWal() +### useRTE() ```ts -useMemWal(enable): MergeInsertBuilder +useRTE(enable): MergeInsertBuilder ``` -Control MemWAL routing for this merge. +Control RTE routing for this merge. -By default (unset), a `mergeInsert` on a table with a MemWAL write spec is +By default (unset), a `mergeInsert` on a table with an RTE write spec is routed through Lance's MemWAL shard writer, and a table without one uses the standard path. #### Parameters * **enable**: `boolean` - `true` forces MemWAL routing and errors if the table has no - MemWAL write spec. `false` forces the standard write path even when a spec is set. + `true` forces RTE routing and errors if the table has no + RTE write spec. `false` forces the standard write path even when a spec is set. #### Returns @@ -108,12 +108,12 @@ validateSingleShard(validateSingleShard): MergeInsertBuilder Controls how an LSM merge checks that its input targets a single shard. -When a table has a MemWAL write spec, every row in a `mergeInsert` call must +When a table has an RTE write spec, every row in a `mergeInsert` call must route to the same shard. When `true` (the default), every row is inspected to verify this. When `false`, only the first row is inspected and the shard it routes to is used for the whole input — a faster path for callers that have already pre-sharded their input. Has no effect on tables without -a MemWAL write spec. +an RTE write spec. #### Parameters diff --git a/docs/src/js/classes/Query.md b/docs/src/js/classes/Query.md index 9d83404f4..8e216d1f4 100644 --- a/docs/src/js/classes/Query.md +++ b/docs/src/js/classes/Query.md @@ -499,16 +499,16 @@ Empty results retain the query's output schema. *** -### useMemWal() +### useRTE() ```ts -useMemWal(enable): this +useRTE(enable): this ``` -Control MemWAL read routing for this query. +Control RTE read routing for this query. -By default (unset), when the table carries a MemWAL write spec (see -[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so +By default (unset), when the table carries an RTE write spec (see +[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and the SSTables), deduplicated by primary key; a table without a spec @@ -518,12 +518,12 @@ reads the base table. * **enable**: `boolean` `true` forces the LSM scanner and errors if the table has no - MemWAL write spec. `false` bypasses the MemWAL and reads the base table only, + RTE write spec. `false` bypasses the RTE and reads the base table only, even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, - hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useMemWal(false)` is set, because a base-only read would silently exclude - un-compacted MemWAL data. + hybrid search, `orderBy`). On an RTE table those shapes error unless + `useRTE(false)` is set, because a base-only read would silently exclude + un-compacted RTE data. #### Returns @@ -531,7 +531,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useMemWal` +`StandardQueryBase.useRTE` *** diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3eaf9c9ca..b41ade302 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -228,13 +228,13 @@ version of the table. *** -### checkpointMemWal() +### checkpointRTE() ```ts -abstract checkpointMemWal(): Promise +abstract checkpointRTE(): Promise ``` -Converge this table's MemWAL write path into its base table. +Converge this table's RTE write path into its base table. Freezes once, then triggers compaction and polls until the SSTables that existed at the start is gone. The target set is fixed at the start, so @@ -254,9 +254,9 @@ is merging. The caller owns the deadline. #### Example ```ts -const before = await table.getMemWalStats(); -await table.checkpointMemWal(); -const after = await table.getMemWalStats(); +const before = await table.getRTEStats(); +await table.checkpointRTE(); +const after = await table.getRTEStats(); ``` *** @@ -279,15 +279,15 @@ Any attempt to use the table after it is closed will result in an error. *** -### closeMemWalWriters() +### closeRTEWriters() ```ts -abstract closeMemWalWriters(): Promise +abstract closeRTEWriters(): Promise ``` -Drain and close any cached MemWAL shard writers held for this table. +Drain and close any cached RTE shard writers held for this table. -When an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) is installed, `mergeInsert` opens MemWAL +When an [RTEWriteSpec](../interfaces/RTEWriteSpec.md) is installed, `mergeInsert` opens RTE shard writers and caches them for reuse across calls. This closes them, flushing pending data; writers reopen lazily on the next `mergeInsert`. It is a no-op when no writers are cached. @@ -298,17 +298,17 @@ It is a no-op when no writers are cached. *** -### compactMemWal() +### compactRTE() ```ts -abstract compactMemWal(): Promise +abstract compactRTE(): Promise ``` Trigger a background SSTable compaction pass per table shard. Returns once the passes are *dispatched*, not once they finish — watch -[Table#getMemWalStats](Table.md#getmemwalstats) for progress, or use -[Table#checkpointMemWal](Table.md#checkpointmemwal) to wait for convergence. +[Table#getRTEStats](Table.md#getrtestats) for progress, or use +[Table#checkpointRTE](Table.md#checkpointrte) to wait for convergence. #### Returns @@ -449,13 +449,13 @@ containing the new version number of the table *** -### disableMemWal() +### disableRTE() ```ts -abstract disableMemWal(): Promise +abstract disableRTE(): Promise ``` -Remove the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) from this table, reverting to the standard +Remove the [RTEWriteSpec](../interfaces/RTEWriteSpec.md) from this table, reverting to the standard `mergeInsert` write path. Errors if no spec is currently set. @@ -531,16 +531,16 @@ Drop an index from the table. *** -### enableMemWal() +### enableRTE() ```ts -abstract enableMemWal(spec): Promise +abstract enableRTE(spec): Promise ``` -Install an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) on this table, selecting Lance's MemWAL +Install an [RTEWriteSpec](../interfaces/RTEWriteSpec.md) on this table, selecting Lance's MemWAL LSM-style write path for future `mergeInsert` calls. -`MemWalWriteSpec` chooses one of three sharding strategies via `specType`: +`RTEWriteSpec` chooses one of three sharding strategies via `specType`: - `"bucket"` — hash-bucket writes by the single-column unenforced primary key (`column` and `numBuckets` required). @@ -552,13 +552,13 @@ All variants require the table to have an unenforced primary key requires it to be the single column being bucketed. Omitting `maintainedIndexes` maintains every index the table has, -including ones created later, and skips a kind the MemWAL cannot maintain. +including ones created later, and skips a kind the RTE cannot maintain. Naming them pins an exact set, and a still-building index is rejected rather than quietly omitted. #### Parameters -* **spec**: [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md) +* **spec**: [`RTEWriteSpec`](../interfaces/RTEWriteSpec.md) The sharding spec to install. #### Returns @@ -569,7 +569,7 @@ rather than quietly omitted. ```ts await table.setUnenforcedPrimaryKey("id"); -await table.enableMemWal({ +await table.enableRTE({ specType: "bucket", column: "id", numBuckets: 16, @@ -647,10 +647,10 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values. *** -### flushMemWal() +### flushRTE() ```ts -abstract flushMemWal(): Promise +abstract flushRTE(): Promise ``` Freeze every table shard's active memtable into a new SSTable. @@ -699,38 +699,38 @@ const { records, truncated } = await table.functionErrors({ column: "embedding" *** -### getMemWalSpec() +### getRTESpec() ```ts -abstract getMemWalSpec(): Promise +abstract getRTESpec(): Promise ``` -Read the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) currently installed on this table. +Read the [RTEWriteSpec](../interfaces/RTEWriteSpec.md) currently installed on this table. -Resolves to `undefined` when the MemWAL write path is not enabled (no -spec has been set, or it was removed with [Table#disableMemWal](Table.md#disablememwal)). +Resolves to `undefined` when the RTE write path is not enabled (no +spec has been set, or it was removed with [Table#disableRTE](Table.md#disablerte)). The spec is the one installed, including its maintained-index selection: an absent `maintainedIndexes` for every index the table has, an empty array for none. #### Returns -`Promise`<`undefined` \| [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)> +`Promise`<`undefined` \| [`RTEWriteSpec`](../interfaces/RTEWriteSpec.md)> *** -### getMemWalStats() +### getRTEStats() ```ts -abstract getMemWalStats(includeSstableRows?): Promise +abstract getRTEStats(includeSstableRows?): Promise ``` -Read live per-table-shard MemWAL state. +Read live per-table-shard RTE state. Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my fresh-tier vector search brute-force". Mutates no table state. -Resolves to `undefined` only when the MemWAL write path is not enabled. +Resolves to `undefined` only when the RTE write path is not enabled. #### Parameters @@ -740,7 +740,7 @@ Resolves to `undefined` only when the MemWAL write path is not enabled. #### Returns -`Promise`<`undefined` \| [`MemWalStats`](../interfaces/MemWalStats.md)> +`Promise`<`undefined` \| [`RTEStats`](../interfaces/RTEStats.md)> *** diff --git a/docs/src/js/classes/TakeQuery.md b/docs/src/js/classes/TakeQuery.md index 9e8d03149..65bc35f08 100644 --- a/docs/src/js/classes/TakeQuery.md +++ b/docs/src/js/classes/TakeQuery.md @@ -275,17 +275,17 @@ Empty results retain the query's output schema. *** -### useMemWal() +### useRTE() ```ts -useMemWal(enable): this +useRTE(enable): this ``` -Control MemWAL read routing for this take query. +Control RTE read routing for this take query. -`false` bypasses the MemWAL and reads the base table only — the escape hatch, +`false` bypasses the RTE and reads the base table only — the escape hatch, since take-by-row-id/offset is not supported on the LSM scanner and, on a -MemWAL table, auto-routes to it and errors otherwise. +RTE table, auto-routes to it and errors otherwise. #### Parameters diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index e329cb254..04ced2ef4 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -659,7 +659,7 @@ minimum; one searches all partitions within the caller maximum. Later probing may continue when too few candidates are found. Any explicit effort, including 0.5, overrides nprobes in either call order. Explicit minimum and maximum bounds still apply. This is not a recall -guarantee and does not change refinement or HNSW ef. The MemWAL LSM +guarantee and does not change refinement or HNSW ef. The RTE LSM scanner does not yet support this option. #### Parameters @@ -781,16 +781,16 @@ Empty results retain the query's output schema. *** -### useMemWal() +### useRTE() ```ts -useMemWal(enable): this +useRTE(enable): this ``` -Control MemWAL read routing for this query. +Control RTE read routing for this query. -By default (unset), when the table carries a MemWAL write spec (see -[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so +By default (unset), when the table carries an RTE write spec (see +[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so they also return data written via the `mergeInsert` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and the SSTables), deduplicated by primary key; a table without a spec @@ -800,12 +800,12 @@ reads the base table. * **enable**: `boolean` `true` forces the LSM scanner and errors if the table has no - MemWAL write spec. `false` bypasses the MemWAL and reads the base table only, + RTE write spec. `false` bypasses the RTE and reads the base table only, even when a spec is present. Note: the LSM scanner does not support every query shape (e.g. reranking, - hybrid search, `orderBy`). On a MemWAL table those shapes error unless - `useMemWal(false)` is set, because a base-only read would silently exclude - un-compacted MemWAL data. + hybrid search, `orderBy`). On an RTE table those shapes error unless + `useRTE(false)` is set, because a base-only read would silently exclude + un-compacted RTE data. #### Returns @@ -813,7 +813,7 @@ reads the base table. #### Inherited from -`StandardQueryBase.useMemWal` +`StandardQueryBase.useRTE` *** diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 7310f3f90..646341190 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -114,8 +114,6 @@ - [ListTablesResponse](interfaces/ListTablesResponse.md) - [ListingOptions](interfaces/ListingOptions.md) - [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) -- [MemWalStats](interfaces/MemWalStats.md) -- [MemWalWriteSpec](interfaces/MemWalWriteSpec.md) - [MemtableStats](interfaces/MemtableStats.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) @@ -124,6 +122,8 @@ - [OptimizeOptions](interfaces/OptimizeOptions.md) - [OptimizeStats](interfaces/OptimizeStats.md) - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md) +- [RTEStats](interfaces/RTEStats.md) +- [RTEWriteSpec](interfaces/RTEWriteSpec.md) - [RefreshColumnResult](interfaces/RefreshColumnResult.md) - [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md) - [RemovalStats](interfaces/RemovalStats.md) diff --git a/docs/src/js/interfaces/MemWalStats.md b/docs/src/js/interfaces/RTEStats.md similarity index 66% rename from docs/src/js/interfaces/MemWalStats.md rename to docs/src/js/interfaces/RTEStats.md index 4221867fc..e279be1ca 100644 --- a/docs/src/js/interfaces/MemWalStats.md +++ b/docs/src/js/interfaces/RTEStats.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MemWalStats +[@lancedb/lancedb](../globals.md) / RTEStats -# Interface: MemWalStats +# Interface: RTEStats -Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`. +Live per-table-shard RTE state, as returned by `Table#getRTEStats`. Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's to compute. diff --git a/docs/src/js/interfaces/MemWalWriteSpec.md b/docs/src/js/interfaces/RTEWriteSpec.md similarity index 80% rename from docs/src/js/interfaces/MemWalWriteSpec.md rename to docs/src/js/interfaces/RTEWriteSpec.md index 50b74da41..57e8c17cc 100644 --- a/docs/src/js/interfaces/MemWalWriteSpec.md +++ b/docs/src/js/interfaces/RTEWriteSpec.md @@ -2,12 +2,12 @@ *** -[@lancedb/lancedb](../globals.md) / MemWalWriteSpec +[@lancedb/lancedb](../globals.md) / RTEWriteSpec -# Interface: MemWalWriteSpec +# Interface: RTEWriteSpec -Specification selecting Lance's MemWAL LSM-style write path for -`mergeInsert`. +Specification enabling the realtime engine (RTE), which routes +`mergeInsert` through Lance's MemWAL LSM-style write path. `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`, `column` and `numBuckets` are required; for `"identity"`, `column` is @@ -34,7 +34,7 @@ Bucket and identity variants: the sharding column. optional maintainedIndexes: string[]; ``` -Indexes the MemWAL keeps up to date. Omit for every index the table has, +Indexes the RTE keeps up to date. Omit for every index the table has, including ones created later. Pass `[]` for none. *** diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 8405852f2..c8a82851f 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -297,7 +297,7 @@ listing a storage directory. ::: lancedb.table.Branches -::: lancedb.MemWalWriteSpec +::: lancedb.RTEWriteSpec ## Functions and Jobs diff --git a/java/.project b/java/.project new file mode 100644 index 000000000..cca1b8cf5 --- /dev/null +++ b/java/.project @@ -0,0 +1,28 @@ + + + lancedb-parent + + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.m2e.core.maven2Nature + + + + 1786980408448 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/java/.settings/org.eclipse.core.resources.prefs b/java/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 000000000..99f26c020 --- /dev/null +++ b/java/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 diff --git a/java/.settings/org.eclipse.m2e.core.prefs b/java/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 000000000..f897a7f1c --- /dev/null +++ b/java/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/java/README.md b/java/README.md index 0e54d20e4..1c20fdda0 100644 --- a/java/README.md +++ b/java/README.md @@ -29,34 +29,34 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder() .build(); ``` -## MemWAL write path +## RTE write path Most table operations reach LanceDB through the `LanceNamespace` above, which is -generated from the Lance Namespace specification. The MemWAL LSM routes are not part +generated from the Lance Namespace specification. The RTE LSM routes are not part of that specification, so they are issued through a separate client: ```java import com.lancedb.LanceDbRestClient; -import com.lancedb.LanceDbTableMemWal; -import com.lancedb.MemWalWriteSpec; +import com.lancedb.LanceDbTableRTE; +import com.lancedb.RTEWriteSpec; LanceDbRestClient client = LanceDbNamespaceClientBuilder.newBuilder() .apiKey("your_lancedb_cloud_api_key") .database("your_database_name") .buildRestClient(); -LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table"); +LanceDbTableRTE rte = new LanceDbTableRTE(client, "my_table"); -// Route future merge_insert upserts through the MemWAL, hash-bucketed by `id`. -memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16)); +// Route future merge_insert upserts through the RTE, hash-bucketed by `id`. +rte.enableRTE(RTEWriteSpec.bucket("id", 16)); // ... merge_insert traffic ... // Converge the fresh tier into the base table. -memWal.checkpointMemWal(); +rte.checkpointRTE(); // Inspect live per-table-shard state. -memWal.getMemWalStats().ifPresent(stats -> stats.tableShards().forEach(tableShard -> +rte.getRTEStats().ifPresent(stats -> stats.tableShards().forEach(tableShard -> System.out.println(tableShard.shardId() + ": " + tableShard.sstables().size() + " SSTables"))); client.close(); diff --git a/java/lancedb-core/.classpath b/java/lancedb-core/.classpath new file mode 100644 index 000000000..8c4583205 --- /dev/null +++ b/java/lancedb-core/.classpath @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/lancedb-core/.project b/java/lancedb-core/.project new file mode 100644 index 000000000..598cec78c --- /dev/null +++ b/java/lancedb-core/.project @@ -0,0 +1,34 @@ + + + lancedb-core + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + + + 1786980408445 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/java/lancedb-core/.settings/org.eclipse.core.resources.prefs b/java/lancedb-core/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 000000000..cdfe4f1b6 --- /dev/null +++ b/java/lancedb-core/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/java/lancedb-core/.settings/org.eclipse.jdt.apt.core.prefs b/java/lancedb-core/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 000000000..d4313d4b2 --- /dev/null +++ b/java/lancedb-core/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=false diff --git a/java/lancedb-core/.settings/org.eclipse.jdt.core.prefs b/java/lancedb-core/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..46235dc07 --- /dev/null +++ b/java/lancedb-core/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,9 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.processAnnotations=disabled +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=11 diff --git a/java/lancedb-core/.settings/org.eclipse.m2e.core.prefs b/java/lancedb-core/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 000000000..f897a7f1c --- /dev/null +++ b/java/lancedb-core/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java index e0a288eaf..60ded9999 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java +++ b/java/lancedb-core/src/main/java/com/lancedb/JsonFields.java @@ -20,11 +20,11 @@ import com.fasterxml.jackson.databind.JsonNode; * *

Every reader fails closed: a missing, null, or wrong-typed field throws rather than * defaulting. That mirrors the serde decoding the Rust client applies to the same payloads in - * {@code rust/lancedb/src/table/memwal_stats.rs}, where a required field has no default and a + * {@code rust/lancedb/src/table/rte_stats.rs}, where a required field has no default and a * malformed response is an error rather than a zero. * *

The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty - * array or a zero — is unsafe here because {@link LanceDbTableMemWal#checkpointMemWal()} decides + * array or a zero — is unsafe here because {@link LanceDbTableRTE#checkpointRTE()} decides * convergence from these numbers. A defaulted {@code sstables} array is indistinguishable from a * drained one, so a malformed response would report a checkpoint that never happened. */ diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java index 05aed8c74..da70b15d2 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java @@ -151,8 +151,8 @@ public class LanceDbNamespaceClientBuilder { * Build a {@link LanceDbRestClient} for the same endpoint. * *

Needed only for LanceDB routes that the Lance Namespace specification does not cover — the - * MemWAL write path, reached through {@link LanceDbTableMemWal}. Every other table - * operation belongs on the {@link LanceNamespace} from {@link #build()}. + * RTE write path, reached through {@link LanceDbTableRTE}. Every other table operation belongs on + * the {@link LanceNamespace} from {@link #build()}. * *

The returned client owns an HTTP connection pool; close it when you are done with it. * diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java index 5275c8346..2b40ef798 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java @@ -31,9 +31,9 @@ import java.io.UncheckedIOException; * specification does not cover. * *

Most table operations reach LanceDB through {@link org.lance.namespace.LanceNamespace}, which - * is generated from the namespace spec. A handful of routes — the MemWAL write path in - * particular — are served by the same endpoint but are not part of that spec, so they are issued - * directly here. See {@link LanceDbTableMemWal}. + * is generated from the namespace spec. A handful of routes — the RTE write path in particular — + * are served by the same endpoint but are not part of that spec, so they are issued directly here. + * See {@link LanceDbTableRTE}. * *

Obtain one from {@link LanceDbNamespaceClientBuilder#buildRestClient()}. */ @@ -50,8 +50,8 @@ public class LanceDbRestClient implements Closeable { this.apiKey = apiKey; this.database = database; // Automatic retries off, deliberately. The default strategy retries 429 and 503 — - // exactly the two statuses LanceDbTableMemWal.checkpointMemWal() acts on — which would - // silently double its explicit retry budget and would also retry compact_memwal in + // exactly the two statuses LanceDbTableRTE.checkpointRTE() acts on — which would + // silently double its explicit retry budget and would also retry compact_rte in // place, where the loop is designed to fall through to a fresh stats poll instead. // The checkpoint loop owns the 421/429/503 transitions; the transport must not. this.http = HttpClients.custom().disableAutomaticRetries().build(); @@ -98,9 +98,8 @@ public class LanceDbRestClient implements Closeable { /** * A non-2xx response. * - *

The status is exposed because callers act on it: {@link - * LanceDbTableMemWal#checkpointMemWal()} treats 429 and 503 as retryable and 421 as a lost node - * claim. + *

The status is exposed because callers act on it: {@link LanceDbTableRTE#checkpointRTE()} + * treats 429 and 503 as retryable and 421 as a lost node claim. */ public static class HttpException extends RuntimeException { private static final long serialVersionUID = 1L; diff --git a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableRTE.java similarity index 73% rename from java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java rename to java/lancedb-core/src/main/java/com/lancedb/LanceDbTableRTE.java index 75c6c33b1..b0a61233c 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java +++ b/java/lancedb-core/src/main/java/com/lancedb/LanceDbTableRTE.java @@ -22,11 +22,11 @@ import java.util.Optional; import java.util.OptionalLong; /** - * The MemWAL write path for one LanceDB Cloud or Enterprise table. + * The RTE write path for one LanceDB Cloud or Enterprise table. * - *

Installing an {@link MemWalWriteSpec} routes {@code mergeInsert} upserts through Lance's - * MemWAL — an LSM-style append — instead of the standard merge path. Rows land in an in-memory - * memtable, freeze into SSTables, and are merged into the base table by compaction. + *

Installing an {@link RTEWriteSpec} routes {@code mergeInsert} upserts through Lance's RTE — an + * LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable, freeze + * into SSTables, and are merged into the base table by compaction. * *

These routes are not part of the Lance Namespace specification, so they are issued directly * rather than through {@link org.lance.namespace.LanceNamespace}. @@ -37,23 +37,23 @@ import java.util.OptionalLong; * .database("your_database_name") * .buildRestClient(); * - * LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table"); - * memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16)); + * LanceDbTableRTE rte = new LanceDbTableRTE(client, "my_table"); + * rte.enableRTE(RTEWriteSpec.bucket("id", 16)); * // ... merge_insert traffic ... - * memWal.checkpointMemWal(); + * rte.checkpointRTE(); * } */ -public class LanceDbTableMemWal { +public class LanceDbTableRTE { /** - * Interval between {@code get_memwal_stats} polls during a checkpoint. One interval is roughly - * one compaction pass, the granularity at which the answer can change. + * Interval between {@code get_rte_stats} polls during a checkpoint. One interval is roughly one + * compaction pass, the granularity at which the answer can change. */ private static final long POLL_INTERVAL_MS = 5_000L; /** - * Cap on re-issues from {@code flushMemWal} after a 421, so a crash-looping node cannot turn - * flush → compact → 421 → flush into a spin. + * Cap on re-issues from {@code flushRTE} after a 421, so a crash-looping node cannot turn flush → + * compact → 421 → flush into a spin. * *

Deliberately not shared with {@link #MAX_RETRIES}: a claim that keeps evaporating is a * broken node, while contention is routine and wants a real budget. @@ -79,7 +79,7 @@ public class LanceDbTableMemWal { * @param tableIdentifier The table's full identifier, {@code $}-delimited when it sits inside a * namespace, such as {@code analytics$events}. */ - public LanceDbTableMemWal(LanceDbRestClient client, String tableIdentifier) { + public LanceDbTableRTE(LanceDbRestClient client, String tableIdentifier) { if (client == null) { throw new IllegalArgumentException("Client cannot be null"); } @@ -91,46 +91,46 @@ public class LanceDbTableMemWal { } /** - * Install an {@link MemWalWriteSpec} on this table, selecting the MemWAL write path for - * future {@code mergeInsert} calls. + * Install an {@link RTEWriteSpec} on this table, selecting the RTE write path for future {@code + * mergeInsert} calls. * *

All variants require the table to have an unenforced primary key; bucket sharding * additionally requires it to be the single column being bucketed. */ - public void enableMemWal(MemWalWriteSpec spec) { + public void enableRTE(RTEWriteSpec spec) { if (spec == null) { throw new IllegalArgumentException("Spec cannot be null"); } - client.post(route("enable_memwal"), spec.toRequestBody()); + client.post(route("enable_rte"), spec.toRequestBody()); } /** - * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard {@code - * mergeInsert} write path. + * Remove the {@link RTEWriteSpec} from this table, reverting to the standard {@code mergeInsert} + * write path. * *

Errors if no spec is currently set. */ - public void disableMemWal() { - client.post(route("disable_memwal"), null); + public void disableRTE() { + client.post(route("disable_rte"), null); } /** - * Read the {@link MemWalWriteSpec} currently installed on this table. + * Read the {@link RTEWriteSpec} currently installed on this table. * - *

Empty when the MemWAL write path is not enabled at all — no spec has been set, or one was - * removed with {@link #disableMemWal()}. That is a different answer from a present spec whose - * {@link MemWalWriteSpec#maintainedIndexes()} is null, which selects indexes automatically. + *

Empty when the RTE write path is not enabled at all — no spec has been set, or one was + * removed with {@link #disableRTE()}. That is a different answer from a present spec whose {@link + * RTEWriteSpec#maintainedIndexes()} is null, which selects indexes automatically. * *

The spec read back is the one that was installed, selection included: null maintains every * supported index the table has now or gains later, an empty list maintains none, and a non-empty * list maintains exactly those. All three round-trip. */ - public Optional getMemWalSpec() { - JsonNode response = client.post(route("get_memwal_spec"), null); - if (response == null || !response.hasNonNull("memwal_spec")) { + public Optional getRTESpec() { + JsonNode response = client.post(route("get_rte_spec"), null); + if (response == null || !response.hasNonNull("rte_spec")) { return Optional.empty(); } - return Optional.of(MemWalWriteSpec.fromJson(response.get("memwal_spec"))); + return Optional.of(RTEWriteSpec.fromJson(response.get("rte_spec"))); } /** @@ -139,56 +139,56 @@ public class LanceDbTableMemWal { *

Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe * to call repeatedly. */ - public void flushMemWal() { - client.post(route("flush_memwal"), null); + public void flushRTE() { + client.post(route("flush_rte"), null); } /** * Trigger a background SSTable compaction pass per table shard. * *

Returns once the passes are dispatched, not once they finish — watch {@link - * #getMemWalStats}, or use {@link #checkpointMemWal} to wait for convergence. + * #getRTEStats}, or use {@link #checkpointRTE} to wait for convergence. */ - public void compactMemWal() { - client.post(route("compact_memwal"), null); + public void compactRTE() { + client.post(route("compact_rte"), null); } /** - * Read live per-table-shard MemWAL state. + * Read live per-table-shard RTE state. * *

Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my * fresh-tier vector search brute-force". Mutates no table state. * - *

Empty only when the MemWAL write path is not enabled — that is, when the server sends an - * absent or null {@code memwal_stats}. A stats object that is present is decoded strictly, and a - * malformed one throws rather than decoding to something empty, because {@link #checkpointMemWal} - * reads convergence out of these numbers and cannot tell a defaulted array from a drained one. + *

Empty only when the RTE write path is not enabled — that is, when the server sends an absent + * or null {@code rte_stats}. A stats object that is present is decoded strictly, and a malformed + * one throws rather than decoding to something empty, because {@link #checkpointRTE} reads + * convergence out of these numbers and cannot tell a defaulted array from a drained one. * * @param includeSstableRows Also count rows per SSTable. Off by default because each count opens * an uncached Lance dataset. * @throws IllegalStateException if the response is absent or does not decode. */ - public Optional getMemWalStats(boolean includeSstableRows) { + public Optional getRTEStats(boolean includeSstableRows) { Map body = new LinkedHashMap(); body.put("include_sstable_rows", includeSstableRows); - JsonNode response = client.post(route("get_memwal_stats"), body); + JsonNode response = client.post(route("get_rte_stats"), body); if (response == null) { - throw new IllegalStateException("get_memwal_stats returned an empty response body"); + throw new IllegalStateException("get_rte_stats returned an empty response body"); } - JsonNode stats = response.get("memwal_stats"); + JsonNode stats = response.get("rte_stats"); if (stats == null || stats.isNull()) { return Optional.empty(); } - return Optional.of(MemWalStats.fromJson(stats)); + return Optional.of(RTEStats.fromJson(stats)); } - /** Equivalent to {@code getMemWalStats(false)}. */ - public Optional getMemWalStats() { - return getMemWalStats(false); + /** Equivalent to {@code getRTEStats(false)}. */ + public Optional getRTEStats() { + return getRTEStats(false); } /** - * Converge this table's MemWAL write path into its base table. + * Converge this table's RTE write path into its base table. * *

Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction * and polls until those SSTables are gone. The target set is fixed at the start, so sstables @@ -196,31 +196,31 @@ public class LanceDbTableMemWal { * load, and what makes it best-effort: it converges the fresh tier as of some instant. * Idempotent, abandonable at any point, safe on a cadence. * - *

The loop runs here, not on the server: {@link #compactMemWal} dispatches a pass and returns, - * so nothing holds a socket and a client can vanish mid-operation with nothing to reconcile. + *

The loop runs here, not on the server: {@link #compactRTE} dispatches a pass and returns, so + * nothing holds a socket and a client can vanish mid-operation with nothing to reconcile. * Completion is read from generation numbers in the shard manifest — durable state, unlike a * count in a compact response, which a concurrent write invalidates. * *

No liveness bound — the caller owns the deadline. The compactor pool is shared across * tables, so a checkpoint queued behind unrelated work looks exactly like one that is merging. */ - public void checkpointMemWal() { + public void checkpointRTE() { for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) { // The freeze turns everything written before this call into an SSTable, so the // watermark has to be read after it. Idempotent: freezing an empty memtable is a // no-op, so a re-issue does not churn empty sstables. - if (issueVoid(this::flushMemWal)) { + if (issueVoid(this::flushRTE)) { backoff(reissue); continue; } - Attempt> stats = issue(() -> getMemWalStats(false)); + Attempt> stats = issue(() -> getRTEStats(false)); if (stats.lostClaim) { backoff(reissue); continue; } if (!stats.value.isPresent()) { - // Not WAL-backed; flushMemWal would have errored first but for a race. + // Not WAL-backed; flushRTE would have errored first but for a race. return; } @@ -235,7 +235,7 @@ public class LanceDbTableMemWal { backoff(reissue); } throw new IllegalStateException( - "checkpointMemWal: the owning node kept losing its claim; re-issued from flush the maximum " + "checkpointRTE: the owning node kept losing its claim; re-issued from flush the maximum " + "number of times"); } @@ -246,7 +246,7 @@ public class LanceDbTableMemWal { */ private boolean drainToTargets(Map targets) { while (true) { - Attempt> stats = issue(() -> getMemWalStats(false)); + Attempt> stats = issue(() -> getRTEStats(false)); if (stats.lostClaim) { return false; } @@ -277,7 +277,7 @@ public class LanceDbTableMemWal { if (!allCompacting) { try { - compactMemWal(); + compactRTE(); } catch (LanceDbRestClient.HttpException e) { if (isLostClaim(e)) { return false; @@ -296,7 +296,7 @@ public class LanceDbTableMemWal { } /** The newest SSTable generation held by each table shard, skipping table shards holding none. */ - private static Map newestSstableGenerations(MemWalStats stats) { + private static Map newestSstableGenerations(RTEStats stats) { Map targets = new HashMap(); for (TableShardStats tableShard : stats.tableShards()) { OptionalLong newest = tableShard.newestSstableGeneration(); diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java b/java/lancedb-core/src/main/java/com/lancedb/RTEStats.java similarity index 80% rename from java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java rename to java/lancedb-core/src/main/java/com/lancedb/RTEStats.java index c84220508..32edd3864 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java +++ b/java/lancedb-core/src/main/java/com/lancedb/RTEStats.java @@ -20,18 +20,18 @@ import java.util.Collections; import java.util.List; /** - * Live per-table-shard MemWAL state, as returned by {@link LanceDbTableMemWal#getMemWalStats()}. + * Live per-table-shard RTE state, as returned by {@link LanceDbTableRTE#getRTEStats()}. * *

Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's * to compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional}, * because a stats object of zeros would read as measurements. */ -public class MemWalStats { - private static final String CONTEXT = "memwal stats"; +public class RTEStats { + private static final String CONTEXT = "rte stats"; private final List tableShards; - MemWalStats(List tableShards) { + RTEStats(List tableShards) { this.tableShards = Collections.unmodifiableList(tableShards); } @@ -40,17 +40,17 @@ public class MemWalStats { return tableShards; } - static MemWalStats fromJson(JsonNode node) { + static RTEStats fromJson(JsonNode node) { JsonFields.requiredObject(node, CONTEXT); List tableShards = new ArrayList(); for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) { tableShards.add(TableShardStats.fromJson(tableShard)); } - return new MemWalStats(tableShards); + return new RTEStats(tableShards); } @Override public String toString() { - return "MemWalStats{tableShards=" + tableShards + "}"; + return "RTEStats{tableShards=" + tableShards + "}"; } } diff --git a/java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java b/java/lancedb-core/src/main/java/com/lancedb/RTEWriteSpec.java similarity index 83% rename from java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java rename to java/lancedb-core/src/main/java/com/lancedb/RTEWriteSpec.java index 95f541214..913440514 100644 --- a/java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java +++ b/java/lancedb-core/src/main/java/com/lancedb/RTEWriteSpec.java @@ -23,22 +23,23 @@ import java.util.List; import java.util.Map; /** - * Specification selecting Lance's MemWAL LSM-style write path for {@code mergeInsert}. + * Specification enabling the realtime engine (RTE), which routes {@code mergeInsert} through + * Lance's MemWAL LSM-style write path. * *

Construct via {@link #bucket}, {@link #identity}, or {@link #unsharded}, then optionally chain * {@link #withMaintainedIndexes} and {@link #withWriterConfigDefaults}. Install it with {@link - * LanceDbTableMemWal#enableMemWal} and remove it with {@link LanceDbTableMemWal#disableMemWal}. + * LanceDbTableRTE#enableRTE} and remove it with {@link LanceDbTableRTE#disableRTE}. * *

This is deliberately not {@code org.lance.memwal.InitializeMemWalParams}. That type is Lance's * own, and its maintained-index default is the opposite of this one: it defaults to maintaining * nothing, while a fresh spec here maintains every index. It also cannot express * the null that asks the server to resolve the set. */ -public class MemWalWriteSpec { +public class RTEWriteSpec { - private static final String CONTEXT = "get_memwal_spec response"; + private static final String CONTEXT = "get_rte_spec response"; - /** How writes are routed to MemWAL shards. */ + /** How writes are routed to RTE shards. */ public enum Sharding { /** Hash-bucket writes by a scalar column. */ BUCKET("bucket"), @@ -73,7 +74,7 @@ public class MemWalWriteSpec { private final List maintainedIndexes; private final Map writerConfigDefaults; - private MemWalWriteSpec( + private RTEWriteSpec( Sharding sharding, String column, Integer numBuckets, @@ -95,11 +96,11 @@ public class MemWalWriteSpec { * @param column A non-nested column with a supported scalar type. * @param numBuckets The number of buckets, in {@code [1, 1024]}. */ - public static MemWalWriteSpec bucket(String column, int numBuckets) { + public static RTEWriteSpec bucket(String column, int numBuckets) { if (column == null || column.trim().isEmpty()) { throw new IllegalArgumentException("Column cannot be null or empty"); } - return new MemWalWriteSpec( + return new RTEWriteSpec( Sharding.BUCKET, column, numBuckets, null, new HashMap()); } @@ -111,29 +112,28 @@ public class MemWalWriteSpec { * with a given primary key must always produce the same {@code column} value, or upserts of that * key can land in different shards and a stale version can win. */ - public static MemWalWriteSpec identity(String column) { + public static RTEWriteSpec identity(String column) { if (column == null || column.trim().isEmpty()) { throw new IllegalArgumentException("Column cannot be null or empty"); } - return new MemWalWriteSpec( - Sharding.IDENTITY, column, null, null, new HashMap()); + return new RTEWriteSpec(Sharding.IDENTITY, column, null, null, new HashMap()); } - /** No sharding — every write goes to a single MemWAL shard — maintaining every index. */ - public static MemWalWriteSpec unsharded() { - return new MemWalWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap()); + /** No sharding — every write goes to a single RTE shard — maintaining every index. */ + public static RTEWriteSpec unsharded() { + return new RTEWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap()); } /** - * Set the indexes the MemWAL keeps up to date as rows are appended. + * Set the indexes the RTE keeps up to date as rows are appended. * *

Pass {@code null} — the default for a fresh spec — to maintain every index the table has, * including ones created later. Pass an empty list to maintain none. * *

Note that {@code null} and the empty list mean opposite things here. */ - public MemWalWriteSpec withMaintainedIndexes(List maintainedIndexes) { - return new MemWalWriteSpec( + public RTEWriteSpec withMaintainedIndexes(List maintainedIndexes) { + return new RTEWriteSpec( sharding, column, numBuckets, @@ -150,11 +150,11 @@ public class MemWalWriteSpec { * max_unflushed_memtable_bytes}, and {@code enable_memtable}. Duration knobs carry an {@code _ms} * suffix, such as {@code max_wal_flush_interval_ms}. */ - public MemWalWriteSpec withWriterConfigDefaults(Map writerConfigDefaults) { + public RTEWriteSpec withWriterConfigDefaults(Map writerConfigDefaults) { if (writerConfigDefaults == null) { throw new IllegalArgumentException("writerConfigDefaults cannot be null"); } - return new MemWalWriteSpec( + return new RTEWriteSpec( sharding, column, numBuckets, @@ -178,7 +178,7 @@ public class MemWalWriteSpec { } /** - * Which indexes the MemWAL maintains. + * Which indexes the RTE maintains. * *

Null selects automatically: every supported index the table has now or gains later. An empty * list maintains none. A non-empty list maintains exactly those. @@ -192,7 +192,7 @@ public class MemWalWriteSpec { return Collections.unmodifiableMap(writerConfigDefaults); } - /** Render this spec as the {@code enable_memwal} request body. */ + /** Render this spec as the {@code enable_rte} request body. */ Map toRequestBody() { Map shardingBody = new LinkedHashMap(); shardingBody.put("mode", sharding.wireName()); @@ -212,15 +212,15 @@ public class MemWalWriteSpec { } /** - * Rebuild a spec from a {@code get_memwal_spec} response body. + * Rebuild a spec from a {@code get_rte_spec} response body. * *

A null {@code maintained_indexes} selects every index the table has; an empty list selects * none. */ - static MemWalWriteSpec fromJson(JsonNode node) { + static RTEWriteSpec fromJson(JsonNode node) { JsonNode shardingNode = node.get("sharding"); if (shardingNode == null || shardingNode.get("mode") == null) { - throw new IllegalStateException("get_memwal_spec response has no sharding mode"); + throw new IllegalStateException("get_rte_spec response has no sharding mode"); } Sharding sharding = Sharding.fromWireName(shardingNode.get("mode").asText()); @@ -253,12 +253,12 @@ public class MemWalWriteSpec { .forEachRemaining(name -> defaults.put(name, defaultsNode.get(name).asText())); } - return new MemWalWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults); + return new RTEWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults); } @Override public String toString() { - return "MemWalWriteSpec{sharding=" + return "RTEWriteSpec{sharding=" + sharding + ", column=" + column diff --git a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableRTETest.java similarity index 67% rename from java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java rename to java/lancedb-core/src/test/java/com/lancedb/LanceDbTableRTETest.java index 349479988..5df3ab49b 100644 --- a/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java +++ b/java/lancedb-core/src/test/java/com/lancedb/LanceDbTableRTETest.java @@ -41,17 +41,17 @@ import java.util.concurrent.ConcurrentHashMap; import static org.junit.jupiter.api.Assertions.*; /** - * Unit tests for the MemWAL LSM routes, run against a scripted local HTTP server. + * Unit tests for the RTE LSM routes, run against a scripted local HTTP server. * *

The wire assertions mirror the Rust mocked-endpoint tests in {@code * rust/lancedb/src/remote/table.rs}, which are the contract these routes have to match. */ -public class LanceDbTableMemWalTest { +public class LanceDbTableRTETest { private static final ObjectMapper MAPPER = new ObjectMapper(); private HttpServer server; private LanceDbRestClient client; - private LanceDbTableMemWal memWal; + private LanceDbTableRTE rte; private final List requestPaths = Collections.synchronizedList(new ArrayList()); private final List requestBodies = Collections.synchronizedList(new ArrayList()); @@ -101,7 +101,7 @@ public class LanceDbTableMemWalTest { .database("test-db") .endpoint("http://127.0.0.1:" + server.getAddress().getPort()) .buildRestClient(); - memWal = new LanceDbTableMemWal(client, "my_table"); + rte = new LanceDbTableRTE(client, "my_table"); } @AfterEach @@ -115,12 +115,12 @@ public class LanceDbTableMemWalTest { // =========================================================================== @Test - public void testEnableMemWalUnsharded() throws Exception { - enqueue("enable_memwal", 200, ""); + public void testEnableRTEUnsharded() throws Exception { + enqueue("enable_rte", 200, ""); - memWal.enableMemWal(MemWalWriteSpec.unsharded()); + rte.enableRTE(RTEWriteSpec.unsharded()); - assertEquals("/v1/table/my_table/enable_memwal/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/enable_rte/", requestPaths.get(0)); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("unsharded", body.get("sharding").get("mode").asText()); assertFalse(body.get("sharding").has("column")); @@ -128,11 +128,10 @@ public class LanceDbTableMemWalTest { } @Test - public void testEnableMemWalBucket() throws Exception { - enqueue("enable_memwal", 200, ""); + public void testEnableRTEBucket() throws Exception { + enqueue("enable_rte", 200, ""); - memWal.enableMemWal( - MemWalWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); + rte.enableRTE(RTEWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx"))); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("bucket", body.get("sharding").get("mode").asText()); @@ -143,10 +142,10 @@ public class LanceDbTableMemWalTest { } @Test - public void testEnableMemWalIdentity() throws Exception { - enqueue("enable_memwal", 200, ""); + public void testEnableRTEIdentity() throws Exception { + enqueue("enable_rte", 200, ""); - memWal.enableMemWal(MemWalWriteSpec.identity("tenant")); + rte.enableRTE(RTEWriteSpec.identity("tenant")); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("identity", body.get("sharding").get("mode").asText()); @@ -160,55 +159,54 @@ public class LanceDbTableMemWalTest { */ @Test public void testMaintainedIndexesNullAndEmptyAreDistinctOnTheWire() throws Exception { - enqueue("enable_memwal", 200, ""); + enqueue("enable_rte", 200, ""); - memWal.enableMemWal(MemWalWriteSpec.unsharded()); + rte.enableRTE(RTEWriteSpec.unsharded()); JsonNode fresh = MAPPER.readTree(requestBodies.get(0)); assertTrue(fresh.has("maintained_indexes"), "the key must be present"); assertTrue(fresh.get("maintained_indexes").isNull(), "a fresh spec sends null, not []"); - memWal.enableMemWal( - MemWalWriteSpec.unsharded().withMaintainedIndexes(Collections.emptyList())); + rte.enableRTE(RTEWriteSpec.unsharded().withMaintainedIndexes(Collections.emptyList())); JsonNode none = MAPPER.readTree(requestBodies.get(1)); assertTrue(none.get("maintained_indexes").isArray()); assertEquals(0, none.get("maintained_indexes").size()); } @Test - public void testEnableMemWalWriterConfigDefaults() throws Exception { - enqueue("enable_memwal", 200, ""); + public void testEnableRTEWriterConfigDefaults() throws Exception { + enqueue("enable_rte", 200, ""); Map defaults = new HashMap(); defaults.put("max_memtable_rows", "50000"); - memWal.enableMemWal(MemWalWriteSpec.unsharded().withWriterConfigDefaults(defaults)); + rte.enableRTE(RTEWriteSpec.unsharded().withWriterConfigDefaults(defaults)); JsonNode body = MAPPER.readTree(requestBodies.get(0)); assertEquals("50000", body.get("writer_config_defaults").get("max_memtable_rows").asText()); } @Test - public void testDisableMemWal() { - enqueue("disable_memwal", 200, ""); + public void testDisableRTE() { + enqueue("disable_rte", 200, ""); - memWal.disableMemWal(); + rte.disableRTE(); - assertEquals("/v1/table/my_table/disable_memwal/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/disable_rte/", requestPaths.get(0)); assertEquals("", requestBodies.get(0)); } @Test - public void testGetMemWalSpec() { + public void testGetRTESpec() { enqueue( - "get_memwal_spec", + "get_rte_spec", 200, - "{\"memwal_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "{\"rte_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\"," + "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"]," + "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}"); - Optional spec = memWal.getMemWalSpec(); + Optional spec = rte.getRTESpec(); assertTrue(spec.isPresent()); - assertEquals(MemWalWriteSpec.Sharding.BUCKET, spec.get().sharding()); + assertEquals(RTEWriteSpec.Sharding.BUCKET, spec.get().sharding()); assertEquals("id", spec.get().column()); assertEquals(Integer.valueOf(16), spec.get().numBuckets()); assertEquals(Arrays.asList("id_idx"), spec.get().maintainedIndexes()); @@ -217,28 +215,28 @@ public class LanceDbTableMemWalTest { /** A null selection, every index the table has, reads back as null rather than an empty list. */ @Test - public void testGetMemWalSpecRoundTripsEveryIndexSelection() { + public void testGetRTESpecRoundTripsEveryIndexSelection() { enqueue( - "get_memwal_spec", + "get_rte_spec", 200, - "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":null,\"writer_config_defaults\":{}}}"); - Optional spec = memWal.getMemWalSpec(); + Optional spec = rte.getRTESpec(); assertTrue(spec.isPresent()); assertNull(spec.get().maintainedIndexes()); } @Test - public void testGetMemWalSpecRoundTripsNoIndexSelection() { + public void testGetRTESpecRoundTripsNoIndexSelection() { enqueue( - "get_memwal_spec", + "get_rte_spec", 200, - "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":[],\"writer_config_defaults\":{}}}"); - Optional spec = memWal.getMemWalSpec(); + Optional spec = rte.getRTESpec(); assertTrue(spec.isPresent()); assertEquals(Collections.emptyList(), spec.get().maintainedIndexes()); @@ -249,32 +247,32 @@ public class LanceDbTableMemWalTest { * rejected: reading a string or an object as null would turn a malformed response into a policy. */ @Test - public void testGetMemWalSpecRejectsANonArrayMaintainedIndexes() { + public void testGetRTESpecRejectsANonArrayMaintainedIndexes() { enqueue( - "get_memwal_spec", + "get_rte_spec", 200, - "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":\"id_idx\",\"writer_config_defaults\":{}}}"); - assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec()); + assertThrows(IllegalStateException.class, () -> rte.getRTESpec()); } @Test - public void testGetMemWalSpecRejectsNonStringIndexNames() { + public void testGetRTESpecRejectsNonStringIndexNames() { enqueue( - "get_memwal_spec", + "get_rte_spec", 200, - "{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"}," + "\"maintained_indexes\":[7],\"writer_config_defaults\":{}}}"); - assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec()); + assertThrows(IllegalStateException.class, () -> rte.getRTESpec()); } @Test - public void testGetMemWalSpecAbsent() { - enqueue("get_memwal_spec", 200, "{\"memwal_spec\":null}"); + public void testGetRTESpecAbsent() { + enqueue("get_rte_spec", 200, "{\"rte_spec\":null}"); - assertFalse(memWal.getMemWalSpec().isPresent()); + assertFalse(rte.getRTESpec().isPresent()); } // =========================================================================== @@ -282,12 +280,12 @@ public class LanceDbTableMemWalTest { // =========================================================================== @Test - public void testGetMemWalStats() throws Exception { - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); + public void testGetRTEStats() throws Exception { + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); - Optional got = memWal.getMemWalStats(true); + Optional got = rte.getRTEStats(true); - assertEquals("/v1/table/my_table/get_memwal_stats/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/get_rte_stats/", requestPaths.get(0)); assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); assertTrue(got.isPresent()); TableShardStats decoded = got.get().tableShards().get(0); @@ -305,18 +303,18 @@ public class LanceDbTableMemWalTest { /** The optional fields decode when the server does send them. */ @Test - public void testGetMemWalStatsDecodesOptionalFields() { + public void testGetRTEStatsDecodesOptionalFields() { enqueue( - "get_memwal_stats", + "get_rte_stats", 200, - "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"rte_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11," + "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}]," + "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5," + "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}"); - TableShardStats decoded = memWal.getMemWalStats(true).get().tableShards().get(0); + TableShardStats decoded = rte.getRTEStats(true).get().tableShards().get(0); assertEquals(3, decoded.replayAfterWalEntryPosition()); assertEquals(11, decoded.walEntryPositionLastSeen()); @@ -332,17 +330,17 @@ public class LanceDbTableMemWalTest { } @Test - public void testGetMemWalStatsAbsentWhenMemWalDisabled() { - enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); + public void testGetRTEStatsAbsentWhenRTEDisabled() { + enqueue("get_rte_stats", 200, "{\"rte_stats\":null}"); - assertFalse(memWal.getMemWalStats().isPresent()); + assertFalse(rte.getRTEStats().isPresent()); } @Test - public void testGetMemWalStatsDefaultsToExcludingGenerationRows() throws Exception { - enqueue("get_memwal_stats", 200, stats()); + public void testGetRTEStatsDefaultsToExcludingGenerationRows() throws Exception { + enqueue("get_rte_stats", 200, stats()); - memWal.getMemWalStats(); + rte.getRTEStats(); assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean()); } @@ -353,22 +351,22 @@ public class LanceDbTableMemWalTest { @Test public void testFlushAndCompactRoutes() { - enqueue("flush_memwal", 200, ""); - enqueue("compact_memwal", 200, ""); + enqueue("flush_rte", 200, ""); + enqueue("compact_rte", 200, ""); - memWal.flushMemWal(); - memWal.compactMemWal(); + rte.flushRTE(); + rte.compactRTE(); - assertEquals("/v1/table/my_table/flush_memwal/", requestPaths.get(0)); - assertEquals("/v1/table/my_table/compact_memwal/", requestPaths.get(1)); + assertEquals("/v1/table/my_table/flush_rte/", requestPaths.get(0)); + assertEquals("/v1/table/my_table/compact_rte/", requestPaths.get(1)); } @Test public void testHttpErrorCarriesStatus() { - enqueue("flush_memwal", 404, "no such table"); + enqueue("flush_rte", 404, "no such table"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.flushMemWal()); + assertThrows(LanceDbRestClient.HttpException.class, () -> rte.flushRTE()); assertEquals(404, e.statusCode()); } @@ -377,98 +375,97 @@ public class LanceDbTableMemWalTest { // =========================================================================== @Test - public void testCheckpointReturnsWhenMemWalDisabled() { - enqueue("flush_memwal", 200, ""); - enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); + public void testCheckpointReturnsWhenRTEDisabled() { + enqueue("flush_rte", 200, ""); + enqueue("get_rte_stats", 200, "{\"rte_stats\":null}"); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(0, countCalls("compact_memwal"), "nothing to compact when the LSM path is off"); + assertEquals(0, countCalls("compact_rte"), "nothing to compact when the LSM path is off"); } @Test public void testCheckpointReturnsWhenNoGenerationsOutstanding() { - enqueue("flush_memwal", 200, ""); + enqueue("flush_rte", 200, ""); // A table shard with no SSTables yields no target, so the drain never starts. - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false))); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(0, countCalls("compact_memwal")); + assertEquals(0, countCalls("compact_rte")); } @Test public void testCheckpointConvergesOnceTargetGenerationsAreGone() { - enqueue("flush_memwal", 200, ""); + enqueue("flush_rte", 200, ""); // Watermark read: shard-0 holds sstables 7 and 8, so target = 8. - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // First drain poll: both still outstanding, nothing compacting -> dispatch a pass. - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 7L, 8L))); // Second drain poll: drained past the target -> done. - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 9L))); - enqueue("compact_memwal", 200, ""); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 9L))); + enqueue("compact_rte", 200, ""); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(1, countCalls("compact_memwal"), "one pass dispatched"); - assertEquals(3, countCalls("get_memwal_stats"), "watermark read plus two drain polls"); + assertEquals(1, countCalls("compact_rte"), "one pass dispatched"); + assertEquals(3, countCalls("get_rte_stats"), "watermark read plus two drain polls"); } @Test public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() { - enqueue("flush_memwal", 200, ""); - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L))); + enqueue("flush_rte", 200, ""); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", true, 4L))); // Still compacting on the first poll, so no pass is dispatched; then it drains. - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L))); - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 5L))); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", true, 4L))); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 5L))); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(0, countCalls("compact_memwal"), "a latched table shard is left alone"); + assertEquals(0, countCalls("compact_rte"), "a latched table shard is left alone"); } @Test public void testCheckpointRetriesFromFlushAfterLostClaim() { // 421 on the watermark read: the node lost its claim, so the whole thing restarts // from flush rather than retrying the read in place. - enqueue("flush_memwal", 200, ""); - enqueue("get_memwal_stats", 421, "no claim"); - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); + enqueue("flush_rte", 200, ""); + enqueue("get_rte_stats", 421, "no claim"); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false))); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(2, countCalls("flush_memwal"), "re-issued from flush"); + assertEquals(2, countCalls("flush_rte"), "re-issued from flush"); } @Test public void testCheckpointRetriesRetryableStatusInPlace() { - enqueue("flush_memwal", 429, "latch held"); - enqueue("flush_memwal", 200, ""); - enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false))); + enqueue("flush_rte", 429, "latch held"); + enqueue("flush_rte", 200, ""); + enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false))); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(2, countCalls("flush_memwal"), "429 retried in place, not re-issued"); + assertEquals(2, countCalls("flush_rte"), "429 retried in place, not re-issued"); } @Test public void testCheckpointPropagatesTerminalStatus() { - enqueue("flush_memwal", 400, "bad request"); + enqueue("flush_rte", 400, "bad request"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal()); + assertThrows(LanceDbRestClient.HttpException.class, () -> rte.checkpointRTE()); assertEquals(400, e.statusCode()); - assertEquals(1, countCalls("flush_memwal"), "a terminal status is not retried"); + assertEquals(1, countCalls("flush_rte"), "a terminal status is not retried"); } @Test public void testCheckpointGivesUpAfterRepeatedLostClaims() { - enqueue("flush_memwal", 421, "no claim"); + enqueue("flush_rte", 421, "no claim"); - IllegalStateException e = - assertThrows(IllegalStateException.class, () -> memWal.checkpointMemWal()); + IllegalStateException e = assertThrows(IllegalStateException.class, () -> rte.checkpointRTE()); assertTrue(e.getMessage().contains("kept losing its claim"), e.getMessage()); - assertEquals(4, countCalls("flush_memwal"), "the initial attempt plus MAX_REISSUES"); + assertEquals(4, countCalls("flush_rte"), "the initial attempt plus MAX_REISSUES"); } // =========================================================================== @@ -478,24 +475,24 @@ public class LanceDbTableMemWalTest { /** * A stats payload that does not decode must fail closed. Every one of these bodies used to be * read as "no table shards", which is indistinguishable from a drained table, so {@code - * checkpointMemWal} reported convergence for a checkpoint that never ran. + * checkpointRTE} reported convergence for a checkpoint that never ran. */ @Test public void testCheckpointRejectsMalformedStats() { Map malformed = new LinkedHashMap(); malformed.put("no response body at all", ""); - malformed.put("stats object with no table shards", "{\"memwal_stats\":{}}"); + malformed.put("stats object with no table shards", "{\"rte_stats\":{}}"); malformed.put( - "table shard missing its required fields", "{\"memwal_stats\":{\"table_shards\":[{}]}}"); + "table shard missing its required fields", "{\"rte_stats\":{\"table_shards\":[{}]}}"); malformed.put( "table shard missing sstables", - "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"rte_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," + "\"compacting\":false}]}}"); malformed.put( "generation with a non-numeric generation number", - "{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "{\"rte_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\"," + "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9," + "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0," + "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}]," @@ -503,25 +500,25 @@ public class LanceDbTableMemWalTest { for (Map.Entry each : malformed.entrySet()) { setUpFresh(); - enqueue("flush_memwal", 200, ""); - enqueue("get_memwal_stats", 200, each.getValue()); + enqueue("flush_rte", 200, ""); + enqueue("get_rte_stats", 200, each.getValue()); assertThrows( IllegalStateException.class, - () -> memWal.checkpointMemWal(), + () -> rte.checkpointRTE(), each.getKey() + " must not report convergence"); } } - /** The one shape that legitimately means "this table has no MemWAL write path". */ + /** The one shape that legitimately means "this table has no RTE write path". */ @Test public void testCheckpointTreatsNullStatsAsNotWalBacked() { - enqueue("flush_memwal", 200, ""); - enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}"); + enqueue("flush_rte", 200, ""); + enqueue("get_rte_stats", 200, "{\"rte_stats\":null}"); - memWal.checkpointMemWal(); + rte.checkpointRTE(); - assertEquals(1, countCalls("get_memwal_stats")); + assertEquals(1, countCalls("get_rte_stats")); } // =========================================================================== @@ -531,19 +528,18 @@ public class LanceDbTableMemWalTest { /** * The transport must not retry on the checkpoint loop's behalf. Apache HttpClient's default * strategy retries exactly 429 and 503 — the two statuses {@code isRetryable} owns — which - * doubled every budget here and also retried {@code compact_memwal} in place, where the loop is + * doubled every budget here and also retried {@code compact_rte} in place, where the loop is * built to fall through to a fresh stats poll instead. */ @Test public void testCheckpointRetryBudgetIsNotDoubledByTheTransport() { - enqueue("flush_memwal", 429, "latch held"); + enqueue("flush_rte", 429, "latch held"); LanceDbRestClient.HttpException e = - assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal()); + assertThrows(LanceDbRestClient.HttpException.class, () -> rte.checkpointRTE()); assertEquals(429, e.statusCode(), "the exhausted budget propagates the last error as itself"); - assertEquals( - 9, countCalls("flush_memwal"), "the initial request plus MAX_RETRIES, and no more"); + assertEquals(9, countCalls("flush_rte"), "the initial request plus MAX_RETRIES, and no more"); } // =========================================================================== @@ -558,9 +554,9 @@ public class LanceDbTableMemWalTest { return numbers; } - /** Build a {@code memwal_stats} response body from table shard fragments. */ + /** Build a {@code rte_stats} response body from table shard fragments. */ private static String stats(String... tableShards) { - return "{\"memwal_stats\":{\"table_shards\":[" + String.join(",", tableShards) + "]}}"; + return "{\"rte_stats\":{\"table_shards\":[" + String.join(",", tableShards) + "]}}"; } private static String tableShard(String shardId, boolean compacting, Long... sstables) { @@ -600,7 +596,7 @@ public class LanceDbTableMemWalTest { return requestPaths.stream().filter(path -> operationOf(path).equals(operation)).count(); } - /** {@code /v1/table/my_table/flush_memwal/} -> {@code flush_memwal}. */ + /** {@code /v1/table/my_table/flush_rte/} -> {@code flush_rte}. */ private static String operationOf(String path) { String[] segments = path.split("/"); return segments.length == 0 ? "" : segments[segments.length - 1]; diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 1c4592a14..7d10f8b59 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -642,11 +642,11 @@ describe.each(arrowVersions)( ); }); - it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => { + it("should expose useRTE on takeRowIds as the base-only escape hatch", async () => { await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]); - // useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables, + // useRTE(false) is reachable on TakeQuery (the escape hatch for RTE tables, // where take-by-row-id auto-routes to the LSM scanner and is rejected). - const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray(); + const res = await table.takeRowIds([0, 2]).useRTE(false).toArray(); expect(res.map((r) => r.id)).toEqual([1, 3]); }); @@ -4998,7 +4998,7 @@ describe("setUnenforcedPrimaryKey", () => { }); }); -describe("enableMemWal / disableMemWal", () => { +describe("enableRTE / disableRTE", () => { let tmpDir: tmp.DirResult; beforeEach(() => { @@ -5018,16 +5018,16 @@ describe("enableMemWal / disableMemWal", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.enableMemWal({ + await table.enableRTE({ specType: "bucket", column: "id", numBuckets: 4, }); - await table.disableMemWal(); + await table.disableRTE(); // A second unset errors — there is no spec left to remove. - await expect(table.disableMemWal()).rejects.toThrow(); + await expect(table.disableRTE()).rejects.toThrow(); // A fresh spec can be installed after unset. - await table.enableMemWal({ + await table.enableRTE({ specType: "bucket", column: "id", numBuckets: 8, @@ -5039,8 +5039,8 @@ describe("enableMemWal / disableMemWal", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.enableMemWal({ specType: "unsharded" }); - await table.disableMemWal(); + await table.enableRTE({ specType: "unsharded" }); + await table.disableRTE(); }); it("installs an identity spec", async () => { @@ -5048,8 +5048,8 @@ describe("enableMemWal / disableMemWal", () => { const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); - await table.enableMemWal({ specType: "identity", column: "id" }); - await table.disableMemWal(); + await table.enableRTE({ specType: "identity", column: "id" }); + await table.disableRTE(); }); it("rejects an invalid spec", async () => { @@ -5059,7 +5059,7 @@ describe("enableMemWal / disableMemWal", () => { await table.setUnenforcedPrimaryKey("id"); // num_buckets out of range. await expect( - table.enableMemWal({ + table.enableRTE({ specType: "bucket", column: "id", numBuckets: 0, @@ -5067,7 +5067,7 @@ describe("enableMemWal / disableMemWal", () => { ).rejects.toThrow(); // Column mismatch. await expect( - table.enableMemWal({ + table.enableRTE({ specType: "bucket", column: "missing", numBuckets: 4, @@ -5075,13 +5075,13 @@ describe("enableMemWal / disableMemWal", () => { ).rejects.toThrow(); }); - it("reads back the installed spec via getMemWalSpec", async () => { + it("reads back the installed spec via getRTESpec", async () => { const conn = await connect(tmpDir.name); const table = await makeTable(conn); await table.setUnenforcedPrimaryKey("id"); // Nothing installed yet. - expect(await table.getMemWalSpec()).toBeUndefined(); + expect(await table.getRTESpec()).toBeUndefined(); // A real scalar index is needed to name it as a maintained index. await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]); @@ -5092,14 +5092,14 @@ describe("enableMemWal / disableMemWal", () => { // defaults. Lance writer-config keys are canonically snake_case. // biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case const writerConfigDefaults = { durable_write: "false" }; - await table.enableMemWal({ + await table.enableRTE({ specType: "bucket", column: "id", numBuckets: 4, maintainedIndexes: [indexName], writerConfigDefaults, }); - const spec = await table.getMemWalSpec(); + const spec = await table.getRTESpec(); expect(spec).toBeDefined(); expect(spec?.specType).toBe("bucket"); expect(spec?.column).toBe("id"); @@ -5108,19 +5108,19 @@ describe("enableMemWal / disableMemWal", () => { expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults); // After unset, undefined again. - await table.disableMemWal(); - expect(await table.getMemWalSpec()).toBeUndefined(); + await table.disableRTE(); + expect(await table.getRTESpec()).toBeUndefined(); // Identity round-trips (column recovered from the schema). - await table.enableMemWal({ specType: "identity", column: "id" }); - const identity = await table.getMemWalSpec(); + await table.enableRTE({ specType: "identity", column: "id" }); + const identity = await table.getRTESpec(); expect(identity?.specType).toBe("identity"); expect(identity?.column).toBe("id"); - await table.disableMemWal(); + await table.disableRTE(); // Unsharded round-trips (no routing column). - await table.enableMemWal({ specType: "unsharded" }); - const unsharded = await table.getMemWalSpec(); + await table.enableRTE({ specType: "unsharded" }); + const unsharded = await table.getRTESpec(); expect(unsharded?.specType).toBe("unsharded"); expect(unsharded?.column).toBeFalsy(); }); @@ -5149,7 +5149,7 @@ describe("LSM merge insert", () => { ]); await table.setUnenforcedPrimaryKey("id"); // numBuckets = 1: every row routes to the single bucket. - await table.enableMemWal({ + await table.enableRTE({ specType: "bucket", column: "id", numBuckets: 1, @@ -5169,22 +5169,22 @@ describe("LSM merge insert", () => { { id: "c", value: 3 }, { id: "d", value: 4 }, ]); - // LSM path: rows go to the MemWAL, so only numRows is populated. + // LSM path: rows go to the RTE, so only numRows is populated. expect(res.numRows).toBe(2); expect(res.version).toBe(0); expect(res.numInsertedRows).toBe(0); - await table.closeMemWalWriters(); + await table.closeRTEWriters(); }); - it("falls back to the standard path with useMemWal(false)", async () => { + it("falls back to the standard path with useRTE(false)", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); const res = await table .mergeInsert("id") .whenNotMatchedInsertAll() - .useMemWal(false) + .useRTE(false) .execute([ { id: "b", value: 9 }, { id: "e", value: 5 }, @@ -5207,7 +5207,7 @@ describe("LSM merge insert", () => { expect(res.numRows).toBe(1); }); - it("rejects a non-upsert merge under a MemWAL spec", async () => { + it("rejects a non-upsert merge under an RTE spec", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); @@ -5219,7 +5219,7 @@ describe("LSM merge insert", () => { ).rejects.toThrow(); }); - it("auto-routes reads through the MemWAL scanner", async () => { + it("auto-routes reads through the RTE scanner", async () => { const conn = await connect(tmpDir.name); const table = await bucketTable(conn); // base ids "a", "b" @@ -5233,24 +5233,22 @@ describe("LSM merge insert", () => { const lsm = await table.query().toArray(); expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]); - // useMemWal(false) bypasses the MemWAL and reads the base table only. - const baseOnly = await table.query().useMemWal(false).toArray(); + // useRTE(false) bypasses the RTE and reads the base table only. + const baseOnly = await table.query().useRTE(false).toArray(); expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]); }); - it("reads the base table when no MemWAL spec is installed", async () => { + it("reads the base table when no RTE spec is installed", async () => { const conn = await connect(tmpDir.name); const table = await conn.createEmptyTable( "plain", new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]), ); - // No spec: default read and useMemWal(false) both succeed against the base table. + // No spec: default read and useRTE(false) both succeed against the base table. await expect(table.query().toArray()).resolves.toBeDefined(); - await expect( - table.query().useMemWal(false).toArray(), - ).resolves.toBeDefined(); - // useMemWal(true) demands MemWAL routing; without a spec it errors. - await expect(table.query().useMemWal(true).toArray()).rejects.toThrow(); + await expect(table.query().useRTE(false).toArray()).resolves.toBeDefined(); + // useRTE(true) demands RTE routing; without a spec it errors. + await expect(table.query().useRTE(true).toArray()).rejects.toThrow(); }); }); @@ -5268,42 +5266,42 @@ describe("LSM convergence and stats", () => { new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]), ); await table.setUnenforcedPrimaryKey("id"); - await table.enableMemWal({ specType: "unsharded" }); + await table.enableRTE({ specType: "unsharded" }); return table; } - // These four route through the server that owns the MemWAL, so a local table + // These four route through the server that owns the RTE, so a local table // rejects them rather than answering. What is asserted here is that the // bindings reach the core at all; the behavior against a real endpoint is // covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs. - it("rejects flushMemWal on a local table", async () => { + it("rejects flushRTE on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.flushMemWal()).rejects.toThrow(/not supported/i); + await expect(table.flushRTE()).rejects.toThrow(/not supported/i); }); - it("rejects compactMemWal on a local table", async () => { + it("rejects compactRTE on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.compactMemWal()).rejects.toThrow(/not supported/i); + await expect(table.compactRTE()).rejects.toThrow(/not supported/i); }); - it("rejects getMemWalStats on a local table", async () => { + it("rejects getRTEStats on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i); - await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i); + await expect(table.getRTEStats()).rejects.toThrow(/not supported/i); + await expect(table.getRTEStats(true)).rejects.toThrow(/not supported/i); }); - it("rejects checkpointMemWal on a local table", async () => { + it("rejects checkpointRTE on a local table", async () => { const conn = await connect(tmpDir.name); const table = await lsmTable(conn); - // checkpointMemWal freezes first, so it surfaces flushMemWal's rejection. - await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i); + // checkpointRTE freezes first, so it surfaces flushRTE's rejection. + await expect(table.checkpointRTE()).rejects.toThrow(/not supported/i); }); }); diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 3096cbe6b..1057e2d53 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -168,8 +168,8 @@ export { WriteProgress, FtsToken, TokenizeTableOptions, - MemWalWriteSpec, - MemWalStats, + RTEWriteSpec, + RTEStats, TableShardStats, SsTableStats, MemtableStats, diff --git a/nodejs/lancedb/merge.ts b/nodejs/lancedb/merge.ts index 0c6cdaa29..7bb36e678 100644 --- a/nodejs/lancedb/merge.ts +++ b/nodejs/lancedb/merge.ts @@ -88,27 +88,27 @@ export class MergeInsertBuilder { ); } /** - * Control MemWAL routing for this merge. + * Control RTE routing for this merge. * - * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is + * By default (unset), a `mergeInsert` on a table with an RTE write spec is * routed through Lance's MemWAL shard writer, and a table without one uses the * standard path. * - * @param enable - `true` forces MemWAL routing and errors if the table has no - * MemWAL write spec. `false` forces the standard write path even when a spec is set. + * @param enable - `true` forces RTE routing and errors if the table has no + * RTE write spec. `false` forces the standard write path even when a spec is set. */ - useMemWal(enable: boolean): MergeInsertBuilder { - return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema); + useRTE(enable: boolean): MergeInsertBuilder { + return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema); } /** * Controls how an LSM merge checks that its input targets a single shard. * - * When a table has a MemWAL write spec, every row in a `mergeInsert` call must + * When a table has an RTE write spec, every row in a `mergeInsert` call must * route to the same shard. When `true` (the default), every row is inspected * to verify this. When `false`, only the first row is inspected and the * shard it routes to is used for the whole input — a faster path for callers * that have already pre-sharded their input. Has no effect on tables without - * a MemWAL write spec. + * an RTE write spec. * * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`. */ diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index ef7443bbb..2abe6db97 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -677,26 +677,26 @@ export class StandardQueryBase< } /** - * Control MemWAL read routing for this query. + * Control RTE read routing for this query. * - * By default (unset), when the table carries a MemWAL write spec (see - * {@link Table#enableMemWal}), reads are routed through the LSM scanner so + * By default (unset), when the table carries an RTE write spec (see + * {@link Table#enableRTE}), reads are routed through the LSM scanner so * they also return data written via the `mergeInsert` LSM path that has not yet * been compacted into the base table (the active/frozen in-memory memtables and * the SSTables), deduplicated by primary key; a table without a spec * reads the base table. * * @param enable - `true` forces the LSM scanner and errors if the table has no - * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only, + * RTE write spec. `false` bypasses the RTE and reads the base table only, * even when a spec is present. * * Note: the LSM scanner does not support every query shape (e.g. reranking, - * hybrid search, `orderBy`). On a MemWAL table those shapes error unless - * `useMemWal(false)` is set, because a base-only read would silently exclude - * un-compacted MemWAL data. + * hybrid search, `orderBy`). On an RTE table those shapes error unless + * `useRTE(false)` is set, because a base-only read would silently exclude + * un-compacted RTE data. */ - useMemWal(enable: boolean): this { - this.doCall((inner: NativeQueryType) => inner.useMemWal(enable)); + useRTE(enable: boolean): this { + this.doCall((inner: NativeQueryType) => inner.useRTE(enable)); return this; } } @@ -776,7 +776,7 @@ export class VectorQuery extends StandardQueryBase { * probing may continue when too few candidates are found. * Any explicit effort, including 0.5, overrides nprobes in either call order. * Explicit minimum and maximum bounds still apply. This is not a recall - * guarantee and does not change refinement or HNSW ef. The MemWAL LSM + * guarantee and does not change refinement or HNSW ef. The RTE LSM * scanner does not yet support this option. * * @example @@ -1082,16 +1082,16 @@ export class TakeQuery extends QueryBase { } /** - * Control MemWAL read routing for this take query. + * Control RTE read routing for this take query. * - * `false` bypasses the MemWAL and reads the base table only — the escape hatch, + * `false` bypasses the RTE and reads the base table only — the escape hatch, * since take-by-row-id/offset is not supported on the LSM scanner and, on a - * MemWAL table, auto-routes to it and errors otherwise. + * RTE table, auto-routes to it and errors otherwise. * * @param enable - `false` reads the base table only. */ - useMemWal(enable: boolean): this { - this.doCall((inner: NativeTakeQuery) => inner.useMemWal(enable)); + useRTE(enable: boolean): this { + this.doCall((inner: NativeTakeQuery) => inner.useRTE(enable)); return this; } } diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 2b5319bec..0c79e6a7e 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,9 +35,9 @@ import { FunctionErrorsOptions, IndexConfig, IndexStatistics, - MemWalStats, Branches as NativeBranches, OptimizeStats, + RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, @@ -61,7 +61,7 @@ export { IndexConfig } from "./native"; export { TableShardStats, SsTableStats, - MemWalStats, + RTEStats, MemtableStats, } from "./native"; @@ -207,8 +207,8 @@ export type TokenizeTableOptions = }; /** - * Specification selecting Lance's MemWAL LSM-style write path for - * `mergeInsert`. + * Specification enabling the realtime engine (RTE), which routes + * `mergeInsert` through Lance's MemWAL LSM-style write path. * * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`, * `column` and `numBuckets` are required; for `"identity"`, `column` is @@ -217,7 +217,7 @@ export type TokenizeTableOptions = * `column` value, or upserts of that key can land in different shards and a * stale version can win). */ -export interface MemWalWriteSpec { +export interface RTEWriteSpec { /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */ specType: "bucket" | "identity" | "unsharded"; /** Bucket and identity variants: the sharding column. */ @@ -225,7 +225,7 @@ export interface MemWalWriteSpec { /** Bucket variant: the number of buckets, in `[1, 1024]`. */ numBuckets?: number; /** - * Indexes the MemWAL keeps up to date. Omit for every index the table has, + * Indexes the RTE keeps up to date. Omit for every index the table has, * including ones created later. Pass `[]` for none. */ maintainedIndexes?: string[]; @@ -758,10 +758,10 @@ export abstract class Table { */ abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise; /** - * Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL + * Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL * LSM-style write path for future `mergeInsert` calls. * - * `MemWalWriteSpec` chooses one of three sharding strategies via `specType`: + * `RTEWriteSpec` chooses one of three sharding strategies via `specType`: * * - `"bucket"` — hash-bucket writes by the single-column unenforced primary * key (`column` and `numBuckets` required). @@ -773,15 +773,15 @@ export abstract class Table { * requires it to be the single column being bucketed. * * Omitting `maintainedIndexes` maintains every index the table has, - * including ones created later, and skips a kind the MemWAL cannot maintain. + * including ones created later, and skips a kind the RTE cannot maintain. * Naming them pins an exact set, and a still-building index is rejected * rather than quietly omitted. - * @param {MemWalWriteSpec} spec The sharding spec to install. + * @param {RTEWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example * ```ts * await table.setUnenforcedPrimaryKey("id"); - * await table.enableMemWal({ + * await table.enableRTE({ * specType: "bucket", * column: "id", * numBuckets: 16, @@ -789,36 +789,36 @@ export abstract class Table { * }); * ``` */ - abstract enableMemWal(spec: MemWalWriteSpec): Promise; + abstract enableRTE(spec: RTEWriteSpec): Promise; /** - * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard + * Remove the {@link RTEWriteSpec} from this table, reverting to the standard * `mergeInsert` write path. * * Errors if no spec is currently set. * @returns {Promise} */ - abstract disableMemWal(): Promise; + abstract disableRTE(): Promise; /** - * Read the {@link MemWalWriteSpec} currently installed on this table. + * Read the {@link RTEWriteSpec} currently installed on this table. * - * Resolves to `undefined` when the MemWAL write path is not enabled (no - * spec has been set, or it was removed with {@link Table#disableMemWal}). + * Resolves to `undefined` when the RTE write path is not enabled (no + * spec has been set, or it was removed with {@link Table#disableRTE}). * The spec is the one installed, including its maintained-index selection: * an absent `maintainedIndexes` for every index the table has, an empty * array for none. - * @returns {Promise} + * @returns {Promise} */ - abstract getMemWalSpec(): Promise; + abstract getRTESpec(): Promise; /** - * Drain and close any cached MemWAL shard writers held for this table. + * Drain and close any cached RTE shard writers held for this table. * - * When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL + * When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE * shard writers and caches them for reuse across calls. This closes them, * flushing pending data; writers reopen lazily on the next `mergeInsert`. * It is a no-op when no writers are cached. * @returns {Promise} */ - abstract closeMemWalWriters(): Promise; + abstract closeRTEWriters(): Promise; /** * Freeze every table shard's active memtable into a new SSTable. * @@ -826,18 +826,18 @@ export abstract class Table { * so this is safe to call repeatedly. * @returns {Promise} */ - abstract flushMemWal(): Promise; + abstract flushRTE(): Promise; /** * Trigger a background SSTable compaction pass per table shard. * * Returns once the passes are *dispatched*, not once they finish — watch - * {@link Table#getMemWalStats} for progress, or use - * {@link Table#checkpointMemWal} to wait for convergence. + * {@link Table#getRTEStats} for progress, or use + * {@link Table#checkpointRTE} to wait for convergence. * @returns {Promise} */ - abstract compactMemWal(): Promise; + abstract compactRTE(): Promise; /** - * Converge this table's MemWAL write path into its base table. + * Converge this table's RTE write path into its base table. * * Freezes once, then triggers compaction and polls until the SSTables that existed * at the start is gone. The target set is fixed at the start, so @@ -852,26 +852,26 @@ export abstract class Table { * @returns {Promise} * @example * ```ts - * const before = await table.getMemWalStats(); - * await table.checkpointMemWal(); - * const after = await table.getMemWalStats(); + * const before = await table.getRTEStats(); + * await table.checkpointRTE(); + * const after = await table.getRTEStats(); * ``` */ - abstract checkpointMemWal(): Promise; + abstract checkpointRTE(): Promise; /** - * Read live per-table-shard MemWAL state. + * Read live per-table-shard RTE state. * * Answers "how far behind is my fresh tier", "which table shard is hot", and * "why is my fresh-tier vector search brute-force". Mutates no table state. * - * Resolves to `undefined` only when the MemWAL write path is not enabled. + * Resolves to `undefined` only when the RTE write path is not enabled. * @param {boolean} includeSstableRows Also count rows per SSTable. * Off by default because each count opens an uncached Lance dataset. - * @returns {Promise} + * @returns {Promise} */ - abstract getMemWalStats( + abstract getRTEStats( includeSstableRows?: boolean, - ): Promise; + ): Promise; /** Retrieve the version of the table */ abstract version(): Promise; @@ -1480,43 +1480,43 @@ export class LocalTable extends Table { return await this.inner.setUnenforcedPrimaryKey(cols); } - async enableMemWal(spec: MemWalWriteSpec): Promise { - return await this.inner.enableMemWal(spec); + async enableRTE(spec: RTEWriteSpec): Promise { + return await this.inner.enableRTE(spec); } - async disableMemWal(): Promise { - return await this.inner.disableMemWal(); + async disableRTE(): Promise { + return await this.inner.disableRTE(); } - async getMemWalSpec(): Promise { + async getRTESpec(): Promise { // The native binding types `specType` as a plain `string`; narrow it back // to the public union. The Rust `From` impl only ever emits one of the // three valid values, so the cast is safe. - return ((await this.inner.getMemWalSpec()) ?? undefined) as - | MemWalWriteSpec + return ((await this.inner.getRTESpec()) ?? undefined) as + | RTEWriteSpec | undefined; } - async closeMemWalWriters(): Promise { - return await this.inner.closeMemWalWriters(); + async closeRTEWriters(): Promise { + return await this.inner.closeRTEWriters(); } - async flushMemWal(): Promise { - return await this.inner.flushMemWal(); + async flushRTE(): Promise { + return await this.inner.flushRTE(); } - async compactMemWal(): Promise { - return await this.inner.compactMemWal(); + async compactRTE(): Promise { + return await this.inner.compactRTE(); } - async checkpointMemWal(): Promise { - return await this.inner.checkpointMemWal(); + async checkpointRTE(): Promise { + return await this.inner.checkpointRTE(); } - async getMemWalStats( + async getRTEStats( includeSstableRows: boolean = false, - ): Promise { - return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined; + ): Promise { + return (await this.inner.getRTEStats(includeSstableRows)) ?? undefined; } async version(): Promise { diff --git a/nodejs/src/merge.rs b/nodejs/src/merge.rs index 3d17a037e..ccff2a689 100644 --- a/nodejs/src/merge.rs +++ b/nodejs/src/merge.rs @@ -50,10 +50,10 @@ impl NativeMergeInsertBuilder { this } - #[napi(js_name = "useMemWal")] - pub fn use_memwal(&self, enable: bool) -> Self { + #[napi(js_name = "useRTE")] + pub fn use_rte(&self, enable: bool) -> Self { let mut this = self.clone(); - this.inner.use_memwal(enable); + this.inner.use_rte(enable); this } diff --git a/nodejs/src/query.rs b/nodejs/src/query.rs index dba8f665a..c2987f259 100644 --- a/nodejs/src/query.rs +++ b/nodejs/src/query.rs @@ -168,9 +168,9 @@ impl Query { self.inner = self.inner.clone().with_row_id(); } - #[napi(js_name = "useMemWal")] - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + #[napi(js_name = "useRTE")] + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } #[napi] @@ -399,9 +399,9 @@ impl VectorQuery { self.inner = self.inner.clone().with_row_id(); } - #[napi(js_name = "useMemWal")] - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + #[napi(js_name = "useRTE")] + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } #[napi] @@ -514,9 +514,9 @@ impl TakeQuery { self.inner = self.inner.clone().with_row_id(); } - #[napi(js_name = "useMemWal")] - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + #[napi(js_name = "useRTE")] + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } #[napi(catch_unwind)] diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 5e29e23b6..0a40e0a13 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -580,60 +580,57 @@ impl Table { .default_error() } - #[napi(catch_unwind, js_name = "enableMemWal")] - pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> { - let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?; + #[napi(catch_unwind, js_name = "enableRTE")] + pub async fn enable_rte(&self, spec: RTEWriteSpec) -> napi::Result<()> { + let native_spec = lancedb::table::RTEWriteSpec::try_from(spec)?; self.inner_ref()? - .enable_memwal(native_spec) + .enable_rte(native_spec) .await .default_error() } - #[napi(catch_unwind, js_name = "disableMemWal")] - pub async fn disable_memwal(&self) -> napi::Result<()> { - self.inner_ref()?.disable_memwal().await.default_error() + #[napi(catch_unwind, js_name = "disableRTE")] + pub async fn disable_rte(&self) -> napi::Result<()> { + self.inner_ref()?.disable_rte().await.default_error() } - #[napi(catch_unwind, js_name = "getMemWalSpec")] - pub async fn get_memwal_spec(&self) -> napi::Result> { - let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?; - Ok(spec.map(MemWalWriteSpec::from)) + #[napi(catch_unwind, js_name = "getRTESpec")] + pub async fn get_rte_spec(&self) -> napi::Result> { + let spec = self.inner_ref()?.get_rte_spec().await.default_error()?; + Ok(spec.map(RTEWriteSpec::from)) } - #[napi(catch_unwind, js_name = "closeMemWalWriters")] - pub async fn close_memwal_writers(&self) -> napi::Result<()> { - self.inner_ref()? - .close_memwal_writers() - .await - .default_error() + #[napi(catch_unwind, js_name = "closeRTEWriters")] + pub async fn close_rte_writers(&self) -> napi::Result<()> { + self.inner_ref()?.close_rte_writers().await.default_error() } - #[napi(catch_unwind, js_name = "flushMemWal")] - pub async fn flush_memwal(&self) -> napi::Result<()> { - self.inner_ref()?.flush_memwal().await.default_error() + #[napi(catch_unwind, js_name = "flushRTE")] + pub async fn flush_rte(&self) -> napi::Result<()> { + self.inner_ref()?.flush_rte().await.default_error() } - #[napi(catch_unwind, js_name = "compactMemWal")] - pub async fn compact_memwal(&self) -> napi::Result<()> { - self.inner_ref()?.compact_memwal().await.default_error() + #[napi(catch_unwind, js_name = "compactRTE")] + pub async fn compact_rte(&self) -> napi::Result<()> { + self.inner_ref()?.compact_rte().await.default_error() } - #[napi(catch_unwind, js_name = "checkpointMemWal")] - pub async fn checkpoint_memwal(&self) -> napi::Result<()> { - self.inner_ref()?.checkpoint_memwal().await.default_error() + #[napi(catch_unwind, js_name = "checkpointRTE")] + pub async fn checkpoint_rte(&self) -> napi::Result<()> { + self.inner_ref()?.checkpoint_rte().await.default_error() } - #[napi(catch_unwind, js_name = "getMemWalStats")] - pub async fn get_memwal_stats( + #[napi(catch_unwind, js_name = "getRTEStats")] + pub async fn get_rte_stats( &self, include_sstable_rows: bool, - ) -> napi::Result> { + ) -> napi::Result> { let stats = self .inner_ref()? - .get_memwal_stats(include_sstable_rows) + .get_rte_stats(include_sstable_rows) .await .default_error()?; - Ok(stats.map(MemWalStats::from)) + Ok(stats.map(RTEStats::from)) } #[napi(catch_unwind)] @@ -946,53 +943,53 @@ impl From for FtsToken { } } -/// Specification selecting Lance's MemWAL LSM-style write path for -/// `mergeInsert`. +/// Specification enabling the realtime engine (RTE), which routes +/// `mergeInsert` through Lance's MemWAL LSM-style write path. /// /// `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For /// `"bucket"`, `column` and `numBuckets` are required; for `"identity"`, /// `column` is required. -#[napi(object)] +#[napi(object, js_name = "RTEWriteSpec")] #[derive(Clone, Debug)] -pub struct MemWalWriteSpec { +pub struct RTEWriteSpec { /// One of `"bucket"`, `"identity"`, or `"unsharded"`. pub spec_type: String, /// Bucket and identity variants: the sharding column. pub column: Option, /// Bucket variant: the number of buckets, in `[1, 1024]`. pub num_buckets: Option, - /// Indexes the MemWAL keeps up to date. Omitted resolves every + /// Indexes the RTE keeps up to date. Omitted resolves every /// maintainable index on install; an empty array means none. pub maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. pub writer_config_defaults: Option>, } -impl TryFrom for lancedb::table::MemWalWriteSpec { +impl TryFrom for lancedb::table::RTEWriteSpec { type Error = napi::Error; - fn try_from(value: MemWalWriteSpec) -> napi::Result { + fn try_from(value: RTEWriteSpec) -> napi::Result { let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { let column = value.column.ok_or_else(|| { - napi::Error::from_reason("MemWalWriteSpec bucket requires `column`") + napi::Error::from_reason("RTEWriteSpec bucket requires `column`") })?; let num_buckets = value.num_buckets.ok_or_else(|| { - napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`") + napi::Error::from_reason("RTEWriteSpec bucket requires `numBuckets`") })?; Self::bucket(column, num_buckets) } "identity" => { let column = value.column.ok_or_else(|| { - napi::Error::from_reason("MemWalWriteSpec identity requires `column`") + napi::Error::from_reason("RTEWriteSpec identity requires `column`") })?; Self::identity(column) } "unsharded" => Self::unsharded(), other => { return Err(napi::Error::from_reason(format!( - "MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'", + "RTEWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'", other ))); } @@ -1003,9 +1000,9 @@ impl TryFrom for lancedb::table::MemWalWriteSpec { } } -impl From for MemWalWriteSpec { - fn from(spec: lancedb::table::MemWalWriteSpec) -> Self { - use lancedb::table::MemWalWriteSpec as Native; +impl From for RTEWriteSpec { + fn from(spec: lancedb::table::RTEWriteSpec) -> Self { + use lancedb::table::RTEWriteSpec as Native; match spec { Native::Bucket { column, @@ -1148,19 +1145,19 @@ impl From for TableShardStats { } } -/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`. +/// Live per-table-shard RTE state, as returned by `Table#getRTEStats`. /// /// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are /// the caller's to compute. -#[napi(object)] +#[napi(object, js_name = "RTEStats")] #[derive(Clone, Debug)] -pub struct MemWalStats { +pub struct RTEStats { /// One entry per table shard backing this table. pub table_shards: Vec, } -impl From for MemWalStats { - fn from(stats: lancedb::table::MemWalStats) -> Self { +impl From for RTEStats { + fn from(stats: lancedb::table::RTEStats) -> Self { Self { table_shards: stats.table_shards.into_iter().map(Into::into).collect(), } diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 4ed64ee31..6b902c466 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -15,7 +15,7 @@ from ._lancedb import FtsToken from ._lancedb import FunctionErrorFragment as FunctionErrorFragment from ._lancedb import FunctionErrorRecord as FunctionErrorRecord from ._lancedb import FunctionErrors as FunctionErrors -from ._lancedb import MemWalWriteSpec +from ._lancedb import RTEWriteSpec from ._lancedb import tokenize as _tokenize from .common import URI, sanitize_uri from urllib.parse import urlparse @@ -610,7 +610,7 @@ __all__ = [ "SqlQuery", "LanceDBConnection", "LanceNamespaceDBConnection", - "MemWalWriteSpec", + "RTEWriteSpec", "RemoteDBConnection", "Session", "Table", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index b9c033d7b..988482a97 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -538,17 +538,17 @@ class Table: async def initial_storage_options(self) -> Optional[Dict[str, str]]: ... async def latest_storage_options(self) -> Optional[Dict[str, str]]: ... async def set_unenforced_primary_key(self, columns: List[str]) -> None: ... - async def enable_memwal(self, spec: MemWalWriteSpec) -> None: ... - async def disable_memwal(self) -> None: ... - async def get_memwal_spec(self) -> Optional[MemWalWriteSpec]: ... + async def enable_rte(self, spec: RTEWriteSpec) -> None: ... + async def disable_rte(self) -> None: ... + async def get_rte_spec(self) -> Optional[RTEWriteSpec]: ... # WAL-PK-FUSION: delete both. def hybrid_pk_fusion_learned(self) -> bool: ... def note_hybrid_pk_fusion(self) -> None: ... - async def checkpoint_memwal(self) -> None: ... - async def flush_memwal(self) -> None: ... - async def compact_memwal(self) -> None: ... - async def get_memwal_stats(self, include_sstable_rows: bool) -> Optional[dict]: ... - async def close_memwal_writers(self) -> None: ... + async def checkpoint_rte(self) -> None: ... + async def flush_rte(self) -> None: ... + async def compact_rte(self) -> None: ... + async def get_rte_stats(self, include_sstable_rows: bool) -> Optional[dict]: ... + async def close_rte_writers(self) -> None: ... @property def tags(self) -> Tags: ... @property @@ -661,7 +661,7 @@ class Query: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_memwal(self, enable: bool): ... + def use_rte(self, enable: bool): ... def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ... def nearest_to_text(self, query: dict) -> FTSQuery: ... def order_by(self, ordering: Optional[List[ColumnOrdering]]): ... @@ -678,7 +678,7 @@ class Query: class TakeQuery: def select(self, columns: List[str]): ... def with_row_id(self): ... - def use_memwal(self, enable: bool): ... + def use_rte(self, enable: bool): ... async def output_schema(self) -> pa.Schema: ... async def execute(self) -> RecordBatchStream: ... async def explain_plan(self, verbose: Optional[bool]) -> str: ... @@ -697,7 +697,7 @@ class FTSQuery: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_memwal(self, enable: bool): ... + def use_rte(self, enable: bool): ... def get_query(self) -> str: ... def add_query_vector(self, query_vec: pa.Array) -> None: ... def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ... @@ -725,7 +725,7 @@ class VectorQuery: def column(self, column: str): ... def distance_type(self, distance_type: str): ... def postfilter(self): ... - def use_memwal(self, enable: bool): ... + def use_rte(self, enable: bool): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... def search_effort(self, search_effort: Optional[float]): ... @@ -750,7 +750,7 @@ class HybridQuery: def fast_search(self): ... def with_row_id(self): ... def postfilter(self): ... - def use_memwal(self, enable: bool): ... + def use_rte(self, enable: bool): ... def distance_type(self, distance_type: str): ... def refine_factor(self, refine_factor: int): ... def nprobes(self, nprobes: int): ... @@ -778,7 +778,7 @@ class PyQueryRequest: select_source_columns: Optional[Dict[str, str]] fast_search: Optional[bool] with_row_id: Optional[bool] - use_memwal: Optional[bool] + use_rte: Optional[bool] column: Optional[str] query_vector: Optional[List[pa.Array]] nprobes: Optional[int] @@ -839,26 +839,22 @@ class FtsToken: text: str position: int -class MemWalWriteSpec: - """Specification selecting Lance's MemWAL LSM-style write path for - `merge_insert`.""" +class RTEWriteSpec: + """Specification enabling the realtime engine (RTE), which routes + `merge_insert` through Lance's MemWAL LSM-style write path.""" @staticmethod - def bucket(column: str, num_buckets: int) -> "MemWalWriteSpec": ... + def bucket(column: str, num_buckets: int) -> "RTEWriteSpec": ... @staticmethod - def identity(column: str) -> "MemWalWriteSpec": ... + def identity(column: str) -> "RTEWriteSpec": ... @staticmethod - def unsharded() -> "MemWalWriteSpec": ... - def with_maintained_indexes( - self, indexes: Optional[List[str]] - ) -> "MemWalWriteSpec": - """Set which indexes the MemWAL keeps up to date. None resolves every + def unsharded() -> "RTEWriteSpec": ... + def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "RTEWriteSpec": + """Set which indexes the RTE keeps up to date. None resolves every index on the table at install, failing if one cannot be maintained; a list is verbatim, empty means none.""" ... - def with_writer_config_defaults( - self, defaults: Dict[str, str] - ) -> "MemWalWriteSpec": + def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "RTEWriteSpec": """Return a copy of this spec recording the given default `ShardWriter` configuration in the MemWAL index.""" ... @@ -872,7 +868,7 @@ class MemWalWriteSpec: def num_buckets(self) -> Optional[int]: ... @property def maintained_indexes(self) -> Optional[List[str]]: - """Indexes the MemWAL keeps up to date, or None for every supported one.""" + """Indexes the RTE keeps up to date, or None for every supported one.""" ... @property def writer_config_defaults(self) -> Dict[str, str]: ... diff --git a/python/python/lancedb/_wal_hybrid.py b/python/python/lancedb/_wal_hybrid.py index 8e5d6ae17..07c4568d7 100644 --- a/python/python/lancedb/_wal_hybrid.py +++ b/python/python/lancedb/_wal_hybrid.py @@ -1,14 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables. +"""WAL-PK-FUSION: hybrid search's primary-key fallback for RTE tables. -Hybrid search joins its vector and full-text legs on `_rowid`, which a MemWAL +Hybrid search joins its vector and full-text legs on `_rowid`, which an RTE table cannot supply: the fresh tier has no stable row id, so the server rejects `with_row_id` before it plans. When that happens the legs are joined on a surrogate `_rowid` built from the table's primary key instead. -This is a stopgap until MemWAL supports `_rowid`, and everything it needs lives +This is a stopgap until RTE supports `_rowid`, and everything it needs lives here so it can be removed in one pass: 1. Delete this file and `tests/test_wal_hybrid.py`. @@ -69,21 +69,21 @@ def _pk_columns(schema: pa.Schema) -> List[str]: # builds it from the same words -- so this substring survives either path and # the wrapping each adds. It is a coupling to a message rather than a code, # because the refusal arrives as a generic 400; if the server ever rewords it -# the fallback stops firing and hybrid on a MemWAL table errors visibly again, +# the fallback stops firing and hybrid on an RTE table errors visibly again, # which is loud rather than silently wrong. _WAL_ROW_ID_REFUSAL = "does not support with_row_id" WAL_ROW_ID_UNSUPPORTED = ( - "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has " + "hybrid search on an RTE table cannot return _rowid: the fresh tier has " "no stable row id, and the ids the fusion joins on are synthesized from " - "the primary key. Set use_memwal(False) to read the base table only (results " - "will exclude un-compacted MemWAL data)" + "the primary key. Set use_rte(False) to read the base table only (results " + "will exclude un-compacted RTE data)" ) def _is_wal_row_id_refusal(error: BaseException) -> bool: """Whether `error` is the server declining `_rowid` because the table is - MemWAL-backed, as opposed to any other failure the query might hit.""" + RTE-backed, as opposed to any other failure the query might hit.""" return _WAL_ROW_ID_REFUSAL in str(error) @@ -112,7 +112,7 @@ def _stamp_surrogate_row_ids( """Give each table a `_rowid` column derived from its primary key. The fusion only ever compares that column for equality — to deduplicate, to - group by in a reranker, and to restore the original scores — so on a MemWAL + group by in a reranker, and to restore the original scores — so on an RTE table, where the server has no stable row id to hand out, any injective mapping of the primary key does the same job. Ids are assigned in first-seen order across `tables`, so passing them in fusion order (vector, then FTS) @@ -124,7 +124,7 @@ def _stamp_surrogate_row_ids( """ if not pk_columns: raise ValueError( - "hybrid search on a MemWAL table needs an unenforced primary key to " + "hybrid search on an RTE table needs an unenforced primary key to " "deduplicate on, and this table declares none" ) @@ -151,13 +151,13 @@ def _stamp_surrogate_row_ids( return stamped -def pk_fusion_learned(table, use_memwal: Optional[bool]) -> bool: +def pk_fusion_learned(table, use_rte: Optional[bool]) -> bool: """Whether a previous hybrid query on `table` was refused `_rowid`, so this one should go straight to the primary key. Learned from a refusal, never probed, so asking is free. """ - if table is None or use_memwal is False: + if table is None or use_rte is False: return False # Only an unambiguous True forks to the key: a table type that answers the # question loosely keeps the `_rowid` path. @@ -165,7 +165,7 @@ def pk_fusion_learned(table, use_memwal: Optional[bool]) -> bool: class PkFusion: - """How the legs of a hybrid query on a MemWAL table are joined: on a + """How the legs of a hybrid query on an RTE table are joined: on a surrogate `_rowid` built from the primary key.""" def __init__(self, pk_columns: List[str]): @@ -190,16 +190,16 @@ class PkFusion: # has turned `_with_row_id` on — including `return_score="all"`. if blob_v2_projection_sources(schema, projection): raise NotImplementedError( - "hybrid search cannot project a blob column on a MemWAL table: " + "hybrid search cannot project a blob column on an RTE table: " "fetching blobs needs a real _rowid, and the fresh tier has no " - "stable one. Set use_memwal(False) to read the base table only " - "(results will exclude un-compacted MemWAL data)" + "stable one. Set use_rte(False) to read the base table only " + "(results will exclude un-compacted RTE data)" ) return cls(_pk_columns(schema)) def inject(self, columns): """`columns` with the key columns it is missing, to project in place of - the `_rowid` a MemWAL table would refuse.""" + the `_rowid` an RTE table would refuse.""" columns, self.injected = _with_pk_columns(columns, self.pk_columns) return columns @@ -226,7 +226,7 @@ class PkFusion: def _check_refusal(error: Exception, *, fused: bool, caller_requested_row_id: bool): - """Re-raise `error` unless it is a MemWAL `_rowid` refusal worth retrying on + """Re-raise `error` unless it is an RTE `_rowid` refusal worth retrying on the primary key.""" if fused or not _is_wal_row_id_refusal(error): raise error @@ -248,7 +248,7 @@ def with_pk_fallback( `_rowid`. `_rowid` is asked for first rather than paying a round trip up front to - find out: only a MemWAL table refuses, and the refusal has to be handled + find out: only an RTE table refuses, and the refusal has to be handled anyway -- a write spec installed elsewhere can arrive between any two queries. It costs nothing server-side: it is raised before the query is planned. `fused` reports whether the attempt already used the key. diff --git a/python/python/lancedb/merge.py b/python/python/lancedb/merge.py index d7c25349e..49f4793ed 100644 --- a/python/python/lancedb/merge.py +++ b/python/python/lancedb/merge.py @@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object): self._when_not_matched_by_source_condition_expr = None self._timeout = None self._use_index = True - self._use_memwal = None + self._use_rte = None self._validate_single_shard = None def when_matched_update_all( @@ -115,22 +115,22 @@ class LanceMergeInsertBuilder(object): self._use_index = use_index return self - def use_memwal(self, enable: bool) -> LanceMergeInsertBuilder: + def use_rte(self, enable: bool) -> LanceMergeInsertBuilder: """ - Control MemWAL routing for this merge. + Control RTE routing for this merge. - By default (unset), a `merge_insert` on a table with a MemWAL write spec is + By default (unset), a `merge_insert` on a table with an RTE write spec is routed through Lance's MemWAL shard writer, and a table without one uses the standard path. Parameters ---------- enable: bool - ``True`` forces MemWAL routing and errors if the table has no MemWAL + ``True`` forces RTE routing and errors if the table has no RTE write spec. ``False`` forces the standard write path even when a spec is set. """ - self._use_memwal = enable + self._use_rte = enable return self def validate_single_shard( @@ -139,13 +139,13 @@ class LanceMergeInsertBuilder(object): """ Controls how an LSM merge checks that its input targets a single shard. - When a table has a MemWAL write spec, every row in a `merge_insert` call + When a table has an RTE write spec, every row in a `merge_insert` call must route to the same shard. When `True` (the default), every row is inspected to verify this. When `False`, only the first row is inspected and the shard it routes to is used for the whole input — a faster path for callers that have already pre-sharded their input. - Has no effect on tables without a MemWAL write spec. + Has no effect on tables without an RTE write spec. Parameters ---------- diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index d97eccc07..103ec0aa4 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -48,7 +48,7 @@ class PermutationBuilder: By default, the permutation builder will create a single split that contains all rows in the same order as the base table. - Tables with a MemWAL write spec are rejected: unflushed rows have no row id. + Tables with an RTE write spec are rejected: unflushed rows have no row id. """ self._async = async_permutation_builder(table) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index ebb4fdf16..c427820e8 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -887,10 +887,10 @@ class Query(pydantic.BaseModel): # if true, will only search the indexed data fast_search: Optional[bool] = None - # MemWAL LSM read routing: None auto-routes when the table carries a write + # RTE LSM read routing: None auto-routes when the table carries a write # spec, True forces the LSM scanner (errors without a spec), False reads the # base table only - use_memwal: Optional[bool] = None + use_rte: Optional[bool] = None # size of the nearest neighbor list maintained during HNSW search ef: Optional[int] = None @@ -910,9 +910,9 @@ class Query(pydantic.BaseModel): query.full_text_query = req.full_text_search query.columns = req.select query.with_row_id = req.with_row_id - # use_memwal is a genuine tri-state (None / True / False); preserve it as-is + # use_rte is a genuine tri-state (None / True / False); preserve it as-is # so a round-tripped query keeps an explicit False. - query.use_memwal = req.use_memwal + query.use_rte = req.use_rte query.vector_column = req.column query.vector = req.query_vector query.distance_type = req.distance_type @@ -1085,7 +1085,7 @@ class LanceQueryBuilder(ABC): self._with_row_address = None self._fragments = None self._fragment_ids = None - self._use_memwal = None + self._use_rte = None self._vector = None self._text = None self._ef = None @@ -1520,10 +1520,10 @@ class LanceQueryBuilder(ABC): self._fragment_ids = fragment_ids return self - def use_memwal(self, enable: bool) -> Self: - """Control MemWAL LSM read routing for this query. + def use_rte(self, enable: bool) -> Self: + """Control RTE LSM read routing for this query. - By default (unset), a query against a table with a MemWAL write spec is + By default (unset), a query against a table with an RTE write spec is routed through the LSM scanner so it also returns data written via the ``merge_insert`` LSM path that has not yet been compacted into the base table (active/frozen memtables + SSTables); a table without a @@ -1532,8 +1532,8 @@ class LanceQueryBuilder(ABC): Parameters ---------- enable : bool - ``True`` forces the LSM scanner and errors if the table has no MemWAL - write spec. ``False`` bypasses the MemWAL and reads the base table + ``True`` forces the LSM scanner and errors if the table has no RTE + write spec. ``False`` bypasses the RTE and reads the base table only, even when a spec is present. Returns @@ -1541,7 +1541,7 @@ class LanceQueryBuilder(ABC): LanceQueryBuilder The LanceQueryBuilder object. """ - self._use_memwal = enable + self._use_rte = enable return self def explain_plan(self, verbose: Optional[bool] = False) -> str: @@ -1878,7 +1878,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): probing may continue when too few candidates are found. Any explicit effort, including 0.5, overrides nprobes in either call order. Explicit minimum and maximum bounds still apply. This is not a recall guarantee - and does not change refinement or HNSW ef. MemWAL LSM queries do not + and does not change refinement or HNSW ef. RTE LSM queries do not yet support this option. Examples @@ -2055,7 +2055,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_memwal=self._use_memwal, + use_rte=self._use_rte, offset=self._offset, fast_search=self._fast_search, ef=self._ef, @@ -2280,7 +2280,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_memwal=self._use_memwal, + use_rte=self._use_rte, full_text_query=FullTextSearchQuery( query=self._query_with_phrase_semantics(), columns=self._fts_columns ), @@ -2362,7 +2362,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder): with_row_address=self._with_row_address, fragments=self._fragments, fragment_ids=self._fragment_ids, - use_memwal=self._use_memwal, + use_rte=self._use_rte, offset=self._offset, order_by=self._order_by, ) @@ -2458,7 +2458,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): """Set whether to return row ids. Recorded separately from `_with_row_id`, which `rerank(return_score= - "all")` also sets for its own use — on a MemWAL table only a caller who + "all")` also sets for its own use — on an RTE table only a caller who asked is refused, and the reranker still falls back to the primary key. """ self._caller_requested_row_id = with_row_id @@ -2728,7 +2728,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): if reranker.score == "all": # WAL-PK-FUSION: without the fallback, `self.with_row_id(True)`. # Not through `with_row_id`, which records caller intent: the - # reranker's need for row ids must not refuse a MemWAL table. + # reranker's need for row ids must not refuse an RTE table. self._with_row_id = True return self @@ -2766,7 +2766,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): probing may continue when too few candidates are found. Any explicit effort, including 0.5, overrides nprobes in either call order. Explicit minimum and maximum bounds still apply. This is not a recall guarantee - and does not change refinement or HNSW ef. MemWAL LSM queries do not + and does not change refinement or HNSW ef. RTE LSM queries do not yet support this option. Examples @@ -3040,7 +3040,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): # WAL-PK-FUSION: without the fallback, select `self._columns` as is. self._pk_fusion = None columns = self._columns - if _wal_hybrid.pk_fusion_learned(self._table, self._use_memwal): + if _wal_hybrid.pk_fusion_learned(self._table, self._use_rte): self._pk_fusion = _wal_hybrid.PkFusion.for_query( self._table.schema, self._columns, @@ -3057,9 +3057,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): if self._with_row_id and self._pk_fusion is None: self._vector_query.with_row_id(True) self._fts_query.with_row_id(True) - if self._use_memwal is not None: - self._vector_query.use_memwal(self._use_memwal) - self._fts_query.use_memwal(self._use_memwal) + if self._use_rte is not None: + self._vector_query.use_rte(self._use_rte) + self._fts_query.use_rte(self._use_rte) if self._phrase_query: self._fts_query.phrase_query(True) if self._distance_type: @@ -3723,12 +3723,12 @@ class AsyncStandardQuery(AsyncQueryBase): self._inner.fast_search() return self - def use_memwal(self, enable: bool) -> Self: + def use_rte(self, enable: bool) -> Self: """ - Control MemWAL LSM read routing for this query. + Control RTE LSM read routing for this query. - By default (unset), a query against a table with a MemWAL write spec (see - [AsyncTable.enable_memwal][lancedb.table.AsyncTable.enable_memwal]) + By default (unset), a query against a table with an RTE write spec (see + [AsyncTable.enable_rte][lancedb.table.AsyncTable.enable_rte]) is routed through the LSM scanner so it also returns data written via the ``merge_insert`` LSM path that has not yet been compacted into the base table (the active/frozen in-memory memtables and the SSTables), @@ -3737,11 +3737,11 @@ class AsyncStandardQuery(AsyncQueryBase): Parameters ---------- enable : bool - ``True`` forces the LSM scanner and errors if the table has no MemWAL - write spec. ``False`` bypasses the MemWAL and reads the base table + ``True`` forces the LSM scanner and errors if the table has no RTE + write spec. ``False`` bypasses the RTE and reads the base table only, even when a spec is present. """ - self._inner.use_memwal(enable) + self._inner.use_rte(enable) return self def postfilter(self) -> Self: @@ -4064,7 +4064,7 @@ class AsyncVectorQueryBase: minimum and maximum bounds remain effective. Zero starts at the caller minimum and one searches all partitions within the maximum. Later probing may continue to obtain enough candidates. This is not a recall - guarantee and does not change refinement or HNSW ef. MemWAL LSM queries + guarantee and does not change refinement or HNSW ef. RTE LSM queries do not yet support this option. Examples @@ -4388,7 +4388,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase): # WAL-PK-FUSION: without the fallback, keep only the two `with_row_id` # calls. self._pk_fusion = None - if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_memwal): + if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_rte): self._pk_fusion = _wal_hybrid.PkFusion.for_query( await self._table.schema(), _query_request_projection(fts_req), @@ -4577,13 +4577,13 @@ class AsyncTakeQuery(AsyncQueryBase): def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None): super().__init__(inner, table) - def use_memwal(self, enable: bool) -> "AsyncTakeQuery": - """Control MemWAL LSM read routing for this take query. + def use_rte(self, enable: bool) -> "AsyncTakeQuery": + """Control RTE LSM read routing for this take query. - ``False`` bypasses the MemWAL and reads the base table only — the escape + ``False`` bypasses the RTE and reads the base table only — the escape hatch, since take-by-row-id/offset is not supported on the LSM scanner. """ - self._inner.use_memwal(enable) + self._inner.use_rte(enable) return self async def _plain_scan_to_pandas( @@ -4644,14 +4644,14 @@ class BaseQueryBuilder(object): self._inner.with_row_id() return self - def use_memwal(self, enable: bool) -> Self: + def use_rte(self, enable: bool) -> Self: """ - Control MemWAL LSM read routing for this query. + Control RTE LSM read routing for this query. - ``False`` bypasses the MemWAL and reads the base table only, the escape + ``False`` bypasses the RTE and reads the base table only, the escape hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id). """ - self._inner.use_memwal(enable) + self._inner.use_rte(enable) return self def with_row_address(self, with_row_address: bool = True) -> Self: diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 2bec79d1e..eec958799 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -31,7 +31,7 @@ from lancedb._lancedb import ( DropColumnsResult, FtsToken, IndexConfig, - MemWalWriteSpec, + RTEWriteSpec, MergeResult, UpdateResult, ) @@ -1037,17 +1037,17 @@ class RemoteTable(Table): """Set the unenforced primary key for this table to a single column.""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) - def enable_memwal(self, spec: "MemWalWriteSpec") -> None: - """Install an MemWalWriteSpec.""" - return LOOP.run(self._table.enable_memwal(spec)) + def enable_rte(self, spec: "RTEWriteSpec") -> None: + """Install an RTEWriteSpec.""" + return LOOP.run(self._table.enable_rte(spec)) - def disable_memwal(self) -> None: - """Remove the MemWalWriteSpec.""" - return LOOP.run(self._table.disable_memwal()) + def disable_rte(self) -> None: + """Remove the RTEWriteSpec.""" + return LOOP.run(self._table.disable_rte()) - def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: - """Read the installed MemWalWriteSpec, or ``None``.""" - return LOOP.run(self._table.get_memwal_spec()) + def get_rte_spec(self) -> Optional["RTEWriteSpec"]: + """Read the installed RTEWriteSpec, or ``None``.""" + return LOOP.run(self._table.get_rte_spec()) # WAL-PK-FUSION: delete both hooks. def _hybrid_pk_fusion_learned(self) -> bool: @@ -1057,31 +1057,31 @@ class RemoteTable(Table): def _note_hybrid_pk_fusion(self) -> None: self._table._note_hybrid_pk_fusion() - def checkpoint_memwal(self) -> None: + def checkpoint_rte(self) -> None: """Synchronous version of - [`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal].""" - return LOOP.run(self._table.checkpoint_memwal()) + [`AsyncTable.checkpoint_rte`][lancedb.AsyncTable.checkpoint_rte].""" + return LOOP.run(self._table.checkpoint_rte()) - def flush_memwal(self) -> None: + def flush_rte(self) -> None: """Synchronous version of - [`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal].""" - return LOOP.run(self._table.flush_memwal()) + [`AsyncTable.flush_rte`][lancedb.AsyncTable.flush_rte].""" + return LOOP.run(self._table.flush_rte()) - def compact_memwal(self) -> None: + def compact_rte(self) -> None: """Synchronous version of - [`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal].""" - return LOOP.run(self._table.compact_memwal()) + [`AsyncTable.compact_rte`][lancedb.AsyncTable.compact_rte].""" + return LOOP.run(self._table.compact_rte()) - def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: + def get_rte_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of - [`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats].""" + [`AsyncTable.get_rte_stats`][lancedb.AsyncTable.get_rte_stats].""" return LOOP.run( - self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows) + self._table.get_rte_stats(include_sstable_rows=include_sstable_rows) ) - def close_memwal_writers(self) -> None: + def close_rte_writers(self) -> None: """No-op for remote tables (no local shard writers).""" - return LOOP.run(self._table.close_memwal_writers()) + return LOOP.run(self._table.close_rte_writers()) def drop_index(self, index_name: str): return LOOP.run(self._table.drop_index(index_name)) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index e0a93f33e..48d5582d1 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -223,7 +223,7 @@ if TYPE_CHECKING: DeleteResult, DropColumnsResult, FtsToken, - MemWalWriteSpec, + RTEWriteSpec, MergeResult, UpdateResult, ) @@ -4849,47 +4849,47 @@ class LanceTable(Table): [`AsyncTable.set_unenforced_primary_key`][lancedb.AsyncTable.set_unenforced_primary_key].""" return LOOP.run(self._table.set_unenforced_primary_key(columns)) - def enable_memwal(self, spec: "MemWalWriteSpec") -> None: - """Install an MemWalWriteSpec. See - [`AsyncTable.enable_memwal`][lancedb.AsyncTable.enable_memwal].""" - return LOOP.run(self._table.enable_memwal(spec)) + def enable_rte(self, spec: "RTEWriteSpec") -> None: + """Install an RTEWriteSpec. See + [`AsyncTable.enable_rte`][lancedb.AsyncTable.enable_rte].""" + return LOOP.run(self._table.enable_rte(spec)) - def disable_memwal(self) -> None: - """Remove the MemWalWriteSpec. See - [`AsyncTable.disable_memwal`][lancedb.AsyncTable.disable_memwal].""" - return LOOP.run(self._table.disable_memwal()) + def disable_rte(self) -> None: + """Remove the RTEWriteSpec. See + [`AsyncTable.disable_rte`][lancedb.AsyncTable.disable_rte].""" + return LOOP.run(self._table.disable_rte()) - def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: - """Read the installed MemWalWriteSpec, or ``None``. See - [`AsyncTable.get_memwal_spec`][lancedb.AsyncTable.get_memwal_spec].""" - return LOOP.run(self._table.get_memwal_spec()) + def get_rte_spec(self) -> Optional["RTEWriteSpec"]: + """Read the installed RTEWriteSpec, or ``None``. See + [`AsyncTable.get_rte_spec`][lancedb.AsyncTable.get_rte_spec].""" + return LOOP.run(self._table.get_rte_spec()) - def checkpoint_memwal(self) -> None: + def checkpoint_rte(self) -> None: """Synchronous version of - [`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal].""" - return LOOP.run(self._table.checkpoint_memwal()) + [`AsyncTable.checkpoint_rte`][lancedb.AsyncTable.checkpoint_rte].""" + return LOOP.run(self._table.checkpoint_rte()) - def flush_memwal(self) -> None: + def flush_rte(self) -> None: """Synchronous version of - [`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal].""" - return LOOP.run(self._table.flush_memwal()) + [`AsyncTable.flush_rte`][lancedb.AsyncTable.flush_rte].""" + return LOOP.run(self._table.flush_rte()) - def compact_memwal(self) -> None: + def compact_rte(self) -> None: """Synchronous version of - [`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal].""" - return LOOP.run(self._table.compact_memwal()) + [`AsyncTable.compact_rte`][lancedb.AsyncTable.compact_rte].""" + return LOOP.run(self._table.compact_rte()) - def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: + def get_rte_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]: """Synchronous version of - [`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats].""" + [`AsyncTable.get_rte_stats`][lancedb.AsyncTable.get_rte_stats].""" return LOOP.run( - self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows) + self._table.get_rte_stats(include_sstable_rows=include_sstable_rows) ) - def close_memwal_writers(self) -> None: - """Close cached MemWAL shard writers. See - [`AsyncTable.close_memwal_writers`][lancedb.AsyncTable.close_memwal_writers].""" - return LOOP.run(self._table.close_memwal_writers()) + def close_rte_writers(self) -> None: + """Close cached RTE shard writers. See + [`AsyncTable.close_rte_writers`][lancedb.AsyncTable.close_rte_writers].""" + return LOOP.run(self._table.close_rte_writers()) def uses_v2_manifest_paths(self) -> bool: """ @@ -5547,24 +5547,24 @@ class AsyncTable: columns = list(columns) await self._inner.set_unenforced_primary_key(columns) - async def enable_memwal(self, spec: "MemWalWriteSpec") -> None: - """Install an MemWalWriteSpec on this table. + async def enable_rte(self, spec: "RTEWriteSpec") -> None: + """Install an RTEWriteSpec on this table. The spec selects Lance's MemWAL LSM-style write path for future - `merge_insert` calls. ``MemWalWriteSpec`` chooses one of three sharding + `merge_insert` calls. ``RTEWriteSpec`` chooses one of three sharding strategies: - - ``MemWalWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by + - ``RTEWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by the single-column unenforced primary key. - - ``MemWalWriteSpec.identity(column)`` — shard by the raw value of a + - ``RTEWriteSpec.identity(column)`` — shard by the raw value of a scalar column. - - ``MemWalWriteSpec.unsharded()`` — route every write to a single shard. + - ``RTEWriteSpec.unsharded()`` — route every write to a single shard. All variants require the table to have an unenforced primary key set via [`set_unenforced_primary_key`]; bucket sharding additionally requires it to be the single column being bucketed. - By default the MemWAL maintains every index on the table, resolved + By default the RTE maintains every index on the table, resolved here — a snapshot, so an index created afterwards needs the spec unset and set again. This fails if one cannot be maintained; name the set with ``with_maintained_indexes`` to install anyway. That pins an exact @@ -5573,30 +5573,30 @@ class AsyncTable: Parameters ---------- - spec : MemWalWriteSpec + spec : RTEWriteSpec The sharding spec to install. Examples -------- - >>> from lancedb import MemWalWriteSpec + >>> from lancedb import RTEWriteSpec >>> # table.set_unenforced_primary_key("id") - >>> # table.enable_memwal(MemWalWriteSpec.bucket("id", 16)) + >>> # table.enable_rte(RTEWriteSpec.bucket("id", 16)) """ - await self._inner.enable_memwal(spec) + await self._inner.enable_rte(spec) - async def disable_memwal(self) -> None: - """Remove the MemWalWriteSpec from this table. + async def disable_rte(self) -> None: + """Remove the RTEWriteSpec from this table. Reverts to the standard `merge_insert` write path. Errors if no spec is currently set. """ - await self._inner.disable_memwal() + await self._inner.disable_rte() - async def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]: - """Read the MemWalWriteSpec currently installed on this table. + async def get_rte_spec(self) -> Optional["RTEWriteSpec"]: + """Read the RTEWriteSpec currently installed on this table. - Returns ``None`` when the MemWAL write path is not enabled at all — no - spec has been set, or one was removed with `disable_memwal`. + Returns ``None`` when the RTE write path is not enabled at all — no + spec has been set, or one was removed with `disable_rte`. That is a different answer from a spec whose ``maintained_indexes`` is ``None``, which is an installed spec selecting indexes automatically. @@ -5606,7 +5606,7 @@ class AsyncTable: or gains later, ``[]`` maintains none, and a non-empty list maintains exactly those. All three round-trip. """ - return await self._inner.get_memwal_spec() + return await self._inner.get_rte_spec() # WAL-PK-FUSION: delete both hooks. def _hybrid_pk_fusion_learned(self) -> bool: @@ -5617,12 +5617,12 @@ class AsyncTable: """See [`Table._note_hybrid_pk_fusion`][lancedb.table.Table].""" self._inner.note_hybrid_pk_fusion() - async def checkpoint_memwal(self) -> None: - """Converge this table's MemWAL write path into its base table. + async def checkpoint_rte(self) -> None: + """Converge this table's RTE write path into its base table. One flush, freezing every memtable into an SSTable, then compaction triggers until every SSTable that existed at that moment has reached base. - The loop runs client-side, reading progress from ``get_memwal_stats``. + The loop runs client-side, reading progress from ``get_rte_stats``. Best-effort: SSTables created *while* it runs are deliberately not waited on, which is what lets it terminate on a table taking writes. @@ -5637,56 +5637,56 @@ class AsyncTable: ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway costs nothing. """ - await self._inner.checkpoint_memwal() + await self._inner.checkpoint_rte() - async def flush_memwal(self) -> None: + async def flush_rte(self) -> None: """Freeze every table shard's active memtable into an SSTable. Does not touch the base table — compacting SSTables into base is - `compact_memwal`. On a node that has not claimed this table, this claims + `compact_rte`. On a node that has not claimed this table, this claims it and replays its WAL log first. """ - await self._inner.flush_memwal() + await self._inner.flush_rte() - async def compact_memwal(self) -> None: + async def compact_rte(self) -> None: """Trigger a background SSTable compaction pass per table shard. Returns once the passes are dispatched, not once they finish: watch - ``get_memwal_stats`` for progress, or use ``checkpoint_memwal`` to loop + ``get_rte_stats`` for progress, or use ``checkpoint_rte`` to loop until the current SSTables have reached base. """ - await self._inner.compact_memwal() + await self._inner.compact_rte() - async def get_memwal_stats( + async def get_rte_stats( self, *, include_sstable_rows: bool = False ) -> Optional[dict]: - """Read live per-table-shard MemWAL state. + """Read live per-table-shard RTE state. Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my fresh-tier vector search brute-force". Mutates no table state, though on a node that has not claimed this table it claims it, exactly as a read would. - Returns ``None`` only when the MemWAL write path is not enabled. + Returns ``None`` only when the RTE write path is not enabled. Parameters ---------- include_sstable_rows Report a row count per SSTable. Off by default: each count - opens an uncached Lance dataset, and ``checkpoint_memwal`` polls this + opens an uncached Lance dataset, and ``checkpoint_rte`` polls this needing only generation numbers. """ - return await self._inner.get_memwal_stats(include_sstable_rows) + return await self._inner.get_rte_stats(include_sstable_rows) - async def close_memwal_writers(self) -> None: - """Drain and close any cached MemWAL shard writers for this table. + async def close_rte_writers(self) -> None: + """Drain and close any cached RTE shard writers for this table. - When a MemWAL write spec is installed, `merge_insert` opens MemWAL shard + When an RTE write spec is installed, `merge_insert` opens RTE shard writers and caches them for reuse across calls. This closes them, flushing pending data; writers reopen lazily on the next `merge_insert`. It is a no-op when no writers are cached. """ - await self._inner.close_memwal_writers() + await self._inner.close_rte_writers() @property def name(self) -> str: @@ -6575,8 +6575,8 @@ class AsyncTable: take_query = self.take_offsets(query.take_offsets) if query.columns: take_query = take_query.select(query.columns) - if query.use_memwal is not None: - take_query = take_query.use_memwal(query.use_memwal) + if query.use_rte is not None: + take_query = take_query.use_rte(query.use_rte) if query.with_row_id: take_query = take_query.with_row_id() return take_query @@ -6592,8 +6592,8 @@ class AsyncTable: async_query = async_query.where(query.filter) if query.fast_search: async_query = async_query.fast_search() - if query.use_memwal is not None: - async_query = async_query.use_memwal(query.use_memwal) + if query.use_rte is not None: + async_query = async_query.use_rte(query.use_rte) if query.with_row_id: async_query = async_query.with_row_id() if query.order_by: @@ -6719,7 +6719,7 @@ class AsyncTable: when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr, timeout=merge._timeout, use_index=merge._use_index, - use_memwal=merge._use_memwal, + use_rte=merge._use_rte, validate_single_shard=merge._validate_single_shard, ), ) diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py index aec25993a..c7d69d779 100644 --- a/python/python/tests/test_merge_insert_lsm.py +++ b/python/python/tests/test_merge_insert_lsm.py @@ -1,14 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Tests for the MemWAL LSM ``merge_insert`` dispatch.""" +"""Tests for the RTE LSM ``merge_insert`` dispatch.""" from datetime import timedelta import lancedb import pyarrow as pa import pytest -from lancedb._lancedb import MemWalWriteSpec +from lancedb._lancedb import RTEWriteSpec from lancedb.index import FTS, IvfPq SCHEMA = pa.schema( @@ -49,12 +49,12 @@ def _region_reader(rows): def _bucket_table(tmp_path): - """A table with ``id`` as the primary key and a single-bucket MemWAL spec.""" + """A table with ``id`` as the primary key and a single-bucket RTE spec.""" db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") # num_buckets = 1: every row routes to the single bucket. - table.enable_memwal(MemWalWriteSpec.bucket("id", 1)) + table.enable_rte(RTEWriteSpec.bucket("id", 1)) return table @@ -67,7 +67,7 @@ def test_lsm_merge_insert_bucket(tmp_path): .when_not_matched_insert_all() .execute(_reader([3, 4, 5])) ) - # LSM path: rows go to the MemWAL, so only num_rows is populated. + # LSM path: rows go to the RTE, so only num_rows is populated. assert result.num_rows == 3 assert result.version == 0 assert result.num_inserted_rows == 0 @@ -78,7 +78,7 @@ def test_lsm_merge_insert_unsharded(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) + table.enable_rte(RTEWriteSpec.unsharded()) result = ( table.merge_insert("id") .when_matched_update_all() @@ -92,7 +92,7 @@ def test_lsm_merge_insert_identity(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _region_reader([(1, "us"), (2, "us")])) table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.identity("region")) + table.enable_rte(RTEWriteSpec.identity("region")) # All rows share one identity value, so they route to one shard. result = ( table.merge_insert([]) @@ -103,31 +103,31 @@ def test_lsm_merge_insert_identity(tmp_path): assert result.num_rows == 2 -def test_lsm_merge_insert_use_memwal_false(tmp_path): +def test_lsm_merge_insert_use_rte_false(tmp_path): table = _bucket_table(tmp_path) # rows id = 1, 2, 3 - # use_memwal(False) opts out: the standard path runs and commits even with a spec. + # use_rte(False) opts out: the standard path runs and commits even with a spec. result = ( table.merge_insert("id") .when_not_matched_insert_all() - .use_memwal(False) + .use_rte(False) .execute(_reader([3, 4, 5])) ) assert result.num_inserted_rows == 2 assert table.count_rows() == 5 -def test_lsm_merge_insert_use_memwal_true_without_spec_errors(tmp_path): - # A table with a primary key but no MemWAL write spec installed. +def test_lsm_merge_insert_use_rte_true_without_spec_errors(tmp_path): + # A table with a primary key but no RTE write spec installed. db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - # use_memwal(True) demands MemWAL routing; without a spec it errors. - with pytest.raises(Exception, match="use_memwal"): + # use_rte(True) demands RTE routing; without a spec it errors. + with pytest.raises(Exception, match="use_rte"): ( table.merge_insert("id") .when_matched_update_all() .when_not_matched_insert_all() - .use_memwal(True) + .use_rte(True) .execute(_reader([3, 4, 5])) ) @@ -145,7 +145,7 @@ def test_lsm_merge_insert_validate_single_shard_off(tmp_path): def test_lsm_merge_insert_no_spec_uses_standard_path(tmp_path): - # A table with a primary key but no MemWAL write spec installed. + # A table with a primary key but no RTE write spec installed. db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") @@ -186,7 +186,7 @@ def test_lsm_close_writers(tmp_path): .when_not_matched_insert_all() .execute(_reader([7, 8])) ) - table.close_memwal_writers() + table.close_rte_writers() # The writer reopens lazily on the next merge_insert. result = ( table.merge_insert([]) @@ -204,14 +204,14 @@ async def test_async_lsm_merge_insert(tmp_path): ) table = await db.create_table("t", _reader([1, 2, 3])) await table.set_unenforced_primary_key("id") - await table.enable_memwal(MemWalWriteSpec.bucket("id", 1)) + await table.enable_rte(RTEWriteSpec.bucket("id", 1)) builder = ( table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all() ) result = await builder.execute(_reader([3, 4, 5])) assert result.num_rows == 3 - await table.close_memwal_writers() + await table.close_rte_writers() def _lsm_upsert(table, ids): @@ -228,7 +228,7 @@ def test_lsm_read_sees_active_memtable(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) # base ids 1,2,3 table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) + table.enable_rte(RTEWriteSpec.unsharded()) _lsm_upsert(table, [4, 5]) # active memtable only, not committed to base @@ -236,8 +236,8 @@ def test_lsm_read_sees_active_memtable(tmp_path): lsm = table.search().to_arrow() assert sorted(lsm["id"].to_pylist()) == [1, 2, 3, 4, 5] - # use_memwal(False) bypasses the MemWAL and reads the base table only. - base_only = table.search().use_memwal(False).to_arrow() + # use_rte(False) bypasses the RTE and reads the base table only. + base_only = table.search().use_rte(False).to_arrow() assert sorted(base_only["id"].to_pylist()) == [1, 2, 3] @@ -245,7 +245,7 @@ def test_lsm_read_dedup_newest_wins(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) # id 2 -> value 1 table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) + table.enable_rte(RTEWriteSpec.unsharded()) _lsm_upsert(table, [2, 3, 4]) # ids 2,3,4 -> values 0,1,2 @@ -258,31 +258,31 @@ def test_lsm_read_dedup_newest_wins(tmp_path): def test_lsm_read_without_spec_reads_base(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) - table.set_unenforced_primary_key("id") # no MemWAL write spec + table.set_unenforced_primary_key("id") # no RTE write spec - # No spec: default read and use_memwal(False) both read the base table, no error. + # No spec: default read and use_rte(False) both read the base table, no error. assert sorted(table.search().to_arrow()["id"].to_pylist()) == [1, 2, 3] - assert sorted(table.search().use_memwal(False).to_arrow()["id"].to_pylist()) == [ + assert sorted(table.search().use_rte(False).to_arrow()["id"].to_pylist()) == [ 1, 2, 3, ] -def test_lsm_read_unsupported_shape_errors_without_use_memwal_false(tmp_path): +def test_lsm_read_unsupported_shape_errors_without_use_rte_false(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) + table.enable_rte(RTEWriteSpec.unsharded()) _lsm_upsert(table, [4]) - # with_row_id is unsupported by the LSM scanner; on a MemWAL table the default + # with_row_id is unsupported by the LSM scanner; on an RTE table the default # (auto-routed) read hard-errors instead of silently reading a stale base. with pytest.raises(Exception): table.search().with_row_id(True).to_arrow() - # use_memwal(False) is the escape hatch: it reads the base table only. - base = table.search().with_row_id(True).use_memwal(False).to_arrow() + # use_rte(False) is the escape hatch: it reads the base table only. + base = table.search().with_row_id(True).use_rte(False).to_arrow() assert sorted(base["id"].to_pylist()) == [1, 2, 3] @@ -293,7 +293,7 @@ async def test_async_lsm_read(tmp_path): ) table = await db.create_table("t", _reader([1, 2, 3])) await table.set_unenforced_primary_key("id") - await table.enable_memwal(MemWalWriteSpec.unsharded()) + await table.enable_rte(RTEWriteSpec.unsharded()) builder = ( table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all() @@ -331,7 +331,7 @@ def _vector_reader(rows): def _vector_table(tmp_path): """Base table whose vector column is indexed so its rows are visible to the LSM vector scanner (the base arm uses ``fast_search`` — indexed data only), plus an - unsharded MemWAL spec that maintains that index for the memtable. + unsharded RTE spec that maintains that index for the memtable. Rows 1,2 are category ``a``, row 3 is ``b``, and 4..60 are filler ``c`` that give the tiny IVF index enough data to train. @@ -353,9 +353,7 @@ def _vector_table(tmp_path): "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4) ) index_name = table.list_indices()[0].name - table.enable_memwal( - MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) - ) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name])) return table @@ -379,9 +377,9 @@ def test_lsm_read_vector_sees_memtable(tmp_path): assert {1, 2, 3} <= ids # indexed base rows assert 1000 in ids # in-flight memtable row - # use_memwal(False) bypasses the MemWAL, so the in-flight row is not visible. + # use_rte(False) bypasses the RTE, so the in-flight row is not visible. base_ids = set( - table.search(query).use_memwal(False).limit(100).to_arrow()["id"].to_pylist() + table.search(query).use_rte(False).limit(100).to_arrow()["id"].to_pylist() ) assert {1, 2, 3} <= base_ids assert 1000 not in base_ids @@ -453,9 +451,7 @@ def test_lsm_read_fts_sees_memtable(tmp_path): # Native FTS index (tantivy is not compatible with the LSM memtable index). table.create_index("text", config=FTS()) index_name = table.list_indices()[0].name - table.enable_memwal( - MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) - ) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name])) # in-flight doc 4 lands in the memtable's maintained FTS index. ( @@ -498,8 +494,8 @@ def test_lsm_read_vector_unsupported_knobs_error(tmp_path): with pytest.raises(Exception, match="use_index"): table.search(query).bypass_vector_index().to_arrow() - # use_memwal(False) is the escape hatch: the base-only standard path honors them. - base = table.search(query).distance_range(0.0, 100.0).use_memwal(False).to_arrow() + # use_rte(False) is the escape hatch: the base-only standard path honors them. + base = table.search(query).distance_range(0.0, 100.0).use_rte(False).to_arrow() assert 1000 not in set(base["id"].to_pylist()) @@ -544,7 +540,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path): table.create_index("text", config=FTS()) # No maintained indexes: the active memtable FTS arm cannot serve un-compacted # docs, so the search would silently omit them — reject instead. - table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search("fox", query_type="fts", fts_columns="text").to_arrow() @@ -553,7 +549,7 @@ def test_lsm_read_time_travel_errors(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) + table.enable_rte(RTEWriteSpec.unsharded()) pinned = table.version table.add(_reader([4, 5])) # standard add commits a newer version table.checkout(pinned) # detached head at the historical version @@ -562,8 +558,8 @@ def test_lsm_read_time_travel_errors(tmp_path): # historical version is rejected. with pytest.raises(Exception, match="time-travel"): table.search().to_arrow() - # use_memwal(False) reads the base table at the pinned version. - base = table.search().use_memwal(False).to_arrow() + # use_rte(False) reads the base table at the pinned version. + base = table.search().use_rte(False).to_arrow() assert sorted(base["id"].to_pylist()) == [1, 2, 3] @@ -571,14 +567,14 @@ def test_lsm_read_take_row_ids_errors(tmp_path): db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0)) table = db.create_table("t", _reader([1, 2, 3])) table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) + table.enable_rte(RTEWriteSpec.unsharded()) _lsm_upsert(table, [4]) # take-by-row-id auto-routes through the LSM scanner, which has no stable _rowid, # so it hard-errors instead of failing with an opaque column-not-found error. with pytest.raises(Exception, match="row id"): table.take_row_ids([0, 1]).to_arrow() - # use_memwal(False) is the escape hatch: it reads the base table. - base = table.take_row_ids([0, 1]).use_memwal(False).to_arrow() + # use_rte(False) is the escape hatch: it reads the base table. + base = table.take_row_ids([0, 1]).use_rte(False).to_arrow() assert base.num_rows == 2 @@ -588,9 +584,7 @@ def test_lsm_read_fts_postfilter_errors(tmp_path): table.set_unenforced_primary_key("id") table.create_index("text", config=FTS()) index_name = table.list_indices()[0].name - table.enable_memwal( - MemWalWriteSpec.unsharded().with_maintained_indexes([index_name]) - ) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name])) # The LSM scanner always prefilters; postfilter on FTS changes result semantics, # so it hard-errors (previously only the vector arm rejected it). with pytest.raises(Exception, match="postfilter"): @@ -608,7 +602,7 @@ def test_lsm_read_fts_multiple_same_type_indexes_errors(tmp_path): table.set_unenforced_primary_key("id") table.create_index("text", config=FTS(), name="fts_a") table.create_index("text", config=FTS(), name="fts_b", replace=False) - table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes(["fts_a"])) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes(["fts_a"])) # Two FTS indexes on the column: the base planner's chosen index is ambiguous, so # the scanner can't pick a catch-up watermark and rejects rather than risk # dropping rows the actually-used index has not caught up to. @@ -629,7 +623,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path): ) # Spec with NO maintained indexes: the base vector index's catch-up is untracked, # so the scanner rejects rather than risk dropping compacted-but-unindexed rows. - table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([])) with pytest.raises(Exception, match="maintained"): table.search([1.0] * VECTOR_DIM).to_arrow() @@ -642,7 +636,7 @@ def test_lsm_read_fts_optimized_index_not_rejected(tmp_path): table.add(_fts_reader([(i, "lazy fox") for i in range(6, 11)])) table.optimize() # may split the FTS index into multiple physical segments name = table.list_indices()[0].name - table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([name])) + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([name])) # Multiple physical segments of one logical index must not be miscounted as # multiple indexes and rejected. ids = set( diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 73fb6ab12..80bd10289 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1844,10 +1844,10 @@ def lsm_test_table(lsm_handler): """A remote table whose LSM routes are served by ``lsm_handler``. ``lsm_handler(request, route)`` is called for ``/v1/table/test//`` - where route is one of flush_memwal, compact_memwal, get_memwal_stats, and is + where route is one of flush_rte, compact_rte, get_rte_stats, and is responsible for writing the response. """ - routes = ("flush_memwal", "compact_memwal", "get_memwal_stats") + routes = ("flush_rte", "compact_rte", "get_rte_stats") def handler(request): match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path) @@ -1879,7 +1879,7 @@ def send_json(request, payload, status=200): request.wfile.write(json.dumps(payload).encode()) -def test_get_memwal_stats_sync(): +def test_get_rte_stats_sync(): """The sync wrapper round-trips the server payload into a dict.""" table_shard = { "shard_id": "b0", @@ -1904,29 +1904,29 @@ def test_get_memwal_stats_sync(): seen_bodies = [] def lsm_handler(request, route): - assert route == "get_memwal_stats" + assert route == "get_rte_stats" seen_bodies.append(read_json_body(request)) - send_json(request, {"memwal_stats": {"table_shards": [table_shard]}}) + send_json(request, {"rte_stats": {"table_shards": [table_shard]}}) with lsm_test_table(lsm_handler) as table: - assert table.get_memwal_stats() == {"table_shards": [table_shard]} + assert table.get_rte_stats() == {"table_shards": [table_shard]} # Off by default, and forwarded when asked for. assert seen_bodies == [{"include_sstable_rows": False}] - table.get_memwal_stats(include_sstable_rows=True) + table.get_rte_stats(include_sstable_rows=True) assert seen_bodies[-1] == {"include_sstable_rows": True} -def test_get_memwal_stats_sync_returns_none_when_memwal_disabled(): - """A null envelope means the MemWAL write path is not enabled, not an error.""" +def test_get_rte_stats_sync_returns_none_when_rte_disabled(): + """A null envelope means the RTE write path is not enabled, not an error.""" def lsm_handler(request, route): - send_json(request, {"memwal_stats": None}) + send_json(request, {"rte_stats": None}) with lsm_test_table(lsm_handler) as table: - assert table.get_memwal_stats() is None + assert table.get_rte_stats() is None -def test_flush_and_compact_memwal_sync(): +def test_flush_and_compact_rte_sync(): """Both are one-shot POSTs answered 202 with no body.""" called = [] @@ -1936,12 +1936,12 @@ def test_flush_and_compact_memwal_sync(): request.end_headers() with lsm_test_table(lsm_handler) as table: - assert table.flush_memwal() is None - assert table.compact_memwal() is None - assert called == ["flush_memwal", "compact_memwal"] + assert table.flush_rte() is None + assert table.compact_rte() is None + assert called == ["flush_rte", "compact_rte"] -def test_checkpoint_memwal_sync(): +def test_checkpoint_rte_sync(): """Freeze, read the watermark, and return once no SSTables remain. The convergence loop itself is covered in Rust; this pins the sync @@ -1951,17 +1951,17 @@ def test_checkpoint_memwal_sync(): def lsm_handler(request, route): called.append(route) - if route == "get_memwal_stats": + if route == "get_rte_stats": # An empty SSTable tier yields no target watermark, so the loop is done # after the freeze without ever polling compaction. - send_json(request, {"memwal_stats": {"table_shards": []}}) + send_json(request, {"rte_stats": {"table_shards": []}}) else: request.send_response(202) request.end_headers() with lsm_test_table(lsm_handler) as table: - assert table.checkpoint_memwal() is None - assert called == ["flush_memwal", "get_memwal_stats"] + assert table.checkpoint_rte() is None + assert called == ["flush_rte", "get_rte_stats"] @contextlib.contextmanager diff --git a/python/python/tests/test_memwal_spec.py b/python/python/tests/test_rte_spec.py similarity index 65% rename from python/python/tests/test_memwal_spec.py rename to python/python/tests/test_rte_spec.py index d292c9185..6b39e0f1d 100644 --- a/python/python/tests/test_memwal_spec.py +++ b/python/python/tests/test_rte_spec.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Tests for installing and clearing an MemWalWriteSpec via -`Table.enable_memwal` / `Table.disable_memwal`. +"""Tests for installing and clearing an RTEWriteSpec via +`Table.enable_rte` / `Table.disable_rte`. """ from datetime import timedelta @@ -10,7 +10,7 @@ from datetime import timedelta import lancedb import pyarrow as pa import pytest -from lancedb._lancedb import MemWalWriteSpec +from lancedb._lancedb import RTEWriteSpec from lancedb.index import BTree SCHEMA = pa.schema( @@ -21,9 +21,9 @@ SCHEMA = pa.schema( ) -def test_memwal_spec_module_metadata(): - assert lancedb.MemWalWriteSpec is MemWalWriteSpec - assert MemWalWriteSpec.__module__ == "lancedb._lancedb" +def test_rte_spec_module_metadata(): + assert lancedb.RTEWriteSpec is RTEWriteSpec + assert RTEWriteSpec.__module__ == "lancedb._lancedb" def _batch(ids, vs): @@ -43,49 +43,49 @@ def _make_table(tmp_path): return db, table -def test_enable_memwal_validates(tmp_path): +def test_enable_rte_validates(tmp_path): _db, table = _make_table(tmp_path) table.set_unenforced_primary_key("id") # Out-of-range num_buckets. with pytest.raises(Exception, match="num_buckets"): - table.enable_memwal(MemWalWriteSpec.bucket("id", 0)) + table.enable_rte(RTEWriteSpec.bucket("id", 0)) with pytest.raises(Exception, match="num_buckets"): - table.enable_memwal(MemWalWriteSpec.bucket("id", 1025)) + table.enable_rte(RTEWriteSpec.bucket("id", 1025)) # Happy path, then a set over the installed spec is refused. - table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) + table.enable_rte(RTEWriteSpec.bucket("id", 4)) with pytest.raises(Exception, match="already set"): - table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) + table.enable_rte(RTEWriteSpec.bucket("id", 8)) -def test_disable_memwal(tmp_path): +def test_disable_rte(tmp_path): _db, table = _make_table(tmp_path) table.set_unenforced_primary_key("id") # unset errors when no spec is set. - with pytest.raises(Exception, match="no MemWAL write spec"): - table.disable_memwal() + with pytest.raises(Exception, match="no RTE write spec"): + table.disable_rte() # Install a spec, then remove it; afterwards a fresh spec can be set. - table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) - table.disable_memwal() + table.enable_rte(RTEWriteSpec.bucket("id", 4)) + table.disable_rte() # A second unset errors — there is no spec left to remove. - with pytest.raises(Exception, match="no MemWAL write spec"): - table.disable_memwal() - table.enable_memwal(MemWalWriteSpec.bucket("id", 8)) + with pytest.raises(Exception, match="no RTE write spec"): + table.disable_rte() + table.enable_rte(RTEWriteSpec.bucket("id", 8)) -def test_enable_memwal_requires_primary_key(tmp_path): +def test_enable_rte_requires_primary_key(tmp_path): _db, table = _make_table(tmp_path) for spec in ( - MemWalWriteSpec.unsharded(), - MemWalWriteSpec.bucket("id", 4), - MemWalWriteSpec.identity("id"), + RTEWriteSpec.unsharded(), + RTEWriteSpec.bucket("id", 4), + RTEWriteSpec.identity("id"), ): with pytest.raises(Exception, match="primary key"): - table.enable_memwal(spec) - assert table.get_memwal_spec() is None + table.enable_rte(spec) + assert table.get_rte_spec() is None def test_set_unsharded_spec(tmp_path): @@ -93,12 +93,12 @@ def test_set_unsharded_spec(tmp_path): # Lance MemWAL still requires a primary key on the dataset; Unsharded # just skips per-row hashing. table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.unsharded()) - table.disable_memwal() + table.enable_rte(RTEWriteSpec.unsharded()) + table.disable_rte() -def test_memwal_spec_repr(): - s = MemWalWriteSpec.bucket("id", 4) +def test_rte_spec_repr(): + s = RTEWriteSpec.bucket("id", 4) assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 @@ -109,20 +109,20 @@ def test_memwal_spec_repr(): assert "id" in repr(s) assert "4" in repr(s) - u = MemWalWriteSpec.unsharded() + u = RTEWriteSpec.unsharded() assert u.spec_type == "unsharded" assert u.column is None assert u.num_buckets is None assert "unsharded" in repr(u) -def test_memwal_spec_with_maintained_indexes(): - s = MemWalWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"]) +def test_rte_spec_with_maintained_indexes(): + s = RTEWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"]) assert s.maintained_indexes == ["idx_a", "idx_b"] @pytest.mark.asyncio -async def test_async_set_disable_memwal(tmp_path): +async def test_async_set_disable_rte(tmp_path): db = await lancedb.connect_async( tmp_path, read_consistency_interval=timedelta(seconds=0) ) @@ -132,11 +132,11 @@ async def test_async_set_disable_memwal(tmp_path): ) await table.set_unenforced_primary_key("id") - await table.enable_memwal(MemWalWriteSpec.bucket("id", 4)) - await table.disable_memwal() + await table.enable_rte(RTEWriteSpec.bucket("id", 4)) + await table.disable_rte() # A second unset errors. - with pytest.raises(Exception, match="no MemWAL write spec"): - await table.disable_memwal() + with pytest.raises(Exception, match="no RTE write spec"): + await table.disable_rte() def test_set_identity_spec(tmp_path): @@ -144,12 +144,12 @@ def test_set_identity_spec(tmp_path): # Identity sharding still requires an unenforced primary key on the # table; it shards by the raw value of the given column. table.set_unenforced_primary_key("id") - table.enable_memwal(MemWalWriteSpec.identity("v")) - table.disable_memwal() + table.enable_rte(RTEWriteSpec.identity("v")) + table.disable_rte() -def test_memwal_spec_identity_and_writer_config_defaults(): - s = MemWalWriteSpec.identity("v") +def test_rte_spec_identity_and_writer_config_defaults(): + s = RTEWriteSpec.identity("v") assert s.spec_type == "identity" assert s.column == "v" assert s.num_buckets is None @@ -160,12 +160,12 @@ def test_memwal_spec_identity_and_writer_config_defaults(): assert "durable_write" in repr(s) -def test_get_memwal_spec(tmp_path): +def test_get_rte_spec(tmp_path): _db, table = _make_table(tmp_path) table.set_unenforced_primary_key("id") # None when nothing is installed. - assert table.get_memwal_spec() is None + assert table.get_rte_spec() is None # A real scalar index is needed to name it as a maintained index. table.create_index("id", config=BTree()) @@ -173,12 +173,12 @@ def test_get_memwal_spec(tmp_path): # Bucket spec round-trips, including maintained indexes and writer config # defaults. - table.enable_memwal( - MemWalWriteSpec.bucket("id", 4) + table.enable_rte( + RTEWriteSpec.bucket("id", 4) .with_maintained_indexes([idx_name]) .with_writer_config_defaults({"durable_write": "false"}) ) - spec = table.get_memwal_spec() + spec = table.get_rte_spec() assert spec is not None assert spec.spec_type == "bucket" assert spec.column == "id" @@ -187,30 +187,30 @@ def test_get_memwal_spec(tmp_path): assert spec.writer_config_defaults == {"durable_write": "false"} # After unset, None again. - table.disable_memwal() - assert table.get_memwal_spec() is None + table.disable_rte() + assert table.get_rte_spec() is None # Identity round-trips (column recovered from the schema). An unnamed # maintained set reads back as None: what is stored is the intent to # maintain every index, not the set it resolves to now. - table.enable_memwal(MemWalWriteSpec.identity("id")) - spec = table.get_memwal_spec() + table.enable_rte(RTEWriteSpec.identity("id")) + spec = table.get_rte_spec() assert spec.spec_type == "identity" assert spec.column == "id" assert spec.maintained_indexes is None - table.disable_memwal() + table.disable_rte() # Unsharded round-trips (no routing column). Opting out is distinct from # the unnamed default. - table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([])) - spec = table.get_memwal_spec() + table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([])) + spec = table.get_rte_spec() assert spec.spec_type == "unsharded" assert spec.column is None assert spec.maintained_indexes == [] @pytest.mark.asyncio -async def test_async_get_memwal_spec(tmp_path): +async def test_async_get_rte_spec(tmp_path): db = await lancedb.connect_async( tmp_path, read_consistency_interval=timedelta(seconds=0) ) @@ -219,21 +219,21 @@ async def test_async_get_memwal_spec(tmp_path): pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]), ) - assert await table.get_memwal_spec() is None + assert await table.get_rte_spec() is None await table.set_unenforced_primary_key("id") # A real scalar index is needed to name it as a maintained index. await table.create_index("id", config=BTree()) idx_name = (await table.list_indices())[0].name - await table.enable_memwal( - MemWalWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name]) + await table.enable_rte( + RTEWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name]) ) - spec = await table.get_memwal_spec() + spec = await table.get_rte_spec() assert spec is not None assert spec.spec_type == "bucket" assert spec.column == "id" assert spec.num_buckets == 8 assert spec.maintained_indexes == [idx_name] - await table.disable_memwal() - assert await table.get_memwal_spec() is None + await table.disable_rte() + assert await table.get_rte_spec() is None diff --git a/python/python/tests/test_wal_hybrid.py b/python/python/tests/test_wal_hybrid.py index 675fc927a..d5030f14e 100644 --- a/python/python/tests/test_wal_hybrid.py +++ b/python/python/tests/test_wal_hybrid.py @@ -14,7 +14,7 @@ from lancedb.rerankers import RRFReranker def _wal_table(schema: pa.Schema) -> mock.Mock: - """A MemWAL-backed table a previous query has already been refused + """An RTE-backed table a previous query has already been refused `_rowid` on, so the fusion goes straight to the primary key.""" table = mock.Mock() table._hybrid_pk_fusion_learned.return_value = True @@ -51,7 +51,7 @@ def test_wal_hybrid_projects_the_key_instead_of_asking_for_row_ids(): assert builder._pk_fusion.injected == ["id"] for leg in (builder._vector_query, builder._fts_query): assert leg._columns == ["text", "id"] - # A MemWAL table rejects with_row_id before it plans. + # An RTE table rejects with_row_id before it plans. assert leg._with_row_id is not True @@ -81,15 +81,15 @@ def test_base_table_hybrid_still_joins_on_row_ids(): assert builder._vector_query._columns is None -def test_wal_hybrid_with_use_memwal_false_joins_on_row_ids(): - builder = _wal_hybrid().use_memwal(False) +def test_wal_hybrid_with_use_rte_false_joins_on_row_ids(): + builder = _wal_hybrid().use_rte(False) builder._create_query_builders() assert builder._pk_fusion is None def test_wal_hybrid_refuses_an_explicit_with_row_id(): builder = _wal_hybrid().with_row_id(True) - with pytest.raises(NotImplementedError, match="use_memwal"): + with pytest.raises(NotImplementedError, match="use_rte"): builder._create_query_builders() @@ -141,7 +141,7 @@ def test_wal_hybrid_refuses_row_id_even_with_return_score_all(row_id_first: bool else: builder = builder.rerank(reranker).with_row_id(True) - with pytest.raises(NotImplementedError, match="use_memwal"): + with pytest.raises(NotImplementedError, match="use_rte"): builder._create_query_builders() @@ -199,7 +199,7 @@ def test_fallback_explains_the_refusal_to_a_caller_who_asked_for_row_ids(): def run(): raise _REFUSAL - with pytest.raises(NotImplementedError, match="use_memwal") as info: + with pytest.raises(NotImplementedError, match="use_rte") as info: with_pk_fallback( run, mock.Mock(), fused=lambda: False, caller_requested_row_id=True ) diff --git a/python/python/tests/utils.py b/python/python/tests/utils.py index b3548b678..ca72a4dfd 100644 --- a/python/python/tests/utils.py +++ b/python/python/tests/utils.py @@ -73,10 +73,10 @@ class MockPermutationServer: }, }, ) - if path == f"/v1/table/{self.name}/get_memwal_spec/": + if path == f"/v1/table/{self.name}/get_rte_spec/": self._read_body(request) - # Null spec: this table has no MemWAL write path. - return self._json(request, {"memwal_spec": None}) + # Null spec: this table has no RTE write path. + return self._json(request, {"rte_spec": None}) if path == f"/v1/table/{self.name}/count_rows/": self._read_body(request) return self._json(request, self.num_rows) diff --git a/python/src/lib.rs b/python/src/lib.rs index 8dd98d019..3814f2caf 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MemWalWriteSpec, MergeResult, - PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table, + FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MergeResult, PyBlobFile, + RTEWriteSpec, RefreshColumnResult, RefreshMaterializedViewResult, Table, UpdateFieldMetadataResult, UpdateResult, }; @@ -104,7 +104,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/query.rs b/python/src/query.rs index 4f23fa1b4..c9c4905c9 100644 --- a/python/src/query.rs +++ b/python/src/query.rs @@ -333,7 +333,7 @@ pub struct PyQueryRequest { pub select_source_columns: Option>, pub fast_search: Option, pub with_row_id: Option, - pub use_memwal: Option, + pub use_rte: Option, pub column: Option, pub query_vector: Option, pub nprobes: Option, @@ -366,7 +366,7 @@ impl From for PyQueryRequest { select: PySelect(query_request.select), fast_search: Some(query_request.fast_search), with_row_id: Some(query_request.with_row_id), - use_memwal: query_request.use_memwal, + use_rte: query_request.use_rte, column: None, query_vector: None, nprobes: None, @@ -395,7 +395,7 @@ impl From for PyQueryRequest { select: PySelect(vector_query.base.select), fast_search: Some(vector_query.base.fast_search), with_row_id: Some(vector_query.base.with_row_id), - use_memwal: vector_query.base.use_memwal, + use_rte: vector_query.base.use_rte, column: vector_query.column, query_vector: Some(PyQueryVectors(vector_query.query_vector)), nprobes: vector_query.nprobes, @@ -540,8 +540,8 @@ impl Query { self.inner = self.inner.clone().fast_search(); } - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } pub fn with_row_id(&mut self) { @@ -706,8 +706,8 @@ impl TakeQuery { self.inner = self.inner.clone().with_row_id(); } - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } #[pyo3(signature = ())] @@ -819,8 +819,8 @@ impl FTSQuery { self.inner = self.inner.clone().fast_search(); } - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } pub fn with_row_id(&mut self) { @@ -970,8 +970,8 @@ impl VectorQuery { self.inner = self.inner.clone().fast_search(); } - pub fn use_memwal(&mut self, enable: bool) { - self.inner = self.inner.clone().use_memwal(enable); + pub fn use_rte(&mut self, enable: bool) { + self.inner = self.inner.clone().use_rte(enable); } pub fn with_row_id(&mut self) { @@ -1178,9 +1178,9 @@ impl HybridQuery { self.inner_fts.postfilter(); } - pub fn use_memwal(&mut self, enable: bool) { - self.inner_vec.use_memwal(enable); - self.inner_fts.use_memwal(enable); + pub fn use_rte(&mut self, enable: bool) { + self.inner_vec.use_rte(enable); + self.inner_fts.use_rte(enable); } pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> { diff --git a/python/src/table.rs b/python/src/table.rs index c4053cc53..6cea168ec 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -39,13 +39,13 @@ use pyo3::{ mod scannable; -/// Convert `MemWalStats` to a Python dict, preserving the per-table-shard list. +/// Convert `RTEStats` to a Python dict, preserving the per-table-shard list. /// /// Deliberately not flattened to a table-level summary: a table is N /// table shards on one node, and the per-shard detail is the reason the /// endpoint exists — flattening hides the single hot table shard someone opened /// it to find. -fn memwal_stats_to_py(py: Python<'_>, stats: &lancedb::table::MemWalStats) -> PyResult> { +fn rte_stats_to_py(py: Python<'_>, stats: &lancedb::table::RTEStats) -> PyResult> { let out = PyDict::new(py); let table_shards = PyList::empty(py); for b in &stats.table_shards { @@ -261,8 +261,8 @@ fn fmt_maintained(maintained: &Option>) -> String { } } -/// Specification selecting Lance's MemWAL LSM-style write path for -/// `merge_insert`. +/// Specification enabling the realtime engine (RTE), which routes +/// `merge_insert` through Lance's MemWAL LSM-style write path. /// /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()` /// classmethods, then optionally chain `with_maintained_indexes(...)` and @@ -270,17 +270,17 @@ fn fmt_maintained(maintained: &Option>) -> String { /// table has, including ones created later. #[pyclass(module = "lancedb._lancedb", from_py_object)] #[derive(Clone, Debug)] -pub struct MemWalWriteSpec { - inner: lancedb::table::MemWalWriteSpec, +pub struct RTEWriteSpec { + inner: lancedb::table::RTEWriteSpec, } #[pymethods] -impl MemWalWriteSpec { +impl RTEWriteSpec { /// Hash-bucket sharding by the unenforced primary key column. #[staticmethod] pub fn bucket(column: String, num_buckets: u32) -> Self { Self { - inner: lancedb::table::MemWalWriteSpec::bucket(column, num_buckets), + inner: lancedb::table::RTEWriteSpec::bucket(column, num_buckets), } } @@ -294,20 +294,20 @@ impl MemWalWriteSpec { #[staticmethod] pub fn identity(column: String) -> Self { Self { - inner: lancedb::table::MemWalWriteSpec::identity(column), + inner: lancedb::table::RTEWriteSpec::identity(column), } } /// No sharding — every `merge_insert` call writes to a single - /// MemWAL shard. + /// RTE shard. #[staticmethod] pub fn unsharded() -> Self { Self { - inner: lancedb::table::MemWalWriteSpec::unsharded(), + inner: lancedb::table::RTEWriteSpec::unsharded(), } } - /// Set which indexes the MemWAL maintains. `None` (the default) is + /// Set which indexes the RTE maintains. `None` (the default) is /// every index the table has, including ones created later; a list is /// verbatim, and an empty list maintains nothing. #[pyo3(signature = (indexes))] @@ -327,33 +327,33 @@ impl MemWalWriteSpec { pub fn __repr__(&self) -> String { match &self.inner { - lancedb::table::MemWalWriteSpec::Bucket { + lancedb::table::RTEWriteSpec::Bucket { column, num_buckets, maintained_indexes, writer_config_defaults, } => format!( - "MemWalWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", + "RTEWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", column, num_buckets, fmt_maintained(maintained_indexes), writer_config_defaults, ), - lancedb::table::MemWalWriteSpec::Identity { + lancedb::table::RTEWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "MemWalWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", + "RTEWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", column, fmt_maintained(maintained_indexes), writer_config_defaults, ), - lancedb::table::MemWalWriteSpec::Unsharded { + lancedb::table::RTEWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "MemWalWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", + "RTEWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", fmt_maintained(maintained_indexes), writer_config_defaults, ), @@ -365,9 +365,9 @@ impl MemWalWriteSpec { #[getter] pub fn spec_type(&self) -> &'static str { match &self.inner { - lancedb::table::MemWalWriteSpec::Bucket { .. } => "bucket", - lancedb::table::MemWalWriteSpec::Identity { .. } => "identity", - lancedb::table::MemWalWriteSpec::Unsharded { .. } => "unsharded", + lancedb::table::RTEWriteSpec::Bucket { .. } => "bucket", + lancedb::table::RTEWriteSpec::Identity { .. } => "identity", + lancedb::table::RTEWriteSpec::Unsharded { .. } => "unsharded", } } @@ -375,9 +375,9 @@ impl MemWalWriteSpec { #[getter] pub fn column(&self) -> Option { match &self.inner { - lancedb::table::MemWalWriteSpec::Bucket { column, .. } - | lancedb::table::MemWalWriteSpec::Identity { column, .. } => Some(column.clone()), - lancedb::table::MemWalWriteSpec::Unsharded { .. } => None, + lancedb::table::RTEWriteSpec::Bucket { column, .. } + | lancedb::table::RTEWriteSpec::Identity { column, .. } => Some(column.clone()), + lancedb::table::RTEWriteSpec::Unsharded { .. } => None, } } @@ -385,12 +385,12 @@ impl MemWalWriteSpec { #[getter] pub fn num_buckets(&self) -> Option { match &self.inner { - lancedb::table::MemWalWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets), + lancedb::table::RTEWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets), _ => None, } } - /// Indexes the MemWAL keeps up to date, or `None` for every supported one. + /// Indexes the RTE keeps up to date, or `None` for every supported one. #[getter] pub fn maintained_indexes(&self) -> Option> { self.inner.maintained_indexes().map(<[String]>::to_vec) @@ -403,14 +403,14 @@ impl MemWalWriteSpec { } } -impl From for lancedb::table::MemWalWriteSpec { - fn from(spec: MemWalWriteSpec) -> Self { +impl From for lancedb::table::RTEWriteSpec { + fn from(spec: RTEWriteSpec) -> Self { spec.inner } } -impl From for MemWalWriteSpec { - fn from(inner: lancedb::table::MemWalWriteSpec) -> Self { +impl From for RTEWriteSpec { + fn from(inner: lancedb::table::RTEWriteSpec) -> Self { Self { inner } } } @@ -1636,8 +1636,8 @@ impl Table { if let Some(use_index) = parameters.use_index { builder.use_index(use_index); } - if let Some(use_memwal) = parameters.use_memwal { - builder.use_memwal(use_memwal); + if let Some(use_rte) = parameters.use_rte { + builder.use_rte(use_rte); } if let Some(validate_single_shard) = parameters.validate_single_shard { builder.validate_single_shard(validate_single_shard); @@ -1662,29 +1662,29 @@ impl Table { }) } - pub fn enable_memwal<'a>( + pub fn enable_rte<'a>( self_: PyRef<'a, Self>, - spec: MemWalWriteSpec, + spec: RTEWriteSpec, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); - let native_spec = lancedb::table::MemWalWriteSpec::from(spec); + let native_spec = lancedb::table::RTEWriteSpec::from(spec); future_into_py(self_.py(), async move { - inner.enable_memwal(native_spec).await.infer_error() + inner.enable_rte(native_spec).await.infer_error() }) } - pub fn disable_memwal(self_: PyRef<'_, Self>) -> PyResult> { + pub fn disable_rte(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.disable_memwal().await.infer_error() + inner.disable_rte().await.infer_error() }) } - pub fn get_memwal_spec(self_: PyRef<'_, Self>) -> PyResult> { + pub fn get_rte_spec(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - let spec = inner.get_memwal_spec().await.infer_error()?; - Ok(spec.map(MemWalWriteSpec::from)) + let spec = inner.get_rte_spec().await.infer_error()?; + Ok(spec.map(RTEWriteSpec::from)) }) } @@ -1703,54 +1703,55 @@ impl Table { Ok(()) } - /// Converge the table's MemWAL write path into its base table. + /// Converge the table's RTE write path into its base table. /// /// Best-effort: with writes flowing, new rows may land after the last /// pass. Errors if the table stops making progress. - pub fn checkpoint_memwal(self_: PyRef<'_, Self>) -> PyResult> { + pub fn checkpoint_rte(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.checkpoint_memwal().await.infer_error() + inner.checkpoint_rte().await.infer_error() }) } /// Freeze every table shard's active memtable into an SSTable. - pub fn flush_memwal(self_: PyRef<'_, Self>) -> PyResult> { + pub fn flush_rte(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); - future_into_py(self_.py(), async move { - inner.flush_memwal().await.infer_error() - }) + future_into_py( + self_.py(), + async move { inner.flush_rte().await.infer_error() }, + ) } /// Trigger a background SSTable compaction pass per table shard. Returns once the - /// passes are dispatched, not once they finish — watch `get_memwal_stats`. - pub fn compact_memwal(self_: PyRef<'_, Self>) -> PyResult> { + /// passes are dispatched, not once they finish — watch `get_rte_stats`. + pub fn compact_rte(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.compact_memwal().await.infer_error() + inner.compact_rte().await.infer_error() }) } - /// Live MemWAL state, or `None` when the MemWAL write path is not enabled. + /// Live RTE state, or `None` when the RTE write path is not enabled. #[pyo3(signature = (include_sstable_rows=false))] - pub fn get_memwal_stats( + pub fn get_rte_stats( self_: PyRef<'_, Self>, include_sstable_rows: bool, ) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let stats = inner - .get_memwal_stats(include_sstable_rows) + .get_rte_stats(include_sstable_rows) .await .infer_error()?; - Python::attach(|py| stats.map(|s| memwal_stats_to_py(py, &s)).transpose()) + Python::attach(|py| stats.map(|s| rte_stats_to_py(py, &s)).transpose()) }) } - pub fn close_memwal_writers(self_: PyRef<'_, Self>) -> PyResult> { + pub fn close_rte_writers(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { - inner.close_memwal_writers().await.infer_error() + inner.close_rte_writers().await.infer_error() }) } @@ -2082,7 +2083,7 @@ pub struct MergeInsertParams { when_not_matched_by_source_condition_expr: Option, timeout: Option, use_index: Option, - use_memwal: Option, + use_rte: Option, validate_single_shard: Option, } diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 8255fef4b..71e9f0ef1 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -540,7 +540,7 @@ impl ListingDatabase { // pairs survive (or none existed in the first place), so an empty // string here must be treated the same as "no query" — otherwise // every table URI ends up with a trailing `?`, which makes downstream - // sub-paths (e.g. MemWAL gen paths) re-parse as path= + + // sub-paths (e.g. RTE gen paths) re-parse as path= + // query=, causing Lance to find the base table dataset // when looking up the sub-path. let query_string = url.query().filter(|q| !q.is_empty()).map(|s| s.to_string()); @@ -3104,7 +3104,7 @@ mod tests { /// /// Earlier, `query_pairs_mut().clear()` left the URL with /// `query=Some("")`, which then propagated as a trailing `?` on - /// every table URI. Sub-path lookups against that URI (e.g. MemWAL + /// every table URI. Sub-path lookups against that URI (e.g. RTE /// `/_mem_wal//_gen_`) re-parsed as /// `path=` + `query=/_mem_wal/...`, causing /// `Dataset::write` to find the base table dataset and falsely @@ -3178,7 +3178,7 @@ mod tests { /// `url::Url::parse` and the `query_pairs_mut()` path) must not /// append a trailing `?` to per-table URIs when the input URI has /// no query string. Sub-path lookups against such a URI (e.g. - /// MemWAL `/_mem_wal//_gen_`) re-parse + /// RTE `/_mem_wal//_gen_`) re-parse /// as `path=` + `query=/_mem_wal/...`, causing /// `Dataset::write` to find the base table dataset and falsely /// report `Dataset already exists`. diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index 3837d9084..32e7063a3 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -263,17 +263,17 @@ impl PermutationBuilder { } // Unflushed rows have no row id, so a permutation cannot address them. - match self.base_table.base_table().get_memwal_spec().await { + match self.base_table.base_table().get_rte_spec().await { Ok(Some(_)) => { return Err(Error::NotSupported { - message: "the data loader does not support tables with a MemWAL write \ + message: "the data loader does not support tables with an RTE write \ spec: rows that have not been flushed to the base table \ have no row id, so a permutation cannot reference them" .to_string(), }); } Ok(None) => {} - // No MemWAL write path means no spec. + // No RTE write path means no spec. Err(Error::NotSupported { .. }) => {} Err(err) => return Err(err), } @@ -502,9 +502,9 @@ mod tests { ) .unwrap() } - "/v1/table/remote_base/get_memwal_spec/" => http::Response::builder() + "/v1/table/remote_base/get_rte_spec/" => http::Response::builder() .status(200) - .body(br#"{"memwal_spec":null}"#.to_vec()) + .body(br#"{"rte_spec":null}"#.to_vec()) .unwrap(), "/v1/table/remote_base/count_rows/" => { let body = body.unwrap(); @@ -750,12 +750,12 @@ mod tests { /// permutation cannot reference them. Reading the base table alone would drop /// them from training without saying so, so the table is refused instead. #[tokio::test] - async fn test_permutation_rejects_memwal_spec() { - use crate::table::MemWalWriteSpec; + async fn test_permutation_rejects_rte_spec() { + use crate::table::RTEWriteSpec; use arrow_array::{Int32Array, RecordBatchIterator}; use arrow_schema::{DataType, Field, Schema}; - // MemWAL needs a real dataset directory and a non-nullable primary key. + // RTE needs a real dataset directory and a non-nullable primary key. let temp_dir = tempfile::tempdir().unwrap(); let db = connect(temp_dir.path().to_str().unwrap()) .execute() @@ -778,14 +778,11 @@ mod tests { .unwrap(); table.set_unenforced_primary_key(["idx"]).await.unwrap(); - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); let err = PermutationBuilder::new(table).build().await.unwrap_err(); assert!( - err.to_string().contains("MemWAL write spec"), + err.to_string().contains("RTE write spec"), "expected the pre-check to refuse the table, got: {err}" ); } diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 68b05c531..e15017e7f 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -167,7 +167,7 @@ //! # } //! ``` -// The MemWAL read path (`table::query::lsm`) deepens the `create_plan` future's +// The RTE read path (`table::query::lsm`) deepens the `create_plan` future's // type graph enough to overflow the default trait-recursion limit while evaluating // auto-traits (`Send`) through the Linux io_uring build's moka cache. Raise it. #![recursion_limit = "256"] diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index b4ff22397..6b4bb6d8a 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -288,15 +288,13 @@ fn refresh_conflict_filter(dataset: &Dataset) -> Result { Ok(filter.build()) } -/// Reject MemWAL/MemWAL state because an ordinary dataset scan cannot see its +/// Reject RTE/RTE state because an ordinary dataset scan cannot see its /// un-compacted tiers. pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> { let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty(); if retained || dataset.mem_wal_index_details().await?.is_some() { return Err(Error::NotSupported { - message: format!( - "{role} '{name}' has a MemWAL write spec or retained un-compacted rows" - ), + message: format!("{role} '{name}' has an RTE write spec or retained un-compacted rows"), }); } Ok(()) diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 5f072cad6..00a2b88f3 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -544,25 +544,25 @@ pub trait QueryBase { /// This allows ordering query results by one or more columns in either ascending or descending order. fn order_by(self, ordering: Option>) -> Self; - /// Control MemWAL read routing for this query. + /// Control RTE read routing for this query. /// - /// By default (unset), when the table carries a MemWAL write spec (see - /// [`crate::Table::enable_memwal`]), reads are routed through the LSM + /// By default (unset), when the table carries an RTE write spec (see + /// [`crate::Table::enable_rte`]), reads are routed through the LSM /// scanner so they also return data written via the `merge_insert` LSM path /// that has not yet been compacted into the base table (active/frozen /// memtables and SSTables); a table without a spec reads the base /// table. /// - /// - `use_memwal(true)` forces MemWAL routing and errors if the table has no - /// MemWAL write spec. - /// - `use_memwal(false)` bypasses the MemWAL and reads the base table only, + /// - `use_rte(true)` forces RTE routing and errors if the table has no + /// RTE write spec. + /// - `use_rte(false)` bypasses the RTE and reads the base table only, /// even when a spec is present. /// /// Note: the LSM scanner does not support every query shape (e.g. reranking, - /// hybrid search, `order_by`). On a MemWAL table those shapes error unless - /// `use_memwal(false)` is set, because a base-only read would silently - /// exclude un-compacted MemWAL data. - fn use_memwal(self, enable: bool) -> Self; + /// hybrid search, `order_by`). On an RTE table those shapes error unless + /// `use_rte(false)` is set, because a base-only read would silently + /// exclude un-compacted RTE data. + fn use_rte(self, enable: bool) -> Self; } pub trait HasQuery { @@ -634,8 +634,8 @@ impl QueryBase for T { self } - fn use_memwal(mut self, enable: bool) -> Self { - self.mut_query().use_memwal = Some(enable); + fn use_rte(mut self, enable: bool) -> Self { + self.mut_query().use_rte = Some(enable); self } } @@ -898,19 +898,19 @@ pub struct QueryRequest { /// This allows ordering query results by one or more columns in either ascending or descending order. pub order_by: Option>, - /// Controls MemWAL read routing. When unset (the default), a query against a - /// table that carries a MemWAL write spec (see - /// [`crate::Table::enable_memwal`]) is routed through the LSM scanner so + /// Controls RTE read routing. When unset (the default), a query against a + /// table that carries an RTE write spec (see + /// [`crate::Table::enable_rte`]) is routed through the LSM scanner so /// it also sees data written via the `merge_insert` LSM path that has not yet /// been compacted into the base table — the active and frozen in-memory /// memtables and the SSTables, deduplicated by primary key /// against the base table (newest generation wins); a table without a spec /// reads the base table. /// - /// - `Some(true)` forces MemWAL routing and errors if the table has no MemWAL + /// - `Some(true)` forces RTE routing and errors if the table has no RTE /// write spec. - /// - `Some(false)` reads only the base table, bypassing the MemWAL. - pub use_memwal: Option, + /// - `Some(false)` reads only the base table, bypassing the RTE. + pub use_rte: Option, } impl Default for QueryRequest { @@ -930,7 +930,7 @@ impl Default for QueryRequest { norm: None, disable_scoring_autoprojection: false, order_by: None, - use_memwal: None, + use_rte: None, } } } diff --git a/rust/lancedb/src/query/wal_fusion.rs b/rust/lancedb/src/query/wal_fusion.rs index e8b6b3269..b174e089c 100644 --- a/rust/lancedb/src/query/wal_fusion.rs +++ b/rust/lancedb/src/query/wal_fusion.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables. +//! WAL-PK-FUSION: hybrid search's primary-key fallback for RTE tables. //! //! Hybrid search joins its vector and full-text legs on `_rowid`, which a -//! MemWAL table cannot supply: the fresh tier has no stable row id, so the +//! RTE table cannot supply: the fresh tier has no stable row id, so the //! server rejects `with_row_id` before it plans. When that happens the legs are //! joined on a surrogate `_rowid` built from the table's primary key instead. //! -//! This is a stopgap until MemWAL supports `_rowid`, and everything it needs +//! This is a stopgap until RTE supports `_rowid`, and everything it needs //! lives here so it can be removed in one pass: //! //! 1. Delete this file and its `mod wal_fusion;` line in `query.rs`. @@ -31,44 +31,44 @@ use lance_core::utils::parse::str_is_truthy; use super::{Select, VectorQuery}; use crate::error::{Error, Result}; -/// The part of the MemWAL `with_row_id` refusal that identifies it. +/// The part of the RTE `with_row_id` refusal that identifies it. /// /// Both layers raise this same sentence verbatim — sophon's /// `LSM_WITH_ROW_ID_UNSUPPORTED` says so in as many words, and the OSS LSM /// scanner builds it from the same words — so this substring survives either /// path and the wrapping each adds. It is a coupling to a message rather than /// a code, because the refusal arrives as a generic 400; if the server ever -/// rewords it the fallback stops firing and hybrid on a MemWAL table goes back +/// rewords it the fallback stops firing and hybrid on an RTE table goes back /// to erroring visibly, which is loud rather than silently wrong. const WAL_ROW_ID_REFUSAL: &str = "does not support with_row_id"; /// Whether `error` is the server declining `_rowid` because the table is -/// MemWAL-backed, as opposed to any other failure the query might hit. +/// RTE-backed, as opposed to any other failure the query might hit. fn is_wal_row_id_refusal(error: &Error) -> bool { error.to_string().contains(WAL_ROW_ID_REFUSAL) } -/// Why a hybrid query on a MemWAL table cannot hand back `_rowid`, raised +/// Why a hybrid query on an RTE table cannot hand back `_rowid`, raised /// whether the refusal was anticipated or came back from the server. fn wal_row_id_unsupported() -> Error { Error::NotSupported { - message: "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has no \ + message: "hybrid search on an RTE table cannot return _rowid: the fresh tier has no \ stable row id, and the ids the fusion joins on are synthesized from the \ - primary key. Set use_memwal(false) to read the base table only (results will \ - exclude un-compacted MemWAL data)" + primary key. Set use_rte(false) to read the base table only (results will \ + exclude un-compacted RTE data)" .to_string(), } } /// Run a hybrid query, joining its legs on the primary key instead of `_rowid` -/// when the table is MemWAL-backed. +/// when the table is RTE-backed. /// /// `run(None)` must join the legs on `_rowid`; `run(Some(fusion))` must build /// them with `fusion` instead. /// /// `_rowid` is asked for first and the primary key used only when the server /// refuses, rather than paying a round trip up front to find out. Only a -/// MemWAL table refuses, so probing would tax every table to learn something +/// RTE table refuses, so probing would tax every table to learn something /// almost none of them need — and the refusal has to be handled regardless, /// since a write spec installed elsewhere can arrive between any two queries. /// The refusal costs nothing server-side: it is raised before the query is @@ -80,7 +80,7 @@ where { // A previous query on this table was already refused, so skip straight to // the key. - if query.request.base.use_memwal != Some(false) && query.parent.hybrid_pk_fusion_learned() { + if query.request.base.use_rte != Some(false) && query.parent.hybrid_pk_fusion_learned() { return run(Some(PkFusion::resolve(query).await?)).await; } match run(None).await { @@ -130,7 +130,7 @@ impl PkFusionMemory { } } -/// How the legs of a hybrid query on a MemWAL table are joined: on a surrogate +/// How the legs of a hybrid query on an RTE table are joined: on a surrogate /// `_rowid` derived client-side from the primary key. pub(super) struct PkFusion { columns: Vec, @@ -152,7 +152,7 @@ impl PkFusion { Ok(Self { columns, injected }) } - /// Make a leg return the key columns, in place of the `_rowid` a MemWAL + /// Make a leg return the key columns, in place of the `_rowid` an RTE /// table would refuse. pub(super) fn prepare_leg(&self, select: &mut Select) { inject_pk_columns(select, &self.columns); @@ -266,7 +266,7 @@ fn pk_columns(schema: &arrow_schema::Schema) -> Vec { /// Replace each batch's join column with a dense surrogate derived from /// `pk_columns`, in place and in the order given. /// -/// The fusion joins its two legs on [`ROW_ID`], which a MemWAL table cannot +/// The fusion joins its two legs on [`ROW_ID`], which an RTE table cannot /// supply — the fresh tier has no stable row id. It only ever compares the /// column for equality, though (`merge_results` dedups, the rerankers group and /// the score restore looks up), so any injective mapping of the primary key @@ -280,7 +280,7 @@ fn pk_columns(schema: &arrow_schema::Schema) -> Vec { fn stamp_surrogate_row_ids(batches: &mut [RecordBatch], pk_columns: &[String]) -> Result<()> { if pk_columns.is_empty() { return Err(Error::InvalidInput { - message: "hybrid search on a MemWAL table needs an unenforced primary key to \ + message: "hybrid search on an RTE table needs an unenforced primary key to \ deduplicate on, and this table declares none" .to_string(), }); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 272ac83f7..3325f6943 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -30,15 +30,15 @@ use crate::table::BranchDiff; use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; -use crate::table::MemWalStats; -use crate::table::MemWalWriteSpec; use crate::table::MergeResult; +use crate::table::RTEStats; +use crate::table::RTEWriteSpec; use crate::table::Tags; use crate::table::UpdateResult; -use crate::table::memwal_stats::GetMemWalStatsResponse; use crate::table::merge::MergeFilter; use crate::table::primary_key; use crate::table::query::create_multi_vector_plan; +use crate::table::rte_stats::GetRTEStatsResponse; use crate::table::write_progress::FinishOnDrop; use crate::table::{ AlterColumnsResult, FieldMetadataUpdate, RefreshColumnResult, UpdateFieldMetadataResult, @@ -1020,10 +1020,10 @@ impl RemoteTable { ) -> Result<()> { params.check_filter()?; body["prefilter"] = params.prefilter.into(); - // Only forward use_memwal when explicitly set; a server that predates it + // Only forward use_rte when explicitly set; a server that predates it // ignores the field and routes as it would by default. - if let Some(use_memwal) = params.use_memwal { - body["use_memwal"] = serde_json::Value::Bool(use_memwal); + if let Some(use_rte) = params.use_rte { + body["use_rte"] = serde_json::Value::Bool(use_rte); } if let Some(offset) = params.offset { body["offset"] = serde_json::Value::Number(serde_json::Number::from(offset)); @@ -3236,43 +3236,42 @@ impl BaseTable for RemoteTable { Ok(()) } - async fn flush_memwal(&self) -> Result<()> { + async fn flush_rte(&self) -> Result<()> { let request = self .client - .post(&format!("/v1/table/{}/flush_memwal/", self.identifier)); + .post(&format!("/v1/table/{}/flush_rte/", self.identifier)); self.send_lsm_route(request).await?; Ok(()) } - async fn compact_memwal(&self) -> Result<()> { + async fn compact_rte(&self) -> Result<()> { let request = self .client - .post(&format!("/v1/table/{}/compact_memwal/", self.identifier)); + .post(&format!("/v1/table/{}/compact_rte/", self.identifier)); self.send_lsm_route(request).await?; Ok(()) } - async fn get_memwal_stats(&self, include_sstable_rows: bool) -> Result> { - // Read-semantics POST, like `get_memwal_spec`. + async fn get_rte_stats(&self, include_sstable_rows: bool) -> Result> { + // Read-semantics POST, like `get_rte_spec`. let request = self .client - .post(&format!("/v1/table/{}/get_memwal_stats/", self.identifier)) + .post(&format!("/v1/table/{}/get_rte_stats/", self.identifier)) .json(&serde_json::json!({ "include_sstable_rows": include_sstable_rows, })); let (request_id, response) = self.send_lsm_route(request).await?; let body = response.text().await.err_to_http(request_id.clone())?; - let parsed: GetMemWalStatsResponse = - serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse get_memwal_stats response: {e}").into(), - request_id, - status_code: None, - })?; - // `null` — and only — when the table has no MemWAL write path. - Ok(parsed.memwal_stats) + let parsed: GetRTEStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse get_rte_stats response: {e}").into(), + request_id, + status_code: None, + })?; + // `null` — and only — when the table has no RTE write path. + Ok(parsed.rte_stats) } - async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { + async fn enable_rte(&self, spec: RTEWriteSpec) -> Result<()> { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally @@ -3280,7 +3279,7 @@ impl BaseTable for RemoteTable { // `maintained_indexes` asks the server to resolve every maintainable // index at HEAD; a list is verbatim, an empty one meaning none. let sharding = match &spec { - MemWalWriteSpec::Bucket { + RTEWriteSpec::Bucket { column, num_buckets, .. @@ -3289,11 +3288,11 @@ impl BaseTable for RemoteTable { "column": column, "num_buckets": num_buckets, }), - MemWalWriteSpec::Identity { column, .. } => serde_json::json!({ + RTEWriteSpec::Identity { column, .. } => serde_json::json!({ "mode": "identity", "column": column, }), - MemWalWriteSpec::Unsharded { .. } => serde_json::json!({ "mode": "unsharded" }), + RTEWriteSpec::Unsharded { .. } => serde_json::json!({ "mode": "unsharded" }), }; let body = serde_json::json!({ "sharding": sharding, @@ -3303,40 +3302,40 @@ impl BaseTable for RemoteTable { let request = self .client - .post(&format!("/v1/table/{}/enable_memwal/", self.identifier)) + .post(&format!("/v1/table/{}/enable_rte/", self.identifier)) .json(&body); let (request_id, response) = self.send(request, true).await?; self.check_table_response(&request_id, response).await?; Ok(()) } - async fn disable_memwal(&self) -> Result<()> { + async fn disable_rte(&self) -> Result<()> { self.check_mutable().await?; let request = self .client - .post(&format!("/v1/table/{}/disable_memwal/", self.identifier)); + .post(&format!("/v1/table/{}/disable_rte/", self.identifier)); let (request_id, response) = self.send(request, true).await?; self.check_table_response(&request_id, response).await?; self.wal_pk_fusion.forget(); // WAL-PK-FUSION: delete. Ok(()) } - async fn get_memwal_spec(&self) -> Result> { + async fn get_rte_spec(&self) -> Result> { // Read counterpart to set/unset, resolved server-side against HEAD. The // server reads the spec from the `__lance_mem_wal` system index (shard // column mapped from its Lance field id against the current schema) and // re-encodes it into the same sophon-owned shape the set endpoint - // accepts — no lance/lancedb types cross the wire. `memwal_spec` is - // null when the MemWAL write path is not enabled for the table. + // accepts — no lance/lancedb types cross the wire. `rte_spec` is + // null when the RTE write path is not enabled for the table. let request = self .client - .post(&format!("/v1/table/{}/get_memwal_spec/", self.identifier)); + .post(&format!("/v1/table/{}/get_rte_spec/", self.identifier)); let (request_id, response) = self.send(request, true).await?; let response = self.check_table_response(&request_id, response).await?; let body = response.text().await.err_to_http(request_id.clone())?; // Mirror of sophon's `Sharding` (internally tagged on `mode`) and - // `MemWalWriteSpecBody` / `GetMemWalSpecResponse`. + // `RTEWriteSpecBody` / `GetRTESpecResponse`. #[derive(Deserialize)] #[serde(tag = "mode", rename_all = "snake_case")] enum Sharding { @@ -3345,7 +3344,7 @@ impl BaseTable for RemoteTable { Identity { column: String }, } #[derive(Deserialize)] - struct MemWalWriteSpecBody { + struct RTEWriteSpecBody { sharding: Sharding, /// `null` selects every index the table has; `[]` selects none. #[serde(default)] @@ -3354,19 +3353,18 @@ impl BaseTable for RemoteTable { writer_config_defaults: std::collections::HashMap, } #[derive(Deserialize)] - struct GetMemWalSpecResponse { - memwal_spec: Option, + struct GetRTESpecResponse { + rte_spec: Option, } - let parsed: GetMemWalSpecResponse = - serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse get_memwal_spec response: {}", e).into(), - request_id, - status_code: None, - })?; + let parsed: GetRTESpecResponse = serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse get_rte_spec response: {}", e).into(), + request_id, + status_code: None, + })?; - let Some(body) = parsed.memwal_spec else { - // The MemWAL write path is not enabled for this table. + let Some(body) = parsed.rte_spec else { + // The RTE write path is not enabled for this table. return Ok(None); }; @@ -3374,9 +3372,9 @@ impl BaseTable for RemoteTable { Sharding::Bucket { column, num_buckets, - } => MemWalWriteSpec::bucket(column, num_buckets), - Sharding::Identity { column } => MemWalWriteSpec::identity(column), - Sharding::Unsharded => MemWalWriteSpec::unsharded(), + } => RTEWriteSpec::bucket(column, num_buckets), + Sharding::Identity { column } => RTEWriteSpec::identity(column), + Sharding::Unsharded => RTEWriteSpec::unsharded(), } .with_maintained_indexes(body.maintained_indexes) .with_writer_config_defaults(body.writer_config_defaults); @@ -3979,10 +3977,10 @@ pub struct MergeInsertRequest { // (the default is true) #[serde(skip_serializing_if = "is_true")] use_index: bool, - // Only serialize use_memwal when explicitly set (Some); a server that predates + // Only serialize use_rte when explicitly set (Some); a server that predates // it ignores the field and routes as it would by default. #[serde(skip_serializing_if = "Option::is_none")] - use_memwal: Option, + use_rte: Option, } impl MergeInsertRequest { @@ -4040,7 +4038,7 @@ impl TryFrom for MergeInsertRequest { when_not_matched_by_source_delete_filt, // Only serialize use_index when it's false for backwards compatibility use_index: value.use_index, - use_memwal: value.use_memwal, + use_rte: value.use_rte, }) } } @@ -10296,10 +10294,10 @@ mod tests { } #[tokio::test] - async fn test_enable_memwal_unsharded() { + async fn test_enable_rte_unsharded() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_rte/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!(body["sharding"], serde_json::json!({ "mode": "unsharded" })); @@ -10313,17 +10311,17 @@ mod tests { .body(r#"{"maintained_indexes":["id_idx"]}"#) .unwrap() }); - let spec = crate::table::MemWalWriteSpec::unsharded() + let spec = crate::table::RTEWriteSpec::unsharded() .with_maintained_indexes(vec!["id_idx".to_string()]) .with_writer_config_defaults([("max_memtable_rows", "1000")]); - table.enable_memwal(spec).await.unwrap(); + table.enable_rte(spec).await.unwrap(); } #[tokio::test] - async fn test_enable_memwal_bucket() { + async fn test_enable_rte_bucket() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_rte/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10335,14 +10333,14 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .enable_memwal(crate::table::MemWalWriteSpec::bucket("id", 16)) + .enable_rte(crate::table::RTEWriteSpec::bucket("id", 16)) .await .unwrap(); } /// `[]` (none) must stay distinguishable on the wire from null (all). #[tokio::test] - async fn test_enable_memwal_no_maintained_indexes() { + async fn test_enable_rte_no_maintained_indexes() { let table = Table::new_with_handler("my_table", |request| { let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); @@ -10350,18 +10348,18 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .enable_memwal( - crate::table::MemWalWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), + .enable_rte( + crate::table::RTEWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), ) .await .unwrap(); } #[tokio::test] - async fn test_enable_memwal_identity() { + async fn test_enable_rte_identity() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!(request.url().path(), "/v1/table/my_table/enable_memwal/"); + assert_eq!(request.url().path(), "/v1/table/my_table/enable_rte/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10371,32 +10369,32 @@ mod tests { http::Response::builder().status(200).body("{}").unwrap() }); table - .enable_memwal(crate::table::MemWalWriteSpec::identity("tenant")) + .enable_rte(crate::table::RTEWriteSpec::identity("tenant")) .await .unwrap(); } #[tokio::test] - async fn test_disable_memwal() { + async fn test_disable_rte() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!(request.url().path(), "/v1/table/my_table/disable_memwal/"); + assert_eq!(request.url().path(), "/v1/table/my_table/disable_rte/"); http::Response::builder().status(200).body("{}").unwrap() }); - table.disable_memwal().await.unwrap(); + table.disable_rte().await.unwrap(); } #[tokio::test] - async fn test_get_memwal_spec() { + async fn test_get_rte_spec() { let table = Table::new_with_handler("my_table", |request| { assert_eq!(request.method(), "POST"); - assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_spec/"); + assert_eq!(request.url().path(), "/v1/table/my_table/get_rte_spec/"); // The server resolves the spec and re-encodes it into the same // sophon-owned shape the set endpoint accepts (`Sharding` internally - // tagged on `mode`, wrapped in `memwal_spec`). + // tagged on `mode`, wrapped in `rte_spec`). let response = serde_json::json!({ - "memwal_spec": { + "rte_spec": { "sharding": { "mode": "bucket", "column": "id", "num_buckets": 4 }, "maintained_indexes": ["id_idx"], "writer_config_defaults": { "durable_write": "false" }, @@ -10409,12 +10407,12 @@ mod tests { }); let spec = table - .get_memwal_spec() + .get_rte_spec() .await .unwrap() .expect("a spec should be reported"); match spec { - crate::table::MemWalWriteSpec::Bucket { + crate::table::RTEWriteSpec::Bucket { column, num_buckets, maintained_indexes, @@ -10441,13 +10439,13 @@ mod tests { #[case::no_index(serde_json::json!([]), Some(vec![]))] #[case::named(serde_json::json!(["id_idx"]), Some(vec!["id_idx".to_string()]))] #[tokio::test] - async fn test_get_memwal_spec_round_trips_the_selection( + async fn test_get_rte_spec_round_trips_the_selection( #[case] reported: serde_json::Value, #[case] expected: Option>, ) { let table = Table::new_with_handler("my_table", move |_| { let response = serde_json::json!({ - "memwal_spec": { + "rte_spec": { "sharding": { "mode": "unsharded" }, "maintained_indexes": reported, "writer_config_defaults": {}, @@ -10460,7 +10458,7 @@ mod tests { }); let spec = table - .get_memwal_spec() + .get_rte_spec() .await .unwrap() .expect("a spec should be reported"); @@ -10472,23 +10470,23 @@ mod tests { } #[tokio::test] - async fn test_get_memwal_spec_absent() { + async fn test_get_rte_spec_absent() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_spec/"); - // Null spec → the MemWAL write path is not enabled. - let response = serde_json::json!({ "memwal_spec": null }); + assert_eq!(request.url().path(), "/v1/table/my_table/get_rte_spec/"); + // Null spec → the RTE write path is not enabled. + let response = serde_json::json!({ "rte_spec": null }); http::Response::builder() .status(200) .body(response.to_string()) .unwrap() }); - assert!(table.get_memwal_spec().await.unwrap().is_none()); + assert!(table.get_rte_spec().await.unwrap().is_none()); } - /// Build a `get_memwal_stats` body for one table shard holding `sstables`. + /// Build a `get_rte_stats` body for one table shard holding `sstables`. fn stats_body(sstables: &[u64], compacting: bool) -> String { serde_json::json!({ - "memwal_stats": { + "rte_stats": { "table_shards": [{ "shard_id": "b0", "status": "Active", @@ -10508,7 +10506,7 @@ mod tests { .to_string() } - /// `flush_memwal` / `compact_memwal` answer 202 with no body at all. + /// `flush_rte` / `compact_rte` answer 202 with no body at all. fn accepted() -> http::Response { http::Response::builder() .status(202) @@ -10529,18 +10527,18 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("compact_memwal") { + if path.contains("compact_rte") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); panic!("an already-converged table must issue no compact calls"); } - if path.contains("flush_memwal") { + if path.contains("flush_rte") { return accepted(); } - assert_eq!(path, "/v1/table/my_table/get_memwal_stats/"); + assert_eq!(path, "/v1/table/my_table/get_rte_stats/"); ok_json(stats_body(&[], false)) }); - table.checkpoint_memwal().await.unwrap(); + table.checkpoint_rte().await.unwrap(); assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0); } @@ -10552,10 +10550,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") { + if path.contains("flush_rte") { return accepted(); } - if path.contains("compact_memwal") { + if path.contains("compact_rte") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10565,7 +10563,7 @@ mod tests { ok_json(stats_body(&left, false)) }); - table.checkpoint_memwal().await.unwrap(); + table.checkpoint_rte().await.unwrap(); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 3, @@ -10582,10 +10580,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") { + if path.contains("flush_rte") { return accepted(); } - if path.contains("compact_memwal") { + if path.contains("compact_rte") { seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10599,7 +10597,7 @@ mod tests { ok_json(body) }); - table.checkpoint_memwal().await.unwrap(); + table.checkpoint_rte().await.unwrap(); assert_eq!( compacts.load(std::sync::atomic::Ordering::SeqCst), 1, @@ -10616,10 +10614,10 @@ mod tests { let seen = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") { + if path.contains("flush_rte") { return accepted(); } - if path.contains("compact_memwal") { + if path.contains("compact_rte") { // First two triggers: every table shard already latched. if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 { return http::Response::builder() @@ -10641,7 +10639,7 @@ mod tests { }); table - .checkpoint_memwal() + .checkpoint_rte() .await .expect("contention must not abort the checkpoint"); assert_eq!( @@ -10661,7 +10659,7 @@ mod tests { let seen = polls.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") || path.contains("compact_memwal") { + if path.contains("flush_rte") || path.contains("compact_rte") { return accepted(); } // The opening read lands; the next two polls are latched out. @@ -10676,7 +10674,7 @@ mod tests { }); table - .checkpoint_memwal() + .checkpoint_rte() .await .expect("a contended poll must be retried, not surfaced"); assert_eq!( @@ -10696,7 +10694,7 @@ mod tests { let seen = flushes.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") { + if path.contains("flush_rte") { if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 { return http::Response::builder() .status(429) @@ -10705,14 +10703,14 @@ mod tests { } return accepted(); } - if path.contains("compact_memwal") { + if path.contains("compact_rte") { return accepted(); } ok_json(stats_body(&[], false)) }); table - .checkpoint_memwal() + .checkpoint_rte() .await .expect("contention must not be reported as a lost claim"); assert_eq!( @@ -10737,7 +10735,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_memwal().await.unwrap_err(); + let err = table.checkpoint_rte().await.unwrap_err(); assert!( matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429), "the fault that spent the budget must be the one reported: {err:?}" @@ -10767,7 +10765,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_memwal().await.unwrap_err(); + let err = table.checkpoint_rte().await.unwrap_err(); let message = err.to_string(); assert!( matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503), @@ -10795,7 +10793,7 @@ mod tests { let seen = polls.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") || path.contains("compact_memwal") { + if path.contains("flush_rte") || path.contains("compact_rte") { return accepted(); } // Flat for far longer than any bound this loop ever had, with @@ -10805,7 +10803,7 @@ mod tests { }); table - .checkpoint_memwal() + .checkpoint_rte() .await .expect("a stall is the server being slow, not the client's call to make"); assert!( @@ -10825,10 +10823,10 @@ mod tests { let seen_compacts = compacts.clone(); let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); - if path.contains("flush_memwal") { + if path.contains("flush_rte") { return accepted(); } - if path.contains("compact_memwal") { + if path.contains("compact_rte") { seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); return accepted(); } @@ -10842,7 +10840,7 @@ mod tests { }); table - .checkpoint_memwal() + .checkpoint_rte() .await .expect("a running pass is progress, not a stall"); assert_eq!( @@ -10856,9 +10854,9 @@ mod tests { /// defaulting to a zero it did not measure. `include_sstable_rows` /// rides in the body and is off unless asked for. #[tokio::test] - async fn test_get_memwal_stats_round_trip() { + async fn test_get_rte_stats_round_trip() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/get_memwal_stats/"); + assert_eq!(request.url().path(), "/v1/table/my_table/get_rte_stats/"); let body = request.body().unwrap().as_bytes().unwrap(); let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!( @@ -10866,7 +10864,7 @@ mod tests { "the flag must reach the server, not be silently dropped" ); let response = serde_json::json!({ - "memwal_stats": { + "rte_stats": { "table_shards": [{ "shard_id": "b0", "status": "Active", @@ -10891,7 +10889,7 @@ mod tests { }); let stats = table - .get_memwal_stats(true) + .get_rte_stats(true) .await .unwrap() .expect("a WAL-backed table reports Some"); @@ -10923,7 +10921,7 @@ mod tests { .unwrap() }); - let err = table.checkpoint_memwal().await.unwrap_err(); + let err = table.checkpoint_rte().await.unwrap_err(); assert!( matches!(err, Error::TableNotFound { .. }), "a missing table must say so: {err:?}" @@ -10944,12 +10942,12 @@ mod tests { let table = Table::new_with_handler("my_table", move |request| { let path = request.url().path().to_string(); let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - if path.contains("flush_memwal") { + if path.contains("flush_rte") { // First flush lands; the claim is then lost, and the // re-issued flush succeeds. return accepted(); } - if path.contains("compact_memwal") { + if path.contains("compact_rte") { if n < 4 { return http::Response::builder() .status(421) @@ -10962,20 +10960,20 @@ mod tests { }); table - .checkpoint_memwal() + .checkpoint_rte() .await .expect("a lost claim must be recovered by re-flushing, not surfaced"); } #[tokio::test] - async fn test_get_memwal_stats_absent_when_wal_off() { + async fn test_get_rte_stats_absent_when_wal_off() { let table = Table::new_with_handler("my_table", |_request| { http::Response::builder() .status(200) - .body(serde_json::json!({ "memwal_stats": null }).to_string()) + .body(serde_json::json!({ "rte_stats": null }).to_string()) .unwrap() }); - assert!(table.get_memwal_stats(false).await.unwrap().is_none()); + assert!(table.get_rte_stats(false).await.unwrap().is_none()); } #[tokio::test] @@ -15006,9 +15004,10 @@ mod tests { // WAL-PK-FUSION: delete everything from here to the end of `mod tests`. - /// Verbatim from sophon's `LSM_WITH_ROW_ID_UNSUPPORTED`; the fallback keys - /// off this sentence, so a test that paraphrased it would prove nothing. - const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: the MemWAL LSM scanner does not support with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid); set use_memwal(false) to read the base table only (results will exclude un-compacted MemWAL data)"}"#; + /// Verbatim from sophon's `wal_unsupported(WITH_ROW_ID_UNSUPPORTED)`; the + /// fallback keys off this sentence, so a test that paraphrased it would prove + /// nothing. + const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: WAL unsupported: the MemWAL fresh tier does not support with_row_id: it has no stable _rowid; set use_rte(false) to read the base table only (results will exclude un-compacted RTE data)"}"#; use lance::dataset::ROW_ID; @@ -15052,7 +15051,7 @@ mod tests { RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap() } - /// A server that refuses `_rowid` the way a MemWAL table does, recording + /// A server that refuses `_rowid` the way an RTE table does, recording /// every query body so a test can see which key each attempt asked for. fn wal_refusing_table(bodies: Arc>>) -> Table { Table::new_with_handler("my_table", move |request| match request.url().path() { @@ -15093,7 +15092,7 @@ mod tests { .unwrap() } - /// The optimistic path: ask for `_rowid`, and when a MemWAL table refuses, + /// The optimistic path: ask for `_rowid`, and when an RTE table refuses, /// fall back to the primary key rather than failing. #[tokio::test] async fn test_hybrid_falls_back_to_the_primary_key_when_row_id_is_refused() { @@ -15267,7 +15266,7 @@ mod tests { // Translated, not passed through: the caller gets the reason and the // way out, which a bare 400 does not carry. assert!(matches!(err, Error::NotSupported { .. }), "{err:?}"); - assert!(err.to_string().contains("use_memwal(false)"), "{err}"); + assert!(err.to_string().contains("use_rte(false)"), "{err}"); assert!( bodies .lock() diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs index e7219280c..bcaf47765 100644 --- a/rust/lancedb/src/remote/table/blobs.rs +++ b/rust/lancedb/src/remote/table/blobs.rs @@ -716,8 +716,8 @@ impl RemoteTable { )), select: Select::columns(&[column]), with_row_id: true, - // Row ids name base-table rows, and the MemWAL scanner has no stable `_rowid`. - use_memwal: Some(false), + // Row ids name base-table rows, and the RTE scanner has no stable `_rowid`. + use_rte: Some(false), ..Default::default() }); let body = self @@ -1153,7 +1153,7 @@ mod tests { .unwrap(); assert_eq!(body["columns"], serde_json::json!(["image"])); assert_eq!(body["with_row_id"], serde_json::json!(true)); - assert_eq!(body["use_memwal"], serde_json::json!(false)); + assert_eq!(body["use_rte"], serde_json::json!(false)); let requested = filter_row_ids(body["filter"].as_str().unwrap()); captured.lock().unwrap().push(TakeMockRequest::Query { row_ids: requested.clone(), diff --git a/rust/lancedb/src/remote/table/insert.rs b/rust/lancedb/src/remote/table/insert.rs index 3a089eb95..8a735fa03 100644 --- a/rust/lancedb/src/remote/table/insert.rs +++ b/rust/lancedb/src/remote/table/insert.rs @@ -1497,7 +1497,7 @@ mod tests { when_not_matched_by_source_delete: false, when_not_matched_by_source_delete_filt: None, use_index: true, - use_memwal: None, + use_rte: None, }; let input: Arc = Arc::new(ErroringExec::new()); diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 802c2f517..d99926ae1 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -80,12 +80,12 @@ pub mod datafusion; pub(crate) mod dataset; pub mod delete; pub mod freshness; -pub mod memwal_stats; pub mod merge; pub mod optimize; pub(crate) mod primary_key; pub mod query; pub mod refresh; +pub mod rte_stats; pub mod schema_evolution; pub mod update; pub mod write_progress; @@ -107,9 +107,9 @@ use futures::future::join_all; pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTags}; pub use lance::dataset::scanner::DatasetRecordBatchStream; pub use lance_index::optimize::OptimizeOptions; -pub use memwal_stats::{MemWalStats, MemtableStats, SsTableStats, TableShardStats}; pub use optimize::{CompactionOptions, OptimizeAction, OptimizeStats}; pub use refresh::RefreshColumnResult; +pub use rte_stats::{MemtableStats, RTEStats, SsTableStats, TableShardStats}; pub use schema_evolution::{ AddColumnsResult, AlterColumnsResult, DropColumnsResult, FieldMetadataUpdate, UpdateFieldMetadataResult, @@ -324,23 +324,23 @@ pub trait Tags: Send + Sync { pub use self::merge::MergeResult; -/// Specification selecting Lance's MemWAL LSM-style write path for -/// `merge_insert`. +/// Specification enabling the realtime engine (RTE), which routes +/// `merge_insert` through Lance's MemWAL LSM-style write path. /// -/// Construct via [`MemWalWriteSpec::bucket`], [`MemWalWriteSpec::identity`], or -/// [`MemWalWriteSpec::unsharded`], then optionally chain -/// [`MemWalWriteSpec::with_maintained_indexes`] (indexes the MemWAL keeps up to -/// date) and [`MemWalWriteSpec::with_writer_config_defaults`] (default +/// Construct via [`RTEWriteSpec::bucket`], [`RTEWriteSpec::identity`], or +/// [`RTEWriteSpec::unsharded`], then optionally chain +/// [`RTEWriteSpec::with_maintained_indexes`] (indexes the RTE keeps up to +/// date) and [`RTEWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// /// A fresh spec maintains every index on the table, including ones created /// after it is installed. /// -/// Install a spec with [`Table::enable_memwal`] and remove it with -/// [`Table::disable_memwal`]. The actual `merge_insert` dispatch -/// onto the MemWAL writer is a follow-up. +/// Install a spec with [`Table::enable_rte`] and remove it with +/// [`Table::disable_rte`]. The actual `merge_insert` dispatch +/// onto the RTE writer is a follow-up. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum MemWalWriteSpec { +pub enum RTEWriteSpec { /// Hash-bucket sharding by a scalar column. /// /// `column` must be a non-nested column with a supported scalar type. @@ -350,7 +350,7 @@ pub enum MemWalWriteSpec { Bucket { column: String, num_buckets: u32, - /// Indexes the MemWAL maintains in-memory as rows are appended. + /// Indexes the RTE maintains in-memory as rows are appended. /// /// `None` means every index the table has, including ones created /// later. `Some([])` maintains nothing. @@ -364,7 +364,7 @@ pub enum MemWalWriteSpec { /// distinct value of `column` becomes its own shard. Identity { column: String, - /// Indexes the MemWAL maintains in-memory as rows are appended. + /// Indexes the RTE maintains in-memory as rows are appended. /// /// `None` means every index the table has, including ones created /// later. `Some([])` maintains nothing. @@ -372,9 +372,9 @@ pub enum MemWalWriteSpec { /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, - /// No sharding — every `merge_insert` call writes to a single MemWAL shard. + /// No sharding — every `merge_insert` call writes to a single RTE shard. Unsharded { - /// Indexes the MemWAL maintains in-memory as rows are appended. + /// Indexes the RTE maintains in-memory as rows are appended. /// /// `None` means every index the table has, including ones created /// later. `Some([])` maintains nothing. @@ -384,7 +384,7 @@ pub enum MemWalWriteSpec { }, } -impl MemWalWriteSpec { +impl RTEWriteSpec { /// Construct a hash-bucket sharding spec maintaining every index on the table. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { @@ -400,7 +400,7 @@ impl MemWalWriteSpec { /// /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same - /// `column` value. MemWAL dedups upserts by primary key but tracks + /// `column` value. RTE dedups upserts by primary key but tracks /// SSTables per shard, so if the same key is written with two /// different `column` values its versions land in different shards and a /// stale value can win. Typically `column` is the primary key itself, or @@ -421,23 +421,23 @@ impl MemWalWriteSpec { } } - /// Set which indexes the MemWAL maintains. + /// Set which indexes the RTE maintains. /// /// `None` (the default) is every index on the table, re-read as the table /// changes, so an index created later is maintained too; one of a kind the - /// MemWAL cannot mirror is skipped rather than failing the table. A list is + /// RTE cannot mirror is skipped rather than failing the table. A list is /// verbatim: each name must already exist and be maintainable, and an empty /// list maintains nothing. The maintainable kinds are Lance's built-ins plus /// any added with [`NativeTable::set_mem_index_registry`]. /// /// ``` - /// # use lancedb::table::MemWalWriteSpec; + /// # use lancedb::table::RTEWriteSpec; /// // Every index the table has, now and later: - /// MemWalWriteSpec::unsharded().with_maintained_indexes(None); + /// RTEWriteSpec::unsharded().with_maintained_indexes(None); /// // Exactly these: - /// MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); + /// RTEWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); /// // None at all: - /// MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new()); + /// RTEWriteSpec::unsharded().with_maintained_indexes(Vec::new()); /// ``` pub fn with_maintained_indexes(mut self, indexes: impl Into>>) -> Self { let indexes = indexes.into(); @@ -455,7 +455,7 @@ impl MemWalWriteSpec { self } - /// Replace the default `ShardWriter` configuration recorded in the MemWAL + /// Replace the default `ShardWriter` configuration recorded in the RTE /// index, so every writer starts from the same defaults. Keys are /// `ShardWriter` config field names (`Duration` knobs use a `_ms` suffix); /// values are their string encodings. @@ -486,7 +486,7 @@ impl MemWalWriteSpec { self } - /// Borrow the list of index names this spec asks MemWAL to maintain, or + /// Borrow the list of index names this spec asks RTE to maintain, or /// `None` when it asks for every index on the table. pub fn maintained_indexes(&self) -> Option<&[String]> { match self { @@ -674,37 +674,37 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "set_unenforced_primary_key is not supported on this table type".into(), }) } - /// Install an [`MemWalWriteSpec`] on this table. + /// Install an [`RTEWriteSpec`] on this table. /// /// The spec selects Lance's MemWAL LSM-style write path for future /// `merge_insert` calls. /// /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL write path must override this. - async fn enable_memwal(&self, _spec: MemWalWriteSpec) -> Result<()> { + /// that support the RTE write path must override this. + async fn enable_rte(&self, _spec: RTEWriteSpec) -> Result<()> { Err(Error::NotSupported { - message: "enable_memwal is not supported on this table type".into(), + message: "enable_rte is not supported on this table type".into(), }) } - /// Remove the [`MemWalWriteSpec`] from this table. + /// Remove the [`RTEWriteSpec`] from this table. /// /// This is a no-op if no spec is currently set. /// /// The default implementation returns `NotSupported`. Implementations - /// that support the MemWAL write path must override this. - async fn disable_memwal(&self) -> Result<()> { + /// that support the RTE write path must override this. + async fn disable_rte(&self) -> Result<()> { Err(Error::NotSupported { - message: "disable_memwal is not supported on this table type".into(), + message: "disable_rte is not supported on this table type".into(), }) } - /// Read the [`MemWalWriteSpec`] currently installed on this table, returning - /// `None` when the MemWAL write path is not enabled. + /// Read the [`RTEWriteSpec`] currently installed on this table, returning + /// `None` when the RTE write path is not enabled. /// /// The default implementation returns `NotSupported`. Implementations that - /// support the MemWAL write path must override this. - async fn get_memwal_spec(&self) -> Result> { + /// support the RTE write path must override this. + async fn get_rte_spec(&self) -> Result> { Err(Error::NotSupported { - message: "get_memwal_spec is not supported on this table type".into(), + message: "get_rte_spec is not supported on this table type".into(), }) } /// Whether a hybrid query on this table has already been told it cannot @@ -713,7 +713,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// WAL-PK-FUSION: delete this and `note_hybrid_pk_fusion`. /// /// Learned, never probed: hybrid optimistically asks for `_rowid` and only - /// a MemWAL table refuses, so paying a round trip up front would tax every + /// an RTE table refuses, so paying a round trip up front would tax every /// table to discover something almost none of them need. Synchronous and /// free by construction — an implementation may only answer from what a /// previous query already learned. @@ -735,33 +735,33 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Freeze every table shard's active memtable into an SSTable. /// /// The default implementation returns `NotSupported`. - async fn flush_memwal(&self) -> Result<()> { + async fn flush_rte(&self) -> Result<()> { Err(Error::NotSupported { - message: "flush_memwal is not supported on this table type".into(), + message: "flush_rte is not supported on this table type".into(), }) } /// Trigger a background SSTable compaction pass per table shard. /// /// The default implementation returns `NotSupported`. - async fn compact_memwal(&self) -> Result<()> { + async fn compact_rte(&self) -> Result<()> { Err(Error::NotSupported { - message: "compact_memwal is not supported on this table type".into(), + message: "compact_rte is not supported on this table type".into(), }) } - /// Read live MemWAL state, or `None` when the MemWAL write path is not + /// Read live RTE state, or `None` when the RTE write path is not /// enabled for this table. /// /// The default implementation returns `NotSupported`. - async fn get_memwal_stats(&self, _include_sstable_rows: bool) -> Result> { + async fn get_rte_stats(&self, _include_sstable_rows: bool) -> Result> { Err(Error::NotSupported { - message: "get_memwal_stats is not supported on this table type".into(), + message: "get_rte_stats is not supported on this table type".into(), }) } - /// Drain and close any cached MemWAL shard writers for this table. + /// Drain and close any cached RTE shard writers for this table. /// /// The default implementation is a no-op; table types that maintain - /// MemWAL shard writers override it. - async fn close_memwal_writers(&self) -> Result<()> { + /// RTE shard writers override it. + async fn close_rte_writers(&self) -> Result<()> { Ok(()) } /// Names of the blob v2 columns in this table, in declaration order. @@ -2008,53 +2008,53 @@ impl Table { self.inner.set_unenforced_primary_key(&borrowed).await } - /// Install an [`MemWalWriteSpec`] on this table, selecting Lance's MemWAL + /// Install an [`RTEWriteSpec`] on this table, selecting Lance's MemWAL /// LSM-style write path for future `merge_insert` calls. /// - /// [`MemWalWriteSpec`] chooses one of three sharding strategies: + /// [`RTEWriteSpec`] chooses one of three sharding strategies: /// - /// - [`MemWalWriteSpec::bucket`] — hash-bucket writes by a scalar column. - /// - [`MemWalWriteSpec::identity`] — shard by the raw value of a scalar column. - /// - [`MemWalWriteSpec::unsharded`] — route every write to a single shard. + /// - [`RTEWriteSpec::bucket`] — hash-bucket writes by a scalar column. + /// - [`RTEWriteSpec::identity`] — shard by the raw value of a scalar column. + /// - [`RTEWriteSpec::unsharded`] — route every write to a single shard. /// /// Every spec requires the table to declare an unenforced primary key /// ([`Table::set_unenforced_primary_key`]); otherwise this fails. /// /// A table carries one spec: this fails while one is installed, since the /// SSTables already written were homed under it. - /// [`Table::disable_memwal`] removes one. + /// [`Table::disable_rte`] removes one. /// /// # Example /// /// ``` - /// # use lancedb::table::{MemWalWriteSpec, Table}; + /// # use lancedb::table::{RTEWriteSpec, Table}; /// # async fn example(table: &Table) -> Result<(), Box> { /// table - /// .enable_memwal( - /// MemWalWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), + /// .enable_rte( + /// RTEWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), /// ) /// .await?; /// # Ok(()) /// # } /// ``` - pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { - self.inner.enable_memwal(spec).await + pub async fn enable_rte(&self, spec: RTEWriteSpec) -> Result<()> { + self.inner.enable_rte(spec).await } - /// Remove the [`MemWalWriteSpec`] from this table, reverting to the standard + /// Remove the [`RTEWriteSpec`] from this table, reverting to the standard /// `merge_insert` write path. /// /// Errors if no spec is currently set. - pub async fn disable_memwal(&self) -> Result<()> { - self.inner.disable_memwal().await + pub async fn disable_rte(&self) -> Result<()> { + self.inner.disable_rte().await } - /// Read the [`MemWalWriteSpec`] currently installed on this table. + /// Read the [`RTEWriteSpec`] currently installed on this table. /// - /// `Ok(None)` means the MemWAL write path is not enabled at all — no spec has - /// been set, or one was removed with [`Table::disable_memwal`]. That + /// `Ok(None)` means the RTE write path is not enabled at all — no spec has + /// been set, or one was removed with [`Table::disable_rte`]. That /// is a different answer from a spec whose - /// [`MemWalWriteSpec::maintained_indexes`] is `None`, which is an installed + /// [`RTEWriteSpec::maintained_indexes`] is `None`, which is an installed /// spec selecting indexes automatically. /// /// The spec read back is the one that was installed, selection included: @@ -2067,21 +2067,21 @@ impl Table { /// ``` /// # use lancedb::table::Table; /// # async fn example(table: &Table) -> Result<(), Box> { - /// if let Some(spec) = table.get_memwal_spec().await? { - /// println!("MemWAL write path enabled: {:?}", spec); + /// if let Some(spec) = table.get_rte_spec().await? { + /// println!("RTE write path enabled: {:?}", spec); /// } /// # Ok(()) /// # } /// ``` - pub async fn get_memwal_spec(&self) -> Result> { - self.inner.get_memwal_spec().await + pub async fn get_rte_spec(&self) -> Result> { + self.inner.get_rte_spec().await } - /// Converge this table's MemWAL write path into its base table. + /// Converge this table's RTE write path into its base table. /// /// One `flush` to freeze every memtable into an SSTable, then compaction triggers /// until every generation that existed at that moment has reached base. - /// The loop runs client-side, reading progress from `get_memwal_stats`, so + /// The loop runs client-side, reading progress from `get_rte_stats`, so /// there is no held socket and nothing to reconcile if you drop this /// future partway through. /// @@ -2104,14 +2104,14 @@ impl Table { /// ```no_run /// # use lancedb::Table; /// # async fn example(table: &Table) -> Result<(), Box> { - /// let before = table.get_memwal_stats(false).await?; - /// table.checkpoint_memwal().await?; - /// let after = table.get_memwal_stats(false).await?; + /// let before = table.get_rte_stats(false).await?; + /// table.checkpoint_rte().await?; + /// let after = table.get_rte_stats(false).await?; /// # Ok(()) /// # } /// ``` - pub async fn checkpoint_memwal(&self) -> Result<()> { - checkpoint::checkpoint_memwal(self).await + pub async fn checkpoint_rte(&self) -> Result<()> { + checkpoint::checkpoint_rte(self).await } /// Freeze every table shard's active memtable into an SSTable without touching the @@ -2121,8 +2121,8 @@ impl Table { /// a lower per-query cost. On a node that has not claimed this table it /// claims it and replays the WAL log first — reporting "nothing to flush" /// without replaying would lie about durable data. - pub async fn flush_memwal(&self) -> Result<()> { - self.inner.flush_memwal().await + pub async fn flush_rte(&self) -> Result<()> { + self.inner.flush_rte().await } /// Run one bounded SSTable compaction pass per table shard, reporting what @@ -2130,11 +2130,11 @@ impl Table { /// /// One pass, not convergence: that bounds each request's cost and gives a /// caller driving its own cadence a progress signal per round trip. - pub async fn compact_memwal(&self) -> Result<()> { - self.inner.compact_memwal().await + pub async fn compact_rte(&self) -> Result<()> { + self.inner.compact_rte().await } - /// Read live per-table-shard MemWAL state. + /// Read live per-table-shard RTE state. /// /// Answers "how far behind is my fresh tier", "which table shard is hot", and /// "why is my fresh-tier vector search brute-force". Mutates no table @@ -2143,30 +2143,27 @@ impl Table { /// /// `include_sstable_rows` reports a row count per SSTable. Off by /// default: each count opens an uncached Lance dataset, and - /// `checkpoint_memwal` polls this needing only generation numbers. + /// `checkpoint_rte` polls this needing only generation numbers. /// - /// `Ok(None)` only when the MemWAL write path is not enabled, matching - /// [`Table::get_memwal_spec`]. Stats is fresh-tier only, so with the + /// `Ok(None)` only when the RTE write path is not enabled, matching + /// [`Table::get_rte_spec`]. Stats is fresh-tier only, so with the /// WAL off there is no manifest to report and a struct of zeros would /// read as measurements. /// /// Do not build a checkpoint's termination on this: the completion /// predicate lives in the `flush` and `compact` responses. - pub async fn get_memwal_stats( - &self, - include_sstable_rows: bool, - ) -> Result> { - self.inner.get_memwal_stats(include_sstable_rows).await + pub async fn get_rte_stats(&self, include_sstable_rows: bool) -> Result> { + self.inner.get_rte_stats(include_sstable_rows).await } - /// Drain and close any cached MemWAL shard writers held for this table. + /// Drain and close any cached RTE shard writers held for this table. /// - /// When an [`MemWalWriteSpec`] is installed, `merge_insert` opens MemWAL shard + /// When an [`RTEWriteSpec`] is installed, `merge_insert` opens RTE shard /// writers and caches them for reuse across calls. This closes them, /// flushing pending data; writers reopen lazily on the next `merge_insert`. /// It is a no-op when no writers are cached. - pub async fn close_memwal_writers(&self) -> Result<()> { - self.inner.close_memwal_writers().await + pub async fn close_rte_writers(&self) -> Result<()> { + self.inner.close_rte_writers().await } /// Retrieve the version of the table @@ -2784,8 +2781,8 @@ impl NativeTable { } /// Maintain the memtable index kinds in `registry` when this table writes - /// through its [`MemWalWriteSpec`], and accept them when - /// [`MemWalWriteSpec::with_maintained_indexes`] names one. + /// through its [`RTEWriteSpec`], and accept them when + /// [`RTEWriteSpec::with_maintained_indexes`] names one. /// /// The registry belongs to this handle, its clones and its branch handles, /// and is not stored with the table: a handle opened separately starts from @@ -2796,7 +2793,7 @@ impl NativeTable { /// ``` /// # use std::sync::Arc; /// # use lancedb::Table; - /// # use lancedb::table::{MemWalWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; + /// # use lancedb::table::{RTEWriteSpec, MemIndexRegistry, mem_index::MemIndexPlugin}; /// # async fn example( /// # table: &Table, /// # plugin: Arc, @@ -2805,8 +2802,8 @@ impl NativeTable { /// if let Some(native) = table.as_native() { /// native.set_mem_index_registry(registry); /// } - /// let spec = MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); - /// table.enable_memwal(spec).await?; + /// let spec = RTEWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); + /// table.enable_rte(spec).await?; /// # Ok(()) /// # } /// ``` @@ -3699,20 +3696,20 @@ impl BaseTable for NativeTable { primary_key::set_unenforced_primary_key(self, columns).await } - async fn enable_memwal(&self, spec: MemWalWriteSpec) -> Result<()> { - merge::lsm::enable_memwal(self, spec).await + async fn enable_rte(&self, spec: RTEWriteSpec) -> Result<()> { + merge::lsm::enable_rte(self, spec).await } - async fn disable_memwal(&self) -> Result<()> { - merge::lsm::disable_memwal(self).await + async fn disable_rte(&self) -> Result<()> { + merge::lsm::disable_rte(self).await } - async fn get_memwal_spec(&self) -> Result> { - merge::lsm::get_memwal_spec(self).await + async fn get_rte_spec(&self) -> Result> { + merge::lsm::get_rte_spec(self).await } - async fn close_memwal_writers(&self) -> Result<()> { - merge::lsm::close_memwal_writers(self).await + async fn close_rte_writers(&self) -> Result<()> { + merge::lsm::close_rte_writers(self).await } async fn blob_columns(&self) -> Result> { @@ -5518,7 +5515,7 @@ mod tests { } #[tokio::test] - async fn test_enable_memwal() { + async fn test_enable_rte() { use arrow_array::StringArray; use lance::dataset::mem_wal::DatasetMemWalExt; @@ -5551,15 +5548,15 @@ mod tests { // Reject num_buckets out of range. for bad in [0u32, 1025] { let err = table - .enable_memwal(MemWalWriteSpec::bucket("id", bad)) + .enable_rte(RTEWriteSpec::bucket("id", bad)) .await .expect_err("should reject"); assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); } - // Happy path: install spec; verify MemWAL details record it. + // Happy path: install spec; verify RTE details record it. table - .enable_memwal(MemWalWriteSpec::bucket("id", 4)) + .enable_rte(RTEWriteSpec::bucket("id", 4)) .await .unwrap(); @@ -5585,7 +5582,7 @@ mod tests { // Mutation rejected. let err = table - .enable_memwal(MemWalWriteSpec::bucket("id", 8)) + .enable_rte(RTEWriteSpec::bucket("id", 8)) .await .expect_err("mutation should be rejected"); assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); @@ -5594,7 +5591,7 @@ mod tests { /// A spec is refused on a table without an unenforced primary key, for every /// sharding mode, and a refused spec leaves nothing installed. #[tokio::test] - async fn test_enable_memwal_requires_primary_key() { + async fn test_enable_rte_requires_primary_key() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5621,11 +5618,11 @@ mod tests { }; for spec in [ - MemWalWriteSpec::unsharded(), - MemWalWriteSpec::bucket("id", 4), - MemWalWriteSpec::identity("id"), + RTEWriteSpec::unsharded(), + RTEWriteSpec::bucket("id", 4), + RTEWriteSpec::identity("id"), ] { - let err = table.enable_memwal(spec).await.unwrap_err(); + let err = table.enable_rte(spec).await.unwrap_err(); assert!( matches!(&err, Error::InvalidInput { message } if message.contains("primary key")), "got {err:?}" @@ -5634,15 +5631,12 @@ mod tests { } table.set_unenforced_primary_key(["id"]).await.unwrap(); - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); assert!(installed(&table).await); } #[tokio::test] - async fn test_enable_memwal_unsharded() { + async fn test_enable_rte_unsharded() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5664,10 +5658,7 @@ mod tests { let table = conn.create_table("t", reader).execute().await.unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); let details = dataset @@ -5683,7 +5674,7 @@ mod tests { } #[tokio::test] - async fn test_enable_memwal_identity() { + async fn test_enable_rte_identity() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5712,8 +5703,8 @@ mod tests { table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .enable_memwal( - MemWalWriteSpec::identity("region") + .enable_rte( + RTEWriteSpec::identity("region") .with_writer_config_defaults([("durable_write", "false")]), ) .await @@ -5741,7 +5732,7 @@ mod tests { } #[tokio::test] - async fn test_disable_memwal() { + async fn test_disable_rte() { use lance::dataset::mem_wal::DatasetMemWalExt; let tmp_dir = tempdir().unwrap(); @@ -5764,11 +5755,11 @@ mod tests { table.set_unenforced_primary_key(["id"]).await.unwrap(); // unset errors when no spec is set. - table.disable_memwal().await.unwrap_err(); + table.disable_rte().await.unwrap_err(); // Install a spec, then unset it. table - .enable_memwal(MemWalWriteSpec::bucket("id", 4)) + .enable_rte(RTEWriteSpec::bucket("id", 4)) .await .unwrap(); { @@ -5776,16 +5767,16 @@ mod tests { assert!(dataset.mem_wal_index_details().await.unwrap().is_some()); } - table.disable_memwal().await.unwrap(); + table.disable_rte().await.unwrap(); { let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); assert!(dataset.mem_wal_index_details().await.unwrap().is_none()); } // A second unset errors; a fresh spec can still be installed afterwards. - table.disable_memwal().await.unwrap_err(); + table.disable_rte().await.unwrap_err(); table - .enable_memwal(MemWalWriteSpec::bucket("id", 8)) + .enable_rte(RTEWriteSpec::bucket("id", 8)) .await .unwrap(); { @@ -5795,7 +5786,7 @@ mod tests { } #[tokio::test] - async fn test_get_memwal_spec() { + async fn test_get_rte_spec() { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); @@ -5822,7 +5813,7 @@ mod tests { table.set_unenforced_primary_key(["id"]).await.unwrap(); // No spec installed yet. - assert_eq!(table.get_memwal_spec().await.unwrap(), None); + assert_eq!(table.get_rte_spec().await.unwrap(), None); // A real scalar index is needed to name it as a maintained index. table @@ -5834,35 +5825,35 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. - let spec = MemWalWriteSpec::bucket("id", 4) + let spec = RTEWriteSpec::bucket("id", 4) .with_maintained_indexes(vec![idx_name.clone()]) .with_writer_config_defaults([("durable_write", "false")]); - table.enable_memwal(spec.clone()).await.unwrap(); - assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); + table.enable_rte(spec.clone()).await.unwrap(); + assert_eq!(table.get_rte_spec().await.unwrap(), Some(spec)); // After unset, no spec is reported. - table.disable_memwal().await.unwrap(); - assert_eq!(table.get_memwal_spec().await.unwrap(), None); + table.disable_rte().await.unwrap(); + assert_eq!(table.get_rte_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). // A spec left at its default round-trips as the default: what is stored // is the intent to maintain everything, not the set it resolves to now. - let spec = MemWalWriteSpec::identity("region"); - table.enable_memwal(spec.clone()).await.unwrap(); - assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); - table.disable_memwal().await.unwrap(); + let spec = RTEWriteSpec::identity("region"); + table.enable_rte(spec.clone()).await.unwrap(); + assert_eq!(table.get_rte_spec().await.unwrap(), Some(spec)); + table.disable_rte().await.unwrap(); // Unsharded round-trips (no routing column). - let spec = MemWalWriteSpec::unsharded(); - table.enable_memwal(spec.clone()).await.unwrap(); - assert_eq!(table.get_memwal_spec().await.unwrap(), Some(spec)); + let spec = RTEWriteSpec::unsharded(); + table.enable_rte(spec.clone()).await.unwrap(); + assert_eq!(table.get_rte_spec().await.unwrap(), Some(spec)); } /// The maintained set defaults to every index on the table. A named index /// the memtable cannot build fails the install, because the caller asked /// for it; an unnamed one is skipped, because it arrived by existing. #[tokio::test] - async fn test_enable_memwal_infers_maintained_indexes() { + async fn test_enable_rte_infers_maintained_indexes() { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); @@ -5903,9 +5894,8 @@ mod tests { // Explicitly naming the bitmap index fails before anything commits. let err = table - .enable_memwal( - MemWalWriteSpec::unsharded() - .with_maintained_indexes(vec!["tag_bitmap".to_string()]), + .enable_rte( + RTEWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]), ) .await .unwrap_err(); @@ -5913,18 +5903,15 @@ mod tests { matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), "expected the bitmap index to be rejected, got {err:?}" ); - assert_eq!(table.get_memwal_spec().await.unwrap(), None); + assert_eq!(table.get_rte_spec().await.unwrap(), None); // The default names nothing, so the bitmap is skipped rather than // refusing the table: it is maintained by existing, not by being asked // for. The intent is what is stored, not the set it resolves to today. - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); assert_eq!( table - .get_memwal_spec() + .get_rte_spec() .await .unwrap() .unwrap() @@ -5935,16 +5922,16 @@ mod tests { // Narrowing to a named subset: unset first, since an installed spec // cannot be set over. - table.disable_memwal().await.unwrap(); + table.disable_rte().await.unwrap(); table - .enable_memwal( - MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), + .enable_rte( + RTEWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), ) .await .unwrap(); assert_eq!( table - .get_memwal_spec() + .get_rte_spec() .await .unwrap() .unwrap() @@ -5953,14 +5940,14 @@ mod tests { ); // Opting out entirely is distinct from the default. - table.disable_memwal().await.unwrap(); + table.disable_rte().await.unwrap(); table - .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .enable_rte(RTEWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await .unwrap(); assert_eq!( table - .get_memwal_spec() + .get_rte_spec() .await .unwrap() .unwrap() @@ -5971,7 +5958,7 @@ mod tests { // An installed spec is never set over, whatever the new one asks for, // and the refused call leaves it alone. let err = table - .enable_memwal(MemWalWriteSpec::bucket("id", 4)) + .enable_rte(RTEWriteSpec::bucket("id", 4)) .await .unwrap_err(); assert!( @@ -5981,7 +5968,7 @@ mod tests { ); assert_eq!( table - .get_memwal_spec() + .get_rte_spec() .await .unwrap() .unwrap() diff --git a/rust/lancedb/src/table/checkpoint.rs b/rust/lancedb/src/table/checkpoint.rs index c9a2f60d6..c8976a234 100644 --- a/rust/lancedb/src/table/checkpoint.rs +++ b/rust/lancedb/src/table/checkpoint.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Converging a table's MemWAL write path into its base table. +//! Converging a table's RTE write path into its base table. //! -//! `checkpoint_memwal` freezes once, then triggers compaction and watches +//! `checkpoint_rte` freezes once, then triggers compaction and watches //! generation numbers until the SSTables that existed at the start are gone. //! -//! The loop runs in the client, not the server: `compact_memwal` dispatches a +//! The loop runs in the client, not the server: `compact_rte` dispatches a //! pass and returns, so nothing holds a socket and a client can vanish //! mid-operation with nothing to reconcile. Completion is read from //! generation numbers in the shard manifest — durable state, unlike a count @@ -68,7 +68,7 @@ fn is_lost_claim(e: &Error) -> bool { status_of(e) == Some(421) } -/// Interval between `get_memwal_stats` polls. One interval is roughly one +/// Interval between `get_rte_stats` polls. One interval is roughly one /// compaction pass, the granularity at which the answer can change. /// /// Fixed rather than configurable, matching `wait_for_index`. It costs @@ -149,15 +149,15 @@ where } } -/// Drive [`Table::checkpoint_memwal`]: freeze once, fix the target watermark +/// Drive [`Table::checkpoint_rte`]: freeze once, fix the target watermark /// from the resulting SSTables, then trigger and poll until they drain. -pub(crate) async fn checkpoint_memwal(table: &Table) -> Result<()> { +pub(crate) async fn checkpoint_rte(table: &Table) -> Result<()> { for reissue in 0..=MAX_REISSUES { // The freeze turns everything written before this call into an // SSTable, so the watermark has to be read after it. Idempotent: // freezing an empty memtable is a no-op, so a re-issue does not churn // empty SSTables. - match issue(|| table.flush_memwal()).await? { + match issue(|| table.flush_rte()).await? { Attempt::Ok(()) => {} Attempt::ReissueFromFlush => { backoff(reissue).await; @@ -165,7 +165,7 @@ pub(crate) async fn checkpoint_memwal(table: &Table) -> Result<()> { } } - let stats = match issue(|| table.get_memwal_stats(false)).await? { + let stats = match issue(|| table.get_rte_stats(false)).await? { Attempt::Ok(stats) => stats, Attempt::ReissueFromFlush => { backoff(reissue).await; @@ -173,7 +173,7 @@ pub(crate) async fn checkpoint_memwal(table: &Table) -> Result<()> { } }; let Some(stats) = stats else { - // Not WAL-backed; `flush_memwal` would have errored first but for a race. + // Not WAL-backed; `flush_rte` would have errored first but for a race. return Ok(()); }; let targets: HashMap = stats @@ -194,7 +194,7 @@ pub(crate) async fn checkpoint_memwal(table: &Table) -> Result<()> { } } Err(Error::Runtime { - message: "checkpoint_memwal: the owning node kept losing its claim; \ + message: "checkpoint_rte: the owning node kept losing its claim; \ re-issued from flush the maximum number of times" .into(), }) @@ -213,7 +213,7 @@ async fn drain_to_targets( targets: &HashMap, ) -> Result { loop { - let stats = match issue(|| table.get_memwal_stats(false)).await? { + let stats = match issue(|| table.get_rte_stats(false)).await? { Attempt::Ok(stats) => stats, Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush), }; @@ -241,7 +241,7 @@ async fn drain_to_targets( } if !all_compacting { - match table.compact_memwal().await { + match table.compact_rte().await { Ok(()) => {} Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush), Err(e) if !is_retryable(&e) => return Err(e), diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 563f185b6..9a3922b2b 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -2197,7 +2197,7 @@ pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result, schema: Arc, fts_query: Option, - use_memwal: Option, + use_rte: Option, } impl BaseTableAdapter { @@ -182,13 +182,13 @@ impl BaseTableAdapter { table, schema: Arc::new(schema), fts_query: None, - use_memwal: None, + use_rte: None, }) } - /// Set the MemWAL read routing of every scan; see [`QueryRequest::use_memwal`]. - pub fn with_use_memwal(mut self, use_memwal: Option) -> Self { - self.use_memwal = use_memwal; + /// Set the RTE read routing of every scan; see [`QueryRequest::use_rte`]. + pub fn with_use_rte(mut self, use_rte: Option) -> Self { + self.use_rte = use_rte; self } @@ -204,7 +204,7 @@ impl BaseTableAdapter { table: self.table.clone(), schema, fts_query: Some(fts_query), - use_memwal: self.use_memwal, + use_rte: self.use_rte, } } } @@ -232,7 +232,7 @@ impl TableProvider for BaseTableAdapter { let mut query = QueryRequest { full_text_search: self.fts_query.clone(), disable_scoring_autoprojection: disable_scoring, - use_memwal: self.use_memwal, + use_rte: self.use_rte, ..Default::default() }; diff --git a/rust/lancedb/src/table/dataset.rs b/rust/lancedb/src/table/dataset.rs index af9fc2563..69d02776f 100644 --- a/rust/lancedb/src/table/dataset.rs +++ b/rust/lancedb/src/table/dataset.rs @@ -19,7 +19,7 @@ use crate::{Error, error::Result, utils::background_cache::BackgroundCache}; pub struct DatasetConsistencyWrapper { state: Arc>, consistency: ConsistencyMode, - /// The single MemWAL `ShardWriter` for this dataset, co-located so it is + /// The single RTE `ShardWriter` for this dataset, co-located so it is /// cached for the session and shares the dataset's lifecycle. A dataset /// writes to one shard at a time. Shared by `Arc` across clones. shard_writer: Arc, @@ -33,7 +33,7 @@ struct DatasetState { /// `None` = tracking latest. pinned_version: Option, /// Whether the pin is an internal query snapshot rather than user-visible - /// time travel. Query snapshots remain read-only but preserve MemWAL read + /// time travel. Query snapshots remain read-only but preserve RTE read /// semantics. query_snapshot: bool, } @@ -99,7 +99,7 @@ impl DatasetConsistencyWrapper { } /// Create an independent read-only wrapper pinned to the current dataset - /// while retaining this wrapper's live MemWAL read context. + /// while retaining this wrapper's live RTE read context. pub async fn new_query_snapshot(&self) -> Result { // Apply the configured consistency policy before taking the snapshot. // The returned dataset is intentionally discarded: a checkout may race @@ -109,7 +109,7 @@ impl DatasetConsistencyWrapper { let (dataset, query_snapshot) = { let state = self.state.lock()?; - // Preserve user time travel so the MemWAL safety guard still sees + // Preserve user time travel so the RTE safety guard still sees // it. Latest and already-internal snapshots remain internal pins. ( state.dataset.clone(), @@ -128,7 +128,7 @@ impl DatasetConsistencyWrapper { }) } - /// The MemWAL `ShardWriter` cache co-located with this dataset. + /// The RTE `ShardWriter` cache co-located with this dataset. pub(crate) fn shard_writer(&self) -> &Arc { &self.shard_writer } diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 589353efa..4b85a6ca8 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -47,7 +47,7 @@ pub struct MergeResult { /// Total number of rows written. /// /// On the standard `merge_insert` path this equals - /// `num_inserted_rows + num_updated_rows`. On the MemWAL write path the + /// `num_inserted_rows + num_updated_rows`. On the RTE write path the /// insert/update breakdown is not known until compaction; in that mode /// `num_inserted_rows`, `num_updated_rows`, `num_deleted_rows`, `version` /// and `num_attempts` are all `0` and this field holds the total number of @@ -76,7 +76,7 @@ pub struct MergeInsertBuilder { pub(crate) when_not_matched_by_source_delete_filt: Option, pub(crate) timeout: Option, pub(crate) use_index: bool, - pub(crate) use_memwal: Option, + pub(crate) use_rte: Option, pub(crate) validate_single_shard: bool, } @@ -92,7 +92,7 @@ impl MergeInsertBuilder { when_not_matched_by_source_delete_filt: None, timeout: None, use_index: true, - use_memwal: None, + use_rte: None, validate_single_shard: true, } } @@ -190,30 +190,30 @@ impl MergeInsertBuilder { self } - /// Control MemWAL routing for this `merge_insert`. + /// Control RTE routing for this `merge_insert`. /// /// By default (unset), a `merge_insert` on a table with an - /// [`MemWalWriteSpec`](super::MemWalWriteSpec) installed is routed through Lance's - /// MemWAL shard writer; a table without one uses the standard path. + /// [`RTEWriteSpec`](super::RTEWriteSpec) installed is routed through Lance's + /// RTE shard writer; a table without one uses the standard path. /// - /// - `use_memwal(true)` forces MemWAL routing and errors if the table has no - /// MemWAL write spec. - /// - `use_memwal(false)` forces the standard write path even when a spec is set. - pub fn use_memwal(&mut self, enable: bool) -> &mut Self { - self.use_memwal = Some(enable); + /// - `use_rte(true)` forces RTE routing and errors if the table has no + /// RTE write spec. + /// - `use_rte(false)` forces the standard write path even when a spec is set. + pub fn use_rte(&mut self, enable: bool) -> &mut Self { + self.use_rte = Some(enable); self } /// Controls how an LSM `merge_insert` checks that its input targets a /// single shard. /// - /// When a table has a MemWAL write spec, every row in a `merge_insert` call + /// When a table has an RTE write spec, every row in a `merge_insert` call /// must route to the same shard. When `true` (the default), every row is /// inspected to verify this. When `false`, only the first row is inspected /// and the shard it routes to is used for the whole input — a faster path /// for callers that have already pre-sharded their input. /// - /// Has no effect on tables without a MemWAL write spec. + /// Has no effect on tables without an RTE write spec. pub fn validate_single_shard(&mut self, validate_single_shard: bool) -> &mut Self { self.validate_single_shard = validate_single_shard; self @@ -903,7 +903,7 @@ mod lsm_tests { use crate::connect; use crate::error::Error; - use crate::table::{MemWalWriteSpec, Table}; + use crate::table::{RTEWriteSpec, Table}; /// A reader of `[id: Int64, value: Int64]` rows; `value` is `0..n`. fn id_value_reader(ids: Vec) -> Box { @@ -987,7 +987,7 @@ mod lsm_tests { let table = id_value_table(&dir).await; // num_buckets = 1: every row routes to the single bucket. table - .enable_memwal(MemWalWriteSpec::bucket("id", 1)) + .enable_rte(RTEWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1001,7 +1001,7 @@ mod lsm_tests { .await .unwrap(); - // LSM path: rows go to the MemWAL, the breakdown is unknown until + // LSM path: rows go to the RTE, the breakdown is unknown until // compaction, so only `num_rows` is populated. assert_eq!(result.num_rows, 3); assert_eq!(result.version, 0); @@ -1013,10 +1013,7 @@ mod lsm_tests { async fn lsm_merge_insert_unsharded() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); let mut builder = table.merge_insert(&["id"]); builder @@ -1043,7 +1040,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .enable_memwal(MemWalWriteSpec::identity("region")) + .enable_rte(RTEWriteSpec::identity("region")) .await .unwrap(); @@ -1060,18 +1057,18 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_merge_insert_use_memwal_false_falls_back() { + async fn lsm_merge_insert_use_rte_false_falls_back() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .enable_memwal(MemWalWriteSpec::bucket("id", 1)) + .enable_rte(RTEWriteSpec::bucket("id", 1)) .await .unwrap(); - // use_memwal(false) opts out: the standard path runs and commits even though + // use_rte(false) opts out: the standard path runs and commits even though // a spec is installed. let mut builder = table.merge_insert(&["id"]); - builder.when_not_matched_insert_all().use_memwal(false); + builder.when_not_matched_insert_all().use_rte(false); let result = builder .execute(id_value_reader(vec![3, 4, 5])) .await @@ -1082,17 +1079,17 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_merge_insert_use_memwal_true_without_spec_errors() { + async fn lsm_merge_insert_use_rte_true_without_spec_errors() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - // use_memwal(true) demands MemWAL routing; without a write spec it errors + // use_rte(true) demands RTE routing; without a write spec it errors // rather than silently falling back to the standard path. let mut builder = table.merge_insert(&["id"]); builder .when_matched_update_all(None) .when_not_matched_insert_all() - .use_memwal(true); + .use_rte(true); let err = builder .execute(id_value_reader(vec![3, 4, 5])) .await @@ -1105,7 +1102,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .enable_memwal(MemWalWriteSpec::bucket("id", 1)) + .enable_rte(RTEWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1122,7 +1119,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .enable_memwal(MemWalWriteSpec::bucket("id", 1)) + .enable_rte(RTEWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1138,7 +1135,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .enable_memwal(MemWalWriteSpec::bucket("id", 1)) + .enable_rte(RTEWriteSpec::bucket("id", 1)) .await .unwrap(); @@ -1148,7 +1145,7 @@ mod lsm_tests { .when_not_matched_insert_all(); builder.execute(id_value_reader(vec![7, 8])).await.unwrap(); - table.close_memwal_writers().await.unwrap(); + table.close_rte_writers().await.unwrap(); // The writer reopens lazily on the next merge_insert. let mut builder = table.merge_insert(&[]); @@ -1173,7 +1170,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .enable_memwal(MemWalWriteSpec::identity("region")) + .enable_rte(RTEWriteSpec::identity("region")) .await .unwrap(); @@ -1210,10 +1207,10 @@ mod lsm_tests { #[tokio::test] async fn lsm_merge_insert_no_spec_uses_standard_path() { let dir = tempdir().unwrap(); - // id_value_table sets a primary key but no MemWAL write spec. + // id_value_table sets a primary key but no RTE write spec. let table = id_value_table(&dir).await; - // Without a spec, a default merge_insert (use_memwal unset) simply uses + // Without a spec, a default merge_insert (use_rte unset) simply uses // the standard path and commits — no opt-out required, no error. let mut builder = table.merge_insert(&["id"]); builder @@ -1241,7 +1238,7 @@ mod lsm_tests { .unwrap(); table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .enable_memwal(MemWalWriteSpec::identity("region")) + .enable_rte(RTEWriteSpec::identity("region")) .await .unwrap(); @@ -1267,7 +1264,7 @@ mod lsm_tests { assert!(matches!(err, Error::InvalidInput { .. }), "got {err:?}"); // After closing the writer, a different shard can be written. - table.close_memwal_writers().await.unwrap(); + table.close_rte_writers().await.unwrap(); let mut builder = table.merge_insert(&[]); builder .when_matched_update_all(None) @@ -1279,7 +1276,7 @@ mod lsm_tests { } // --------------------------------------------------------------------- - // MemWAL read path + // RTE read path // --------------------------------------------------------------------- use crate::arrow::SendableRecordBatchStream; @@ -1337,10 +1334,7 @@ mod lsm_tests { async fn lsm_read_sees_active_memtable() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: ids 1,2,3 (value 0,1,2) - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); // Insert ids 4,5 into the active memtable (not committed to base). lsm_upsert(&table, vec![4, 5]).await; @@ -1353,8 +1347,8 @@ mod lsm_tests { vec![1, 2, 3, 4, 5] ); - // use_memwal(false) bypasses the MemWAL and reads the base table only. - let base_only = table.query().use_memwal(false).execute().await.unwrap(); + // use_rte(false) bypasses the RTE and reads the base table only. + let base_only = table.query().use_rte(false).execute().await.unwrap(); let rows = collect_id_value(base_only).await; assert_eq!( rows.iter().map(|(id, _)| *id).collect::>(), @@ -1363,23 +1357,20 @@ mod lsm_tests { } #[tokio::test] - async fn base_table_adapter_use_memwal_false_scans_the_base_only() { + async fn base_table_adapter_use_rte_false_scans_the_base_only() { use crate::table::datafusion::BaseTableAdapter; use datafusion::prelude::SessionContext; let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: ids 1,2,3 - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); // Ids 4,5 land in the active memtable, not the base table. lsm_upsert(&table, vec![4, 5]).await; let adapter = BaseTableAdapter::try_new(table.base_table().clone()) .await .unwrap() - .with_use_memwal(Some(false)); + .with_use_rte(Some(false)); let ctx = SessionContext::new(); ctx.register_table("base_only", Arc::new(adapter)).unwrap(); let batches = ctx @@ -1401,10 +1392,7 @@ mod lsm_tests { async fn query_snapshot_preserves_lsm_read_semantics() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); lsm_upsert(&table, vec![4, 5]).await; let snapshot = table.query_snapshot().await.unwrap(); @@ -1419,10 +1407,7 @@ mod lsm_tests { async fn query_snapshot_preserves_time_travel_lsm_guard() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); lsm_upsert(&table, vec![4]).await; let version = table.version().await.unwrap(); @@ -1439,10 +1424,7 @@ mod lsm_tests { async fn lsm_read_dedup_newest_wins() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; // base: id 2 -> value 1 - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); // Upsert ids 2,3,4 with values 0,1,2. id 2 and 3 shadow the base rows. lsm_upsert(&table, vec![2, 3, 4]).await; @@ -1457,10 +1439,7 @@ mod lsm_tests { async fn lsm_read_point_lookup_filter() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); lsm_upsert(&table, vec![2, 3, 4]).await; // id 2 -> value 0 (shadows base) let lsm = table.query().only_if("id = 2").execute().await.unwrap(); @@ -1473,14 +1452,14 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; table - .enable_memwal(MemWalWriteSpec::bucket("id", 8)) + .enable_rte(RTEWriteSpec::bucket("id", 8)) .await .unwrap(); // Two single-row upserts that route to (likely) different shards; each // closes the writer so the next opens a fresh shard. lsm_upsert(&table, vec![10]).await; - table.close_memwal_writers().await.unwrap(); + table.close_rte_writers().await.unwrap(); lsm_upsert(&table, vec![11]).await; let lsm = table.query().execute().await.unwrap(); @@ -1494,15 +1473,12 @@ mod lsm_tests { async fn lsm_read_after_close_sees_flushed() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); lsm_upsert(&table, vec![4, 5]).await; // close flushes the active memtable to an on-disk generation and drops // the cached writer; the read must still see those rows via the shard // manifest snapshot. - table.close_memwal_writers().await.unwrap(); + table.close_rte_writers().await.unwrap(); let lsm = table.query().execute().await.unwrap(); let ids: Vec = collect_id_value(lsm) @@ -1516,11 +1492,11 @@ mod lsm_tests { #[tokio::test] async fn lsm_read_without_spec_reads_base() { let dir = tempdir().unwrap(); - let table = id_value_table(&dir).await; // no MemWAL write spec + let table = id_value_table(&dir).await; // no RTE write spec // With no spec installed there is nothing to route: the default read and - // an explicit use_memwal(false) both read the base table without error. - for query in [table.query(), table.query().use_memwal(false)] { + // an explicit use_rte(false) both read the base table without error. + for query in [table.query(), table.query().use_rte(false)] { let rows = collect_id_value(query.execute().await.unwrap()).await; assert_eq!( rows.iter().map(|(id, _)| *id).collect::>(), @@ -1530,16 +1506,13 @@ mod lsm_tests { } #[tokio::test] - async fn lsm_read_unsupported_shape_errors_without_use_memwal_false() { + async fn lsm_read_unsupported_shape_errors_without_use_rte_false() { let dir = tempdir().unwrap(); let table = id_value_table(&dir).await; - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); lsm_upsert(&table, vec![4]).await; - // `with_row_id` is a shape the LSM scanner cannot honor. On a MemWAL + // `with_row_id` is a shape the LSM scanner cannot honor. On an RTE // table the default (auto-routed) read hard-errors rather than silently // reading a stale base-only result that would exclude un-compacted row 4. let err = table @@ -1548,15 +1521,15 @@ mod lsm_tests { .execute() .await .err() - .expect("unsupported shape on a MemWAL table must error"); + .expect("unsupported shape on an RTE table must error"); assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); - // use_memwal(false) is the escape hatch: it reads the base table only. + // use_rte(false) is the escape hatch: it reads the base table only. let rows = collect_id_value( table .query() .with_row_id() - .use_memwal(false) + .use_rte(false) .execute() .await .unwrap(), @@ -1568,7 +1541,7 @@ mod lsm_tests { ); } - /// A table with `id`/`text` and a primary key, ready for a MemWAL write spec. + /// A table with `id`/`text` and a primary key, ready for an RTE write spec. async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table { let conn = connect(dir.path().to_str().unwrap()) .execute() @@ -1583,7 +1556,7 @@ mod lsm_tests { table } - /// Upsert `rows` through the MemWAL write path. + /// Upsert `rows` through the RTE write path. async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) { let mut builder = table.merge_insert(&[]); builder @@ -1626,10 +1599,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = lsm_text_table(&dir).await; // The default set, on a table with no FTS index yet. - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); // Opens the writer, whose MemTable carries no FTS index. upsert_text(&table, vec![(99, "zebra")]).await; @@ -1661,7 +1631,7 @@ mod lsm_tests { /// A spec is replaced by unsetting it and setting the new one. /// - /// `disable_memwal` drains the open writer, so the rows it held + /// `disable_rte` drains the open writer, so the rows it held /// survive the replacement, and the writer the next write opens builds its /// MemTables from the new set. #[tokio::test] @@ -1681,25 +1651,23 @@ mod lsm_tests { // Installed maintaining nothing, then written to: the writer opens with // no FTS index. table - .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(Vec::new())) + .enable_rte(RTEWriteSpec::unsharded().with_maintained_indexes(Vec::new())) .await .unwrap(); upsert_text(&table, vec![(99, "zebra")]).await; // Setting over an installed spec is refused; unset, then set. let err = table - .enable_memwal( - MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]), - ) + .enable_rte(RTEWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()])) .await .expect_err("an installed spec cannot be set over"); assert!( err.to_string().contains("already set"), "unexpected error: {err}" ); - table.disable_memwal().await.unwrap(); + table.disable_rte().await.unwrap(); table - .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .enable_rte(RTEWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1739,10 +1707,7 @@ mod lsm_tests { let dir = tempdir().unwrap(); let table = lsm_text_table(&dir).await; // No index yet, and the default set: maintain whatever the table has. - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); // Opens the writer, whose MemTable therefore carries no FTS index. upsert_text(&table, vec![(99, "zebra")]).await; @@ -1796,7 +1761,7 @@ mod lsm_tests { .unwrap(); let fts_index = table.list_indices().await.unwrap()[0].name.clone(); table - .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .enable_rte(RTEWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) .await .unwrap(); @@ -1891,7 +1856,7 @@ mod lsm_tests { } let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table - .enable_memwal(MemWalWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) + .enable_rte(RTEWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) .await .unwrap(); @@ -2163,11 +2128,10 @@ mod lsm_tests { .execute() .await .unwrap(); - let spec = || { - MemWalWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]) - }; + let spec = + || RTEWriteSpec::unsharded().with_maintained_indexes(vec!["region_bitmap".to_string()]); - let refused = table.enable_memwal(spec()).await.unwrap_err(); + let refused = table.enable_rte(spec()).await.unwrap_err(); assert!( refused.to_string().contains("region_bitmap") && refused @@ -2180,7 +2144,7 @@ mod lsm_tests { .with_plugin(Arc::new(BitmapAsBTree)) .unwrap(); table.as_native().unwrap().set_mem_index_registry(registry); - table.enable_memwal(spec()).await.unwrap(); + table.enable_rte(spec()).await.unwrap(); // The registry is not stored with the table. let other = conn.open_table("t").execute().await.unwrap(); @@ -2238,7 +2202,7 @@ mod lsm_tests { active.index_store.index_names() ); - table.close_memwal_writers().await.unwrap(); + table.close_rte_writers().await.unwrap(); assert_eq!(fresh().await, vec![10, 11]); assert_sstables_answer_through_region_bitmap(&table, vec![10, 11]).await; } @@ -2345,10 +2309,7 @@ mod lsm_tests { .is_some(), "a clone shares the handle's registry" ); - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); let upsert = |rows: Vec<(i64, &'static str)>| { let table = table.clone(); @@ -2389,7 +2350,7 @@ mod lsm_tests { active.index_store.index_names() ); - table.close_memwal_writers().await.unwrap(); + table.close_rte_writers().await.unwrap(); // Only the SSTable written after the index exists carries it. let sstables = published_sstables(&table).await; let mut carrying = 0; diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index d8877428d..1596679de 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! MemWAL write path for `merge_insert`. +//! RTE write path for `merge_insert`. //! -//! [`enable_memwal`] installs an [`MemWalWriteSpec`] on a table by creating -//! Lance's MemWAL index; [`disable_memwal`] removes it. Once a spec is +//! [`enable_rte`] installs an [`RTEWriteSpec`] on a table by creating +//! Lance's MemWAL index; [`disable_rte`] removes it. Once a spec is //! installed, `merge_insert` upsert calls are dispatched through Lance's MemWAL //! `ShardWriter` (LSM-style append) instead of the standard merge path — see //! [`lsm_dispatch_decision`] and [`execute_lsm_merge_insert`]. @@ -14,7 +14,7 @@ //! unsharded). [`MergeInsertBuilder::validate_single_shard`] controls whether //! every row is checked or only the first. A dataset writes to one shard at a //! time; its writer is cached in the [`ShardWriterCache`] held alongside the -//! dataset, and [`close_memwal_writers`] closes it. +//! dataset, and [`close_rte_writers`] closes it. use std::collections::HashMap; use std::sync::Arc; @@ -42,9 +42,9 @@ use uuid::Uuid; use crate::error::{Error, Result}; use crate::index::IndexConfig; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{BaseTable, MemWalWriteSpec, NativeTable}; +use crate::table::{BaseTable, NativeTable, RTEWriteSpec}; -/// Spec id of the sole sharding spec installed by [`enable_memwal`]. +/// Spec id of the sole sharding spec installed by [`enable_rte`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). const SHARDING_SPEC_ID: u32 = 1; @@ -63,23 +63,23 @@ const NUM_BUCKETS_PARAM: &str = "num_buckets"; const SHARD_NAMESPACE: Uuid = Uuid::from_u128(0x4c53_4d57_5249_5445_5f53_4841_5244_3031); // ============================================================================= -// enable_memwal +// enable_rte // ============================================================================= -/// Install an [`MemWalWriteSpec`] on the table. +/// Install an [`RTEWriteSpec`] on the table. /// /// The bucket / identity / unsharded sharding spec is constructed and validated /// by Lance's /// [`InitializeMemWalBuilder`](lance::dataset::mem_wal::InitializeMemWalBuilder). #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> Result<()> { +pub(crate) async fn enable_rte(table: &NativeTable, spec: RTEWriteSpec) -> Result<()> { table.dataset.ensure_mutable()?; { let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { return Err(Error::InvalidInput { - message: "enable_memwal: a MemWAL write spec is already set on this \ + message: "enable_rte: an RTE write spec is already set on this \ table and cannot be changed" .into(), }); @@ -117,25 +117,25 @@ pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> let schema = arrow_schema::Schema::from(dataset.schema()); if !crate::table::computed_columns::computed_columns(&schema).is_empty() { return Err(Error::NotSupported { - message: "a MemWAL write spec cannot be installed on a table with computed \ + message: "an RTE write spec cannot be installed on a table with computed \ columns: rows in un-compacted tiers are invisible to refresh" .into(), }); } if crate::materialized_view::read_definition(&dataset.schema().metadata)?.is_some() { return Err(Error::NotSupported { - message: "a MemWAL write spec cannot be installed on a materialized view: \ + message: "an RTE write spec cannot be installed on a materialized view: \ rows in un-compacted tiers are invisible to refresh" .into(), }); } - // Every MemWAL write and read resolves rows by the unenforced primary key, + // Every RTE write and read resolves rows by the unenforced primary key, // so a spec on a table without one leaves the table unwritable through the - // MemWAL path. A key cannot be removed once set, and an overwrite or restore + // RTE path. A key cannot be removed once set, and an overwrite or restore // of the table conflicts with the commit below. if dataset.schema().unenforced_primary_key().is_empty() { return Err(Error::InvalidInput { - message: "a MemWAL write spec requires the table to declare an unenforced \ + message: "an RTE write spec requires the table to declare an unenforced \ primary key; set one with set_unenforced_primary_key first" .into(), }); @@ -144,7 +144,7 @@ pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> .initialize_mem_wal() .mem_index_registry(table.mem_index_registry()); let writer_config_defaults = match spec { - MemWalWriteSpec::Bucket { + RTEWriteSpec::Bucket { column, num_buckets, writer_config_defaults, @@ -153,7 +153,7 @@ pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> builder = builder.bucket_sharding(column, num_buckets); writer_config_defaults } - MemWalWriteSpec::Identity { + RTEWriteSpec::Identity { column, writer_config_defaults, .. @@ -161,7 +161,7 @@ pub(crate) async fn enable_memwal(table: &NativeTable, spec: MemWalWriteSpec) -> builder = builder.identity_sharding(column); writer_config_defaults } - MemWalWriteSpec::Unsharded { + RTEWriteSpec::Unsharded { writer_config_defaults, .. } => { @@ -191,22 +191,22 @@ fn index_name_list(indices: &[IndexConfig]) -> String { } // ============================================================================= -// disable_memwal +// disable_rte // ============================================================================= -/// Remove the [`MemWalWriteSpec`] from the table by dropping the MemWAL index. +/// Remove the [`RTEWriteSpec`] from the table by dropping the MemWAL index. /// /// Any cached shard writers are drained and closed first. Errors if no spec is /// currently set. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn disable_memwal(table: &NativeTable) -> Result<()> { +pub(crate) async fn disable_rte(table: &NativeTable) -> Result<()> { table.dataset.ensure_mutable()?; { let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_none() { return Err(Error::InvalidInput { - message: "disable_memwal: no MemWAL write spec is set on this table".into(), + message: "disable_rte: no RTE write spec is set on this table".into(), }); } } @@ -222,46 +222,46 @@ pub(crate) async fn disable_memwal(table: &NativeTable) -> Result<()> { } // ============================================================================= -// get_memwal_spec +// get_rte_spec // ============================================================================= -/// Read the [`MemWalWriteSpec`] currently installed on the table, if any. +/// Read the [`RTEWriteSpec`] currently installed on the table, if any. /// /// Reconstructs the spec from the MemWAL index's stored details — the same -/// metadata [`enable_memwal`] writes — read directly via +/// metadata [`enable_rte`] writes — read directly via /// [`mem_wal_index_details`](DatasetMemWalExt::mem_wal_index_details) (the raw /// manifest record, not the `describe_indices` enrichment path, so it is /// unaffected by system-index filtering there). Returns `Ok(None)` when no /// MemWAL index is installed. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn get_memwal_spec(table: &NativeTable) -> Result> { +pub(crate) async fn get_rte_spec(table: &NativeTable) -> Result> { let dataset = table.dataset.get().await?; let Some(details) = dataset.mem_wal_index_details().await? else { return Ok(None); }; - let spec = memwal_spec_from_details(&details, dataset.schema())?; + let spec = rte_spec_from_details(&details, dataset.schema())?; Ok(Some(spec)) } -/// Reconstruct the public [`MemWalWriteSpec`] from stored MemWAL index details. +/// Reconstruct the public [`RTEWriteSpec`] from stored MemWAL index details. /// /// The sharding transform and its parameters recover the mode; the routing /// column is resolved from the sharding field's source id back to its schema /// name (only bucket / identity carry a column). `maintained_indexes` and /// `writer_config_defaults` are copied verbatim, so the result round-trips -/// what `enable_memwal` installed. -fn memwal_spec_from_details( +/// what `enable_rte` installed. +fn rte_spec_from_details( details: &MemWalIndexDetails, schema: &LanceSchema, -) -> Result { +) -> Result { let spec = details .sharding_specs .first() .ok_or_else(|| Error::Runtime { - message: "get_memwal_spec: MemWAL index has no sharding spec".to_string(), + message: "get_rte_spec: MemWAL index has no sharding spec".to_string(), })?; let field = spec.fields.first().ok_or_else(|| Error::Runtime { - message: "get_memwal_spec: MemWAL index has an empty sharding spec".to_string(), + message: "get_rte_spec: MemWAL index has an empty sharding spec".to_string(), })?; let base = match field.transform.as_deref() { @@ -272,16 +272,16 @@ fn memwal_spec_from_details( .and_then(|s| s.parse::().ok()) .filter(|n| *n > 0) .ok_or_else(|| Error::Runtime { - message: "get_memwal_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(), + message: "get_rte_spec: RTE bucket spec has a missing or invalid num_buckets parameter".to_string(), })?; - MemWalWriteSpec::bucket(sharding_column(field, schema)?, num_buckets) + RTEWriteSpec::bucket(sharding_column(field, schema)?, num_buckets) } - Some(IDENTITY_TRANSFORM) => MemWalWriteSpec::identity(sharding_column(field, schema)?), - Some(UNSHARDED_TRANSFORM) => MemWalWriteSpec::unsharded(), + Some(IDENTITY_TRANSFORM) => RTEWriteSpec::identity(sharding_column(field, schema)?), + Some(UNSHARDED_TRANSFORM) => RTEWriteSpec::unsharded(), other => { return Err(Error::Runtime { message: format!( - "get_memwal_spec: MemWAL index has an unsupported sharding transform {:?}", + "get_rte_spec: MemWAL index has an unsupported sharding transform {:?}", other ), }); @@ -297,30 +297,30 @@ fn memwal_spec_from_details( /// Resolve the single routing column name from a sharding field's source id. /// -/// `enable_memwal` records the shard column by its Lance field id, so the +/// `enable_rte` records the shard column by its Lance field id, so the /// name is recovered by looking that id up in the current schema. fn sharding_column(field: &ShardingField, schema: &LanceSchema) -> Result { let source_id = *field.source_ids.first().ok_or_else(|| Error::Runtime { - message: "get_memwal_spec: sharding field has no source column".to_string(), + message: "get_rte_spec: sharding field has no source column".to_string(), })?; schema .field_by_id(source_id) .map(|f| f.name.clone()) .ok_or_else(|| Error::Runtime { message: format!( - "get_memwal_spec: sharding source field id {} not found in schema", + "get_rte_spec: sharding source field id {} not found in schema", source_id ), }) } // ============================================================================= -// close_memwal_writers +// close_rte_writers // ============================================================================= -/// Drain and close every cached MemWAL shard writer for the table. +/// Drain and close every cached RTE shard writer for the table. #[allow(clippy::redundant_pub_crate)] -pub(crate) async fn close_memwal_writers(table: &NativeTable) -> Result<()> { +pub(crate) async fn close_rte_writers(table: &NativeTable) -> Result<()> { table.dataset.shard_writer().drain_and_close().await } @@ -328,13 +328,13 @@ pub(crate) async fn close_memwal_writers(table: &NativeTable) -> Result<()> { // ShardWriter cache // ============================================================================= -/// Per-table cache holding the single open MemWAL `ShardWriter`. +/// Per-table cache holding the single open RTE `ShardWriter`. /// /// Held by [`DatasetConsistencyWrapper`](crate::table::dataset::DatasetConsistencyWrapper) /// so the writer lives where the dataset lives — cached for the session and /// reused across `merge_insert` calls. A dataset writes to one shard at a /// time; routing a `merge_insert` to a different shard requires closing the -/// current writer first via [`close_memwal_writers`]. `ShardWriter::put` takes +/// current writer first via [`close_rte_writers`]. `ShardWriter::put` takes /// `&self`, so concurrent puts on the cached writer are safe; `close` consumes /// the writer, so the entry wraps it in `RwLock>`. #[derive(Default)] @@ -384,7 +384,7 @@ impl ShardWriterEntry { /// The cached writer's latest in-memory manifest (current generation + /// SSTables). `Ok(None)` if the writer was already closed. - /// Used by the MemWAL read path to snapshot this shard authoritatively + /// Used by the RTE read path to snapshot this shard authoritatively /// without re-reading the on-disk manifest. async fn manifest(&self) -> Result> { let guard = self.inner.read().await; @@ -444,7 +444,7 @@ impl ShardWriterCache { .await .map_err(|e| Error::Runtime { message: format!( - "merge_insert: failed to open MemWAL shard writer for shard {}: {}", + "merge_insert: failed to open RTE shard writer for shard {}: {}", shard_id, e ), })?; @@ -502,7 +502,7 @@ impl ShardWriterCache { } } - /// Snapshot the cached writer's shard for the MemWAL read path: its shard id, + /// Snapshot the cached writer's shard for the RTE read path: its shard id, /// authoritative in-memory manifest, and active + frozen memtable refs. /// Returns `None` when no writer is currently cached (e.g. nothing has been /// written this session, or the writer was closed). @@ -550,7 +550,7 @@ fn check_shard_match(cached: Uuid, wanted: Uuid) -> Result<()> { } Err(Error::InvalidInput { message: format!( - "merge_insert: a shard writer is already open for shard {} but this input routes to shard {}; call close_memwal_writers before writing to a different shard", + "merge_insert: a shard writer is already open for shard {} but this input routes to shard {}; call close_rte_writers before writing to a different shard", cached, wanted ), }) @@ -571,7 +571,7 @@ enum LsmMode { Unsharded, } -/// Resolved plan for routing a `merge_insert` through the MemWAL write path. +/// Resolved plan for routing a `merge_insert` through the RTE write path. #[derive(Debug)] #[allow(clippy::redundant_pub_crate)] pub(crate) struct LsmPlan { @@ -582,13 +582,13 @@ pub(crate) struct LsmPlan { /// Outcome of [`lsm_dispatch_decision`]. #[allow(clippy::redundant_pub_crate)] pub(crate) enum LsmDispatch { - /// No MemWAL write spec applies; use the standard `merge_insert` path. + /// No RTE write spec applies; use the standard `merge_insert` path. Standard, - /// Route the `merge_insert` through the MemWAL shard writer. + /// Route the `merge_insert` through the RTE shard writer. Lsm(LsmPlan), } -/// Decide whether a `merge_insert` should be routed through the MemWAL write +/// Decide whether a `merge_insert` should be routed through the RTE write /// path, validating the builder against the installed spec. #[allow(clippy::redundant_pub_crate)] pub(crate) async fn lsm_dispatch_decision( @@ -596,18 +596,18 @@ pub(crate) async fn lsm_dispatch_decision( params: &MergeInsertBuilder, ) -> Result { // Explicit opt-out: use the standard path regardless of any installed spec. - if params.use_memwal == Some(false) { + if params.use_rte == Some(false) { return Ok(LsmDispatch::Standard); } let dataset = table.dataset.get().await?; let Some(details) = dataset.mem_wal_index_details().await? else { - // No write spec installed. `use_memwal(true)` demanded MemWAL routing, so + // No write spec installed. `use_rte(true)` demanded RTE routing, so // that is an error; otherwise fall back to the standard path. - if params.use_memwal == Some(true) { + if params.use_rte == Some(true) { return Err(Error::InvalidInput { - message: "use_memwal(true) was set but the table has no MemWAL write spec; \ - install one with enable_memwal or leave use_memwal unset" + message: "use_rte(true) was set but the table has no RTE write spec; \ + install one with enable_rte or leave use_rte unset" .to_string(), }); } @@ -629,7 +629,7 @@ pub(crate) async fn lsm_dispatch_decision( if !params.on.is_empty() && params.on != pk_cols { return Err(Error::InvalidInput { message: format!( - "merge_insert: `on` columns {:?} must match the table's unenforced primary key {:?} when a MemWAL write spec is set; pass an empty `on` to default to the primary key", + "merge_insert: `on` columns {:?} must match the table's unenforced primary key {:?} when an RTE write spec is set; pass an empty `on` to default to the primary key", params.on, pk_cols ), }); @@ -637,7 +637,7 @@ pub(crate) async fn lsm_dispatch_decision( if !is_upsert_only(params) { return Err(Error::InvalidInput { - message: "merge_insert: when a MemWAL write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_memwal(false) to use the standard merge_insert path".to_string(), + message: "merge_insert: when an RTE write spec is set, only the upsert form (when_matched_update_all without a filter + when_not_matched_insert_all, no by-source delete) is supported; call use_rte(false) to use the standard merge_insert path".to_string(), }); } @@ -678,7 +678,7 @@ fn resolve_lsm_mode(details: &MemWalIndexDetails) -> Result { .and_then(|s| s.parse::().ok()) .filter(|n| *n > 0) .ok_or_else(|| Error::Runtime { - message: "merge_insert: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(), + message: "merge_insert: RTE bucket spec has a missing or invalid num_buckets parameter".to_string(), })?; Ok(LsmMode::Bucket { spec }) } @@ -697,7 +697,7 @@ fn resolve_lsm_mode(details: &MemWalIndexDetails) -> Result { // LSM merge_insert execution // ============================================================================= -/// Execute a `merge_insert` through the MemWAL shard writer cache. +/// Execute a `merge_insert` through the RTE shard writer cache. /// /// The entire input is collected, schema-aligned, and shard-validated before /// anything is written, then issued as a single atomic `ShardWriter::put` — so @@ -716,7 +716,7 @@ pub(crate) async fn execute_lsm_merge_insert( // Collect, align and shard-validate the whole input before writing // anything. `ShardWriter::put` is atomic over the batch vector, so any - // failure raised here leaves the MemWAL untouched. + // failure raised here leaves the RTE untouched. let mut batches: Vec = Vec::new(); let mut total_rows: u64 = 0; @@ -803,7 +803,7 @@ fn resolve_batch_shard( .downcast_ref::() .ok_or_else(|| Error::Runtime { message: format!( - "merge_insert: MemWAL bucket evaluator returned {:?}; expected Int32", + "merge_insert: RTE bucket evaluator returned {:?}; expected Int32", values.data_type() ), })?; @@ -824,7 +824,7 @@ fn resolve_batch_shard( Ok(bucket_shard_id(u32::try_from(first).map_err(|_| { Error::Runtime { message: format!( - "merge_insert: MemWAL bucket evaluator returned negative bucket {}", + "merge_insert: RTE bucket evaluator returned negative bucket {}", first ), } @@ -856,7 +856,7 @@ fn evaluate_lsm_shard_values( if values.num_columns() != 1 { return Err(Error::Runtime { message: format!( - "merge_insert: MemWAL sharding spec evaluated to {} fields; expected exactly one", + "merge_insert: RTE sharding spec evaluated to {} fields; expected exactly one", values.num_columns() ), }); @@ -996,7 +996,7 @@ fn shard_writer_config_from_defaults(defaults: &HashMap) -> Shar } /// Re-attach the dataset's Arrow schema (including field metadata) to a -/// user-supplied input batch. The MemWAL `ShardWriter` checks batch schemas +/// user-supplied input batch. The RTE `ShardWriter` checks batch schemas /// against the dataset schema by exact equality, so input readers built /// without the primary-key metadata must be rewrapped before being put. /// diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 90a1d3217..cf70c6626 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -94,9 +94,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R { return Ok(false); } - // A MemWAL write spec means reads auto-route through the LSM scanner in - // `create_plan` even when `use_memwal` is unset. The namespace request has no - // use_memwal field, so pushing the default query down would silently omit + // An RTE write spec means reads auto-route through the LSM scanner in + // `create_plan` even when `use_rte` is unset. The namespace request has no + // use_rte field, so pushing the default query down would silently omit // un-compacted rows — force local execution whenever a spec is installed. let dataset = table.dataset.get().await?; if dataset.mem_wal_index_details().await?.is_some() { @@ -106,10 +106,10 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R } fn requires_local_namespace_execution(query: &AnyQuery) -> bool { - // The namespace QueryTable request cannot represent approx_mode, search_effort or use_memwal. - // For use_memwal that is worse than a tuning miss: MemWAL read routing lives only + // The namespace QueryTable request cannot represent approx_mode, search_effort or use_rte. + // For use_rte that is worse than a tuning miss: RTE read routing lives only // in `create_plan`, so a pushed-down query would return stale base-only data. - if query.base().use_memwal.is_some() || query.base().take_offsets.is_some() { + if query.base().use_rte.is_some() || query.base().take_offsets.is_some() { return true; } matches!( @@ -167,25 +167,25 @@ pub async fn create_plan( let ds_ref = table.dataset.get().await?; - // MemWAL read routing driven by `use_memwal`: + // RTE read routing driven by `use_rte`: // * unset — route through the LSM scanner iff the table carries a write spec - // * Some(true) — force MemWAL routing; error if the table has no write spec - // * Some(false) — read the base table only, bypassing the MemWAL + // * Some(true) — force RTE routing; error if the table has no write spec + // * Some(false) — read the base table only, bypassing the RTE // The LSM scanner surfaces in-flight `merge_insert` data (active/frozen // memtables + SSTables); validation and dispatch live in `lsm`. let has_spec = ds_ref.mem_wal_index_details().await?.is_some(); - let use_memwal = match query.base.use_memwal { + let use_rte = match query.base.use_rte { Some(true) if !has_spec => { return Err(Error::InvalidInput { - message: "use_memwal(true) was set but the table has no MemWAL write spec; \ - install one with enable_memwal or leave use_memwal unset" + message: "use_rte(true) was set but the table has no RTE write spec; \ + install one with enable_rte or leave use_rte unset" .to_string(), }); } Some(enable) => enable, None => has_spec, }; - if use_memwal { + if use_rte { return lsm::create_lsm_plan(table, ds_ref, query).await; } @@ -1195,7 +1195,7 @@ mod tests { table.set_unenforced_primary_key(["id"]).await.unwrap(); table - .enable_memwal(crate::table::MemWalWriteSpec::unsharded()) + .enable_rte(crate::table::RTEWriteSpec::unsharded()) .await .unwrap(); let lsm_error = table @@ -1395,7 +1395,7 @@ mod tests { } #[tokio::test] - async fn test_execute_query_use_memwal_with_namespace_pushdown_runs_locally() { + async fn test_execute_query_use_rte_with_namespace_pushdown_runs_locally() { use crate::connect; use crate::table::query::execute_query; use arrow_array::{Int32Array, RecordBatch}; @@ -1418,7 +1418,7 @@ mod tests { .unwrap(); let table = conn - .create_table("test_use_memwal_namespace_fallback", batch) + .create_table("test_use_rte_namespace_fallback", batch) .execute() .await .unwrap(); @@ -1429,13 +1429,13 @@ mod tests { .pushdown_operations .insert(NamespaceClientPushdownOperation::QueryTable); - // `use_memwal` set (even to false) must force local execution — the namespace - // request has no use_memwal field, so a pushdown would silently ignore it. + // `use_rte` set (even to false) must force local execution — the namespace + // request has no use_rte field, so a pushdown would silently ignore it. let query_vector = Arc::new(Float32Array::from(vec![0.0, 0.0])); let query = AnyQuery::VectorQuery(VectorQueryRequest { base: QueryRequest { limit: Some(1), - use_memwal: Some(false), + use_rte: Some(false), ..Default::default() }, column: Some("vector".to_string()), diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 32966e4a6..0cc240291 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! MemWAL read path. +//! RTE read path. //! -//! When a table has a MemWAL write spec installed (see [`enable_memwal`]), +//! When a table has an RTE write spec installed (see [`enable_rte`]), //! reads are routed through Lance's [`LsmScanner`] instead of the plain //! base-table scan unless the query sets -//! [`use_memwal(false)`](crate::query::QueryBase::use_memwal). This makes data +//! [`use_rte(false)`](crate::query::QueryBase::use_rte). This makes data //! written via the LSM `merge_insert` path — which lives in the active/frozen //! in-memory memtables and the SSTables until an external //! compaction merges it into the base table — visible to queries, deduplicated by @@ -17,10 +17,10 @@ //! three run through a single [`LsmScanner`], so a `where` filter is honored as a //! prefilter uniformly — including for vector search, where `LsmScanner` threads //! it into the vector planner's prefilter. Shapes the LSM path cannot honor are -//! rejected with [`Error::NotSupported`]; the caller must set `use_memwal(false)` to +//! rejected with [`Error::NotSupported`]; the caller must set `use_rte(false)` to //! run those against the base table. //! -//! [`enable_memwal`]: crate::Table::enable_memwal +//! [`enable_rte`]: crate::Table::enable_rte use std::collections::HashMap; use std::sync::Arc; @@ -52,12 +52,12 @@ use crate::utils::default_vector_column; /// widen the per-source fetch to keep result pages filled. const LSM_OVERFETCH_FACTOR: f64 = 2.0; -/// Build the LSM read plan for a MemWAL-routed query. +/// Build the LSM read plan for an RTE-routed query. /// -/// The caller guarantees `ds_ref` carries a MemWAL write spec (routing is decided +/// The caller guarantees `ds_ref` carries an RTE write spec (routing is decided /// in [`create_plan`](super::create_plan)). Errors with [`Error::NotSupported`] /// for query shapes the LSM scanner cannot honor — the caller must set -/// `use_memwal(false)` to run those against the base table. +/// `use_rte(false)` to run those against the base table. pub(super) async fn create_lsm_plan( table: &NativeTable, ds_ref: Arc, @@ -67,11 +67,11 @@ pub(super) async fn create_lsm_plan( // A time-traveled (checked-out) handle pins an older dataset version, but the // WAL manifests and cached writer expose current live state — mixing them would - // surface WAL rows written after the requested version. `use_memwal(false)` reads + // surface WAL rows written after the requested version. `use_rte(false)` reads // the base table at the pinned version. if table.dataset.time_travel_version().is_some() { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_memwal(false) to read the base table at this version".to_string(), + message: "the MemWAL LSM scanner cannot read from a time-traveled dataset version; set use_rte(false) to read the base table at this version".to_string(), }); } @@ -80,7 +80,7 @@ pub(super) async fn create_lsm_plan( .mem_wal_index_details() .await? .ok_or_else(|| Error::Runtime { - message: "the MemWAL write spec disappeared during read planning".to_string(), + message: "the RTE write spec disappeared during read planning".to_string(), })?; let pk_columns = pk_columns(&ds_ref)?; @@ -135,12 +135,12 @@ pub(super) async fn create_lsm_plan( restore_projection(plan, &query, &pk_columns) } -/// Reject query shapes the MemWAL read path does not implement. On a MemWAL table +/// Reject query shapes the RTE read path does not implement. On an RTE table /// reads route through the LSM scanner by default, so an unsupported shape is a /// hard error rather than a silent fallback to the base-only scan — which would -/// exclude un-compacted MemWAL data. The caller must set `use_memwal(false)` to +/// exclude un-compacted RTE data. The caller must set `use_rte(false)` to /// run these against the base table, accepting that the results omit un-compacted -/// MemWAL data. +/// RTE data. /// /// A `where` filter is intentionally *not* rejected: every arm routes through /// [`LsmScanner`], which applies it as a prefilter (see [`base_scanner`]). @@ -148,7 +148,7 @@ fn reject_unsupported(query: &VectorQueryRequest) -> Result<()> { let unsupported = |what: &str| { Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner does not support {what}; set use_memwal(false) to read the base table only (results will exclude un-compacted MemWAL data)" + "the MemWAL LSM scanner does not support {what}; set use_rte(false) to read the base table only (results will exclude un-compacted RTE data)" ), }) }; @@ -413,7 +413,7 @@ fn base_scanner( QueryFilter::Datafusion(expr) => scanner.filter_expr(expr.clone()), QueryFilter::Substrait(_) => { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner does not support Substrait filters; set use_memwal(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner does not support Substrait filters; set use_rte(false) to read the base table only".to_string(), }); } }; @@ -458,11 +458,11 @@ async fn fts_plan( offset: Option, ) -> Result> { // Pre-check for a lancedb-flavored error; `LsmScanner` also validates the - // single-column requirement, but without the `use_memwal(false)` guidance. + // single-column requirement, but without the `use_rte(false)` guidance. let columns: Vec = fts.columns().into_iter().collect(); if columns.len() > 1 { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner full-text search supports a single column; set use_memwal(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner full-text search supports a single column; set use_rte(false) to read the base table only".to_string(), }); } let column = columns.first().ok_or_else(|| Error::NotSupported { @@ -482,7 +482,7 @@ async fn fts_plan( { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (MemWalWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_memwal(false) to read the base table only" + "the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (RTEWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_rte(false) to read the base table only" ), }); } @@ -515,7 +515,7 @@ fn resident_memtables_carry( .all(|tables| carries(&tables.active) && tables.frozen.iter().all(&carries)) } -/// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's +/// Whether an index of `type_url_suffix` covering `column` is in the RTE spec's /// maintained set. Only a maintained index has its catch-up tracked (so exclusion is /// gated correctly) and its in-memory arm kept current; an unmaintained base index /// falls back to the compaction watermark and can drop rows it has not re-indexed. @@ -633,7 +633,7 @@ fn resolve_single_index( if names.len() > 1 { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_memwal(false) to read the base table only" + "the MemWAL LSM scanner cannot resolve the {arm} index catch-up watermark for '{column}': it has multiple {arm} indexes; set use_rte(false) to read the base table only" ), }); } @@ -682,7 +682,7 @@ fn restore_projection( /// /// Note: the base and SSTable arms use `fast_search` (indexed data only), so a /// base-table row not covered by a vector index is invisible here — it surfaces -/// only via the memtable or `use_memwal(false)`. +/// only via the memtable or `use_rte(false)`. #[allow(clippy::too_many_arguments)] async fn vector_plan( dataset: &Dataset, @@ -717,7 +717,7 @@ async fn vector_plan( if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? { return Err(Error::NotSupported { message: format!( - "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (MemWalWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_memwal(false) to read the base table only" + "the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (RTEWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_rte(false) to read the base table only" ), }); } @@ -726,7 +726,7 @@ async fn vector_plan( // clear error rather than failing deep in the planner. if is_binary_vector_column(&arrow_schema, &column) { return Err(Error::NotSupported { - message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_memwal(false) to read the base table only".to_string(), + message: "the MemWAL LSM scanner does not support binary (uint8) vector search; set use_rte(false) to read the base table only".to_string(), }); } @@ -813,7 +813,7 @@ mod tests { }; assert!( matches!(reject_unsupported(&query), Err(Error::NotSupported { message }) - if message.contains("search_effort") && message.contains("un-compacted MemWAL")) + if message.contains("search_effort") && message.contains("un-compacted RTE")) ); } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 0aebcaf2d..8f7f576ff 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -87,7 +87,7 @@ async fn execute_refresh_column_with_source( column: &str, ) -> Result { table.dataset.ensure_mutable()?; - ensure_no_memwal_spec(table).await?; + ensure_no_rte_spec(table).await?; let dataset = table.dataset.get().await?; let expression = declared_expression(&dataset, column)?; @@ -303,7 +303,7 @@ pub(crate) async fn execute_refresh_column_async( // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; - ensure_no_memwal_spec(table).await?; + ensure_no_rte_spec(table).await?; let dataset = table.dataset.get().await?; declared_expression(&dataset, column)?; drop(dataset); @@ -323,12 +323,12 @@ pub(crate) async fn execute_refresh_column_async( }))) } -/// Refuse to refresh under a MemWAL write spec. +/// Refuse to refresh under an RTE write spec. /// /// Refresh enumerates base fragments, and a write spec keeps visible rows in -/// un-compacted MemWAL tiers it cannot reach -- success would silently omit +/// un-compacted RTE tiers it cannot reach -- success would silently omit /// readable rows. -async fn ensure_no_memwal_spec(table: &NativeTable) -> Result<()> { +async fn ensure_no_rte_spec(table: &NativeTable) -> Result<()> { use lance::dataset::mem_wal::DatasetMemWalExt; // Unset drops the MemWAL index, so the spec alone stops describing a table @@ -341,7 +341,7 @@ async fn ensure_no_memwal_spec(table: &NativeTable) -> Result<()> { .list_mem_wal_latest_shard_ids() .await? .is_empty(); - if retained_sstables || table.get_memwal_spec().await?.is_some() { + if retained_sstables || table.get_rte_spec().await?.is_some() { return Err(Error::NotSupported { message: "refresh_column is not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" @@ -1637,7 +1637,7 @@ mod tests { /// writer left in that state. #[tokio::test] async fn test_refresh_refuses_a_foreign_lsm_state() { - use crate::table::MemWalWriteSpec; + use crate::table::RTEWriteSpec; let tmp_dir = tempfile::tempdir().unwrap(); let conn = connect(tmp_dir.path().to_str().unwrap()) @@ -1654,10 +1654,7 @@ mod tests { .unwrap(); let table = conn.create_table("lsm", batch).execute().await.unwrap(); table.set_unenforced_primary_key(["x"]).await.unwrap(); - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await; let err = table.refresh_column("doubled").await.unwrap_err(); @@ -1673,7 +1670,7 @@ mod tests { /// flag still marks retained SSTable rows; refresh refuses on the flag. #[tokio::test] async fn test_refresh_refuses_retained_catchup_state() { - use crate::table::MemWalWriteSpec; + use crate::table::RTEWriteSpec; let tmp_dir = tempfile::tempdir().unwrap(); let conn = connect(tmp_dir.path().to_str().unwrap()) @@ -1696,15 +1693,12 @@ mod tests { .await .unwrap(); table.set_unenforced_primary_key(["x"]).await.unwrap(); - table - .enable_memwal(MemWalWriteSpec::unsharded()) - .await - .unwrap(); + table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap(); let mut merge = table.merge_insert(&["x"]); merge .when_matched_update_all(None) .when_not_matched_insert_all() - .use_memwal(true); + .use_rte(true); merge .execute(Box::new(arrow_array::RecordBatchIterator::new( vec![Ok(batch)], @@ -1712,7 +1706,7 @@ mod tests { ))) .await .unwrap(); - table.disable_memwal().await.unwrap(); + table.disable_rte().await.unwrap(); super::super::computed_columns::add_foreign_kind(&table, "doubled", "sql").await; let err = table.refresh_column("doubled").await.unwrap_err(); diff --git a/rust/lancedb/src/table/memwal_stats.rs b/rust/lancedb/src/table/rte_stats.rs similarity index 93% rename from rust/lancedb/src/table/memwal_stats.rs rename to rust/lancedb/src/table/rte_stats.rs index d9747d257..46743be93 100644 --- a/rust/lancedb/src/table/memwal_stats.rs +++ b/rust/lancedb/src/table/rte_stats.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Live per-table-shard MemWAL state — the shape [`crate::Table::get_memwal_stats`] +//! Live per-table-shard RTE state — the shape [`crate::Table::get_rte_stats`] //! returns and [`super::checkpoint`] polls. //! //! Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) @@ -79,18 +79,18 @@ impl TableShardStats { } } -/// Live MemWAL state, one entry per table shard. +/// Live RTE state, one entry per table shard. #[derive(Debug, Clone, Deserialize)] -pub struct MemWalStats { +pub struct RTEStats { pub table_shards: Vec, } -/// Server-side JSON envelope for `get_memwal_stats`. `memwal_stats` is null when -/// the table has no MemWAL write path. +/// Server-side JSON envelope for `get_rte_stats`. `rte_stats` is null when +/// the table has no RTE write path. #[derive(Debug, Deserialize)] -pub(crate) struct GetMemWalStatsResponse { +pub(crate) struct GetRTEStatsResponse { #[serde(default)] - pub memwal_stats: Option, + pub rte_stats: Option, } #[cfg(test)] diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 1bb57d6c0..c541a9a43 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -131,7 +131,7 @@ pub(crate) async fn execute_declare( ) -> Result { use lance::dataset::mem_wal::DatasetMemWalExt; - // A MemWAL write spec keeps visible rows in tiers refresh cannot reach; + // An RTE write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. table.checkout_latest().await?; computed_columns::ensure_not_function_bound( @@ -149,7 +149,7 @@ pub(crate) async fn execute_declare( .list_mem_wal_latest_shard_ids() .await? .is_empty(); - if retained_sstables || table.get_memwal_spec().await?.is_some() { + if retained_sstables || table.get_rte_spec().await?.is_some() { return Err(Error::NotSupported { message: "computed columns are not supported on a table with an LSM write \ spec: rows in un-compacted tiers are invisible to refresh" diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index 91f3a978c..8559a0e43 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -1981,8 +1981,8 @@ async fn blob_layout_overwrite_replaces_schema_in_both_directions() -> Result<() } #[tokio::test] -async fn blob_layout_memwal_merge_normalizes_before_writing() -> Result<()> { - use lancedb::{query::Select, table::MemWalWriteSpec}; +async fn blob_layout_rte_merge_normalizes_before_writing() -> Result<()> { + use lancedb::{query::Select, table::RTEWriteSpec}; for wide in [false, true] { let tmp = tempdir().unwrap(); let db = connect(tmp.path().to_str().unwrap()).execute().await?; @@ -1991,7 +1991,7 @@ async fn blob_layout_memwal_merge_normalizes_before_writing() -> Result<()> { .execute() .await?; table.set_unenforced_primary_key(["id"]).await?; - table.enable_memwal(MemWalWriteSpec::unsharded()).await?; + table.enable_rte(RTEWriteSpec::unsharded()).await?; let mut merge = table.merge_insert(&["id"]); merge .when_matched_update_all(None) From 9e54c8cf27b726929d040a971b31a8132bcfc927 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 20 Oct 2026 17:06:22 +0000 Subject: [PATCH 22/22] =?UTF-8?q?Bump=20version:=200.41.0-beta.9=20?= =?UTF-8?q?=E2=86=92=200.41.0-beta.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index f5be5a9ab..413d11f82 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.41.0-beta.9" +current_version = "0.41.0-beta.10" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index b339cf454..61aee21be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5698,7 +5698,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.41.0-beta.9" +version = "0.41.0-beta.10" dependencies = [ "ahash", "anyhow", @@ -5796,7 +5796,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.41.0-beta.9" +version = "0.41.0-beta.10" dependencies = [ "arrow-array", "arrow-buffer", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.41.0-beta.9" +version = "0.41.0-beta.10" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 7a9a89641..b6cbd8586 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.41.0-beta.9 + 0.41.0-beta.10 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 83b27fefd..e64f65852 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.9 + 0.41.0-beta.10 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 4dd145b31..4e4a82b63 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.41.0-beta.9 + 0.41.0-beta.10 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 3ef5438de..79a594f14 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.41.0-beta.9" +version = "0.41.0-beta.10" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index da0158b7d..403b1d2c2 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 451766110..ef2d4ca40 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index d818cb60f..5b247be2c 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 4fe1c12cc..0c36b7213 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 7dc63ccaa..274877c83 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index dc97bbed8..67f0176f5 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 235bb0e55..5f3cf6032 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 22f51ce61..df083600a 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.41.0-beta.9", + "version": "0.41.0-beta.10", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index ce4230f9d..615b7c5aa 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.41.0-beta.9" +version = "0.41.0-beta.10" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 1daeef984..4649e560d 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.41.0-beta.9" +version = "0.41.0-beta.10" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true