rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-22 19:52:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4260-1

Browse Source 
Keep main's Lance beta.22 update and align the patched Arrow crate metadata while preserving the sliced-child validity fix.

Validated workspace Clippy with warnings denied, 112 Arrow tests and 6 doctests, the nullable struct merge regression, 26 Python tests and the original reproducer, Node build/docs/lint with 518 tests, and source archive contents and dependency resolution. Root Ruff findings remain confined to the unchanged docs/web-tests directory.
This commit is contained in:
Gatefixer
2026-10-21 09:12:43 +00:00
parent eb4535398a 46ccbe4e2c
commit 6366bab6a3
117 changed files with 5276 additions and 1884 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.41.0-beta.9"
current_version = "0.41.0-beta.11"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
AGENTS.md
+2 -2
View File
@@ -102,8 +102,8 @@ TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `pnpm build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
3. Add typescript method on abstract class `Table` in `nodejs/lancedb/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/lancedb/table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `pnpm run docs` to generate TypeScript documentation.
Cargo.lock
Generated
+57 -56
View File
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4441,7 +4441,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots 1.0.7",
"webpki-roots 1.0.10",
]
[[package]]
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arc-swap",
"arrow",
@@ -5110,6 +5110,7 @@ dependencies = [
"lance-tokenizer",
"log",
"moka",
"num-traits",
"object_store 0.14.2",
"permutation",
"pin-project",
@@ -5134,7 +5135,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "14.0.0-beta.21"
version = "14.0.0-beta.22"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5156,7 +5157,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5170,7 +5171,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5179,8 +5180,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrayref",
"crunchy",
@@ -5190,8 +5191,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5228,8 +5229,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow",
"arrow-array",
@@ -5260,8 +5261,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow",
"arrow-array",
@@ -5278,8 +5279,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"proc-macro2",
"quote",
@@ -5288,8 +5289,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5322,8 +5323,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5355,8 +5356,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"datafusion",
"geo-traits",
@@ -5370,8 +5371,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arc-swap",
"arrow",
@@ -5442,8 +5443,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5466,8 +5467,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow",
"arrow-array",
@@ -5510,8 +5511,8 @@ dependencies = [
[[package]]
name = "lance-jsonb"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"byteorder",
"ethnum",
@@ -5528,8 +5529,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5543,8 +5544,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow",
"async-trait",
@@ -5558,8 +5559,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arc-swap",
"arrow",
@@ -5585,7 +5586,7 @@ dependencies = [
"object_store 0.14.2",
"quick-xml 0.40.1",
"rand 0.9.5",
"reqwest 0.12.28",
"reqwest 0.13.4",
"roaring",
"serde",
"serde_json",
@@ -5613,8 +5614,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5628,8 +5629,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow",
"arrow-array",
@@ -5669,8 +5670,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5683,8 +5684,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "14.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.22#72c5b645600bde5d2e0e0b629ed986e2018c62bf"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5697,7 +5698,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.41.0-beta.9"
version = "0.41.0-beta.11"
dependencies = [
"ahash",
"anyhow",
@@ -5795,7 +5796,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.41.0-beta.9"
version = "0.41.0-beta.11"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5820,7 +5821,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.41.0-beta.9"
version = "0.41.0-beta.11"
dependencies = [
"arc-swap",
"arrow",
@@ -8909,7 +8910,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams 0.4.2",
"web-sys",
"webpki-roots 1.0.7",
"webpki-roots 1.0.10",
]
[[package]]
@@ -9239,9 +9240,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"aws-lc-rs",
"ring",
@@ -11156,14 +11157,14 @@ version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
"webpki-roots 1.0.7",
"webpki-roots 1.0.10",
]
[[package]]
name = "webpki-roots"
version = "1.0.7"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
dependencies = [
"rustls-pki-types",
]
Cargo.toml
+15 -15
View File
@@ -13,21 +13,21 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=14.0.0-beta.21", tag = "v14.0.0-beta.21", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=14.0.0-beta.21", path = "python/vendor/lance-arrow" }
lance = { "version" = "=14.0.0-beta.22", default-features = false, "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.22", default-features = false, "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=14.0.0-beta.22", tag = "v14.0.0-beta.22", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.22", default-features = false, "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.22", "tag" = "v14.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=14.0.0-beta.22", path = "python/vendor/lance-arrow" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.41.0-beta.9</version>
<version>0.41.0-beta.11</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+10 -10
View File
@@ -425,16 +425,16 @@ Empty results retain the query's output schema.
***
### useMemWal()
### useRTE()
```ts
useMemWal(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this query.
Control RTE read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so
By default (unset), when the table carries an RTE write spec (see
[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the SSTables), deduplicated by primary key; a table without a spec
@@ -444,12 +444,12 @@ reads the base table.
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
RTE write spec. `false` bypasses the RTE and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useMemWal(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
hybrid search, `orderBy`). On an RTE table those shapes error unless
`useRTE(false)` is set, because a base-only read would silently exclude
un-compacted RTE data.
#### Returns
@@ -457,7 +457,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useMemWal`
`StandardQueryBase.useRTE`
***
docs/src/js/classes/MergeInsertBuilder.md
+8 -8
View File
@@ -76,23 +76,23 @@ the query optimizer chooses a suboptimal path.
***
### useMemWal()
### useRTE()
```ts
useMemWal(enable): MergeInsertBuilder
useRTE(enable): MergeInsertBuilder
```
Control MemWAL routing for this merge.
Control RTE routing for this merge.
By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
By default (unset), a `mergeInsert` on a table with an RTE write spec is
routed through Lance's MemWAL shard writer, and a table without one uses the
standard path.
#### Parameters
* **enable**: `boolean`
`true` forces MemWAL routing and errors if the table has no
MemWAL write spec. `false` forces the standard write path even when a spec is set.
`true` forces RTE routing and errors if the table has no
RTE write spec. `false` forces the standard write path even when a spec is set.
#### Returns
@@ -108,12 +108,12 @@ validateSingleShard(validateSingleShard): MergeInsertBuilder
Controls how an LSM merge checks that its input targets a single shard.
When a table has a MemWAL write spec, every row in a `mergeInsert` call must
When a table has an RTE write spec, every row in a `mergeInsert` call must
route to the same shard. When `true` (the default), every row is inspected
to verify this. When `false`, only the first row is inspected and the
shard it routes to is used for the whole input — a faster path for callers
that have already pre-sharded their input. Has no effect on tables without
a MemWAL write spec.
an RTE write spec.
#### Parameters
docs/src/js/classes/Query.md
+10 -10
View File
@@ -499,16 +499,16 @@ Empty results retain the query's output schema.
***
### useMemWal()
### useRTE()
```ts
useMemWal(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this query.
Control RTE read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so
By default (unset), when the table carries an RTE write spec (see
[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the SSTables), deduplicated by primary key; a table without a spec
@@ -518,12 +518,12 @@ reads the base table.
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
RTE write spec. `false` bypasses the RTE and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useMemWal(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
hybrid search, `orderBy`). On an RTE table those shapes error unless
`useRTE(false)` is set, because a base-only read would silently exclude
un-compacted RTE data.
#### Returns
@@ -531,7 +531,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useMemWal`
`StandardQueryBase.useRTE`
***
docs/src/js/classes/Table.md
+37 -37
View File
@@ -228,13 +228,13 @@ version of the table.
***
### checkpointMemWal()
### checkpointRTE()
```ts
abstract checkpointMemWal(): Promise<void>
abstract checkpointRTE(): Promise<void>
```
Converge this table's MemWAL write path into its base table.
Converge this table's RTE write path into its base table.
Freezes once, then triggers compaction and polls until the SSTables that existed
at the start is gone. The target set is fixed at the start, so
@@ -254,9 +254,9 @@ is merging. The caller owns the deadline.
#### Example
```ts
const before = await table.getMemWalStats();
await table.checkpointMemWal();
const after = await table.getMemWalStats();
const before = await table.getRTEStats();
await table.checkpointRTE();
const after = await table.getRTEStats();
```
***
@@ -279,15 +279,15 @@ Any attempt to use the table after it is closed will result in an error.
***
### closeMemWalWriters()
### closeRTEWriters()
```ts
abstract closeMemWalWriters(): Promise<void>
abstract closeRTEWriters(): Promise<void>
```
Drain and close any cached MemWAL shard writers held for this table.
Drain and close any cached RTE shard writers held for this table.
When an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) is installed, `mergeInsert` opens MemWAL
When an [RTEWriteSpec](../interfaces/RTEWriteSpec.md) is installed, `mergeInsert` opens RTE
shard writers and caches them for reuse across calls. This closes them,
flushing pending data; writers reopen lazily on the next `mergeInsert`.
It is a no-op when no writers are cached.
@@ -298,17 +298,17 @@ It is a no-op when no writers are cached.
***
### compactMemWal()
### compactRTE()
```ts
abstract compactMemWal(): Promise<void>
abstract compactRTE(): Promise<void>
```
Trigger a background SSTable compaction pass per table shard.
Returns once the passes are *dispatched*, not once they finish — watch
[Table#getMemWalStats](Table.md#getmemwalstats) for progress, or use
[Table#checkpointMemWal](Table.md#checkpointmemwal) to wait for convergence.
[Table#getRTEStats](Table.md#getrtestats) for progress, or use
[Table#checkpointRTE](Table.md#checkpointrte) to wait for convergence.
#### Returns
@@ -449,13 +449,13 @@ containing the new version number of the table
***
### disableMemWal()
### disableRTE()
```ts
abstract disableMemWal(): Promise<void>
abstract disableRTE(): Promise<void>
```
Remove the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) from this table, reverting to the standard
Remove the [RTEWriteSpec](../interfaces/RTEWriteSpec.md) from this table, reverting to the standard
`mergeInsert` write path.
Errors if no spec is currently set.
@@ -531,16 +531,16 @@ Drop an index from the table.
***
### enableMemWal()
### enableRTE()
```ts
abstract enableMemWal(spec): Promise<void>
abstract enableRTE(spec): Promise<void>
```
Install an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) on this table, selecting Lance's MemWAL
Install an [RTEWriteSpec](../interfaces/RTEWriteSpec.md) on this table, selecting Lance's MemWAL
LSM-style write path for future `mergeInsert` calls.
`MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
`RTEWriteSpec` chooses one of three sharding strategies via `specType`:
- `"bucket"` — hash-bucket writes by the single-column unenforced primary
key (`column` and `numBuckets` required).
@@ -552,13 +552,13 @@ All variants require the table to have an unenforced primary key
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
including ones created later, and skips a kind the RTE cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)
* **spec**: [`RTEWriteSpec`](../interfaces/RTEWriteSpec.md)
The sharding spec to install.
#### Returns
@@ -569,7 +569,7 @@ rather than quietly omitted.
```ts
await table.setUnenforcedPrimaryKey("id");
await table.enableMemWal({
await table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 16,
@@ -647,10 +647,10 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
***
### flushMemWal()
### flushRTE()
```ts
abstract flushMemWal(): Promise<void>
abstract flushRTE(): Promise<void>
```
Freeze every table shard's active memtable into a new SSTable.
@@ -699,38 +699,38 @@ const { records, truncated } = await table.functionErrors({ column: "embedding"
***
### getMemWalSpec()
### getRTESpec()
```ts
abstract getMemWalSpec(): Promise<undefined | MemWalWriteSpec>
abstract getRTESpec(): Promise<undefined | RTEWriteSpec>
```
Read the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) currently installed on this table.
Read the [RTEWriteSpec](../interfaces/RTEWriteSpec.md) currently installed on this table.
Resolves to `undefined` when the MemWAL write path is not enabled (no
spec has been set, or it was removed with [Table#disableMemWal](Table.md#disablememwal)).
Resolves to `undefined` when the RTE write path is not enabled (no
spec has been set, or it was removed with [Table#disableRTE](Table.md#disablerte)).
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
`Promise`&lt;`undefined` \| [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)&gt;
`Promise`&lt;`undefined` \| [`RTEWriteSpec`](../interfaces/RTEWriteSpec.md)&gt;
***
### getMemWalStats()
### getRTEStats()
```ts
abstract getMemWalStats(includeSstableRows?): Promise<undefined | MemWalStats>
abstract getRTEStats(includeSstableRows?): Promise<undefined | RTEStats>
```
Read live per-table-shard MemWAL state.
Read live per-table-shard RTE state.
Answers "how far behind is my fresh tier", "which table shard is hot", and
"why is my fresh-tier vector search brute-force". Mutates no table state.
Resolves to `undefined` only when the MemWAL write path is not enabled.
Resolves to `undefined` only when the RTE write path is not enabled.
#### Parameters
@@ -740,7 +740,7 @@ Resolves to `undefined` only when the MemWAL write path is not enabled.
#### Returns
`Promise`&lt;`undefined` \| [`MemWalStats`](../interfaces/MemWalStats.md)&gt;
`Promise`&lt;`undefined` \| [`RTEStats`](../interfaces/RTEStats.md)&gt;
***
docs/src/js/classes/TakeQuery.md
+5 -5
View File
@@ -275,17 +275,17 @@ Empty results retain the query's output schema.
***
### useMemWal()
### useRTE()
```ts
useMemWal(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this take query.
Control RTE read routing for this take query.
`false` bypasses the MemWAL and reads the base table only — the escape hatch,
`false` bypasses the RTE and reads the base table only — the escape hatch,
since take-by-row-id/offset is not supported on the LSM scanner and, on a
MemWAL table, auto-routes to it and errors otherwise.
RTE table, auto-routes to it and errors otherwise.
#### Parameters
docs/src/js/classes/VectorQuery.md
+11 -11
View File
@@ -659,7 +659,7 @@ minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The MemWAL LSM
guarantee and does not change refinement or HNSW ef. The RTE LSM
scanner does not yet support this option.
#### Parameters
@@ -781,16 +781,16 @@ Empty results retain the query's output schema.
***
### useMemWal()
### useRTE()
```ts
useMemWal(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this query.
Control RTE read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so
By default (unset), when the table carries an RTE write spec (see
[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the SSTables), deduplicated by primary key; a table without a spec
@@ -800,12 +800,12 @@ reads the base table.
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
RTE write spec. `false` bypasses the RTE and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useMemWal(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
hybrid search, `orderBy`). On an RTE table those shapes error unless
`useRTE(false)` is set, because a base-only read would silently exclude
un-compacted RTE data.
#### Returns
@@ -813,7 +813,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useMemWal`
`StandardQueryBase.useRTE`
***
docs/src/js/functions/blob.md
+9
View File
@@ -11,6 +11,15 @@ function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
whole external files, and external byte ranges. Set both `position` and
positive `size` for a range, or omit both for a whole file.
Existing two-field columns remain writable. Appends and merge inserts fill
missing range fields with nulls for four-field targets and drop all-null
range fields for two-field targets. Non-null ranges require a four-field
target. Overwrite with Arrow input replaces the table's schema; row objects
are converted using the current table schema.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
docs/src/js/globals.md
+2 -2
View File
@@ -114,8 +114,6 @@
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemWalStats](interfaces/MemWalStats.md)
- [MemWalWriteSpec](interfaces/MemWalWriteSpec.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeResult](interfaces/MergeResult.md)
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
@@ -124,6 +122,8 @@
- [OptimizeOptions](interfaces/OptimizeOptions.md)
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RTEStats](interfaces/RTEStats.md)
- [RTEWriteSpec](interfaces/RTEWriteSpec.md)
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
- [RemovalStats](interfaces/RemovalStats.md)
docs/src/js/interfaces/MemWalStats.md → docs/src/js/interfaces/RTEStats.md
+3 -3
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MemWalStats
[@lancedb/lancedb](../globals.md) / RTEStats
# Interface: MemWalStats
# Interface: RTEStats
Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
the caller's to compute.
docs/src/js/interfaces/MemWalWriteSpec.md → docs/src/js/interfaces/RTEWriteSpec.md
+5 -5
View File
@@ -2,12 +2,12 @@
***
[@lancedb/lancedb](../globals.md) / MemWalWriteSpec
[@lancedb/lancedb](../globals.md) / RTEWriteSpec
# Interface: MemWalWriteSpec
# Interface: RTEWriteSpec
Specification selecting Lance's MemWAL LSM-style write path for
`mergeInsert`.
Specification enabling the realtime engine (RTE), which routes
`mergeInsert` through Lance's MemWAL LSM-style write path.
`specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
`column` and `numBuckets` are required; for `"identity"`, `column` is
@@ -34,7 +34,7 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit for every index the table has,
Indexes the RTE keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/type-aliases/BlobInput.md
+4
View File
@@ -12,6 +12,7 @@ type BlobInput:
| BlobUri
| object
| object
| object
| null
| undefined;
```
@@ -20,6 +21,9 @@ A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md))
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
External byte ranges require both
`position` (non-negative) and `size` (positive). Use safe integer numbers or
`bigint` values for these unsigned 64-bit byte counts.
## Example
docs/src/python/python.md
+1 -1
View File
@@ -297,7 +297,7 @@ listing a storage directory.
::: lancedb.table.Branches
::: lancedb.MemWalWriteSpec
::: lancedb.RTEWriteSpec
## Functions and Jobs
docs/web/docs.json
+1
View File
@@ -327,6 +327,7 @@
"management/namespace",
"management/table",
"tables/branching",
"tables/realtime-engine",
"management/materialized-view",
"management/view"
]
docs/web/feature-engineering/materialized-views.mdx
+1 -1
View File
@@ -52,7 +52,7 @@ In Python SDK 0.40.0, create the source with
above does. Stable row IDs let the view track source rows across updates and
compaction. Set this option when creating the table; it cannot be added later.
A source with a MemWAL write spec or retained, uncompacted write-ahead-log
A source with the [realtime engine](/tables/realtime-engine) enabled or retained, uncompacted write-ahead-log
rows is not accepted by the local materialized-view implementation.
## Refresh behavior {#incremental}
docs/web/maintenance/background-jobs.mdx
+3 -3
View File
@@ -27,13 +27,13 @@ also start cache-warming jobs through the admin API.
## Write-ahead log maintenance {#wal}
For a table with a [write-ahead log](/sql/ddl/table#properties) enabled, two jobs keep the
For a table with the [realtime engine](/tables/realtime-engine) enabled, two jobs keep the
log from growing without bound:
| Job type | What it does |
| --- | --- |
| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments. The two are unrelated. |
| `wal_index_catchup` | Extends a base-table index over the rows `sstable_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. |
| `memwal_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments. The two are unrelated. |
| `memwal_catchup_<type>` | Extends a base-table index over the rows `memwal_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. The suffix names the index type, for example `memwal_catchup_btree`, `memwal_catchup_fts`, or `memwal_catchup_ivfpq`. |
## Index maintenance {#index}
docs/web/maintenance/cleanup.mdx
+25 -4
View File
@@ -65,13 +65,34 @@ files, so a tag on what was the latest version can become an older one by the
time cleanup runs. `optimize()` then raises after compacting, and its index
update does not run.
A deployment's background cleanup follows the same rules. Its own background
jobs, such as compaction and index updates, also commit new versions, so a tag
on what is now the latest version can become an older one without a call from
you. If you rely on reading old versions, set the window to cover the range you
Enterprise background cleanup keeps tagged snapshots while pruning unrelated
expired versions. If you rely on reading untagged old versions, set the window
to cover the range you
need before running this, not after.
</Warning>
## Managed blob objects {#managed-blobs}
[Managed blobs](/management/table#managed-blobs) store out-of-line payloads
under the table's `_blobs/` directory. Cleanup follows the references from
retained snapshots, including protected tags and branches, before selecting
objects for deletion. Replacing a blob or deleting a row makes its old payload
eligible only after no retained snapshot references it.
Cleanup removes whole objects. If a packed object contains several blobs, it
remains while any of them is live. Compaction preserves Managed payload objects
rather than copying their bytes, so compaction alone does not reclaim the
unused portion of a packed object.
Use table maintenance to remove Managed objects; deleting `_blobs/` files
directly can corrupt current or historical snapshots. Upgrade all clients and
maintenance workers to a version that understands Managed blobs before using
them, and reopen handles created before that upgrade.
For local tables, `optimize()` performs compaction and cleanup. Enterprise runs
these as background jobs; the remote Python table does not support `optimize()`.
See [Background jobs](/maintenance/background-jobs) for the Enterprise path.
## Disk usage does not drop immediately {#disk}
Compaction writes new, merged files before the old ones are removed, so disk
docs/web/management/function.mdx
+62
View File
@@ -143,6 +143,68 @@ rows added since the last one and recomputes rows whose inputs changed, and
leaves filled rows with unchanged inputs alone. See
[Refresh a column](/feature-engineering/refresh-column).
## Read and produce blobs {#blobs}
A Function can bind a [Blob v2 column](/management/table#managed-blobs) and
return another blob. In row mode, a top-level blob parameter arrives as Python
`bytes`, or `None` for a nullable input. The worker reads the payload before
calling your function. Return bytes and let the destination table store the
Managed payload and descriptor.
Declare blob inputs and outputs explicitly. This example reverses the bytes
to demonstrate the round trip without requiring an image library. Here `db`
is an Enterprise database and `table` contains an `image` Blob v2 column
with table schema metadata `lance.blob.managed=true`, as in the
[table creation example](/management/table#managed-blobs-read-write):
```python
from datetime import timedelta
import lancedb
from lancedb import col
from lancedb.functions import udf
import pyarrow as pa
@udf(
name="reverse_image",
python_version="3.12",
pip=["pyarrow==26.0.0"],
input_schema=pa.schema([lancedb.blob("image")]),
output_schema=lancedb.blob("reversed", nullable=False),
)
def reverse_image(image):
return b"missing" if image is None else image[::-1]
function = db.create_function_async(reverse_image).wait(
timeout=timedelta(minutes=15)
)
table.add_columns({"reversed": function(image=col("image"))})
result = table.refresh_column_async("reversed").wait(
timeout=timedelta(minutes=15)
)
assert result.rows_failed == 0
hits = table.search().select(["id", "reversed"]).with_row_id(True).to_arrow()
values = table.fetch_blobs("reversed", hits).to_pylist()
```
An empty byte string is a filled result. A scalar output cannot be nullable,
because a physical NULL marks an unfilled computed row. To return a nullable
blob as a value, use a named-struct output schema with a nullable blob field;
the struct itself records whether the result has been filled. Read a nested
blob with `table.fetch_blobs("payload.image", hits)`.
On deployments that support computed-column inputs, another Function can bind
to `col("reversed")`. Refresh the producer first, then the consumer. Once
outputs are current, a later refresh fills appended rows. A filtered refresh
preserves payload references for unselected Managed outputs. An unfiltered
refresh can recompute a whole fragment when it cannot establish that the
fragment's outputs are current.
Function output blobs follow the same compaction and cleanup lifecycle as
ingested Managed blobs. Retained versions continue to own their payloads;
cleanup reclaims an object only after its retained references are gone.
## Declare the environment {#environment}
The function runs remotely, so its dependencies are part of its definition:
docs/web/management/table.mdx
+85 -1
View File
@@ -427,7 +427,91 @@ Once you have the bytes back from the search result, you can decode them into th
</CodeBlock>
</CodeGroup>
### Large Blobs (Blob API) {#multimodal-large-blobs-blob-api}
### Managed blobs (Blob v2) {#managed-blobs}
Use `lancedb.blob()` for images, audio, videos, and other binary values whose
payloads should load separately from the rest of a row. Write Python `bytes`
normally. Set the table schema metadata `lance.blob.managed` to `true` to
store small values inline and larger values in managed objects under the table's `_blobs/` directory. Medium values can share a packed
object; large values use a dedicated object.
The table owns these objects. You do not upload them separately, keep an object
catalog, or delete their files yourself. A descriptor records where to read the
bytes. Compaction preserves the descriptor and its payload object, so merging
table fragments does not rewrite the blob payloads.
<Warning>
Use an SDK and Enterprise deployment that support Blob v2 Managed storage.
Upgrade every reader, writer, and maintenance worker before activating Managed
blobs on a table, and reopen existing table handles after the upgrade. Older
clients cannot safely maintain a table containing Managed blobs.
</Warning>
#### Write and read blobs {#managed-blobs-read-write}
This example uses a local connection. The same schema, `add`, query, and
`fetch_blobs` calls work with an Enterprise database connection:
```python
import lancedb
import pyarrow as pa
db = lancedb.connect("./media")
table = db.create_table(
"images",
schema=pa.schema([
pa.field("id", pa.int64(), nullable=False),
lancedb.blob("image"),
], metadata={"lance.blob.managed": "true"}),
)
table.add([
{"id": 1, "image": b""},
{"id": 2, "image": bytes(range(256)) * 400},
{"id": 3, "image": None},
])
# Query descriptors first, then fetch only the selected rows' payloads.
hits = (
table.search()
.where("id <= 2")
.select(["id", "image"])
.with_row_id(True)
.to_arrow()
)
images = table.fetch_blobs("image", hits).to_pylist()
by_id = dict(zip(hits["id"].to_pylist(), images, strict=True))
assert by_id[1] == b""
assert len(by_id[2]) == 102400
```
`None` is a null value; `b""` is a present, empty blob. `fetch_blobs` returns
bytes in the order of the supplied row IDs. It materializes the selected
payloads, so fetch bounded batches when blobs are large. Row IDs belong to
their table; fetch them from the same table and snapshot that produced the
query result.
Keep descriptors opaque in application code. Their representation can differ
between inline values, Managed objects, and externally referenced blobs. For
streaming and range reads, see the
[Lance blob guide](https://lance.org/guide/blob/).
#### Derived blobs and maintenance {#managed-blobs-lifecycle}
Enterprise Functions can consume blob columns and
return new blobs. Their output payloads are managed by the destination table.
Local tables can also derive blob columns with
[SQL computed columns](/feature-engineering/add-computed-column).
Deleting a row or replacing its blob does not immediately remove the old
payload: retained table versions may still reference it. [Cleanup](/maintenance/cleanup#managed-blobs)
reclaims objects after those references expire. A packed object remains while
any of its contained blobs is still referenced.
### Legacy large-binary blobs {#multimodal-large-blobs-blob-api}
The schema below describes the earlier large-binary Blob API. It is distinct
from the `lancedb.blob()` schema above; changing the field's metadata alone does
not migrate an existing column to Blob v2.
For larger files like high-resolution images or videos, Lance provides a specialized **Blob API**. By using a large-binary Arrow type (`pa.large_binary()` in Python, `LargeBinary` in TypeScript, and `DataType::LargeBinary` in Rust) and specific metadata, you enable **lazy loading** and optimized encoding. This allows you to work with massive datasets without loading all binary data into memory upfront.
docs/web/sql/ddl/table.mdx
+8 -6
View File
@@ -102,18 +102,20 @@ duplicates impossible. It tells the planner what to assume. The column must be
declared `NOT NULL`.
```sql
ALTER TABLE t SET MEMWAL
ALTER TABLE t SET MEMWAL WITH (sharding = 'bucket', column = 'id', buckets = 4)
ALTER TABLE t SET MEMWAL WITH (maintained_indexes = '')
ALTER TABLE t UNSET MEMWAL
ALTER TABLE t SET RTE
ALTER TABLE t SET RTE WITH (sharding = 'bucket', column = 'id', buckets = 4)
ALTER TABLE t SET RTE WITH (maintained_indexes = '')
ALTER TABLE t UNSET RTE
```
A write-ahead log makes small frequent writes cheap by batching them out of the
The [realtime engine](/tables/realtime-engine) puts a write-ahead log in front
of the table, making small frequent writes cheap by batching them out of the
commit path. `sharding` and `buckets` spread the log so concurrent writers do
not contend on one place; `bucket` sharding names its `column`, which must be
the table's unenforced primary key, and inserts into the table then need a
scalar index on that key. `maintained_indexes` chooses which indexes are kept
current against WAL rows. A table with a computed column cannot take a
current against WAL rows; omitted, every supported index is kept current,
including ones created later. A table with a computed column cannot take a
write-ahead log. See
[Background jobs](/maintenance/background-jobs#wal) for how the log gets
folded back into the table.
docs/web/tables/realtime-engine.mdx
+229
View File
@@ -0,0 +1,229 @@
---
title: "Realtime engine (RTE)"
sidebarTitle: "Realtime Engine"
description: "Absorb small, frequent writes cheaply and make them queryable within milliseconds."
icon: "bolt"
keywords: ["rte", "realtime engine", "wal", "memwal", "lsm", "write-ahead log", "streaming writes", "upsert"]
---
<Badge color="red">Enterprise</Badge>
## What is the realtime engine? {#what-is-rte}
Every write to a LanceDB table is normally a commit to the underlying Lance
dataset. That is ideal for bulk loads, but a stream of small upserts becomes a
stream of small commits, and each write is only visible once its commit lands.
The realtime engine (RTE) puts an LSM-style write tier in front of the table.
It turns many small writes into a few large commits, and makes each write
**searchable within milliseconds** of being acknowledged. It is enabled per
table, and nothing about how you query that table changes.
## How it works {#how-it-works}
- **Writes.** A `merge_insert` or `delete` lands in an in-memory **memtable**
and is acknowledged once it is durable in the write-ahead log on object
storage.
- **Flush and compaction.** Memtables freeze into **SSTables**, which background
jobs compact into the base table, catching its indexes up as they go. See
[Background jobs](/maintenance/background-jobs#wal).
- **Reads.** Every query runs against the base table and the fresh tier
(memtables and SSTables) in parallel. Fresh rows supersede base rows with the
same primary key, and the two are merged into one result.
- **Indexes stay fresh.** Vector, full-text, BTree, bitmap, and most other
scalar indexes are maintained on the memtable, so new rows are searchable
through the index as soon as they are written.
## Getting started {#getting-started}
### 1. Create a table with a primary key {#primary-key}
RTE deduplicates writes by primary key. Create the table with a non-nullable
key column, then declare it with `set_unenforced_primary_key`. The key is a
single `int32`, `int64`, string, or binary column, and cannot be changed once
set.
```python
from datetime import timedelta
import lancedb
import numpy as np
import pyarrow as pa
from lancedb import RTEWriteSpec
from lancedb.index import FTS, BTree, IvfPq
db = lancedb.connect("db://my_db", api_key="...", host_override="https://...")
n, dim = 1024, 128
schema = pa.schema([
pa.field("id", pa.int64(), nullable=False),
pa.field("vector", pa.list_(pa.float32(), dim)),
pa.field("text", pa.utf8()),
])
initial_data = pa.table({
"id": pa.array(range(n), pa.int64()),
"vector": pa.FixedSizeListArray.from_arrays(
pa.array(np.random.rand(n * dim).astype("float32")), dim),
"text": pa.array([f"doc {i} alpha bravo" for i in range(n)]),
}, schema=schema)
table = db.create_table("docs", initial_data, schema=schema)
table.set_unenforced_primary_key("id")
```
Or in SQL:
```sql
ALTER TABLE docs SET UNENFORCED PRIMARY KEY (id)
```
### 2. Create indexes {#indexes}
By default, RTE maintains every supported index on the table, including
indexes created after it is enabled. A scalar index on the primary key is
required for partial updates and deletes.
```python
table.create_index("vector", config=IvfPq())
table.create_index("text", config=FTS())
table.create_index("id", config=BTree())
table.wait_for_index(["vector_idx", "text_idx", "id_idx"], timeout=timedelta(minutes=5))
```
### 3. Enable RTE {#enable}
```python
table.enable_rte(RTEWriteSpec.unsharded())
```
Or in SQL:
```sql
ALTER TABLE docs SET RTE
```
### 4. Write and query {#write-and-query}
<Warning>
On an RTE table, an upsert through `merge_insert` is the only way to write
rows. Use the form below exactly: `when_matched_update_all()` with no filter,
plus `when_not_matched_insert_all()`. Other `merge_insert` forms, such as
conditional updates or deleting rows missing from the source, are rejected.
</Warning>
Writes go through `merge_insert` on the primary key, and are visible to queries
immediately:
```python
new_rows = pa.table({
"id": pa.array([2000, 2001], pa.int64()),
"vector": pa.FixedSizeListArray.from_arrays(
pa.array(np.random.rand(2 * dim).astype("float32")), dim),
"text": pa.array(["fresh charlie delta", "fresh echo foxtrot"]),
}, schema=schema)
query_vector = np.random.rand(dim).astype("float32")
table.merge_insert("id") \
.when_matched_update_all() \
.when_not_matched_insert_all() \
.execute(new_rows)
table.search(query_vector).limit(10).to_list()
table.search("charlie").limit(10).to_list()
```
**Partial updates.** Send the primary key plus only the columns you want to
change; omitted columns keep their current values. This requires a scalar
index on the primary key.
```python
patch = pa.table(
{"id": [0, 1], "text": ["doc 0 revised", "doc 1 revised"]},
schema=pa.schema([schema.field("id"), schema.field("text")]),
)
table.merge_insert("id").when_matched_update_all().when_not_matched_insert_all().execute(patch)
```
**Deletes.** The predicate must bind the primary key to literal values. Ranges,
`OR`, and filters on other columns are rejected. Deletes require a scalar index
on the primary key.
```python
table.delete("id IN (2000, 2001)")
```
### Disabling RTE {#disable}
Stop writes and wait for in-flight writes to finish, then flush pending writes
into the base table and disable. `checkpoint_rte()` only drains writes that
existed when it started, so anything written during the checkpoint can be left
behind. `disable_rte()` is rejected while the write-ahead log still holds rows,
so run `checkpoint_rte()` first.
```python
table.checkpoint_rte()
table.disable_rte()
```
## Performance {#performance}
### Writes {#write-performance}
RTE acknowledges a write as soon as it is durable in the write-ahead log, so
ingest is no longer limited by how fast the table can commit.
**Compared with writing directly to the table.** On tables ranging from
hundreds of thousands to tens of millions of rows, RTE delivered 4–12× the
write throughput and 24–52× lower p99 write latency, depending on the mix of
inserts, updates, and deletes.
**Scaling with shards.** An RTE table can be split into
[shards](/sql/ddl/table#properties) that write independently, and a single
table's ingest grows with its shard count. In our testing on AWS, with every
index on the table (vector, full-text, and BTree) kept current, a single table
reached these peak rates:
| Shards | Peak ingest |
| --: | --: |
| 1 | 90 MB/s |
| 2 | 170 MB/s |
| 4 | 340 MB/s |
| 8 | 625 MB/s |
| 16 | 690 MB/s |
| 32 | 885 MB/s |
Adding shards scales a table's ingest nearly linearly.
### Reads {#read-performance}
Reads on an RTE table stay close to those on a fully compacted table, even
with the fresh tier at its fullest before compaction.
The table below compares a 5M-row table with a 128-dimension IVF-PQ index plus
BTree and FTS indexes, with and without RTE. The RTE table holds one
unflushed memtable and three SSTables, each from a 100k-row write wave, half of
which are updates to existing rows. Latencies are p50 in milliseconds.
| Query | Without RTE | With RTE |
| --- | --: | --: |
| Primary-key lookup | 62 | 63 |
| Vector search | 68 | 71 |
| Vector search with prefilter | 90 | 91 |
| Full-text term | 63 | 64 |
| Full-text phrase | 68 | 82 |
| Hybrid search (RRF) | 68 | 83 |
| SQL point lookup | 189 | 187 |
Most queries are within a few milliseconds of the base table. Hybrid and phrase
queries add roughly 15 ms to merge the two tiers.
<Note>
Support for all table operations on RTE tables is coming soon. Time travel,
branching, `count_rows`, row IDs (`with_row_id`), and `TABLESAMPLE` are not yet
available.
</Note>
## Next steps {#next-steps}
- [Merge insert](/ingestion/merge-insert)
- [Background jobs](/maintenance/background-jobs#wal)
- [`ALTER TABLE ... SET RTE`](/sql/ddl/table#properties)
docs/web/training/torch.mdx
+135 -28
View File
@@ -1,7 +1,7 @@
---
title: "PyTorch Integration"
sidebarTitle: "PyTorch"
description: Learn how to use LanceDB with PyTorch for training and inference.
description: Use a LanceDB table as a PyTorch IterableDataset, mapped onto the `torch.utils.data` concepts you already know.
icon: fire
availability:
oss: available
@@ -11,44 +11,151 @@ availability:
tables in the Python SDK.
---
Use LanceDB tables as input to a PyTorch `DataLoader`. Select the columns you
need, choose a batch format, and read data in parallel for training or inference.
`StreamingDataset` is a LanceDB table exposed as a PyTorch
[`IterableDataset`](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset),
so it drops into a stock [`DataLoader`](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader).
It handles per-epoch shuffling, sharding across ranks and workers, prefetching, and exact resume, so you
don't write those yourself. The shuffle is fixed when the dataset is constructed, so each epoch needs a new
`StreamingDataset` built with that epoch's number.
## Quickstart {#quickstart}
The `Table` class in LanceDB implements a contract for a PyTorch
[Dataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset).
This means you can simply use a LanceDB table in a PyTorch dataloader directly.
```py Python icon=Python
import lancedb, pyarrow as pa, torch
from torch import nn
from lancedb.streaming import StreamingDataset
```py Python icon=Python
import lancedb
import torch
import pyarrow as pa
from lancedb.util import tbl_to_tensor
def to_samples(rb: pa.RecordBatch):
f = lambda c: torch.from_dlpack(rb[c].flatten()).view(len(rb), -1)
rows = zip(f("observation_state"), f("action"))
return [{"state": s, "action": a} for s, a in rows]
mem_db = lancedb.connect("memory://")
table = mem_db.create_table("test_table", pa.table({"a": range(1000)}))
uri = "hf://datasets/lance-format/lerobot-xvla-soft-fold/data"
db = lancedb.connect(uri, storage_options={"hf_enable_resolve_cache": "true"})
table = db.open_table("frames")
train = StreamingDataset(
table, columns=["observation_state", "action"],
transform=to_samples)
# Any LanceDB table can be used as a PyTorch Dataset
dataloader = torch.utils.data.DataLoader(
table, batch_size=1024, shuffle=True, collate_fn=tbl_to_tensor
)
for batch in dataloader:
print(batch)
model = nn.Sequential(nn.Linear(96, 256), nn.ReLU(), nn.Linear(256, 14))
opt = torch.optim.AdamW(model.parameters(), lr=1e-3)
for batch in torch.utils.data.DataLoader(train, batch_size=256):
loss = nn.MSELoss()(model(batch["state"]), batch["action"])
opt.zero_grad()
loss.backward()
opt.step()
```
Although the `Table` class in LanceDB implements the `torch.utils.data.Dataset` interface, you may find that using
a table [Permutation](/training/data-loading) is more flexible.
`StreamingDataset` is a standard PyTorch `IterableDataset`, so it works with everything built for one: a stock
`DataLoader`, its default `collate_fn`, and worker processes. The transform returns plain tensors, so the
model, loss and optimizer are unmodified PyTorch. Here it streams the
[`lerobot-xvla-soft-fold`](https://huggingface.co/datasets/lance-format/lerobot-xvla-soft-fold) dataset
straight from the Hugging Face Hub to train a policy that predicts each `action` from the robot's
`observation_state`.
```py Python icon=Python
from lancedb.permutation import Permutation
## PyTorch concepts in LanceDB {#concept-map}
permutation = Permutation.identity(table)
dataloader = torch.utils.data.DataLoader(permutation)
| In PyTorch you would write | In LanceDB you use |
| --- | --- |
| An `IterableDataset` that reads files in `__iter__` | `StreamingDataset(table)` |
| `DistributedSampler`, or sharding by `get_worker_info()` in `__iter__` | `rank=` and `world_size=`. Each rank and each DataLoader worker reads its own disjoint splits |
| Decoding in `__getitem__` or `collate_fn` | `transform=`, which receives a whole Arrow `RecordBatch` and runs in a thread pool |
| `random_split`, `Subset`, or reading only the fields a model needs | `filter="<SQL predicate>"` and `columns=[...]`. Unselected columns are never read from storage |
| `StatefulDataLoader` | `state_dict()` and `load_state_dict()`, plus `StreamingDataLoader` for workers |
The dataset does not copy your data. At construction it computes a shuffled assignment of row IDs to
`num_splits` fixed splits, then reads rows from the table as it iterates. It reads the table version
that exists at construction, so rows appended later do not appear mid-epoch.
## Optimizations {#optimizations}
The quickstart works as written. These sections cover what to tune as training scales up: the transform that
builds each sample, running across many GPUs, and the output format of a `Table` or `Permutation`.
### Transform {#transform}
A `transform` turns a batch of Arrow rows into the per-sample items your `DataLoader` collates. It receives a
whole `pyarrow.RecordBatch` (`read_batch_size` rows, 64 by default) rather than one sample at a time, so
vectorized code is cheap, and it must return exactly one item per row. The quickstart's transform is the
whole pattern: columns become tensors and each row becomes a dict, which the default `collate_fn` stacks into
a batch. Without a transform, rows arrive as plain Python dicts.
Heavier per-sample work belongs in the same function, such as decoding video. Store each clip in a blob v2
column (`lancedb.blob("video")`). A scan returns small descriptors instead of the video bytes, so reading
other columns never pulls video in. `fetch_blob_files` opens the clips for a batch of rows as file handles,
and torchcodec's `VideoDecoder` decodes just the frames you ask for. `ExtractFrames` samples 8 evenly spaced
frames from each clip:
```py Python icon=Python
import functools
from torchcodec.decoders import VideoDecoder
class ExtractFrames:
def __init__(self, uri: str, name: str, num_frames: int = 8) -> None:
self.uri, self.name, self.num_frames = uri, name, num_frames
@functools.cached_property
def table(self) -> lancedb.table.Table:
return lancedb.connect(self.uri).open_table(self.name)
def __call__(self, rb: pa.RecordBatch) -> list[dict[str, torch.Tensor]]:
samples = []
for blob in self.table.fetch_blob_files("video", rb["_rowid"]):
decoder = VideoDecoder(blob)
idx = torch.linspace(0, len(decoder) - 1, self.num_frames).long()
samples.append({"frames": decoder.get_frames_at(idx.tolist()).data})
return samples
# `clips` is the table "clips" in the database at `uri`, with a blob v2 column "video"
train = StreamingDataset(clips, columns=["_rowid"], transform=ExtractFrames(uri, "clips"))
```
## Output Formats {#output-formats}
Transforms run in a thread pool, one thread per CPU by default (`transform_parallelism`), overlapping with
storage reads and your training step. NumPy, PyArrow and PyTorch release the GIL, so vectorized
work runs in parallel. A pure-Python loop does not. If a profile shows the transform is the bottleneck, move
it into [DataLoader workers](#using-multiple-dataloader-workers).
Real data has bad rows, such as a truncated frame. Raise inside the transform and set
`on_transform_error="skip"` (or `"warn"` to log each failure). LanceDB re-runs the failing batch one row at a
time, drops only the rows that raise, and counts them in `dataset.rows_skipped`. When the bad rows match a SQL
predicate, prefer `filter=`: it runs before the splits are built, so every split keeps the same length.
### Distributed Training {#distributed-training}
`StreamingDataset` takes the place of `DistributedSampler`. Pass the rank and world size, and each rank reads
its own disjoint part of the table, together covering it once per epoch:
```py Python icon=Python
import torch.distributed as dist
dist.init_process_group("nccl")
train = StreamingDataset(
table, columns=["observation_state", "action"], transform=to_samples,
num_splits=48, shuffle_seed=42, epoch=epoch,
rank=dist.get_rank(), world_size=dist.get_world_size())
```
Everything after the dataset is standard distributed PyTorch. Wrap the model in `DistributedDataParallel`
and pass `batch_size` to the `DataLoader` per GPU, so the global batch is `batch_size * world_size`. Nothing
in the data path changes.
Three arguments matter in distributed runs:
- **`num_splits`** is how many fixed streams the table is divided into. It must be divisible by `world_size`,
and by `world_size * num_workers` when you use DataLoader workers. Choose a number with many divisors, such
as 48, and runs on 8, 4 or 2 GPUs see the same data at every step, provided the global batch size is a multiple of
`num_splits`.
- **`shuffle_seed` and `epoch`** must be identical on every rank. Pass a fixed `shuffle_seed` and increase
`epoch` each epoch. `shuffle_seed=None` draws a different random seed in each process, so ranks would read
overlapping rows and miss others.
- **`rank` and `world_size`** come from your launcher, for example `torchrun`.
To resume after a failure, save `train.state_dict()` next to the model. It records progress per split, so the
resumed run continues from the same row even on a different number of GPUs. See
[Parallelism & Elasticity](/training/data-loading#parallelism-&-elasticity) and
[Checkpointing](/training/data-loading#checkpointing) for the batch-size rules and the multi-worker case.
### Output Formats {#output-formats}
By default, a `Table` data loader will emit Arrow data. `collate_fn` is PyTorch's batching hook: PyTorch calls it to
turn the fetched items into one batch. PyTorch's default collate function only knows how to combine tensors, NumPy
@@ -68,7 +175,7 @@ the Arrow data in different ways. The `arrow` and `polars` formats will always
`pandas`, and `torch_col` formats will also avoid data copies in most cases. The `python`, `python_col`, and
`torch` formats will all require at least one full copy of the data and are the slowest options.
### Using the torch column format with a torch data loader {#using-the-torch-column-format-with-a-torch-data-loader}
#### Using the torch column format with a torch data loader {#using-the-torch-column-format-with-a-torch-data-loader}
The `torch_col` format is the most efficient way to convert from Arrow to a `torch.Tensor`. It will convert the
entire Arrow batch to a _column-major_ `torch.Tensor`. In other words, given C columns and R rows, the resulting
java/.project
+28
View File
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>lancedb-parent</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
<filteredResources>
<filter>
<id>1786980408448</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>
java/.settings/org.eclipse.core.resources.prefs
+2
View File
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
encoding/<project>=UTF-8
java/.settings/org.eclipse.m2e.core.prefs
+4
View File
@@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1
java/README.md
+9 -9
View File
@@ -29,34 +29,34 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder()
.build();
```
## MemWAL write path
## RTE write path
Most table operations reach LanceDB through the `LanceNamespace` above, which is
generated from the Lance Namespace specification. The MemWAL LSM routes are not part
generated from the Lance Namespace specification. The RTE LSM routes are not part
of that specification, so they are issued through a separate client:
```java
import com.lancedb.LanceDbRestClient;
import com.lancedb.LanceDbTableMemWal;
import com.lancedb.MemWalWriteSpec;
import com.lancedb.LanceDbTableRTE;
import com.lancedb.RTEWriteSpec;
LanceDbRestClient client = LanceDbNamespaceClientBuilder.newBuilder()
.apiKey("your_lancedb_cloud_api_key")
.database("your_database_name")
.buildRestClient();
LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table");
LanceDbTableRTE rte = new LanceDbTableRTE(client, "my_table");
// Route future merge_insert upserts through the MemWAL, hash-bucketed by `id`.
memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16));
// Route future merge_insert upserts through the RTE, hash-bucketed by `id`.
rte.enableRTE(RTEWriteSpec.bucket("id", 16));
// ... merge_insert traffic ...
// Converge the fresh tier into the base table.
memWal.checkpointMemWal();
rte.checkpointRTE();
// Inspect live per-table-shard state.
memWal.getMemWalStats().ifPresent(stats -> stats.tableShards().forEach(tableShard ->
rte.getRTEStats().ifPresent(stats -> stats.tableShards().forEach(tableShard ->
System.out.println(tableShard.shardId() + ": " + tableShard.sstables().size() + " SSTables")));
client.close();
java/lancedb-core/.classpath
+57
View File
@@ -0,0 +1,57 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
<attribute name="test" value="true"/>
<attribute name="optional" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" path="target/generated-sources/annotations">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="ignore_optional_problems" value="true"/>
<attribute name="m2e-apt" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="target/generated-test-sources/test-annotations">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
<attribute name="ignore_optional_problems" value="true"/>
<attribute name="m2e-apt" value="true"/>
<attribute name="test" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
java/lancedb-core/.project
+34
View File
@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>lancedb-core</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
<filteredResources>
<filter>
<id>1786980408445</id>
<name></name>
<type>30</type>
<matcher>
<id>org.eclipse.core.resources.regexFilterMatcher</id>
<arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>
</matcher>
</filter>
</filteredResources>
</projectDescription>
java/lancedb-core/.settings/org.eclipse.core.resources.prefs
+5
View File
@@ -0,0 +1,5 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/test/java=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8
java/lancedb-core/.settings/org.eclipse.jdt.apt.core.prefs
+2
View File
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
org.eclipse.jdt.apt.aptEnabled=false
java/lancedb-core/.settings/org.eclipse.jdt.core.prefs
+9
View File
@@ -0,0 +1,9 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=11
org.eclipse.jdt.core.compiler.compliance=11
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.processAnnotations=disabled
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=11
java/lancedb-core/.settings/org.eclipse.m2e.core.prefs
+4
View File
@@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.9</version>
<version>0.41.0-beta.11</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/lancedb-core/src/main/java/com/lancedb/JsonFields.java
+2 -2
View File
@@ -20,11 +20,11 @@ import com.fasterxml.jackson.databind.JsonNode;
*
* <p>Every reader fails closed: a missing, null, or wrong-typed field throws rather than
* defaulting. That mirrors the serde decoding the Rust client applies to the same payloads in
* {@code rust/lancedb/src/table/memwal_stats.rs}, where a required field has no default and a
* {@code rust/lancedb/src/table/rte_stats.rs}, where a required field has no default and a
* malformed response is an error rather than a zero.
*
* <p>The alternative — Jackson's {@code path()}, which yields a missing node that reads as an empty
* array or a zero — is unsafe here because {@link LanceDbTableMemWal#checkpointMemWal()} decides
* array or a zero — is unsafe here because {@link LanceDbTableRTE#checkpointRTE()} decides
* convergence from these numbers. A defaulted {@code sstables} array is indistinguishable from a
* drained one, so a malformed response would report a checkpoint that never happened.
*/
java/lancedb-core/src/main/java/com/lancedb/LanceDbNamespaceClientBuilder.java
+2 -2
View File
@@ -151,8 +151,8 @@ public class LanceDbNamespaceClientBuilder {
* Build a {@link LanceDbRestClient} for the same endpoint.
*
* <p>Needed only for LanceDB routes that the Lance Namespace specification does not cover — the
* MemWAL write path, reached through {@link LanceDbTableMemWal}. Every other table
* operation belongs on the {@link LanceNamespace} from {@link #build()}.
* RTE write path, reached through {@link LanceDbTableRTE}. Every other table operation belongs on
* the {@link LanceNamespace} from {@link #build()}.
*
* <p>The returned client owns an HTTP connection pool; close it when you are done with it.
*
java/lancedb-core/src/main/java/com/lancedb/LanceDbRestClient.java
+7 -8
View File
@@ -31,9 +31,9 @@ import java.io.UncheckedIOException;
* specification does not cover.
*
* <p>Most table operations reach LanceDB through {@link org.lance.namespace.LanceNamespace}, which
* is generated from the namespace spec. A handful of routes — the MemWAL write path in
* particular — are served by the same endpoint but are not part of that spec, so they are issued
* directly here. See {@link LanceDbTableMemWal}.
* is generated from the namespace spec. A handful of routes — the RTE write path in particular —
* are served by the same endpoint but are not part of that spec, so they are issued directly here.
* See {@link LanceDbTableRTE}.
*
* <p>Obtain one from {@link LanceDbNamespaceClientBuilder#buildRestClient()}.
*/
@@ -50,8 +50,8 @@ public class LanceDbRestClient implements Closeable {
this.apiKey = apiKey;
this.database = database;
// Automatic retries off, deliberately. The default strategy retries 429 and 503 —
// exactly the two statuses LanceDbTableMemWal.checkpointMemWal() acts on — which would
// silently double its explicit retry budget and would also retry compact_memwal in
// exactly the two statuses LanceDbTableRTE.checkpointRTE() acts on — which would
// silently double its explicit retry budget and would also retry compact_rte in
// place, where the loop is designed to fall through to a fresh stats poll instead.
// The checkpoint loop owns the 421/429/503 transitions; the transport must not.
this.http = HttpClients.custom().disableAutomaticRetries().build();
@@ -98,9 +98,8 @@ public class LanceDbRestClient implements Closeable {
/**
* A non-2xx response.
*
* <p>The status is exposed because callers act on it: {@link
* LanceDbTableMemWal#checkpointMemWal()} treats 429 and 503 as retryable and 421 as a lost node
* claim.
* <p>The status is exposed because callers act on it: {@link LanceDbTableRTE#checkpointRTE()}
* treats 429 and 503 as retryable and 421 as a lost node claim.
*/
public static class HttpException extends RuntimeException {
private static final long serialVersionUID = 1L;
java/lancedb-core/src/main/java/com/lancedb/LanceDbTableMemWal.java → java/lancedb-core/src/main/java/com/lancedb/LanceDbTableRTE.java
+58 -58
View File
@@ -22,11 +22,11 @@ import java.util.Optional;
import java.util.OptionalLong;
/**
* The MemWAL write path for one LanceDB Cloud or Enterprise table.
* The RTE write path for one LanceDB Cloud or Enterprise table.
*
* <p>Installing an {@link MemWalWriteSpec} routes {@code mergeInsert} upserts through Lance's
* MemWAL — an LSM-style append — instead of the standard merge path. Rows land in an in-memory
* memtable, freeze into SSTables, and are merged into the base table by compaction.
* <p>Installing an {@link RTEWriteSpec} routes {@code mergeInsert} upserts through Lance's RTE — an
* LSM-style append — instead of the standard merge path. Rows land in an in-memory memtable, freeze
* into SSTables, and are merged into the base table by compaction.
*
* <p>These routes are not part of the Lance Namespace specification, so they are issued directly
* rather than through {@link org.lance.namespace.LanceNamespace}.
@@ -37,23 +37,23 @@ import java.util.OptionalLong;
* .database("your_database_name")
* .buildRestClient();
*
* LanceDbTableMemWal memWal = new LanceDbTableMemWal(client, "my_table");
* memWal.enableMemWal(MemWalWriteSpec.bucket("id", 16));
* LanceDbTableRTE rte = new LanceDbTableRTE(client, "my_table");
* rte.enableRTE(RTEWriteSpec.bucket("id", 16));
* // ... merge_insert traffic ...
* memWal.checkpointMemWal();
* rte.checkpointRTE();
* }</pre>
*/
public class LanceDbTableMemWal {
public class LanceDbTableRTE {
/**
* Interval between {@code get_memwal_stats} polls during a checkpoint. One interval is roughly
* one compaction pass, the granularity at which the answer can change.
* Interval between {@code get_rte_stats} polls during a checkpoint. One interval is roughly one
* compaction pass, the granularity at which the answer can change.
*/
private static final long POLL_INTERVAL_MS = 5_000L;
/**
* Cap on re-issues from {@code flushMemWal} after a 421, so a crash-looping node cannot turn
* flush → compact → 421 → flush into a spin.
* Cap on re-issues from {@code flushRTE} after a 421, so a crash-looping node cannot turn flush →
* compact → 421 → flush into a spin.
*
* <p>Deliberately not shared with {@link #MAX_RETRIES}: a claim that keeps evaporating is a
* broken node, while contention is routine and wants a real budget.
@@ -79,7 +79,7 @@ public class LanceDbTableMemWal {
* @param tableIdentifier The table's full identifier, {@code $}-delimited when it sits inside a
* namespace, such as {@code analytics$events}.
*/
public LanceDbTableMemWal(LanceDbRestClient client, String tableIdentifier) {
public LanceDbTableRTE(LanceDbRestClient client, String tableIdentifier) {
if (client == null) {
throw new IllegalArgumentException("Client cannot be null");
}
@@ -91,46 +91,46 @@ public class LanceDbTableMemWal {
}
/**
* Install an {@link MemWalWriteSpec} on this table, selecting the MemWAL write path for
* future {@code mergeInsert} calls.
* Install an {@link RTEWriteSpec} on this table, selecting the RTE write path for future {@code
* mergeInsert} calls.
*
* <p>All variants require the table to have an unenforced primary key; bucket sharding
* additionally requires it to be the single column being bucketed.
*/
public void enableMemWal(MemWalWriteSpec spec) {
public void enableRTE(RTEWriteSpec spec) {
if (spec == null) {
throw new IllegalArgumentException("Spec cannot be null");
}
client.post(route("enable_memwal"), spec.toRequestBody());
client.post(route("enable_rte"), spec.toRequestBody());
}
/**
* Remove the {@link MemWalWriteSpec} from this table, reverting to the standard {@code
* mergeInsert} write path.
* Remove the {@link RTEWriteSpec} from this table, reverting to the standard {@code mergeInsert}
* write path.
*
* <p>Errors if no spec is currently set.
*/
public void disableMemWal() {
client.post(route("disable_memwal"), null);
public void disableRTE() {
client.post(route("disable_rte"), null);
}
/**
* Read the {@link MemWalWriteSpec} currently installed on this table.
* Read the {@link RTEWriteSpec} currently installed on this table.
*
* <p>Empty when the MemWAL write path is not enabled at all — no spec has been set, or one was
* removed with {@link #disableMemWal()}. That is a different answer from a present spec whose
* {@link MemWalWriteSpec#maintainedIndexes()} is null, which selects indexes automatically.
* <p>Empty when the RTE write path is not enabled at all — no spec has been set, or one was
* removed with {@link #disableRTE()}. That is a different answer from a present spec whose {@link
* RTEWriteSpec#maintainedIndexes()} is null, which selects indexes automatically.
*
* <p>The spec read back is the one that was installed, selection included: null maintains every
* supported index the table has now or gains later, an empty list maintains none, and a non-empty
* list maintains exactly those. All three round-trip.
*/
public Optional<MemWalWriteSpec> getMemWalSpec() {
JsonNode response = client.post(route("get_memwal_spec"), null);
if (response == null || !response.hasNonNull("memwal_spec")) {
public Optional<RTEWriteSpec> getRTESpec() {
JsonNode response = client.post(route("get_rte_spec"), null);
if (response == null || !response.hasNonNull("rte_spec")) {
return Optional.empty();
}
return Optional.of(MemWalWriteSpec.fromJson(response.get("memwal_spec")));
return Optional.of(RTEWriteSpec.fromJson(response.get("rte_spec")));
}
/**
@@ -139,56 +139,56 @@ public class LanceDbTableMemWal {
* <p>Returns once the freeze is committed. Freezing an empty memtable is a no-op, so this is safe
* to call repeatedly.
*/
public void flushMemWal() {
client.post(route("flush_memwal"), null);
public void flushRTE() {
client.post(route("flush_rte"), null);
}
/**
* Trigger a background SSTable compaction pass per table shard.
*
* <p>Returns once the passes are <em>dispatched</em>, not once they finish — watch {@link
* #getMemWalStats}, or use {@link #checkpointMemWal} to wait for convergence.
* #getRTEStats}, or use {@link #checkpointRTE} to wait for convergence.
*/
public void compactMemWal() {
client.post(route("compact_memwal"), null);
public void compactRTE() {
client.post(route("compact_rte"), null);
}
/**
* Read live per-table-shard MemWAL state.
* Read live per-table-shard RTE state.
*
* <p>Answers "how far behind is my fresh tier", "which table shard is hot", and "why is my
* fresh-tier vector search brute-force". Mutates no table state.
*
* <p>Empty only when the MemWAL write path is not enabled — that is, when the server sends an
* absent or null {@code memwal_stats}. A stats object that is present is decoded strictly, and a
* malformed one throws rather than decoding to something empty, because {@link #checkpointMemWal}
* reads convergence out of these numbers and cannot tell a defaulted array from a drained one.
* <p>Empty only when the RTE write path is not enabled — that is, when the server sends an absent
* or null {@code rte_stats}. A stats object that is present is decoded strictly, and a malformed
* one throws rather than decoding to something empty, because {@link #checkpointRTE} reads
* convergence out of these numbers and cannot tell a defaulted array from a drained one.
*
* @param includeSstableRows Also count rows per SSTable. Off by default because each count opens
* an uncached Lance dataset.
* @throws IllegalStateException if the response is absent or does not decode.
*/
public Optional<MemWalStats> getMemWalStats(boolean includeSstableRows) {
public Optional<RTEStats> getRTEStats(boolean includeSstableRows) {
Map<String, Object> body = new LinkedHashMap<String, Object>();
body.put("include_sstable_rows", includeSstableRows);
JsonNode response = client.post(route("get_memwal_stats"), body);
JsonNode response = client.post(route("get_rte_stats"), body);
if (response == null) {
throw new IllegalStateException("get_memwal_stats returned an empty response body");
throw new IllegalStateException("get_rte_stats returned an empty response body");
}
JsonNode stats = response.get("memwal_stats");
JsonNode stats = response.get("rte_stats");
if (stats == null || stats.isNull()) {
return Optional.empty();
}
return Optional.of(MemWalStats.fromJson(stats));
return Optional.of(RTEStats.fromJson(stats));
}
/** Equivalent to {@code getMemWalStats(false)}. */
public Optional<MemWalStats> getMemWalStats() {
return getMemWalStats(false);
/** Equivalent to {@code getRTEStats(false)}. */
public Optional<RTEStats> getRTEStats() {
return getRTEStats(false);
}
/**
* Converge this table's MemWAL write path into its base table.
* Converge this table's RTE write path into its base table.
*
* <p>Freezes once, fixes a target watermark from the resulting SSTables, then triggers compaction
* and polls until those SSTables are gone. The target set is fixed at the start, so sstables
@@ -196,31 +196,31 @@ public class LanceDbTableMemWal {
* load, and what makes it best-effort: it converges the fresh tier as of some instant.
* Idempotent, abandonable at any point, safe on a cadence.
*
* <p>The loop runs here, not on the server: {@link #compactMemWal} dispatches a pass and returns,
* so nothing holds a socket and a client can vanish mid-operation with nothing to reconcile.
* <p>The loop runs here, not on the server: {@link #compactRTE} dispatches a pass and returns, so
* nothing holds a socket and a client can vanish mid-operation with nothing to reconcile.
* Completion is read from generation numbers in the shard manifest — durable state, unlike a
* count in a compact response, which a concurrent write invalidates.
*
* <p>No liveness bound — the caller owns the deadline. The compactor pool is shared across
* tables, so a checkpoint queued behind unrelated work looks exactly like one that is merging.
*/
public void checkpointMemWal() {
public void checkpointRTE() {
for (int reissue = 0; reissue <= MAX_REISSUES; reissue++) {
// The freeze turns everything written before this call into an SSTable, so the
// watermark has to be read after it. Idempotent: freezing an empty memtable is a
// no-op, so a re-issue does not churn empty sstables.
if (issueVoid(this::flushMemWal)) {
if (issueVoid(this::flushRTE)) {
backoff(reissue);
continue;
}
Attempt<Optional<MemWalStats>> stats = issue(() -> getMemWalStats(false));
Attempt<Optional<RTEStats>> stats = issue(() -> getRTEStats(false));
if (stats.lostClaim) {
backoff(reissue);
continue;
}
if (!stats.value.isPresent()) {
// Not WAL-backed; flushMemWal would have errored first but for a race.
// Not WAL-backed; flushRTE would have errored first but for a race.
return;
}
@@ -235,7 +235,7 @@ public class LanceDbTableMemWal {
backoff(reissue);
}
throw new IllegalStateException(
"checkpointMemWal: the owning node kept losing its claim; re-issued from flush the maximum "
"checkpointRTE: the owning node kept losing its claim; re-issued from flush the maximum "
+ "number of times");
}
@@ -246,7 +246,7 @@ public class LanceDbTableMemWal {
*/
private boolean drainToTargets(Map<String, Long> targets) {
while (true) {
Attempt<Optional<MemWalStats>> stats = issue(() -> getMemWalStats(false));
Attempt<Optional<RTEStats>> stats = issue(() -> getRTEStats(false));
if (stats.lostClaim) {
return false;
}
@@ -277,7 +277,7 @@ public class LanceDbTableMemWal {
if (!allCompacting) {
try {
compactMemWal();
compactRTE();
} catch (LanceDbRestClient.HttpException e) {
if (isLostClaim(e)) {
return false;
@@ -296,7 +296,7 @@ public class LanceDbTableMemWal {
}
/** The newest SSTable generation held by each table shard, skipping table shards holding none. */
private static Map<String, Long> newestSstableGenerations(MemWalStats stats) {
private static Map<String, Long> newestSstableGenerations(RTEStats stats) {
Map<String, Long> targets = new HashMap<String, Long>();
for (TableShardStats tableShard : stats.tableShards()) {
OptionalLong newest = tableShard.newestSstableGeneration();
java/lancedb-core/src/main/java/com/lancedb/MemWalStats.java → java/lancedb-core/src/main/java/com/lancedb/RTEStats.java
+7 -7
View File
@@ -20,18 +20,18 @@ import java.util.Collections;
import java.util.List;
/**
* Live per-table-shard MemWAL state, as returned by {@link LanceDbTableMemWal#getMemWalStats()}.
* Live per-table-shard RTE state, as returned by {@link LanceDbTableRTE#getRTEStats()}.
*
* <p>Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are the caller's
* to compute. There is no "LSM is off" shape — that case is an empty {@link java.util.Optional},
* because a stats object of zeros would read as measurements.
*/
public class MemWalStats {
private static final String CONTEXT = "memwal stats";
public class RTEStats {
private static final String CONTEXT = "rte stats";
private final List<TableShardStats> tableShards;
MemWalStats(List<TableShardStats> tableShards) {
RTEStats(List<TableShardStats> tableShards) {
this.tableShards = Collections.unmodifiableList(tableShards);
}
@@ -40,17 +40,17 @@ public class MemWalStats {
return tableShards;
}
static MemWalStats fromJson(JsonNode node) {
static RTEStats fromJson(JsonNode node) {
JsonFields.requiredObject(node, CONTEXT);
List<TableShardStats> tableShards = new ArrayList<TableShardStats>();
for (JsonNode tableShard : JsonFields.requiredArray(node, "table_shards", CONTEXT)) {
tableShards.add(TableShardStats.fromJson(tableShard));
}
return new MemWalStats(tableShards);
return new RTEStats(tableShards);
}
@Override
public String toString() {
return "MemWalStats{tableShards=" + tableShards + "}";
return "RTEStats{tableShards=" + tableShards + "}";
}
}
java/lancedb-core/src/main/java/com/lancedb/MemWalWriteSpec.java → java/lancedb-core/src/main/java/com/lancedb/RTEWriteSpec.java
+26 -26
View File
@@ -23,22 +23,23 @@ import java.util.List;
import java.util.Map;
/**
* Specification selecting Lance's MemWAL LSM-style write path for {@code mergeInsert}.
* Specification enabling the realtime engine (RTE), which routes {@code mergeInsert} through
* Lance's MemWAL LSM-style write path.
*
* <p>Construct via {@link #bucket}, {@link #identity}, or {@link #unsharded}, then optionally chain
* {@link #withMaintainedIndexes} and {@link #withWriterConfigDefaults}. Install it with {@link
* LanceDbTableMemWal#enableMemWal} and remove it with {@link LanceDbTableMemWal#disableMemWal}.
* LanceDbTableRTE#enableRTE} and remove it with {@link LanceDbTableRTE#disableRTE}.
*
* <p>This is deliberately not {@code org.lance.memwal.InitializeMemWalParams}. That type is Lance's
* own, and its maintained-index default is the opposite of this one: it defaults to maintaining
* <em>nothing</em>, while a fresh spec here maintains <em>every</em> index. It also cannot express
* the null that asks the server to resolve the set.
*/
public class MemWalWriteSpec {
public class RTEWriteSpec {
private static final String CONTEXT = "get_memwal_spec response";
private static final String CONTEXT = "get_rte_spec response";
/** How writes are routed to MemWAL shards. */
/** How writes are routed to RTE shards. */
public enum Sharding {
/** Hash-bucket writes by a scalar column. */
BUCKET("bucket"),
@@ -73,7 +74,7 @@ public class MemWalWriteSpec {
private final List<String> maintainedIndexes;
private final Map<String, String> writerConfigDefaults;
private MemWalWriteSpec(
private RTEWriteSpec(
Sharding sharding,
String column,
Integer numBuckets,
@@ -95,11 +96,11 @@ public class MemWalWriteSpec {
* @param column A non-nested column with a supported scalar type.
* @param numBuckets The number of buckets, in {@code [1, 1024]}.
*/
public static MemWalWriteSpec bucket(String column, int numBuckets) {
public static RTEWriteSpec bucket(String column, int numBuckets) {
if (column == null || column.trim().isEmpty()) {
throw new IllegalArgumentException("Column cannot be null or empty");
}
return new MemWalWriteSpec(
return new RTEWriteSpec(
Sharding.BUCKET, column, numBuckets, null, new HashMap<String, String>());
}
@@ -111,29 +112,28 @@ public class MemWalWriteSpec {
* with a given primary key must always produce the same {@code column} value, or upserts of that
* key can land in different shards and a stale version can win.
*/
public static MemWalWriteSpec identity(String column) {
public static RTEWriteSpec identity(String column) {
if (column == null || column.trim().isEmpty()) {
throw new IllegalArgumentException("Column cannot be null or empty");
}
return new MemWalWriteSpec(
Sharding.IDENTITY, column, null, null, new HashMap<String, String>());
return new RTEWriteSpec(Sharding.IDENTITY, column, null, null, new HashMap<String, String>());
}
/** No sharding — every write goes to a single MemWAL shard — maintaining every index. */
public static MemWalWriteSpec unsharded() {
return new MemWalWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap<String, String>());
/** No sharding — every write goes to a single RTE shard — maintaining every index. */
public static RTEWriteSpec unsharded() {
return new RTEWriteSpec(Sharding.UNSHARDED, null, null, null, new HashMap<String, String>());
}
/**
* Set the indexes the MemWAL keeps up to date as rows are appended.
* Set the indexes the RTE keeps up to date as rows are appended.
*
* <p>Pass {@code null} — the default for a fresh spec — to maintain every index the table has,
* including ones created later. Pass an empty list to maintain none.
*
* <p>Note that {@code null} and the empty list mean opposite things here.
*/
public MemWalWriteSpec withMaintainedIndexes(List<String> maintainedIndexes) {
return new MemWalWriteSpec(
public RTEWriteSpec withMaintainedIndexes(List<String> maintainedIndexes) {
return new RTEWriteSpec(
sharding,
column,
numBuckets,
@@ -150,11 +150,11 @@ public class MemWalWriteSpec {
* max_unflushed_memtable_bytes}, and {@code enable_memtable}. Duration knobs carry an {@code _ms}
* suffix, such as {@code max_wal_flush_interval_ms}.
*/
public MemWalWriteSpec withWriterConfigDefaults(Map<String, String> writerConfigDefaults) {
public RTEWriteSpec withWriterConfigDefaults(Map<String, String> writerConfigDefaults) {
if (writerConfigDefaults == null) {
throw new IllegalArgumentException("writerConfigDefaults cannot be null");
}
return new MemWalWriteSpec(
return new RTEWriteSpec(
sharding,
column,
numBuckets,
@@ -178,7 +178,7 @@ public class MemWalWriteSpec {
}
/**
* Which indexes the MemWAL maintains.
* Which indexes the RTE maintains.
*
* <p>Null selects automatically: every supported index the table has now or gains later. An empty
* list maintains none. A non-empty list maintains exactly those.
@@ -192,7 +192,7 @@ public class MemWalWriteSpec {
return Collections.unmodifiableMap(writerConfigDefaults);
}
/** Render this spec as the {@code enable_memwal} request body. */
/** Render this spec as the {@code enable_rte} request body. */
Map<String, Object> toRequestBody() {
Map<String, Object> shardingBody = new LinkedHashMap<String, Object>();
shardingBody.put("mode", sharding.wireName());
@@ -212,15 +212,15 @@ public class MemWalWriteSpec {
}
/**
* Rebuild a spec from a {@code get_memwal_spec} response body.
* Rebuild a spec from a {@code get_rte_spec} response body.
*
* <p>A null {@code maintained_indexes} selects every index the table has; an empty list selects
* none.
*/
static MemWalWriteSpec fromJson(JsonNode node) {
static RTEWriteSpec fromJson(JsonNode node) {
JsonNode shardingNode = node.get("sharding");
if (shardingNode == null || shardingNode.get("mode") == null) {
throw new IllegalStateException("get_memwal_spec response has no sharding mode");
throw new IllegalStateException("get_rte_spec response has no sharding mode");
}
Sharding sharding = Sharding.fromWireName(shardingNode.get("mode").asText());
@@ -253,12 +253,12 @@ public class MemWalWriteSpec {
.forEachRemaining(name -> defaults.put(name, defaultsNode.get(name).asText()));
}
return new MemWalWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults);
return new RTEWriteSpec(sharding, column, numBuckets, maintainedIndexes, defaults);
}
@Override
public String toString() {
return "MemWalWriteSpec{sharding="
return "RTEWriteSpec{sharding="
+ sharding
+ ", column="
+ column
java/lancedb-core/src/test/java/com/lancedb/LanceDbTableMemWalTest.java → java/lancedb-core/src/test/java/com/lancedb/LanceDbTableRTETest.java
+129 -133
View File
@@ -41,17 +41,17 @@ import java.util.concurrent.ConcurrentHashMap;
import static org.junit.jupiter.api.Assertions.*;
/**
* Unit tests for the MemWAL LSM routes, run against a scripted local HTTP server.
* Unit tests for the RTE LSM routes, run against a scripted local HTTP server.
*
* <p>The wire assertions mirror the Rust mocked-endpoint tests in {@code
* rust/lancedb/src/remote/table.rs}, which are the contract these routes have to match.
*/
public class LanceDbTableMemWalTest {
public class LanceDbTableRTETest {
private static final ObjectMapper MAPPER = new ObjectMapper();
private HttpServer server;
private LanceDbRestClient client;
private LanceDbTableMemWal memWal;
private LanceDbTableRTE rte;
private final List<String> requestPaths = Collections.synchronizedList(new ArrayList<String>());
private final List<String> requestBodies = Collections.synchronizedList(new ArrayList<String>());
@@ -101,7 +101,7 @@ public class LanceDbTableMemWalTest {
.database("test-db")
.endpoint("http://127.0.0.1:" + server.getAddress().getPort())
.buildRestClient();
memWal = new LanceDbTableMemWal(client, "my_table");
rte = new LanceDbTableRTE(client, "my_table");
}
@AfterEach
@@ -115,12 +115,12 @@ public class LanceDbTableMemWalTest {
// ===========================================================================
@Test
public void testEnableMemWalUnsharded() throws Exception {
enqueue("enable_memwal", 200, "");
public void testEnableRTEUnsharded() throws Exception {
enqueue("enable_rte", 200, "");
memWal.enableMemWal(MemWalWriteSpec.unsharded());
rte.enableRTE(RTEWriteSpec.unsharded());
assertEquals("/v1/table/my_table/enable_memwal/", requestPaths.get(0));
assertEquals("/v1/table/my_table/enable_rte/", requestPaths.get(0));
JsonNode body = MAPPER.readTree(requestBodies.get(0));
assertEquals("unsharded", body.get("sharding").get("mode").asText());
assertFalse(body.get("sharding").has("column"));
@@ -128,11 +128,10 @@ public class LanceDbTableMemWalTest {
}
@Test
public void testEnableMemWalBucket() throws Exception {
enqueue("enable_memwal", 200, "");
public void testEnableRTEBucket() throws Exception {
enqueue("enable_rte", 200, "");
memWal.enableMemWal(
MemWalWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
rte.enableRTE(RTEWriteSpec.bucket("id", 16).withMaintainedIndexes(Arrays.asList("id_idx")));
JsonNode body = MAPPER.readTree(requestBodies.get(0));
assertEquals("bucket", body.get("sharding").get("mode").asText());
@@ -143,10 +142,10 @@ public class LanceDbTableMemWalTest {
}
@Test
public void testEnableMemWalIdentity() throws Exception {
enqueue("enable_memwal", 200, "");
public void testEnableRTEIdentity() throws Exception {
enqueue("enable_rte", 200, "");
memWal.enableMemWal(MemWalWriteSpec.identity("tenant"));
rte.enableRTE(RTEWriteSpec.identity("tenant"));
JsonNode body = MAPPER.readTree(requestBodies.get(0));
assertEquals("identity", body.get("sharding").get("mode").asText());
@@ -160,55 +159,54 @@ public class LanceDbTableMemWalTest {
*/
@Test
public void testMaintainedIndexesNullAndEmptyAreDistinctOnTheWire() throws Exception {
enqueue("enable_memwal", 200, "");
enqueue("enable_rte", 200, "");
memWal.enableMemWal(MemWalWriteSpec.unsharded());
rte.enableRTE(RTEWriteSpec.unsharded());
JsonNode fresh = MAPPER.readTree(requestBodies.get(0));
assertTrue(fresh.has("maintained_indexes"), "the key must be present");
assertTrue(fresh.get("maintained_indexes").isNull(), "a fresh spec sends null, not []");
memWal.enableMemWal(
MemWalWriteSpec.unsharded().withMaintainedIndexes(Collections.<String>emptyList()));
rte.enableRTE(RTEWriteSpec.unsharded().withMaintainedIndexes(Collections.<String>emptyList()));
JsonNode none = MAPPER.readTree(requestBodies.get(1));
assertTrue(none.get("maintained_indexes").isArray());
assertEquals(0, none.get("maintained_indexes").size());
}
@Test
public void testEnableMemWalWriterConfigDefaults() throws Exception {
enqueue("enable_memwal", 200, "");
public void testEnableRTEWriterConfigDefaults() throws Exception {
enqueue("enable_rte", 200, "");
Map<String, String> defaults = new HashMap<String, String>();
defaults.put("max_memtable_rows", "50000");
memWal.enableMemWal(MemWalWriteSpec.unsharded().withWriterConfigDefaults(defaults));
rte.enableRTE(RTEWriteSpec.unsharded().withWriterConfigDefaults(defaults));
JsonNode body = MAPPER.readTree(requestBodies.get(0));
assertEquals("50000", body.get("writer_config_defaults").get("max_memtable_rows").asText());
}
@Test
public void testDisableMemWal() {
enqueue("disable_memwal", 200, "");
public void testDisableRTE() {
enqueue("disable_rte", 200, "");
memWal.disableMemWal();
rte.disableRTE();
assertEquals("/v1/table/my_table/disable_memwal/", requestPaths.get(0));
assertEquals("/v1/table/my_table/disable_rte/", requestPaths.get(0));
assertEquals("", requestBodies.get(0));
}
@Test
public void testGetMemWalSpec() {
public void testGetRTESpec() {
enqueue(
"get_memwal_spec",
"get_rte_spec",
200,
"{\"memwal_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\","
"{\"rte_spec\":{\"sharding\":{\"mode\":\"bucket\",\"column\":\"id\","
+ "\"num_buckets\":16},\"maintained_indexes\":[\"id_idx\"],"
+ "\"writer_config_defaults\":{\"durable_write\":\"true\"}}}");
Optional<MemWalWriteSpec> spec = memWal.getMemWalSpec();
Optional<RTEWriteSpec> spec = rte.getRTESpec();
assertTrue(spec.isPresent());
assertEquals(MemWalWriteSpec.Sharding.BUCKET, spec.get().sharding());
assertEquals(RTEWriteSpec.Sharding.BUCKET, spec.get().sharding());
assertEquals("id", spec.get().column());
assertEquals(Integer.valueOf(16), spec.get().numBuckets());
assertEquals(Arrays.asList("id_idx"), spec.get().maintainedIndexes());
@@ -217,28 +215,28 @@ public class LanceDbTableMemWalTest {
/** A null selection, every index the table has, reads back as null rather than an empty list. */
@Test
public void testGetMemWalSpecRoundTripsEveryIndexSelection() {
public void testGetRTESpecRoundTripsEveryIndexSelection() {
enqueue(
"get_memwal_spec",
"get_rte_spec",
200,
"{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
"{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
+ "\"maintained_indexes\":null,\"writer_config_defaults\":{}}}");
Optional<MemWalWriteSpec> spec = memWal.getMemWalSpec();
Optional<RTEWriteSpec> spec = rte.getRTESpec();
assertTrue(spec.isPresent());
assertNull(spec.get().maintainedIndexes());
}
@Test
public void testGetMemWalSpecRoundTripsNoIndexSelection() {
public void testGetRTESpecRoundTripsNoIndexSelection() {
enqueue(
"get_memwal_spec",
"get_rte_spec",
200,
"{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
"{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
+ "\"maintained_indexes\":[],\"writer_config_defaults\":{}}}");
Optional<MemWalWriteSpec> spec = memWal.getMemWalSpec();
Optional<RTEWriteSpec> spec = rte.getRTESpec();
assertTrue(spec.isPresent());
assertEquals(Collections.emptyList(), spec.get().maintainedIndexes());
@@ -249,32 +247,32 @@ public class LanceDbTableMemWalTest {
* rejected: reading a string or an object as null would turn a malformed response into a policy.
*/
@Test
public void testGetMemWalSpecRejectsANonArrayMaintainedIndexes() {
public void testGetRTESpecRejectsANonArrayMaintainedIndexes() {
enqueue(
"get_memwal_spec",
"get_rte_spec",
200,
"{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
"{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
+ "\"maintained_indexes\":\"id_idx\",\"writer_config_defaults\":{}}}");
assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec());
assertThrows(IllegalStateException.class, () -> rte.getRTESpec());
}
@Test
public void testGetMemWalSpecRejectsNonStringIndexNames() {
public void testGetRTESpecRejectsNonStringIndexNames() {
enqueue(
"get_memwal_spec",
"get_rte_spec",
200,
"{\"memwal_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
"{\"rte_spec\":{\"sharding\":{\"mode\":\"unsharded\"},"
+ "\"maintained_indexes\":[7],\"writer_config_defaults\":{}}}");
assertThrows(IllegalStateException.class, () -> memWal.getMemWalSpec());
assertThrows(IllegalStateException.class, () -> rte.getRTESpec());
}
@Test
public void testGetMemWalSpecAbsent() {
enqueue("get_memwal_spec", 200, "{\"memwal_spec\":null}");
public void testGetRTESpecAbsent() {
enqueue("get_rte_spec", 200, "{\"rte_spec\":null}");
assertFalse(memWal.getMemWalSpec().isPresent());
assertFalse(rte.getRTESpec().isPresent());
}
// ===========================================================================
@@ -282,12 +280,12 @@ public class LanceDbTableMemWalTest {
// ===========================================================================
@Test
public void testGetMemWalStats() throws Exception {
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
public void testGetRTEStats() throws Exception {
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
Optional<MemWalStats> got = memWal.getMemWalStats(true);
Optional<RTEStats> got = rte.getRTEStats(true);
assertEquals("/v1/table/my_table/get_memwal_stats/", requestPaths.get(0));
assertEquals("/v1/table/my_table/get_rte_stats/", requestPaths.get(0));
assertTrue(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean());
assertTrue(got.isPresent());
TableShardStats decoded = got.get().tableShards().get(0);
@@ -305,18 +303,18 @@ public class LanceDbTableMemWalTest {
/** The optional fields decode when the server does send them. */
@Test
public void testGetMemWalStatsDecodesOptionalFields() {
public void testGetRTEStatsDecodesOptionalFields() {
enqueue(
"get_memwal_stats",
"get_rte_stats",
200,
"{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
"{\"rte_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
+ "\"replay_after_wal_entry_position\":3,\"wal_entry_position_last_seen\":11,"
+ "\"sstables\":[{\"generation\":7,\"bytes\":1024,\"rows\":42}],"
+ "\"compacting\":true,\"memtables\":[{\"generation\":8,\"rows\":5,"
+ "\"bytes\":64,\"batches\":2,\"indexes\":[\"id_idx\"]}]}]}}");
TableShardStats decoded = memWal.getMemWalStats(true).get().tableShards().get(0);
TableShardStats decoded = rte.getRTEStats(true).get().tableShards().get(0);
assertEquals(3, decoded.replayAfterWalEntryPosition());
assertEquals(11, decoded.walEntryPositionLastSeen());
@@ -332,17 +330,17 @@ public class LanceDbTableMemWalTest {
}
@Test
public void testGetMemWalStatsAbsentWhenMemWalDisabled() {
enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}");
public void testGetRTEStatsAbsentWhenRTEDisabled() {
enqueue("get_rte_stats", 200, "{\"rte_stats\":null}");
assertFalse(memWal.getMemWalStats().isPresent());
assertFalse(rte.getRTEStats().isPresent());
}
@Test
public void testGetMemWalStatsDefaultsToExcludingGenerationRows() throws Exception {
enqueue("get_memwal_stats", 200, stats());
public void testGetRTEStatsDefaultsToExcludingGenerationRows() throws Exception {
enqueue("get_rte_stats", 200, stats());
memWal.getMemWalStats();
rte.getRTEStats();
assertFalse(MAPPER.readTree(requestBodies.get(0)).get("include_sstable_rows").asBoolean());
}
@@ -353,22 +351,22 @@ public class LanceDbTableMemWalTest {
@Test
public void testFlushAndCompactRoutes() {
enqueue("flush_memwal", 200, "");
enqueue("compact_memwal", 200, "");
enqueue("flush_rte", 200, "");
enqueue("compact_rte", 200, "");
memWal.flushMemWal();
memWal.compactMemWal();
rte.flushRTE();
rte.compactRTE();
assertEquals("/v1/table/my_table/flush_memwal/", requestPaths.get(0));
assertEquals("/v1/table/my_table/compact_memwal/", requestPaths.get(1));
assertEquals("/v1/table/my_table/flush_rte/", requestPaths.get(0));
assertEquals("/v1/table/my_table/compact_rte/", requestPaths.get(1));
}
@Test
public void testHttpErrorCarriesStatus() {
enqueue("flush_memwal", 404, "no such table");
enqueue("flush_rte", 404, "no such table");
LanceDbRestClient.HttpException e =
assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.flushMemWal());
assertThrows(LanceDbRestClient.HttpException.class, () -> rte.flushRTE());
assertEquals(404, e.statusCode());
}
@@ -377,98 +375,97 @@ public class LanceDbTableMemWalTest {
// ===========================================================================
@Test
public void testCheckpointReturnsWhenMemWalDisabled() {
enqueue("flush_memwal", 200, "");
enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}");
public void testCheckpointReturnsWhenRTEDisabled() {
enqueue("flush_rte", 200, "");
enqueue("get_rte_stats", 200, "{\"rte_stats\":null}");
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(0, countCalls("compact_memwal"), "nothing to compact when the LSM path is off");
assertEquals(0, countCalls("compact_rte"), "nothing to compact when the LSM path is off");
}
@Test
public void testCheckpointReturnsWhenNoGenerationsOutstanding() {
enqueue("flush_memwal", 200, "");
enqueue("flush_rte", 200, "");
// A table shard with no SSTables yields no target, so the drain never starts.
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false)));
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false)));
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(0, countCalls("compact_memwal"));
assertEquals(0, countCalls("compact_rte"));
}
@Test
public void testCheckpointConvergesOnceTargetGenerationsAreGone() {
enqueue("flush_memwal", 200, "");
enqueue("flush_rte", 200, "");
// Watermark read: shard-0 holds sstables 7 and 8, so target = 8.
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
// First drain poll: both still outstanding, nothing compacting -> dispatch a pass.
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 7L, 8L)));
// Second drain poll: drained past the target -> done.
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 9L)));
enqueue("compact_memwal", 200, "");
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 9L)));
enqueue("compact_rte", 200, "");
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(1, countCalls("compact_memwal"), "one pass dispatched");
assertEquals(3, countCalls("get_memwal_stats"), "watermark read plus two drain polls");
assertEquals(1, countCalls("compact_rte"), "one pass dispatched");
assertEquals(3, countCalls("get_rte_stats"), "watermark read plus two drain polls");
}
@Test
public void testCheckpointDoesNotPileOnWhileEveryTargetBucketIsCompacting() {
enqueue("flush_memwal", 200, "");
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L)));
enqueue("flush_rte", 200, "");
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", true, 4L)));
// Still compacting on the first poll, so no pass is dispatched; then it drains.
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", true, 4L)));
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false, 5L)));
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", true, 4L)));
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false, 5L)));
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(0, countCalls("compact_memwal"), "a latched table shard is left alone");
assertEquals(0, countCalls("compact_rte"), "a latched table shard is left alone");
}
@Test
public void testCheckpointRetriesFromFlushAfterLostClaim() {
// 421 on the watermark read: the node lost its claim, so the whole thing restarts
// from flush rather than retrying the read in place.
enqueue("flush_memwal", 200, "");
enqueue("get_memwal_stats", 421, "no claim");
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false)));
enqueue("flush_rte", 200, "");
enqueue("get_rte_stats", 421, "no claim");
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false)));
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(2, countCalls("flush_memwal"), "re-issued from flush");
assertEquals(2, countCalls("flush_rte"), "re-issued from flush");
}
@Test
public void testCheckpointRetriesRetryableStatusInPlace() {
enqueue("flush_memwal", 429, "latch held");
enqueue("flush_memwal", 200, "");
enqueue("get_memwal_stats", 200, stats(tableShard("shard-0", false)));
enqueue("flush_rte", 429, "latch held");
enqueue("flush_rte", 200, "");
enqueue("get_rte_stats", 200, stats(tableShard("shard-0", false)));
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(2, countCalls("flush_memwal"), "429 retried in place, not re-issued");
assertEquals(2, countCalls("flush_rte"), "429 retried in place, not re-issued");
}
@Test
public void testCheckpointPropagatesTerminalStatus() {
enqueue("flush_memwal", 400, "bad request");
enqueue("flush_rte", 400, "bad request");
LanceDbRestClient.HttpException e =
assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal());
assertThrows(LanceDbRestClient.HttpException.class, () -> rte.checkpointRTE());
assertEquals(400, e.statusCode());
assertEquals(1, countCalls("flush_memwal"), "a terminal status is not retried");
assertEquals(1, countCalls("flush_rte"), "a terminal status is not retried");
}
@Test
public void testCheckpointGivesUpAfterRepeatedLostClaims() {
enqueue("flush_memwal", 421, "no claim");
enqueue("flush_rte", 421, "no claim");
IllegalStateException e =
assertThrows(IllegalStateException.class, () -> memWal.checkpointMemWal());
IllegalStateException e = assertThrows(IllegalStateException.class, () -> rte.checkpointRTE());
assertTrue(e.getMessage().contains("kept losing its claim"), e.getMessage());
assertEquals(4, countCalls("flush_memwal"), "the initial attempt plus MAX_REISSUES");
assertEquals(4, countCalls("flush_rte"), "the initial attempt plus MAX_REISSUES");
}
// ===========================================================================
@@ -478,24 +475,24 @@ public class LanceDbTableMemWalTest {
/**
* A stats payload that does not decode must fail closed. Every one of these bodies used to be
* read as "no table shards", which is indistinguishable from a drained table, so {@code
* checkpointMemWal} reported convergence for a checkpoint that never ran.
* checkpointRTE} reported convergence for a checkpoint that never ran.
*/
@Test
public void testCheckpointRejectsMalformedStats() {
Map<String, String> malformed = new LinkedHashMap<String, String>();
malformed.put("no response body at all", "");
malformed.put("stats object with no table shards", "{\"memwal_stats\":{}}");
malformed.put("stats object with no table shards", "{\"rte_stats\":{}}");
malformed.put(
"table shard missing its required fields", "{\"memwal_stats\":{\"table_shards\":[{}]}}");
"table shard missing its required fields", "{\"rte_stats\":{\"table_shards\":[{}]}}");
malformed.put(
"table shard missing sstables",
"{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
"{\"rte_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
+ "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
+ "\"compacting\":false}]}}");
malformed.put(
"generation with a non-numeric generation number",
"{\"memwal_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
"{\"rte_stats\":{\"table_shards\":[{\"shard_id\":\"shard-0\",\"status\":\"Active\","
+ "\"writer_epoch\":1,\"manifest_version\":2,\"current_generation\":9,"
+ "\"replay_after_wal_entry_position\":0,\"wal_entry_position_last_seen\":0,"
+ "\"sstables\":[{\"generation\":\"7\",\"bytes\":1024}],"
@@ -503,25 +500,25 @@ public class LanceDbTableMemWalTest {
for (Map.Entry<String, String> each : malformed.entrySet()) {
setUpFresh();
enqueue("flush_memwal", 200, "");
enqueue("get_memwal_stats", 200, each.getValue());
enqueue("flush_rte", 200, "");
enqueue("get_rte_stats", 200, each.getValue());
assertThrows(
IllegalStateException.class,
() -> memWal.checkpointMemWal(),
() -> rte.checkpointRTE(),
each.getKey() + " must not report convergence");
}
}
/** The one shape that legitimately means "this table has no MemWAL write path". */
/** The one shape that legitimately means "this table has no RTE write path". */
@Test
public void testCheckpointTreatsNullStatsAsNotWalBacked() {
enqueue("flush_memwal", 200, "");
enqueue("get_memwal_stats", 200, "{\"memwal_stats\":null}");
enqueue("flush_rte", 200, "");
enqueue("get_rte_stats", 200, "{\"rte_stats\":null}");
memWal.checkpointMemWal();
rte.checkpointRTE();
assertEquals(1, countCalls("get_memwal_stats"));
assertEquals(1, countCalls("get_rte_stats"));
}
// ===========================================================================
@@ -531,19 +528,18 @@ public class LanceDbTableMemWalTest {
/**
* The transport must not retry on the checkpoint loop's behalf. Apache HttpClient's default
* strategy retries exactly 429 and 503 — the two statuses {@code isRetryable} owns — which
* doubled every budget here and also retried {@code compact_memwal} in place, where the loop is
* doubled every budget here and also retried {@code compact_rte} in place, where the loop is
* built to fall through to a fresh stats poll instead.
*/
@Test
public void testCheckpointRetryBudgetIsNotDoubledByTheTransport() {
enqueue("flush_memwal", 429, "latch held");
enqueue("flush_rte", 429, "latch held");
LanceDbRestClient.HttpException e =
assertThrows(LanceDbRestClient.HttpException.class, () -> memWal.checkpointMemWal());
assertThrows(LanceDbRestClient.HttpException.class, () -> rte.checkpointRTE());
assertEquals(429, e.statusCode(), "the exhausted budget propagates the last error as itself");
assertEquals(
9, countCalls("flush_memwal"), "the initial request plus MAX_RETRIES, and no more");
assertEquals(9, countCalls("flush_rte"), "the initial request plus MAX_RETRIES, and no more");
}
// ===========================================================================
@@ -558,9 +554,9 @@ public class LanceDbTableMemWalTest {
return numbers;
}
/** Build a {@code memwal_stats} response body from table shard fragments. */
/** Build a {@code rte_stats} response body from table shard fragments. */
private static String stats(String... tableShards) {
return "{\"memwal_stats\":{\"table_shards\":[" + String.join(",", tableShards) + "]}}";
return "{\"rte_stats\":{\"table_shards\":[" + String.join(",", tableShards) + "]}}";
}
private static String tableShard(String shardId, boolean compacting, Long... sstables) {
@@ -600,7 +596,7 @@ public class LanceDbTableMemWalTest {
return requestPaths.stream().filter(path -> operationOf(path).equals(operation)).count();
}
/** {@code /v1/table/my_table/flush_memwal/} -> {@code flush_memwal}. */
/** {@code /v1/table/my_table/flush_rte/} -> {@code flush_rte}. */
private static String operationOf(String path) {
String[] segments = path.split("/");
return segments.length == 0 ? "" : segments[segments.length - 1];
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.9</version>
<version>0.41.0-beta.11</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>14.0.0-beta.21</lance-core.version>
<lance-core.version>14.0.0-beta.22</lance-core.version>
<jackson.version>2.18.11</jackson.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.41.0-beta.9"
version = "0.41.0-beta.11"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/blob.test.ts
+187 -1
View File
@@ -21,7 +21,41 @@ import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
import { getRegistry } from "../lancedb/embedding/registry";
function legacyBlob(name: string): Field {
const field = blob(name);
return new Field(
name,
new Struct((field.type as Struct).children.slice(0, 2)),
field.nullable,
field.metadata,
);
}
describe("blob()", () => {
it("declares the four-field range layout", () => {
const field = blob("image", {
nullable: false,
inlineSizeThreshold: 12,
});
expect(field.nullable).toBe(false);
expect(isBlobField(field)).toBe(true);
expect(
field.metadata.get("lance-encoding:blob-inline-size-threshold"),
).toBe("12");
expect(
(field.type as Struct).children.map((f) => [
f.name,
f.type.toString(),
f.nullable,
]),
).toEqual([
["data", "LargeBinary", true],
["uri", "Utf8", true],
["position", "Uint64", true],
["size", "Uint64", true],
]);
});
it("marks the field as lance.blob.v2", () => {
const field = blob("image", { nullable: false });
expect(field.nullable).toBe(false);
@@ -156,7 +190,7 @@ describe("coerceBlobValue", () => {
[
"unknown struct field",
{ data: Buffer.from("y"), mime: "image/png" },
/Blob struct values only support 'data' and 'uri', got 'mime'/,
/Blob struct values only support .*got 'mime'/,
],
[
"misspelled struct field",
@@ -174,6 +208,158 @@ describe("coerceBlobValue", () => {
});
});
describe("external blob ranges", () => {
const uri = "s3://bucket/key";
it.each([0, 4, 0n, 4n, 9007199254740993n])(
"preserves position %s",
(position) => {
expect(coerceBlobValue({ uri, position, size: 6 })).toEqual({
data: null,
uri,
position: BigInt(position),
size: 6n,
});
},
);
it.each([
[{ uri, position: 0 }, /both position and size/],
[{ uri, size: 6 }, /both position and size/],
[{ uri, position: null, size: 6 }, /both position and size/],
[{ uri, position: 0, size: 0 }, /size must be positive/],
[{ uri, position: -1, size: 6 }, /position must be non-negative/],
[{ uri, position: 0, size: -1n }, /size must be positive/],
[{ uri, position: 0.5, size: 6 }, /safe integer or bigint/],
[
{ uri, position: Number.MAX_SAFE_INTEGER + 1, size: 6 },
/safe integer or bigint/,
],
[{ uri, position: 0, size: Number.NaN }, /safe integer or bigint/],
[{ uri, position: "4", size: 6 }, /safe integer or bigint/],
[{ uri, position: false, size: 6 }, /safe integer or bigint/],
[{ uri, position: 1n << 64n, size: 6 }, /fit in uint64/],
[{ uri, position: 0xffffffffffffffffn, size: 1 }, /overflows uint64/],
[{ data: Buffer.from("x"), position: 0, size: 6 }, /require a uri/],
])("rejects malformed input %p", (value, message) => {
expect(() => coerceBlobValue(value)).toThrow(message);
});
it("preserves ranges while normalizing a URL held by getters", () => {
class Range {
get uri() {
return new URL(uri);
}
get position() {
return 4;
}
get size() {
return 6;
}
}
expect(coerceBlobValue(new Range())).toEqual({
data: null,
uri,
position: 4n,
size: 6n,
});
});
it("writes ranges and mixed whole-file, inline, and null values to Arrow", () => {
const schema = new Schema([blob("image")]);
const rows = [
null,
Buffer.from("x"),
uri,
{ uri, position: 4, size: 6 },
].map((image) => ({ image }));
const column = makeArrowTable(rows, { schema }).getChild("image")!;
expect(column.get(0)).toBeNull();
expect(Buffer.from(column.getChild("data")!.get(1))).toEqual(
Buffer.from("x"),
);
expect(column.getChild("uri")!.get(2)).toBe(uri);
expect(column.getChild("position")!.toArray()).toEqual(
new BigUint64Array([0n, 0n, 0n, 4n]),
);
expect(column.getChild("position")!.get(2)).toBeNull();
expect(column.getChild("size")!.get(3)).toBe(6n);
});
it.each([
Buffer.from("inline"),
uri,
{ uri, position: null, size: null },
null,
])("retains the full declared range layout when seeded with %p", (image) => {
const schema = new Schema([blob("image")]);
const table = makeArrowTable([{ image }], { schema });
expect(table.schema).toEqual(schema);
expect(table.getChild("image")!.getChild("position")!.get(0)).toBeNull();
});
it.each([0, 4])(
"rejects position %s on a minimal schema before dropping keys",
(position) => {
expect(() =>
makeArrowTable(
[
{ image: Buffer.from("inline") },
{ image: { uri, position, size: 6 } },
],
{ schema: new Schema([legacyBlob("image")]) },
),
).toThrow(
/field image at row 1: Blob storage schema cannot store byte ranges/,
);
},
);
it("accepts omitted or null range fields on the minimal schema", () => {
const column = makeArrowTable(
[{ image: { uri, position: null, size: null } }],
{
schema: new Schema([legacyBlob("image")]),
},
).getChild("image")!;
expect(column.getChild("uri")!.get(0)).toBe(uri);
});
it("validates ranges nested in structs and lists against their storage fields", () => {
const value = { uri, position: 4, size: 6 };
for (const field of [legacyBlob("image"), blob("image")]) {
const schema = new Schema([
new Field("info", new Struct([field]), true),
new Field("images", new List(field), true),
]);
if ((field.type as Struct).children.length === 2) {
expect(() =>
makeArrowTable([{ info: { image: value } }], { schema }),
).toThrow(/field info.image.*storage schema/);
expect(() => makeArrowTable([{ images: [value] }], { schema })).toThrow(
/field images\[0\].*storage schema/,
);
} else {
const table = makeArrowTable(
[{ info: { image: value }, images: [value] }],
{ schema },
);
const info = table.getChild("info")!.get(0) as {
image: { position: bigint; size: bigint };
};
expect(info.image.position).toBe(4n);
const images = Array.from(
table.getChild("images")!.get(0) as Iterable<{
position: bigint;
size: bigint;
}>,
);
expect(images[0].size).toBe(6n);
}
}
});
});
describe("resolveBlobInputs", () => {
it("reads Blob and File values in blob columns only", async () => {
const schema = new Schema([
nodejs/__test__/remote.test.ts
+135
View File
@@ -12,6 +12,7 @@ import {
OAuthConfig,
OAuthFlowType,
TlsConfig,
blob,
connect,
} from "../lancedb";
import {
@@ -19,7 +20,10 @@ import {
Field,
Int64,
RecordBatch,
RecordBatchReader,
Schema,
Struct,
makeArrowTable,
tableToIPC,
} from "../lancedb/arrow";
import {
@@ -94,6 +98,137 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it.each([false, true])(
"normalizes remote blob writes (range target: %s)",
async (wide) => {
const field = (full: boolean) => {
const f = blob("image");
return full
? f
: new Field(
f.name,
new Struct((f.type as Struct).children.slice(0, 2)),
f.nullable,
f.metadata,
);
};
const schema = (full: boolean) =>
new Schema([new Field("id", new Int64(), true), field(full)]);
const writes: { path: string; schema: Schema }[] = [];
let storedWide = wide;
const describe = () => ({
version: writes.length + 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: true },
{
name: "image",
nullable: true,
metadata: { "ARROW:extension:name": "lance.blob.v2" },
type: {
type: "struct",
fields: [
{
name: "data",
nullable: true,
type: { type: "large_binary" },
},
{ name: "uri", nullable: true, type: { type: "string" } },
...(storedWide
? ["position", "size"].map((name) => ({
name,
nullable: true,
type: { type: "uint64" },
}))
: []),
],
},
},
],
},
});
await withMockDatabase(
(req, res) => {
if (req.url?.includes("/describe/")) {
res
.writeHead(200, { "content-type": "application/json" })
.end(JSON.stringify(describe()));
return;
}
const chunks: Buffer[] = [];
// Failed streaming input aborts before end; only complete bodies commit.
req.on("error", () => {
/* Aborted inputs must not commit. */
});
req.on("data", (chunk) => chunks.push(chunk));
req.on("end", () => {
const reader = RecordBatchReader.from(Buffer.concat(chunks)).open();
// Only read the schema: Rust separately checks compressed wire values.
writes.push({ path: req.url ?? "", schema: reader.schema });
storedWide =
(reader.schema.fields[1].type as Struct).children.length === 4;
res.writeHead(200, { "content-type": "application/json" }).end(
JSON.stringify({
version: writes.length + 1,
// biome-ignore lint/style/useNamingConvention: server wire format
num_inserted_rows: 1,
}),
);
});
},
async (db) => {
const table = await db.openTable("test");
const input = makeArrowTable(
[{ id: 1n, image: Buffer.from("payload") }],
{ schema: schema(!wide) },
);
await table.add(input);
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(input);
expect(writes).toHaveLength(2);
for (const write of writes)
expect(write.schema).toEqual(schema(wide));
expect(writes[0].path).toContain("/insert/");
expect(writes[1].path).toContain("/merge_insert/");
if (!wide) {
const invalid = makeArrowTable(
[
{
id: 1n,
image: {
uri: "s3://registered/payload",
position: 4,
size: 6,
},
},
],
{ schema: schema(true) },
);
await expect(table.add(invalid)).rejects.toThrow(
/cannot store byte ranges/,
);
await expect(
table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(invalid),
).rejects.toThrow(/cannot store byte ranges/);
expect(writes).toHaveLength(2);
}
await table.add(input, { mode: "overwrite" });
expect(writes).toHaveLength(3);
expect(writes[2].path).toContain("mode=overwrite");
expect(writes[2].schema).toEqual(schema(!wide));
expect(await table.schema()).toEqual(schema(!wide));
},
);
},
);
it("reports unsupported operations with backend-neutral errors", async () => {
await withMockDatabase(
(req, res) => {
nodejs/__test__/table.test.ts
+238 -53
View File
@@ -642,11 +642,11 @@ describe.each(arrowVersions)(
);
});
it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => {
it("should expose useRTE on takeRowIds as the base-only escape hatch", async () => {
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
// useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
// useRTE(false) is reachable on TakeQuery (the escape hatch for RTE tables,
// where take-by-row-id auto-routes to the LSM scanner and is rejected).
const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray();
const res = await table.takeRowIds([0, 2]).useRTE(false).toArray();
expect(res.map((r) => r.id)).toEqual([1, 3]);
});
@@ -2583,6 +2583,16 @@ describe("when dealing with versioning", () => {
});
});
function legacyBlob(name: string): Field {
const field = blob(name);
return new Field(
name,
new Struct((field.type as Struct).children.slice(0, 2)),
field.nullable,
field.metadata,
);
}
describe("when dealing with blob columns", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -2592,6 +2602,89 @@ describe("when dealing with blob columns", () => {
tmpDir.removeCallback();
});
it.each([false, true])(
"normalizes Arrow blob layouts for append and merge (range target: %s)",
async (wide) => {
const db = await connect(tmpDir.name);
const schema = (full: boolean) =>
new Schema([
new Field("id", new Int64(), true),
full ? blob("image") : legacyBlob("image"),
]);
const table = await db.createTable(
"layouts",
[{ id: 1n, image: Buffer.from("old") }],
{ schema: schema(wide) },
);
await table.add(
makeArrowTable([{ id: 2n, image: Buffer.from("appended") }], {
schema: schema(!wide),
}),
);
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(
makeArrowTable(
[
{ id: 1n, image: Buffer.from("updated") },
{ id: 3n, image: Buffer.from("inserted") },
],
{ schema: schema(!wide) },
),
);
expect(await table.schema()).toEqual(schema(wide));
const rows = await table.query().withRowId().toArray();
rows.sort((a, b) => Number(a.id - b.id));
expect(
await table.fetchBlobs(
"image",
rows.map((row) => row._rowid as bigint),
),
).toEqual([
Buffer.from("updated"),
Buffer.from("appended"),
Buffer.from("inserted"),
]);
if (!wide) {
const bad = makeArrowTable(
[
{
id: 1n,
image: { uri: "s3://registered/file", position: 0, size: 6 },
},
],
{ schema: schema(true) },
);
const version = await table.version();
await expect(table.add(bad)).rejects.toThrow(
/cannot store byte ranges/,
);
await expect(
table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute(bad),
).rejects.toThrow(/cannot store byte ranges/);
expect(await table.version()).toBe(version);
expect(await table.countRows()).toBe(3);
}
const replacement = makeArrowTable(
[{ id: 4n, image: Buffer.from("replacement") }],
{ schema: schema(!wide) },
);
await table.add(replacement, { mode: "overwrite" });
expect(await table.schema()).toEqual(schema(!wide));
expect(await table.countRows()).toBe(1);
const [row] = await table.query().withRowId().toArray();
expect(await table.fetchBlobs("image", [row._rowid as bigint])).toEqual([
Buffer.from("replacement"),
]);
},
);
it("rejects unsupported blob values in createTable and add", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
@@ -2668,6 +2761,100 @@ describe("when dealing with blob columns", () => {
expect(bytes[0]).toEqual(payload);
});
it.each([0, 4])(
"fetches the exact external range at position %s after reopening",
async (position) => {
const { db, uri, payload } = await openExternalBlobTable();
const schema = new Schema([
new Field("id", new Int64(), true),
blob("payload"),
]);
const rows: { id: bigint; payload: BlobInput }[] = [
{ id: 1n, payload: Buffer.from("inline") },
{ id: 2n, payload: { uri: new URL(uri), position, size: 6 } },
{ id: 3n, payload: uri },
{ id: 4n, payload: null },
];
const table = await db.createTable("ranges", rows.slice(0, 1), {
schema,
});
await table.add(rows.slice(1), { allowExternalBlobOutsideBases: true });
table.close();
const reopened = await db.openTable("ranges");
const results = await reopened.query().withRowId().toArray();
results.sort((a, b) => Number(a.id - b.id));
expect(results[1].payload.position).toBe(BigInt(position));
expect(results[1].payload.size).toBe(6n);
expect(
await reopened.fetchBlobs(
"payload",
results.map((r) => r._rowid as bigint),
),
).toEqual([
Buffer.from("inline"),
payload.subarray(position, position + 6),
payload,
null,
]);
},
);
it.each([0, 4])(
"rejects ranges at position %s on reopened minimal tables without writing",
async (position) => {
const { db, uri, payload } = await openExternalBlobTable();
const table = await db.createEmptyTable(
"legacy_blobs",
new Schema([new Field("id", new Int64(), true), legacyBlob("payload")]),
);
table.close();
const reopened = await db.openTable("legacy_blobs");
await expect(
reopened.add(
[
{ id: 1n, payload: Buffer.from("inline") },
{ id: 2n, payload: { uri, position, size: 6 } },
],
{ allowExternalBlobOutsideBases: true },
),
).rejects.toThrow(/storage schema cannot store byte ranges/);
expect(await reopened.countRows()).toBe(0);
await reopened.add(
[
{ id: 3n, payload: uri },
{ id: 4n, payload: Buffer.from("inline") },
],
{ allowExternalBlobOutsideBases: true },
);
const rows = await reopened.query().withRowId().toArray();
rows.sort((a, b) => Number(a.id - b.id));
expect(
await reopened.fetchBlobs(
"payload",
rows.map((r) => r._rowid as bigint),
),
).toEqual([payload, Buffer.from("inline")]);
},
);
it.each([{ position: 0 }, { size: 6 }, { position: 0, size: 0 }])(
"rejects partial or empty ranges without writing: %p",
async (range) => {
const { db, uri } = await openExternalBlobTable();
const table = await db.createEmptyTable(
"invalid_ranges",
new Schema([blob("payload")]),
);
await expect(
table.add(
[{ payload: Buffer.from("inline") }, { payload: { uri, ...range } }],
{ allowExternalBlobOutsideBases: true },
),
).rejects.toThrow(/both position and size|size must be positive/);
expect(await table.countRows()).toBe(0);
},
);
it("applies the external blob opt-in only to the current add", async () => {
const { table, uri } = await openExternalBlobTable();
@@ -4811,7 +4998,7 @@ describe("setUnenforcedPrimaryKey", () => {
});
});
describe("enableMemWal / disableMemWal", () => {
describe("enableRTE / disableRTE", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -4831,16 +5018,16 @@ describe("enableMemWal / disableMemWal", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.enableMemWal({
await table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 4,
});
await table.disableMemWal();
await table.disableRTE();
// A second unset errors — there is no spec left to remove.
await expect(table.disableMemWal()).rejects.toThrow();
await expect(table.disableRTE()).rejects.toThrow();
// A fresh spec can be installed after unset.
await table.enableMemWal({
await table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 8,
@@ -4852,8 +5039,8 @@ describe("enableMemWal / disableMemWal", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.enableMemWal({ specType: "unsharded" });
await table.disableMemWal();
await table.enableRTE({ specType: "unsharded" });
await table.disableRTE();
});
it("installs an identity spec", async () => {
@@ -4861,8 +5048,8 @@ describe("enableMemWal / disableMemWal", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.enableMemWal({ specType: "identity", column: "id" });
await table.disableMemWal();
await table.enableRTE({ specType: "identity", column: "id" });
await table.disableRTE();
});
it("rejects an invalid spec", async () => {
@@ -4872,7 +5059,7 @@ describe("enableMemWal / disableMemWal", () => {
await table.setUnenforcedPrimaryKey("id");
// num_buckets out of range.
await expect(
table.enableMemWal({
table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 0,
@@ -4880,7 +5067,7 @@ describe("enableMemWal / disableMemWal", () => {
).rejects.toThrow();
// Column mismatch.
await expect(
table.enableMemWal({
table.enableRTE({
specType: "bucket",
column: "missing",
numBuckets: 4,
@@ -4888,13 +5075,13 @@ describe("enableMemWal / disableMemWal", () => {
).rejects.toThrow();
});
it("reads back the installed spec via getMemWalSpec", async () => {
it("reads back the installed spec via getRTESpec", async () => {
const conn = await connect(tmpDir.name);
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
// Nothing installed yet.
expect(await table.getMemWalSpec()).toBeUndefined();
expect(await table.getRTESpec()).toBeUndefined();
// A real scalar index is needed to name it as a maintained index.
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
@@ -4905,14 +5092,14 @@ describe("enableMemWal / disableMemWal", () => {
// defaults. Lance writer-config keys are canonically snake_case.
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
const writerConfigDefaults = { durable_write: "false" };
await table.enableMemWal({
await table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 4,
maintainedIndexes: [indexName],
writerConfigDefaults,
});
const spec = await table.getMemWalSpec();
const spec = await table.getRTESpec();
expect(spec).toBeDefined();
expect(spec?.specType).toBe("bucket");
expect(spec?.column).toBe("id");
@@ -4921,19 +5108,19 @@ describe("enableMemWal / disableMemWal", () => {
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
// After unset, undefined again.
await table.disableMemWal();
expect(await table.getMemWalSpec()).toBeUndefined();
await table.disableRTE();
expect(await table.getRTESpec()).toBeUndefined();
// Identity round-trips (column recovered from the schema).
await table.enableMemWal({ specType: "identity", column: "id" });
const identity = await table.getMemWalSpec();
await table.enableRTE({ specType: "identity", column: "id" });
const identity = await table.getRTESpec();
expect(identity?.specType).toBe("identity");
expect(identity?.column).toBe("id");
await table.disableMemWal();
await table.disableRTE();
// Unsharded round-trips (no routing column).
await table.enableMemWal({ specType: "unsharded" });
const unsharded = await table.getMemWalSpec();
await table.enableRTE({ specType: "unsharded" });
const unsharded = await table.getRTESpec();
expect(unsharded?.specType).toBe("unsharded");
expect(unsharded?.column).toBeFalsy();
});
@@ -4962,7 +5149,7 @@ describe("LSM merge insert", () => {
]);
await table.setUnenforcedPrimaryKey("id");
// numBuckets = 1: every row routes to the single bucket.
await table.enableMemWal({
await table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 1,
@@ -4982,22 +5169,22 @@ describe("LSM merge insert", () => {
{ id: "c", value: 3 },
{ id: "d", value: 4 },
]);
// LSM path: rows go to the MemWAL, so only numRows is populated.
// LSM path: rows go to the RTE, so only numRows is populated.
expect(res.numRows).toBe(2);
expect(res.version).toBe(0);
expect(res.numInsertedRows).toBe(0);
await table.closeMemWalWriters();
await table.closeRTEWriters();
});
it("falls back to the standard path with useMemWal(false)", async () => {
it("falls back to the standard path with useRTE(false)", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
const res = await table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.useMemWal(false)
.useRTE(false)
.execute([
{ id: "b", value: 9 },
{ id: "e", value: 5 },
@@ -5020,7 +5207,7 @@ describe("LSM merge insert", () => {
expect(res.numRows).toBe(1);
});
it("rejects a non-upsert merge under a MemWAL spec", async () => {
it("rejects a non-upsert merge under an RTE spec", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
@@ -5032,7 +5219,7 @@ describe("LSM merge insert", () => {
).rejects.toThrow();
});
it("auto-routes reads through the MemWAL scanner", async () => {
it("auto-routes reads through the RTE scanner", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn); // base ids "a", "b"
@@ -5046,24 +5233,22 @@ describe("LSM merge insert", () => {
const lsm = await table.query().toArray();
expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
// useMemWal(false) bypasses the MemWAL and reads the base table only.
const baseOnly = await table.query().useMemWal(false).toArray();
// useRTE(false) bypasses the RTE and reads the base table only.
const baseOnly = await table.query().useRTE(false).toArray();
expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
});
it("reads the base table when no MemWAL spec is installed", async () => {
it("reads the base table when no RTE spec is installed", async () => {
const conn = await connect(tmpDir.name);
const table = await conn.createEmptyTable(
"plain",
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
// No spec: default read and useMemWal(false) both succeed against the base table.
// No spec: default read and useRTE(false) both succeed against the base table.
await expect(table.query().toArray()).resolves.toBeDefined();
await expect(
table.query().useMemWal(false).toArray(),
).resolves.toBeDefined();
// useMemWal(true) demands MemWAL routing; without a spec it errors.
await expect(table.query().useMemWal(true).toArray()).rejects.toThrow();
await expect(table.query().useRTE(false).toArray()).resolves.toBeDefined();
// useRTE(true) demands RTE routing; without a spec it errors.
await expect(table.query().useRTE(true).toArray()).rejects.toThrow();
});
});
@@ -5081,42 +5266,42 @@ describe("LSM convergence and stats", () => {
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
await table.setUnenforcedPrimaryKey("id");
await table.enableMemWal({ specType: "unsharded" });
await table.enableRTE({ specType: "unsharded" });
return table;
}
// These four route through the server that owns the MemWAL, so a local table
// These four route through the server that owns the RTE, so a local table
// rejects them rather than answering. What is asserted here is that the
// bindings reach the core at all; the behavior against a real endpoint is
// covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
it("rejects flushMemWal on a local table", async () => {
it("rejects flushRTE on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.flushMemWal()).rejects.toThrow(/not supported/i);
await expect(table.flushRTE()).rejects.toThrow(/not supported/i);
});
it("rejects compactMemWal on a local table", async () => {
it("rejects compactRTE on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.compactMemWal()).rejects.toThrow(/not supported/i);
await expect(table.compactRTE()).rejects.toThrow(/not supported/i);
});
it("rejects getMemWalStats on a local table", async () => {
it("rejects getRTEStats on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i);
await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i);
await expect(table.getRTEStats()).rejects.toThrow(/not supported/i);
await expect(table.getRTEStats(true)).rejects.toThrow(/not supported/i);
});
it("rejects checkpointMemWal on a local table", async () => {
it("rejects checkpointRTE on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
// checkpointMemWal freezes first, so it surfaces flushMemWal's rejection.
await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i);
// checkpointRTE freezes first, so it surfaces flushRTE's rejection.
await expect(table.checkpointRTE()).rejects.toThrow(/not supported/i);
});
});
nodejs/lancedb/arrow.ts
+27 -7
View File
@@ -456,14 +456,29 @@ export function makeArrowTable(
if (schema !== undefined) {
// Validate blob values up front and give every one the full
// `{ data, uri }` shape, so inference sees the same struct whether a row
// storage shape, so inference sees the same struct whether a row
// passed bytes, a URI, or a partial struct.
data = mapBlobInputs(data, schema, (value, field, row) => {
data = mapBlobInputs(data, schema, (value, field, row, storageField) => {
if (value === undefined) {
return value;
}
try {
return coerceBlobValue(value);
const normalized = coerceBlobValue(value, storageField);
if (
normalized === null ||
!(storageField.type instanceof Struct) ||
storageField.type.children.length === 2
) {
return normalized;
}
// Keep even unused range children during schema inference. Otherwise
// creating a range-capable table with inline data would lose them.
return Object.fromEntries(
storageField.type.children.map((child) => [
child.name,
normalized[child.name as keyof typeof normalized] ?? null,
]),
);
} catch (e) {
throw new Error(
`Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`,
@@ -533,7 +548,12 @@ function containsBlobField(field: Field): boolean {
);
}
type BlobInputVisitor = (value: unknown, field: string, row: number) => unknown;
type BlobInputVisitor = (
value: unknown,
field: string,
row: number,
storageField: Field,
) => unknown;
/**
* Calls `visit` on every value that lands in a blob column of `schema`,
@@ -592,7 +612,7 @@ function mapBlobField(
visit: BlobInputVisitor,
): unknown {
if (isBlobField(field)) {
return visit(value, label, row);
return visit(value, label, row, field);
}
if (field.type instanceof Struct && isObject(value)) {
return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
@@ -697,11 +717,11 @@ function transposeData(
const valuesPath = [...path, field.name];
if (isBlobField(field) && field.type instanceof Struct) {
const blobRows = data.map((datum) =>
coerceBlobValue(valueAtPath(datum, valuesPath)),
coerceBlobValue(valueAtPath(datum, valuesPath), field),
);
const childVectors = field.type.children.map((child) => {
const values = blobRows.map((row) =>
row == null ? null : (row[child.name as "data" | "uri"] ?? null),
row == null ? null : (row[child.name as keyof typeof row] ?? null),
);
return makeVector(values, child.type, undefined, child.nullable);
});
nodejs/lancedb/blob.ts
+91 -8
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Field, LargeBinary, Struct, Utf8 } from "apache-arrow";
import { Field, LargeBinary, Struct, Uint64, Utf8 } from "apache-arrow";
import { BlobFile as NativeBlobFile } from "./native";
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
@@ -28,6 +28,9 @@ export type BlobUri = string | URL;
*
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
* External byte ranges require both
* `position` (non-negative) and `size` (positive). Use safe integer numbers or
* `bigint` values for these unsigned 64-bit byte counts.
*
* @example
* ```ts
@@ -45,8 +48,14 @@ export type BlobUri = string | URL;
export type BlobInput =
| BlobData
| BlobUri
| { data: BlobData; uri?: null }
| { data?: null; uri: BlobUri }
| { data: BlobData; uri?: null; position?: null; size?: null }
| { data?: null; uri: BlobUri; position?: null; size?: null }
| {
data?: null;
uri: BlobUri;
position: number | bigint;
size: number | bigint;
}
| null
| undefined;
@@ -54,6 +63,8 @@ export type BlobInput =
export type BlobValue = {
data: Buffer | Uint8Array | null;
uri: string | null;
position?: bigint;
size?: bigint;
};
export type BlobOptions = {
@@ -78,6 +89,15 @@ export type BlobOptions = {
/**
* Declares a `lance.blob.v2` column.
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
* whole external files, and external byte ranges. Set both `position` and
* positive `size` for a range, or omit both for a whole file.
*
* Existing two-field columns remain writable. Appends and merge inserts fill
* missing range fields with nulls for four-field targets and drop all-null
* range fields for two-field targets. Non-null ranges require a four-field
* target. Overwrite with Arrow input replaces the table's schema; row objects
* are converted using the current table schema.
*
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
* or {@link Table.fetchBlobFiles} to read bytes.
@@ -142,6 +162,8 @@ export function blob(name: string, options: BlobOptions = {}): Field {
new Struct([
new Field("data", new LargeBinary(), true),
new Field("uri", new Utf8(), true),
new Field("position", new Uint64(), true),
new Field("size", new Uint64(), true),
]),
options.nullable ?? true,
metadata,
@@ -317,6 +339,8 @@ export function normalizeBlobInput(value: unknown): unknown {
...value,
data: value.data,
uri: value.uri,
position: value.position,
size: value.size,
};
let changed = false;
if (value.data instanceof ArrayBuffer) {
@@ -355,10 +379,19 @@ export function withBlobBytes(value: unknown, bytes: Uint8Array): unknown {
return bytes;
}
const record = value as Record<string, unknown>;
return { ...record, uri: record.uri, data: bytes };
return {
...record,
uri: record.uri,
position: record.position,
size: record.size,
data: bytes,
};
}
export function coerceBlobValue(input: unknown): BlobValue | null {
export function coerceBlobValue(
input: unknown,
field?: Field,
): BlobValue | null {
const value = normalizeBlobInput(input);
if (value == null) {
return null;
@@ -385,11 +418,11 @@ export function coerceBlobValue(input: unknown): BlobValue | null {
);
}
const unknownKeys = Object.keys(record).filter(
(key) => key !== "data" && key !== "uri",
(key) => !["data", "uri", "position", "size"].includes(key),
);
if (unknownKeys.length > 0) {
throw new Error(
`Blob struct values only support 'data' and 'uri', got ${unknownKeys
`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
.map((key) => `'${key}'`)
.join(", ")}`,
);
@@ -416,13 +449,63 @@ export function coerceBlobValue(input: unknown): BlobValue | null {
"Blob struct values must set exactly one of 'data' or 'uri'",
);
}
const hasPosition = record.position != null;
const hasSize = record.size != null;
if (hasPosition !== hasSize) {
throw new Error(
"External blob must set both position and size, or neither",
);
}
if (hasPosition) {
if (uriValue == null) {
throw new Error("External blob byte ranges require a uri");
}
const position = blobRangeInteger(record.position, "position", 0n);
const size = blobRangeInteger(record.size, "size", 1n);
if (position + size > 0xffffffffffffffffn) {
throw new Error("External blob range overflows uint64");
}
if (
field !== undefined &&
(!(field.type instanceof Struct) ||
["position", "size"].some(
(name) =>
!field.type.children.some((child: Field) => child.name === name),
))
) {
throw new Error(
"Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size",
);
}
return { data: null, uri: uriValue, position, size };
}
return { data: bytes, uri: uriValue };
}
throw new Error(
"Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }",
"Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }",
);
}
function blobRangeInteger(
value: unknown,
name: string,
minimum: bigint,
): bigint {
if (
typeof value !== "bigint" &&
(typeof value !== "number" || !Number.isSafeInteger(value))
) {
throw new Error(`Blob ${name} must be a safe integer or bigint`);
}
const integer = BigInt(value);
if (integer < minimum || integer > 0xffffffffffffffffn) {
throw new Error(
`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`,
);
}
return integer;
}
function isBlobLike(value: unknown): value is Blob {
return typeof Blob !== "undefined" && value instanceof Blob;
}
nodejs/lancedb/index.ts
+2 -2
View File
@@ -168,8 +168,8 @@ export {
WriteProgress,
FtsToken,
TokenizeTableOptions,
MemWalWriteSpec,
MemWalStats,
RTEWriteSpec,
RTEStats,
TableShardStats,
SsTableStats,
MemtableStats,
nodejs/lancedb/merge.ts
+8 -8
View File
@@ -88,27 +88,27 @@ export class MergeInsertBuilder {
);
}
/**
* Control MemWAL routing for this merge.
* Control RTE routing for this merge.
*
* By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
* By default (unset), a `mergeInsert` on a table with an RTE write spec is
* routed through Lance's MemWAL shard writer, and a table without one uses the
* standard path.
*
* @param enable - `true` forces MemWAL routing and errors if the table has no
* MemWAL write spec. `false` forces the standard write path even when a spec is set.
* @param enable - `true` forces RTE routing and errors if the table has no
* RTE write spec. `false` forces the standard write path even when a spec is set.
*/
useMemWal(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema);
useRTE(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
}
/**
* Controls how an LSM merge checks that its input targets a single shard.
*
* When a table has a MemWAL write spec, every row in a `mergeInsert` call must
* When a table has an RTE write spec, every row in a `mergeInsert` call must
* route to the same shard. When `true` (the default), every row is inspected
* to verify this. When `false`, only the first row is inspected and the
* shard it routes to is used for the whole input — a faster path for callers
* that have already pre-sharded their input. Has no effect on tables without
* a MemWAL write spec.
* an RTE write spec.
*
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
*/
nodejs/lancedb/query.ts
+15 -15
View File
@@ -677,26 +677,26 @@ export class StandardQueryBase<
}
/**
* Control MemWAL read routing for this query.
* Control RTE read routing for this query.
*
* By default (unset), when the table carries a MemWAL write spec (see
* {@link Table#enableMemWal}), reads are routed through the LSM scanner so
* By default (unset), when the table carries an RTE write spec (see
* {@link Table#enableRTE}), reads are routed through the LSM scanner so
* they also return data written via the `mergeInsert` LSM path that has not yet
* been compacted into the base table (the active/frozen in-memory memtables and
* the SSTables), deduplicated by primary key; a table without a spec
* reads the base table.
*
* @param enable - `true` forces the LSM scanner and errors if the table has no
* MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
* RTE write spec. `false` bypasses the RTE and reads the base table only,
* even when a spec is present.
*
* Note: the LSM scanner does not support every query shape (e.g. reranking,
* hybrid search, `orderBy`). On a MemWAL table those shapes error unless
* `useMemWal(false)` is set, because a base-only read would silently exclude
* un-compacted MemWAL data.
* hybrid search, `orderBy`). On an RTE table those shapes error unless
* `useRTE(false)` is set, because a base-only read would silently exclude
* un-compacted RTE data.
*/
useMemWal(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useMemWal(enable));
useRTE(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useRTE(enable));
return this;
}
}
@@ -776,7 +776,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* probing may continue when too few candidates are found.
* Any explicit effort, including 0.5, overrides nprobes in either call order.
* Explicit minimum and maximum bounds still apply. This is not a recall
* guarantee and does not change refinement or HNSW ef. The MemWAL LSM
* guarantee and does not change refinement or HNSW ef. The RTE LSM
* scanner does not yet support this option.
*
* @example
@@ -1082,16 +1082,16 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
}
/**
* Control MemWAL read routing for this take query.
* Control RTE read routing for this take query.
*
* `false` bypasses the MemWAL and reads the base table only — the escape hatch,
* `false` bypasses the RTE and reads the base table only — the escape hatch,
* since take-by-row-id/offset is not supported on the LSM scanner and, on a
* MemWAL table, auto-routes to it and errors otherwise.
* RTE table, auto-routes to it and errors otherwise.
*
* @param enable - `false` reads the base table only.
*/
useMemWal(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useMemWal(enable));
useRTE(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useRTE(enable));
return this;
}
}
nodejs/lancedb/table.ts
+54 -54
View File
@@ -35,9 +35,9 @@ import {
FunctionErrorsOptions,
IndexConfig,
IndexStatistics,
MemWalStats,
Branches as NativeBranches,
OptimizeStats,
RTEStats,
RefreshColumnResult,
RefreshMaterializedViewResult,
TableStatistics,
@@ -61,7 +61,7 @@ export { IndexConfig } from "./native";
export {
TableShardStats,
SsTableStats,
MemWalStats,
RTEStats,
MemtableStats,
} from "./native";
@@ -207,8 +207,8 @@ export type TokenizeTableOptions =
};
/**
* Specification selecting Lance's MemWAL LSM-style write path for
* `mergeInsert`.
* Specification enabling the realtime engine (RTE), which routes
* `mergeInsert` through Lance's MemWAL LSM-style write path.
*
* `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
* `column` and `numBuckets` are required; for `"identity"`, `column` is
@@ -217,7 +217,7 @@ export type TokenizeTableOptions =
* `column` value, or upserts of that key can land in different shards and a
* stale version can win).
*/
export interface MemWalWriteSpec {
export interface RTEWriteSpec {
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
specType: "bucket" | "identity" | "unsharded";
/** Bucket and identity variants: the sharding column. */
@@ -225,7 +225,7 @@ export interface MemWalWriteSpec {
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
numBuckets?: number;
/**
* Indexes the MemWAL keeps up to date. Omit for every index the table has,
* Indexes the RTE keeps up to date. Omit for every index the table has,
* including ones created later. Pass `[]` for none.
*/
maintainedIndexes?: string[];
@@ -758,10 +758,10 @@ export abstract class Table {
*/
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
/**
* Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL
* Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
* LSM-style write path for future `mergeInsert` calls.
*
* `MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
* `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
*
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
* key (`column` and `numBuckets` required).
@@ -773,15 +773,15 @@ export abstract class Table {
* requires it to be the single column being bucketed.
*
* Omitting `maintainedIndexes` maintains every index the table has,
* including ones created later, and skips a kind the MemWAL cannot maintain.
* including ones created later, and skips a kind the RTE cannot maintain.
* Naming them pins an exact set, and a still-building index is rejected
* rather than quietly omitted.
* @param {MemWalWriteSpec} spec The sharding spec to install.
* @param {RTEWriteSpec} spec The sharding spec to install.
* @returns {Promise<void>}
* @example
* ```ts
* await table.setUnenforcedPrimaryKey("id");
* await table.enableMemWal({
* await table.enableRTE({
* specType: "bucket",
* column: "id",
* numBuckets: 16,
@@ -789,36 +789,36 @@ export abstract class Table {
* });
* ```
*/
abstract enableMemWal(spec: MemWalWriteSpec): Promise<void>;
abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
/**
* Remove the {@link MemWalWriteSpec} from this table, reverting to the standard
* Remove the {@link RTEWriteSpec} from this table, reverting to the standard
* `mergeInsert` write path.
*
* Errors if no spec is currently set.
* @returns {Promise<void>}
*/
abstract disableMemWal(): Promise<void>;
abstract disableRTE(): Promise<void>;
/**
* Read the {@link MemWalWriteSpec} currently installed on this table.
* Read the {@link RTEWriteSpec} currently installed on this table.
*
* Resolves to `undefined` when the MemWAL write path is not enabled (no
* spec has been set, or it was removed with {@link Table#disableMemWal}).
* Resolves to `undefined` when the RTE write path is not enabled (no
* spec has been set, or it was removed with {@link Table#disableRTE}).
* The spec is the one installed, including its maintained-index selection:
* an absent `maintainedIndexes` for every index the table has, an empty
* array for none.
* @returns {Promise<MemWalWriteSpec | undefined>}
* @returns {Promise<RTEWriteSpec | undefined>}
*/
abstract getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
/**
* Drain and close any cached MemWAL shard writers held for this table.
* Drain and close any cached RTE shard writers held for this table.
*
* When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL
* When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
* shard writers and caches them for reuse across calls. This closes them,
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
* It is a no-op when no writers are cached.
* @returns {Promise<void>}
*/
abstract closeMemWalWriters(): Promise<void>;
abstract closeRTEWriters(): Promise<void>;
/**
* Freeze every table shard's active memtable into a new SSTable.
*
@@ -826,18 +826,18 @@ export abstract class Table {
* so this is safe to call repeatedly.
* @returns {Promise<void>}
*/
abstract flushMemWal(): Promise<void>;
abstract flushRTE(): Promise<void>;
/**
* Trigger a background SSTable compaction pass per table shard.
*
* Returns once the passes are *dispatched*, not once they finish — watch
* {@link Table#getMemWalStats} for progress, or use
* {@link Table#checkpointMemWal} to wait for convergence.
* {@link Table#getRTEStats} for progress, or use
* {@link Table#checkpointRTE} to wait for convergence.
* @returns {Promise<void>}
*/
abstract compactMemWal(): Promise<void>;
abstract compactRTE(): Promise<void>;
/**
* Converge this table's MemWAL write path into its base table.
* Converge this table's RTE write path into its base table.
*
* Freezes once, then triggers compaction and polls until the SSTables that existed
* at the start is gone. The target set is fixed at the start, so
@@ -852,26 +852,26 @@ export abstract class Table {
* @returns {Promise<void>}
* @example
* ```ts
* const before = await table.getMemWalStats();
* await table.checkpointMemWal();
* const after = await table.getMemWalStats();
* const before = await table.getRTEStats();
* await table.checkpointRTE();
* const after = await table.getRTEStats();
* ```
*/
abstract checkpointMemWal(): Promise<void>;
abstract checkpointRTE(): Promise<void>;
/**
* Read live per-table-shard MemWAL state.
* Read live per-table-shard RTE state.
*
* Answers "how far behind is my fresh tier", "which table shard is hot", and
* "why is my fresh-tier vector search brute-force". Mutates no table state.
*
* Resolves to `undefined` only when the MemWAL write path is not enabled.
* Resolves to `undefined` only when the RTE write path is not enabled.
* @param {boolean} includeSstableRows Also count rows per SSTable.
* Off by default because each count opens an uncached Lance dataset.
* @returns {Promise<MemWalStats | undefined>}
* @returns {Promise<RTEStats | undefined>}
*/
abstract getMemWalStats(
abstract getRTEStats(
includeSstableRows?: boolean,
): Promise<MemWalStats | undefined>;
): Promise<RTEStats | undefined>;
/** Retrieve the version of the table */
abstract version(): Promise<number>;
@@ -1480,43 +1480,43 @@ export class LocalTable extends Table {
return await this.inner.setUnenforcedPrimaryKey(cols);
}
async enableMemWal(spec: MemWalWriteSpec): Promise<void> {
return await this.inner.enableMemWal(spec);
async enableRTE(spec: RTEWriteSpec): Promise<void> {
return await this.inner.enableRTE(spec);
}
async disableMemWal(): Promise<void> {
return await this.inner.disableMemWal();
async disableRTE(): Promise<void> {
return await this.inner.disableRTE();
}
async getMemWalSpec(): Promise<MemWalWriteSpec | undefined> {
async getRTESpec(): Promise<RTEWriteSpec | undefined> {
// The native binding types `specType` as a plain `string`; narrow it back
// to the public union. The Rust `From` impl only ever emits one of the
// three valid values, so the cast is safe.
return ((await this.inner.getMemWalSpec()) ?? undefined) as
| MemWalWriteSpec
return ((await this.inner.getRTESpec()) ?? undefined) as
| RTEWriteSpec
| undefined;
}
async closeMemWalWriters(): Promise<void> {
return await this.inner.closeMemWalWriters();
async closeRTEWriters(): Promise<void> {
return await this.inner.closeRTEWriters();
}
async flushMemWal(): Promise<void> {
return await this.inner.flushMemWal();
async flushRTE(): Promise<void> {
return await this.inner.flushRTE();
}
async compactMemWal(): Promise<void> {
return await this.inner.compactMemWal();
async compactRTE(): Promise<void> {
return await this.inner.compactRTE();
}
async checkpointMemWal(): Promise<void> {
return await this.inner.checkpointMemWal();
async checkpointRTE(): Promise<void> {
return await this.inner.checkpointRTE();
}
async getMemWalStats(
async getRTEStats(
includeSstableRows: boolean = false,
): Promise<MemWalStats | undefined> {
return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined;
): Promise<RTEStats | undefined> {
return (await this.inner.getRTEStats(includeSstableRows)) ?? undefined;
}
async version(): Promise<number> {
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.41.0-beta.9",
"version": "0.41.0-beta.11",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/merge.rs
+3 -3
View File
@@ -50,10 +50,10 @@ impl NativeMergeInsertBuilder {
this
}
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&self, enable: bool) -> Self {
#[napi(js_name = "useRTE")]
pub fn use_rte(&self, enable: bool) -> Self {
let mut this = self.clone();
this.inner.use_memwal(enable);
this.inner.use_rte(enable);
this
}
nodejs/src/query.rs
+9 -9
View File
@@ -168,9 +168,9 @@ impl Query {
self.inner = self.inner.clone().with_row_id();
}
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
#[napi(js_name = "useRTE")]
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[napi]
@@ -399,9 +399,9 @@ impl VectorQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
#[napi(js_name = "useRTE")]
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[napi]
@@ -514,9 +514,9 @@ impl TakeQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
#[napi(js_name = "useRTE")]
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[napi(catch_unwind)]
nodejs/src/table.rs
+47 -50
View File
@@ -580,60 +580,57 @@ impl Table {
.default_error()
}
#[napi(catch_unwind, js_name = "enableMemWal")]
pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> {
let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?;
#[napi(catch_unwind, js_name = "enableRTE")]
pub async fn enable_rte(&self, spec: RTEWriteSpec) -> napi::Result<()> {
let native_spec = lancedb::table::RTEWriteSpec::try_from(spec)?;
self.inner_ref()?
.enable_memwal(native_spec)
.enable_rte(native_spec)
.await
.default_error()
}
#[napi(catch_unwind, js_name = "disableMemWal")]
pub async fn disable_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.disable_memwal().await.default_error()
#[napi(catch_unwind, js_name = "disableRTE")]
pub async fn disable_rte(&self) -> napi::Result<()> {
self.inner_ref()?.disable_rte().await.default_error()
}
#[napi(catch_unwind, js_name = "getMemWalSpec")]
pub async fn get_memwal_spec(&self) -> napi::Result<Option<MemWalWriteSpec>> {
let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?;
Ok(spec.map(MemWalWriteSpec::from))
#[napi(catch_unwind, js_name = "getRTESpec")]
pub async fn get_rte_spec(&self) -> napi::Result<Option<RTEWriteSpec>> {
let spec = self.inner_ref()?.get_rte_spec().await.default_error()?;
Ok(spec.map(RTEWriteSpec::from))
}
#[napi(catch_unwind, js_name = "closeMemWalWriters")]
pub async fn close_memwal_writers(&self) -> napi::Result<()> {
self.inner_ref()?
.close_memwal_writers()
.await
.default_error()
#[napi(catch_unwind, js_name = "closeRTEWriters")]
pub async fn close_rte_writers(&self) -> napi::Result<()> {
self.inner_ref()?.close_rte_writers().await.default_error()
}
#[napi(catch_unwind, js_name = "flushMemWal")]
pub async fn flush_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.flush_memwal().await.default_error()
#[napi(catch_unwind, js_name = "flushRTE")]
pub async fn flush_rte(&self) -> napi::Result<()> {
self.inner_ref()?.flush_rte().await.default_error()
}
#[napi(catch_unwind, js_name = "compactMemWal")]
pub async fn compact_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.compact_memwal().await.default_error()
#[napi(catch_unwind, js_name = "compactRTE")]
pub async fn compact_rte(&self) -> napi::Result<()> {
self.inner_ref()?.compact_rte().await.default_error()
}
#[napi(catch_unwind, js_name = "checkpointMemWal")]
pub async fn checkpoint_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_memwal().await.default_error()
#[napi(catch_unwind, js_name = "checkpointRTE")]
pub async fn checkpoint_rte(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_rte().await.default_error()
}
#[napi(catch_unwind, js_name = "getMemWalStats")]
pub async fn get_memwal_stats(
#[napi(catch_unwind, js_name = "getRTEStats")]
pub async fn get_rte_stats(
&self,
include_sstable_rows: bool,
) -> napi::Result<Option<MemWalStats>> {
) -> napi::Result<Option<RTEStats>> {
let stats = self
.inner_ref()?
.get_memwal_stats(include_sstable_rows)
.get_rte_stats(include_sstable_rows)
.await
.default_error()?;
Ok(stats.map(MemWalStats::from))
Ok(stats.map(RTEStats::from))
}
#[napi(catch_unwind)]
@@ -946,53 +943,53 @@ impl From<LanceDbFtsToken> for FtsToken {
}
}
/// Specification selecting Lance's MemWAL LSM-style write path for
/// `mergeInsert`.
/// Specification enabling the realtime engine (RTE), which routes
/// `mergeInsert` through Lance's MemWAL LSM-style write path.
///
/// `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
/// `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
/// `column` is required.
#[napi(object)]
#[napi(object, js_name = "RTEWriteSpec")]
#[derive(Clone, Debug)]
pub struct MemWalWriteSpec {
pub struct RTEWriteSpec {
/// One of `"bucket"`, `"identity"`, or `"unsharded"`.
pub spec_type: String,
/// Bucket and identity variants: the sharding column.
pub column: Option<String>,
/// Bucket variant: the number of buckets, in `[1, 1024]`.
pub num_buckets: Option<u32>,
/// Indexes the MemWAL keeps up to date. Omitted resolves every
/// Indexes the RTE keeps up to date. Omitted resolves every
/// maintainable index on install; an empty array means none.
pub maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
pub writer_config_defaults: Option<HashMap<String, String>>,
}
impl TryFrom<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
impl TryFrom<RTEWriteSpec> for lancedb::table::RTEWriteSpec {
type Error = napi::Error;
fn try_from(value: MemWalWriteSpec) -> napi::Result<Self> {
fn try_from(value: RTEWriteSpec) -> napi::Result<Self> {
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
let spec = match value.spec_type.as_str() {
"bucket" => {
let column = value.column.ok_or_else(|| {
napi::Error::from_reason("MemWalWriteSpec bucket requires `column`")
napi::Error::from_reason("RTEWriteSpec bucket requires `column`")
})?;
let num_buckets = value.num_buckets.ok_or_else(|| {
napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`")
napi::Error::from_reason("RTEWriteSpec bucket requires `numBuckets`")
})?;
Self::bucket(column, num_buckets)
}
"identity" => {
let column = value.column.ok_or_else(|| {
napi::Error::from_reason("MemWalWriteSpec identity requires `column`")
napi::Error::from_reason("RTEWriteSpec identity requires `column`")
})?;
Self::identity(column)
}
"unsharded" => Self::unsharded(),
other => {
return Err(napi::Error::from_reason(format!(
"MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
"RTEWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
other
)));
}
@@ -1003,9 +1000,9 @@ impl TryFrom<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
}
}
impl From<lancedb::table::MemWalWriteSpec> for MemWalWriteSpec {
fn from(spec: lancedb::table::MemWalWriteSpec) -> Self {
use lancedb::table::MemWalWriteSpec as Native;
impl From<lancedb::table::RTEWriteSpec> for RTEWriteSpec {
fn from(spec: lancedb::table::RTEWriteSpec) -> Self {
use lancedb::table::RTEWriteSpec as Native;
match spec {
Native::Bucket {
column,
@@ -1148,19 +1145,19 @@ impl From<lancedb::table::TableShardStats> for TableShardStats {
}
}
/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
/// Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
///
/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
/// the caller's to compute.
#[napi(object)]
#[napi(object, js_name = "RTEStats")]
#[derive(Clone, Debug)]
pub struct MemWalStats {
pub struct RTEStats {
/// One entry per table shard backing this table.
pub table_shards: Vec<TableShardStats>,
}
impl From<lancedb::table::MemWalStats> for MemWalStats {
fn from(stats: lancedb::table::MemWalStats) -> Self {
impl From<lancedb::table::RTEStats> for RTEStats {
fn from(stats: lancedb::table::RTEStats) -> Self {
Self {
table_shards: stats.table_shards.into_iter().map(Into::into).collect(),
}
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.9"
version = "0.41.0-beta.11"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/__init__.py
+2 -2
View File
@@ -15,7 +15,7 @@ from ._lancedb import FtsToken
from ._lancedb import FunctionErrorFragment as FunctionErrorFragment
from ._lancedb import FunctionErrorRecord as FunctionErrorRecord
from ._lancedb import FunctionErrors as FunctionErrors
from ._lancedb import MemWalWriteSpec
from ._lancedb import RTEWriteSpec
from ._lancedb import tokenize as _tokenize
from .common import URI, sanitize_uri
from urllib.parse import urlparse
@@ -610,7 +610,7 @@ __all__ = [
"SqlQuery",
"LanceDBConnection",
"LanceNamespaceDBConnection",
"MemWalWriteSpec",
"RTEWriteSpec",
"RemoteDBConnection",
"Session",
"Table",
python/python/lancedb/_lancedb.pyi
+26 -30
View File
@@ -538,17 +538,17 @@ class Table:
async def initial_storage_options(self) -> Optional[Dict[str, str]]: ...
async def latest_storage_options(self) -> Optional[Dict[str, str]]: ...
async def set_unenforced_primary_key(self, columns: List[str]) -> None: ...
async def enable_memwal(self, spec: MemWalWriteSpec) -> None: ...
async def disable_memwal(self) -> None: ...
async def get_memwal_spec(self) -> Optional[MemWalWriteSpec]: ...
async def enable_rte(self, spec: RTEWriteSpec) -> None: ...
async def disable_rte(self) -> None: ...
async def get_rte_spec(self) -> Optional[RTEWriteSpec]: ...
# WAL-PK-FUSION: delete both.
def hybrid_pk_fusion_learned(self) -> bool: ...
def note_hybrid_pk_fusion(self) -> None: ...
async def checkpoint_memwal(self) -> None: ...
async def flush_memwal(self) -> None: ...
async def compact_memwal(self) -> None: ...
async def get_memwal_stats(self, include_sstable_rows: bool) -> Optional[dict]: ...
async def close_memwal_writers(self) -> None: ...
async def checkpoint_rte(self) -> None: ...
async def flush_rte(self) -> None: ...
async def compact_rte(self) -> None: ...
async def get_rte_stats(self, include_sstable_rows: bool) -> Optional[dict]: ...
async def close_rte_writers(self) -> None: ...
@property
def tags(self) -> Tags: ...
@property
@@ -661,7 +661,7 @@ class Query:
def fast_search(self): ...
def with_row_id(self): ...
def postfilter(self): ...
def use_memwal(self, enable: bool): ...
def use_rte(self, enable: bool): ...
def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
def nearest_to_text(self, query: dict) -> FTSQuery: ...
def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
@@ -678,7 +678,7 @@ class Query:
class TakeQuery:
def select(self, columns: List[str]): ...
def with_row_id(self): ...
def use_memwal(self, enable: bool): ...
def use_rte(self, enable: bool): ...
async def output_schema(self) -> pa.Schema: ...
async def execute(self) -> RecordBatchStream: ...
async def explain_plan(self, verbose: Optional[bool]) -> str: ...
@@ -697,7 +697,7 @@ class FTSQuery:
def fast_search(self): ...
def with_row_id(self): ...
def postfilter(self): ...
def use_memwal(self, enable: bool): ...
def use_rte(self, enable: bool): ...
def get_query(self) -> str: ...
def add_query_vector(self, query_vec: pa.Array) -> None: ...
def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
@@ -725,7 +725,7 @@ class VectorQuery:
def column(self, column: str): ...
def distance_type(self, distance_type: str): ...
def postfilter(self): ...
def use_memwal(self, enable: bool): ...
def use_rte(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
@@ -750,7 +750,7 @@ class HybridQuery:
def fast_search(self): ...
def with_row_id(self): ...
def postfilter(self): ...
def use_memwal(self, enable: bool): ...
def use_rte(self, enable: bool): ...
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
@@ -778,7 +778,7 @@ class PyQueryRequest:
select_source_columns: Optional[Dict[str, str]]
fast_search: Optional[bool]
with_row_id: Optional[bool]
use_memwal: Optional[bool]
use_rte: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
nprobes: Optional[int]
@@ -839,26 +839,22 @@ class FtsToken:
text: str
position: int
class MemWalWriteSpec:
"""Specification selecting Lance's MemWAL LSM-style write path for
`merge_insert`."""
class RTEWriteSpec:
"""Specification enabling the realtime engine (RTE), which routes
`merge_insert` through Lance's MemWAL LSM-style write path."""
@staticmethod
def bucket(column: str, num_buckets: int) -> "MemWalWriteSpec": ...
def bucket(column: str, num_buckets: int) -> "RTEWriteSpec": ...
@staticmethod
def identity(column: str) -> "MemWalWriteSpec": ...
def identity(column: str) -> "RTEWriteSpec": ...
@staticmethod
def unsharded() -> "MemWalWriteSpec": ...
def with_maintained_indexes(
self, indexes: Optional[List[str]]
) -> "MemWalWriteSpec":
"""Set which indexes the MemWAL keeps up to date. None resolves every
index on the table at install, failing if one cannot be maintained;
a list is verbatim, empty means none."""
def unsharded() -> "RTEWriteSpec": ...
def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "RTEWriteSpec":
"""Set which indexes the RTE keeps up to date. None is every index on
the table, now and later, skipping any it cannot maintain; a list is
verbatim, empty means none."""
...
def with_writer_config_defaults(
self, defaults: Dict[str, str]
) -> "MemWalWriteSpec":
def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "RTEWriteSpec":
"""Return a copy of this spec recording the given default
`ShardWriter` configuration in the MemWAL index."""
...
@@ -872,7 +868,7 @@ class MemWalWriteSpec:
def num_buckets(self) -> Optional[int]: ...
@property
def maintained_indexes(self) -> Optional[List[str]]:
"""Indexes the MemWAL keeps up to date, or None for every supported one."""
"""Indexes the RTE keeps up to date, or None for every supported one."""
...
@property
def writer_config_defaults(self) -> Dict[str, str]: ...
python/python/lancedb/_wal_hybrid.py
+19 -19
View File
@@ -1,14 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables.
"""WAL-PK-FUSION: hybrid search's primary-key fallback for RTE tables.
Hybrid search joins its vector and full-text legs on `_rowid`, which a MemWAL
Hybrid search joins its vector and full-text legs on `_rowid`, which an RTE
table cannot supply: the fresh tier has no stable row id, so the server rejects
`with_row_id` before it plans. When that happens the legs are joined on a
surrogate `_rowid` built from the table's primary key instead.
This is a stopgap until MemWAL supports `_rowid`, and everything it needs lives
This is a stopgap until RTE supports `_rowid`, and everything it needs lives
here so it can be removed in one pass:
1. Delete this file and `tests/test_wal_hybrid.py`.
@@ -69,21 +69,21 @@ def _pk_columns(schema: pa.Schema) -> List[str]:
# builds it from the same words -- so this substring survives either path and
# the wrapping each adds. It is a coupling to a message rather than a code,
# because the refusal arrives as a generic 400; if the server ever rewords it
# the fallback stops firing and hybrid on a MemWAL table errors visibly again,
# the fallback stops firing and hybrid on an RTE table errors visibly again,
# which is loud rather than silently wrong.
_WAL_ROW_ID_REFUSAL = "does not support with_row_id"
WAL_ROW_ID_UNSUPPORTED = (
"hybrid search on a MemWAL table cannot return _rowid: the fresh tier has "
"hybrid search on an RTE table cannot return _rowid: the fresh tier has "
"no stable row id, and the ids the fusion joins on are synthesized from "
"the primary key. Set use_memwal(False) to read the base table only (results "
"will exclude un-compacted MemWAL data)"
"the primary key. Set use_rte(False) to read the base table only (results "
"will exclude un-compacted RTE data)"
)
def _is_wal_row_id_refusal(error: BaseException) -> bool:
"""Whether `error` is the server declining `_rowid` because the table is
MemWAL-backed, as opposed to any other failure the query might hit."""
RTE-backed, as opposed to any other failure the query might hit."""
return _WAL_ROW_ID_REFUSAL in str(error)
@@ -112,7 +112,7 @@ def _stamp_surrogate_row_ids(
"""Give each table a `_rowid` column derived from its primary key.
The fusion only ever compares that column for equality — to deduplicate, to
group by in a reranker, and to restore the original scores — so on a MemWAL
group by in a reranker, and to restore the original scores — so on an RTE
table, where the server has no stable row id to hand out, any injective
mapping of the primary key does the same job. Ids are assigned in first-seen
order across `tables`, so passing them in fusion order (vector, then FTS)
@@ -124,7 +124,7 @@ def _stamp_surrogate_row_ids(
"""
if not pk_columns:
raise ValueError(
"hybrid search on a MemWAL table needs an unenforced primary key to "
"hybrid search on an RTE table needs an unenforced primary key to "
"deduplicate on, and this table declares none"
)
@@ -151,13 +151,13 @@ def _stamp_surrogate_row_ids(
return stamped
def pk_fusion_learned(table, use_memwal: Optional[bool]) -> bool:
def pk_fusion_learned(table, use_rte: Optional[bool]) -> bool:
"""Whether a previous hybrid query on `table` was refused `_rowid`, so this
one should go straight to the primary key.
Learned from a refusal, never probed, so asking is free.
"""
if table is None or use_memwal is False:
if table is None or use_rte is False:
return False
# Only an unambiguous True forks to the key: a table type that answers the
# question loosely keeps the `_rowid` path.
@@ -165,7 +165,7 @@ def pk_fusion_learned(table, use_memwal: Optional[bool]) -> bool:
class PkFusion:
"""How the legs of a hybrid query on a MemWAL table are joined: on a
"""How the legs of a hybrid query on an RTE table are joined: on a
surrogate `_rowid` built from the primary key."""
def __init__(self, pk_columns: List[str]):
@@ -190,16 +190,16 @@ class PkFusion:
# has turned `_with_row_id` on — including `return_score="all"`.
if blob_v2_projection_sources(schema, projection):
raise NotImplementedError(
"hybrid search cannot project a blob column on a MemWAL table: "
"hybrid search cannot project a blob column on an RTE table: "
"fetching blobs needs a real _rowid, and the fresh tier has no "
"stable one. Set use_memwal(False) to read the base table only "
"(results will exclude un-compacted MemWAL data)"
"stable one. Set use_rte(False) to read the base table only "
"(results will exclude un-compacted RTE data)"
)
return cls(_pk_columns(schema))
def inject(self, columns):
"""`columns` with the key columns it is missing, to project in place of
the `_rowid` a MemWAL table would refuse."""
the `_rowid` an RTE table would refuse."""
columns, self.injected = _with_pk_columns(columns, self.pk_columns)
return columns
@@ -226,7 +226,7 @@ class PkFusion:
def _check_refusal(error: Exception, *, fused: bool, caller_requested_row_id: bool):
"""Re-raise `error` unless it is a MemWAL `_rowid` refusal worth retrying on
"""Re-raise `error` unless it is an RTE `_rowid` refusal worth retrying on
the primary key."""
if fused or not _is_wal_row_id_refusal(error):
raise error
@@ -248,7 +248,7 @@ def with_pk_fallback(
`_rowid`.
`_rowid` is asked for first rather than paying a round trip up front to
find out: only a MemWAL table refuses, and the refusal has to be handled
find out: only an RTE table refuses, and the refusal has to be handled
anyway -- a write spec installed elsewhere can arrive between any two
queries. It costs nothing server-side: it is raised before the query is
planned. `fused` reports whether the attempt already used the key.
python/python/lancedb/merge.py
+8 -8
View File
@@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object):
self._when_not_matched_by_source_condition_expr = None
self._timeout = None
self._use_index = True
self._use_memwal = None
self._use_rte = None
self._validate_single_shard = None
def when_matched_update_all(
@@ -115,22 +115,22 @@ class LanceMergeInsertBuilder(object):
self._use_index = use_index
return self
def use_memwal(self, enable: bool) -> LanceMergeInsertBuilder:
def use_rte(self, enable: bool) -> LanceMergeInsertBuilder:
"""
Control MemWAL routing for this merge.
Control RTE routing for this merge.
By default (unset), a `merge_insert` on a table with a MemWAL write spec is
By default (unset), a `merge_insert` on a table with an RTE write spec is
routed through Lance's MemWAL shard writer, and a table without one uses
the standard path.
Parameters
----------
enable: bool
``True`` forces MemWAL routing and errors if the table has no MemWAL
``True`` forces RTE routing and errors if the table has no RTE
write spec. ``False`` forces the standard write path even when a spec
is set.
"""
self._use_memwal = enable
self._use_rte = enable
return self
def validate_single_shard(
@@ -139,13 +139,13 @@ class LanceMergeInsertBuilder(object):
"""
Controls how an LSM merge checks that its input targets a single shard.
When a table has a MemWAL write spec, every row in a `merge_insert` call
When a table has an RTE write spec, every row in a `merge_insert` call
must route to the same shard. When `True` (the default), every row is
inspected to verify this. When `False`, only the first row is inspected
and the shard it routes to is used for the whole input — a faster path
for callers that have already pre-sharded their input.
Has no effect on tables without a MemWAL write spec.
Has no effect on tables without an RTE write spec.
Parameters
----------
python/python/lancedb/permutation.py
+1 -1
View File
@@ -48,7 +48,7 @@ class PermutationBuilder:
By default, the permutation builder will create a single split that contains all
rows in the same order as the base table.
Tables with a MemWAL write spec are rejected: unflushed rows have no row id.
Tables with an RTE write spec are rejected: unflushed rows have no row id.
"""
self._async = async_permutation_builder(table)
python/python/lancedb/query.py
+39 -39
View File
@@ -887,10 +887,10 @@ class Query(pydantic.BaseModel):
# if true, will only search the indexed data
fast_search: Optional[bool] = None
# MemWAL LSM read routing: None auto-routes when the table carries a write
# RTE LSM read routing: None auto-routes when the table carries a write
# spec, True forces the LSM scanner (errors without a spec), False reads the
# base table only
use_memwal: Optional[bool] = None
use_rte: Optional[bool] = None
# size of the nearest neighbor list maintained during HNSW search
ef: Optional[int] = None
@@ -910,9 +910,9 @@ class Query(pydantic.BaseModel):
query.full_text_query = req.full_text_search
query.columns = req.select
query.with_row_id = req.with_row_id
# use_memwal is a genuine tri-state (None / True / False); preserve it as-is
# use_rte is a genuine tri-state (None / True / False); preserve it as-is
# so a round-tripped query keeps an explicit False.
query.use_memwal = req.use_memwal
query.use_rte = req.use_rte
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
@@ -1085,7 +1085,7 @@ class LanceQueryBuilder(ABC):
self._with_row_address = None
self._fragments = None
self._fragment_ids = None
self._use_memwal = None
self._use_rte = None
self._vector = None
self._text = None
self._ef = None
@@ -1520,10 +1520,10 @@ class LanceQueryBuilder(ABC):
self._fragment_ids = fragment_ids
return self
def use_memwal(self, enable: bool) -> Self:
"""Control MemWAL LSM read routing for this query.
def use_rte(self, enable: bool) -> Self:
"""Control RTE LSM read routing for this query.
By default (unset), a query against a table with a MemWAL write spec is
By default (unset), a query against a table with an RTE write spec is
routed through the LSM scanner so it also returns data written via the
``merge_insert`` LSM path that has not yet been compacted into the base
table (active/frozen memtables + SSTables); a table without a
@@ -1532,8 +1532,8 @@ class LanceQueryBuilder(ABC):
Parameters
----------
enable : bool
``True`` forces the LSM scanner and errors if the table has no MemWAL
write spec. ``False`` bypasses the MemWAL and reads the base table
``True`` forces the LSM scanner and errors if the table has no RTE
write spec. ``False`` bypasses the RTE and reads the base table
only, even when a spec is present.
Returns
@@ -1541,7 +1541,7 @@ class LanceQueryBuilder(ABC):
LanceQueryBuilder
The LanceQueryBuilder object.
"""
self._use_memwal = enable
self._use_rte = enable
return self
def explain_plan(self, verbose: Optional[bool] = False) -> str:
@@ -1878,7 +1878,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
and does not change refinement or HNSW ef. RTE LSM queries do not
yet support this option.
Examples
@@ -2055,7 +2055,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
with_row_address=self._with_row_address,
fragments=self._fragments,
fragment_ids=self._fragment_ids,
use_memwal=self._use_memwal,
use_rte=self._use_rte,
offset=self._offset,
fast_search=self._fast_search,
ef=self._ef,
@@ -2280,7 +2280,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
with_row_address=self._with_row_address,
fragments=self._fragments,
fragment_ids=self._fragment_ids,
use_memwal=self._use_memwal,
use_rte=self._use_rte,
full_text_query=FullTextSearchQuery(
query=self._query_with_phrase_semantics(), columns=self._fts_columns
),
@@ -2362,7 +2362,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
with_row_address=self._with_row_address,
fragments=self._fragments,
fragment_ids=self._fragment_ids,
use_memwal=self._use_memwal,
use_rte=self._use_rte,
offset=self._offset,
order_by=self._order_by,
)
@@ -2458,7 +2458,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
"""Set whether to return row ids.
Recorded separately from `_with_row_id`, which `rerank(return_score=
"all")` also sets for its own use — on a MemWAL table only a caller who
"all")` also sets for its own use — on an RTE table only a caller who
asked is refused, and the reranker still falls back to the primary key.
"""
self._caller_requested_row_id = with_row_id
@@ -2728,7 +2728,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
if reranker.score == "all":
# WAL-PK-FUSION: without the fallback, `self.with_row_id(True)`.
# Not through `with_row_id`, which records caller intent: the
# reranker's need for row ids must not refuse a MemWAL table.
# reranker's need for row ids must not refuse an RTE table.
self._with_row_id = True
return self
@@ -2766,7 +2766,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
and does not change refinement or HNSW ef. RTE LSM queries do not
yet support this option.
Examples
@@ -3040,7 +3040,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
# WAL-PK-FUSION: without the fallback, select `self._columns` as is.
self._pk_fusion = None
columns = self._columns
if _wal_hybrid.pk_fusion_learned(self._table, self._use_memwal):
if _wal_hybrid.pk_fusion_learned(self._table, self._use_rte):
self._pk_fusion = _wal_hybrid.PkFusion.for_query(
self._table.schema,
self._columns,
@@ -3057,9 +3057,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
if self._with_row_id and self._pk_fusion is None:
self._vector_query.with_row_id(True)
self._fts_query.with_row_id(True)
if self._use_memwal is not None:
self._vector_query.use_memwal(self._use_memwal)
self._fts_query.use_memwal(self._use_memwal)
if self._use_rte is not None:
self._vector_query.use_rte(self._use_rte)
self._fts_query.use_rte(self._use_rte)
if self._phrase_query:
self._fts_query.phrase_query(True)
if self._distance_type:
@@ -3723,12 +3723,12 @@ class AsyncStandardQuery(AsyncQueryBase):
self._inner.fast_search()
return self
def use_memwal(self, enable: bool) -> Self:
def use_rte(self, enable: bool) -> Self:
"""
Control MemWAL LSM read routing for this query.
Control RTE LSM read routing for this query.
By default (unset), a query against a table with a MemWAL write spec (see
[AsyncTable.enable_memwal][lancedb.table.AsyncTable.enable_memwal])
By default (unset), a query against a table with an RTE write spec (see
[AsyncTable.enable_rte][lancedb.table.AsyncTable.enable_rte])
is routed through the LSM scanner so it also returns data written via the
``merge_insert`` LSM path that has not yet been compacted into the base
table (the active/frozen in-memory memtables and the SSTables),
@@ -3737,11 +3737,11 @@ class AsyncStandardQuery(AsyncQueryBase):
Parameters
----------
enable : bool
``True`` forces the LSM scanner and errors if the table has no MemWAL
write spec. ``False`` bypasses the MemWAL and reads the base table
``True`` forces the LSM scanner and errors if the table has no RTE
write spec. ``False`` bypasses the RTE and reads the base table
only, even when a spec is present.
"""
self._inner.use_memwal(enable)
self._inner.use_rte(enable)
return self
def postfilter(self) -> Self:
@@ -4064,7 +4064,7 @@ class AsyncVectorQueryBase:
minimum and maximum bounds remain effective. Zero starts at the caller
minimum and one searches all partitions within the maximum. Later
probing may continue to obtain enough candidates. This is not a recall
guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
guarantee and does not change refinement or HNSW ef. RTE LSM queries
do not yet support this option.
Examples
@@ -4388,7 +4388,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
# WAL-PK-FUSION: without the fallback, keep only the two `with_row_id`
# calls.
self._pk_fusion = None
if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_memwal):
if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_rte):
self._pk_fusion = _wal_hybrid.PkFusion.for_query(
await self._table.schema(),
_query_request_projection(fts_req),
@@ -4577,13 +4577,13 @@ class AsyncTakeQuery(AsyncQueryBase):
def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None):
super().__init__(inner, table)
def use_memwal(self, enable: bool) -> "AsyncTakeQuery":
"""Control MemWAL LSM read routing for this take query.
def use_rte(self, enable: bool) -> "AsyncTakeQuery":
"""Control RTE LSM read routing for this take query.
``False`` bypasses the MemWAL and reads the base table only — the escape
``False`` bypasses the RTE and reads the base table only — the escape
hatch, since take-by-row-id/offset is not supported on the LSM scanner.
"""
self._inner.use_memwal(enable)
self._inner.use_rte(enable)
return self
async def _plain_scan_to_pandas(
@@ -4644,14 +4644,14 @@ class BaseQueryBuilder(object):
self._inner.with_row_id()
return self
def use_memwal(self, enable: bool) -> Self:
def use_rte(self, enable: bool) -> Self:
"""
Control MemWAL LSM read routing for this query.
Control RTE LSM read routing for this query.
``False`` bypasses the MemWAL and reads the base table only, the escape
``False`` bypasses the RTE and reads the base table only, the escape
hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id).
"""
self._inner.use_memwal(enable)
self._inner.use_rte(enable)
return self
def with_row_address(self, with_row_address: bool = True) -> Self:
python/python/lancedb/remote/table.py
+24 -24
View File
@@ -31,7 +31,7 @@ from lancedb._lancedb import (
DropColumnsResult,
FtsToken,
IndexConfig,
MemWalWriteSpec,
RTEWriteSpec,
MergeResult,
UpdateResult,
)
@@ -1037,17 +1037,17 @@ class RemoteTable(Table):
"""Set the unenforced primary key for this table to a single column."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def enable_memwal(self, spec: "MemWalWriteSpec") -> None:
"""Install an MemWalWriteSpec."""
return LOOP.run(self._table.enable_memwal(spec))
def enable_rte(self, spec: "RTEWriteSpec") -> None:
"""Install an RTEWriteSpec."""
return LOOP.run(self._table.enable_rte(spec))
def disable_memwal(self) -> None:
"""Remove the MemWalWriteSpec."""
return LOOP.run(self._table.disable_memwal())
def disable_rte(self) -> None:
"""Remove the RTEWriteSpec."""
return LOOP.run(self._table.disable_rte())
def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]:
"""Read the installed MemWalWriteSpec, or ``None``."""
return LOOP.run(self._table.get_memwal_spec())
def get_rte_spec(self) -> Optional["RTEWriteSpec"]:
"""Read the installed RTEWriteSpec, or ``None``."""
return LOOP.run(self._table.get_rte_spec())
# WAL-PK-FUSION: delete both hooks.
def _hybrid_pk_fusion_learned(self) -> bool:
@@ -1057,31 +1057,31 @@ class RemoteTable(Table):
def _note_hybrid_pk_fusion(self) -> None:
self._table._note_hybrid_pk_fusion()
def checkpoint_memwal(self) -> None:
def checkpoint_rte(self) -> None:
"""Synchronous version of
[`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal]."""
return LOOP.run(self._table.checkpoint_memwal())
[`AsyncTable.checkpoint_rte`][lancedb.AsyncTable.checkpoint_rte]."""
return LOOP.run(self._table.checkpoint_rte())
def flush_memwal(self) -> None:
def flush_rte(self) -> None:
"""Synchronous version of
[`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal]."""
return LOOP.run(self._table.flush_memwal())
[`AsyncTable.flush_rte`][lancedb.AsyncTable.flush_rte]."""
return LOOP.run(self._table.flush_rte())
def compact_memwal(self) -> None:
def compact_rte(self) -> None:
"""Synchronous version of
[`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal]."""
return LOOP.run(self._table.compact_memwal())
[`AsyncTable.compact_rte`][lancedb.AsyncTable.compact_rte]."""
return LOOP.run(self._table.compact_rte())
def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
def get_rte_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
"""Synchronous version of
[`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats]."""
[`AsyncTable.get_rte_stats`][lancedb.AsyncTable.get_rte_stats]."""
return LOOP.run(
self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows)
self._table.get_rte_stats(include_sstable_rows=include_sstable_rows)
)
def close_memwal_writers(self) -> None:
def close_rte_writers(self) -> None:
"""No-op for remote tables (no local shard writers)."""
return LOOP.run(self._table.close_memwal_writers())
return LOOP.run(self._table.close_rte_writers())
def drop_index(self, index_name: str):
return LOOP.run(self._table.drop_index(index_name))
python/python/lancedb/table.py
+101 -89
View File
@@ -223,7 +223,7 @@ if TYPE_CHECKING:
DeleteResult,
DropColumnsResult,
FtsToken,
MemWalWriteSpec,
RTEWriteSpec,
MergeResult,
UpdateResult,
)
@@ -588,7 +588,7 @@ def _cast_to_target_schema(
def _preserve_blob_v2_fields(
inferred_schema: pa.Schema, existing_schema: pa.Schema
) -> pa.Schema:
"""Keep existing blob fields when an overwrite infers a replacement schema."""
"""Keep inferred blob semantics without overriding explicit replacement types."""
existing_fields = {field.name: field for field in existing_schema}
fields = [
_preserve_blob_v2_field(field, existing_fields[field.name])
@@ -600,6 +600,9 @@ def _preserve_blob_v2_fields(
def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field:
if is_blob_v2_field(inferred):
# An explicitly declared blob layout is part of the replacement schema.
return inferred
if is_blob_v2_field(existing):
return existing
@@ -637,12 +640,10 @@ def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field:
def _coerce_blob_write_columns(
batch: pa.RecordBatch, target_schema: pa.Schema
) -> pa.RecordBatch:
"""Materialize blob storage structs before the stream leaves Python.
"""Materialize Python bytes and URIs as blob structs before Arrow casting.
merge_insert requires its source reader to already match the table's
physical schema. Unlike add and insert, it does not pass through
LanceDB's Rust blob coercion, so preserving binary input here would
reach Lance as binary and fail the schema check.
Existing blob structs retain their input layout. Rust normalizes them to
the stored schema and rejects populated ranges on two-field targets.
"""
columns = []
fields = []
@@ -973,6 +974,17 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
"(Utf8, LargeUtf8, Utf8View), or a Struct with a "
"'data' or 'uri' child"
)
if pa.types.is_struct(source_type) or (
isinstance(source_type, pa.ExtensionType)
and (
not isinstance(target_field.type, pa.ExtensionType)
or type(source_type) is type(target_field.type)
)
):
# Keep the source layout until Rust can fill missing range fields or
# validate every batch before dropping all-null range fields. Arrow
# casting here would reject differing layouts before that validation.
return field.with_metadata(target_field.metadata)
# LanceDB exposes stored JSON columns as lance.json (JSONB-backed LargeBinary), but
# casting the input to that storage type here merely relabels the raw JSON bytes as
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
@@ -1832,8 +1844,8 @@ class Table(ABC):
- pyarrow.Table or pyarrow.RecordBatch
mode: str
The mode to use when writing the data. Valid values are
"append" and "overwrite". Overwrite infers a replacement schema
but preserves existing blob v2 fields present in the new data.
"append" and "overwrite". Overwrite adopts explicitly declared blob
layouts, while inferred bytes retain the existing blob v2 fields.
on_bad_vectors: str, default "error"
What to do if any of the vectors are not the same size or contains NaNs.
One of "error", "drop", "fill".
@@ -4111,8 +4123,8 @@ class LanceTable(Table):
The data to insert into the table.
mode: str
The mode to use when writing the data. Valid values are
"append" and "overwrite". Overwrite infers a replacement schema
but preserves existing blob v2 fields present in the new data.
"append" and "overwrite". Overwrite adopts explicitly declared blob
layouts, while inferred bytes retain the existing blob v2 fields.
on_bad_vectors: str, default "error"
What to do if any of the vectors are not the same size or contains NaNs.
One of "error", "drop", "fill", "null".
@@ -4837,47 +4849,47 @@ class LanceTable(Table):
[`AsyncTable.set_unenforced_primary_key`][lancedb.AsyncTable.set_unenforced_primary_key]."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def enable_memwal(self, spec: "MemWalWriteSpec") -> None:
"""Install an MemWalWriteSpec. See
[`AsyncTable.enable_memwal`][lancedb.AsyncTable.enable_memwal]."""
return LOOP.run(self._table.enable_memwal(spec))
def enable_rte(self, spec: "RTEWriteSpec") -> None:
"""Install an RTEWriteSpec. See
[`AsyncTable.enable_rte`][lancedb.AsyncTable.enable_rte]."""
return LOOP.run(self._table.enable_rte(spec))
def disable_memwal(self) -> None:
"""Remove the MemWalWriteSpec. See
[`AsyncTable.disable_memwal`][lancedb.AsyncTable.disable_memwal]."""
return LOOP.run(self._table.disable_memwal())
def disable_rte(self) -> None:
"""Remove the RTEWriteSpec. See
[`AsyncTable.disable_rte`][lancedb.AsyncTable.disable_rte]."""
return LOOP.run(self._table.disable_rte())
def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]:
"""Read the installed MemWalWriteSpec, or ``None``. See
[`AsyncTable.get_memwal_spec`][lancedb.AsyncTable.get_memwal_spec]."""
return LOOP.run(self._table.get_memwal_spec())
def get_rte_spec(self) -> Optional["RTEWriteSpec"]:
"""Read the installed RTEWriteSpec, or ``None``. See
[`AsyncTable.get_rte_spec`][lancedb.AsyncTable.get_rte_spec]."""
return LOOP.run(self._table.get_rte_spec())
def checkpoint_memwal(self) -> None:
def checkpoint_rte(self) -> None:
"""Synchronous version of
[`AsyncTable.checkpoint_memwal`][lancedb.AsyncTable.checkpoint_memwal]."""
return LOOP.run(self._table.checkpoint_memwal())
[`AsyncTable.checkpoint_rte`][lancedb.AsyncTable.checkpoint_rte]."""
return LOOP.run(self._table.checkpoint_rte())
def flush_memwal(self) -> None:
def flush_rte(self) -> None:
"""Synchronous version of
[`AsyncTable.flush_memwal`][lancedb.AsyncTable.flush_memwal]."""
return LOOP.run(self._table.flush_memwal())
[`AsyncTable.flush_rte`][lancedb.AsyncTable.flush_rte]."""
return LOOP.run(self._table.flush_rte())
def compact_memwal(self) -> None:
def compact_rte(self) -> None:
"""Synchronous version of
[`AsyncTable.compact_memwal`][lancedb.AsyncTable.compact_memwal]."""
return LOOP.run(self._table.compact_memwal())
[`AsyncTable.compact_rte`][lancedb.AsyncTable.compact_rte]."""
return LOOP.run(self._table.compact_rte())
def get_memwal_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
def get_rte_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
"""Synchronous version of
[`AsyncTable.get_memwal_stats`][lancedb.AsyncTable.get_memwal_stats]."""
[`AsyncTable.get_rte_stats`][lancedb.AsyncTable.get_rte_stats]."""
return LOOP.run(
self._table.get_memwal_stats(include_sstable_rows=include_sstable_rows)
self._table.get_rte_stats(include_sstable_rows=include_sstable_rows)
)
def close_memwal_writers(self) -> None:
"""Close cached MemWAL shard writers. See
[`AsyncTable.close_memwal_writers`][lancedb.AsyncTable.close_memwal_writers]."""
return LOOP.run(self._table.close_memwal_writers())
def close_rte_writers(self) -> None:
"""Close cached RTE shard writers. See
[`AsyncTable.close_rte_writers`][lancedb.AsyncTable.close_rte_writers]."""
return LOOP.run(self._table.close_rte_writers())
def uses_v2_manifest_paths(self) -> bool:
"""
@@ -5535,56 +5547,56 @@ class AsyncTable:
columns = list(columns)
await self._inner.set_unenforced_primary_key(columns)
async def enable_memwal(self, spec: "MemWalWriteSpec") -> None:
"""Install an MemWalWriteSpec on this table.
async def enable_rte(self, spec: "RTEWriteSpec") -> None:
"""Install an RTEWriteSpec on this table.
The spec selects Lance's MemWAL LSM-style write path for future
`merge_insert` calls. ``MemWalWriteSpec`` chooses one of three sharding
`merge_insert` calls. ``RTEWriteSpec`` chooses one of three sharding
strategies:
- ``MemWalWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by
- ``RTEWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by
the single-column unenforced primary key.
- ``MemWalWriteSpec.identity(column)`` — shard by the raw value of a
- ``RTEWriteSpec.identity(column)`` — shard by the raw value of a
scalar column.
- ``MemWalWriteSpec.unsharded()`` — route every write to a single shard.
- ``RTEWriteSpec.unsharded()`` — route every write to a single shard.
All variants require the table to have an unenforced primary key set
via [`set_unenforced_primary_key`]; bucket sharding additionally
requires it to be the single column being bucketed.
By default the MemWAL maintains every index on the table, resolved
here — a snapshot, so an index created afterwards needs the spec unset
and set again. This fails if one cannot be maintained; name the set
with ``with_maintained_indexes`` to install anyway. That pins an exact
set (a still-building index is rejected, not omitted); ``[]`` maintains
none.
By default the RTE maintains every index on the table, re-read as the
table changes, so an index created later is maintained too; one of a
kind the RTE cannot mirror is skipped. Name the set with
``with_maintained_indexes`` to pin it instead: each name must already
exist and be maintainable (a still-building index is rejected, not
omitted); ``[]`` maintains none.
Parameters
----------
spec : MemWalWriteSpec
spec : RTEWriteSpec
The sharding spec to install.
Examples
--------
>>> from lancedb import MemWalWriteSpec
>>> from lancedb import RTEWriteSpec
>>> # table.set_unenforced_primary_key("id")
>>> # table.enable_memwal(MemWalWriteSpec.bucket("id", 16))
>>> # table.enable_rte(RTEWriteSpec.bucket("id", 16))
"""
await self._inner.enable_memwal(spec)
await self._inner.enable_rte(spec)
async def disable_memwal(self) -> None:
"""Remove the MemWalWriteSpec from this table.
async def disable_rte(self) -> None:
"""Remove the RTEWriteSpec from this table.
Reverts to the standard `merge_insert` write path. Errors if no spec
is currently set.
"""
await self._inner.disable_memwal()
await self._inner.disable_rte()
async def get_memwal_spec(self) -> Optional["MemWalWriteSpec"]:
"""Read the MemWalWriteSpec currently installed on this table.
async def get_rte_spec(self) -> Optional["RTEWriteSpec"]:
"""Read the RTEWriteSpec currently installed on this table.
Returns ``None`` when the MemWAL write path is not enabled at all — no
spec has been set, or one was removed with `disable_memwal`.
Returns ``None`` when the RTE write path is not enabled at all — no
spec has been set, or one was removed with `disable_rte`.
That is a different answer from a spec whose ``maintained_indexes``
is ``None``, which is an installed spec selecting indexes
automatically.
@@ -5594,7 +5606,7 @@ class AsyncTable:
or gains later, ``[]`` maintains none, and a non-empty list maintains
exactly those. All three round-trip.
"""
return await self._inner.get_memwal_spec()
return await self._inner.get_rte_spec()
# WAL-PK-FUSION: delete both hooks.
def _hybrid_pk_fusion_learned(self) -> bool:
@@ -5605,12 +5617,12 @@ class AsyncTable:
"""See [`Table._note_hybrid_pk_fusion`][lancedb.table.Table]."""
self._inner.note_hybrid_pk_fusion()
async def checkpoint_memwal(self) -> None:
"""Converge this table's MemWAL write path into its base table.
async def checkpoint_rte(self) -> None:
"""Converge this table's RTE write path into its base table.
One flush, freezing every memtable into an SSTable, then compaction triggers
until every SSTable that existed at that moment has reached base.
The loop runs client-side, reading progress from ``get_memwal_stats``.
The loop runs client-side, reading progress from ``get_rte_stats``.
Best-effort: SSTables created *while* it runs are deliberately not
waited on, which is what lets it terminate on a table taking writes.
@@ -5625,56 +5637,56 @@ class AsyncTable:
``asyncio.wait_for`` for a wall-clock bound; abandoning it partway
costs nothing.
"""
await self._inner.checkpoint_memwal()
await self._inner.checkpoint_rte()
async def flush_memwal(self) -> None:
async def flush_rte(self) -> None:
"""Freeze every table shard's active memtable into an SSTable.
Does not touch the base table — compacting SSTables into base is
`compact_memwal`. On a node that has not claimed this table, this claims
`compact_rte`. On a node that has not claimed this table, this claims
it and replays its WAL log first.
"""
await self._inner.flush_memwal()
await self._inner.flush_rte()
async def compact_memwal(self) -> None:
async def compact_rte(self) -> None:
"""Trigger a background SSTable compaction pass per table shard.
Returns once the passes are dispatched, not once they finish: watch
``get_memwal_stats`` for progress, or use ``checkpoint_memwal`` to loop
``get_rte_stats`` for progress, or use ``checkpoint_rte`` to loop
until the current SSTables have reached base.
"""
await self._inner.compact_memwal()
await self._inner.compact_rte()
async def get_memwal_stats(
async def get_rte_stats(
self, *, include_sstable_rows: bool = False
) -> Optional[dict]:
"""Read live per-table-shard MemWAL state.
"""Read live per-table-shard RTE state.
Answers "how far behind is my fresh tier", "which table shard is hot", and
"why is my fresh-tier vector search brute-force". Mutates no table
state, though on a node that has not claimed this table it claims it,
exactly as a read would.
Returns ``None`` only when the MemWAL write path is not enabled.
Returns ``None`` only when the RTE write path is not enabled.
Parameters
----------
include_sstable_rows
Report a row count per SSTable. Off by default: each count
opens an uncached Lance dataset, and ``checkpoint_memwal`` polls this
opens an uncached Lance dataset, and ``checkpoint_rte`` polls this
needing only generation numbers.
"""
return await self._inner.get_memwal_stats(include_sstable_rows)
return await self._inner.get_rte_stats(include_sstable_rows)
async def close_memwal_writers(self) -> None:
"""Drain and close any cached MemWAL shard writers for this table.
async def close_rte_writers(self) -> None:
"""Drain and close any cached RTE shard writers for this table.
When a MemWAL write spec is installed, `merge_insert` opens MemWAL shard
When an RTE write spec is installed, `merge_insert` opens RTE shard
writers and caches them for reuse across calls. This closes them,
flushing pending data; writers reopen lazily on the next
`merge_insert`. It is a no-op when no writers are cached.
"""
await self._inner.close_memwal_writers()
await self._inner.close_rte_writers()
@property
def name(self) -> str:
@@ -6145,8 +6157,8 @@ class AsyncTable:
- pyarrow.Table or pyarrow.RecordBatch
mode: str
The mode to use when writing the data. Valid values are
"append" and "overwrite". Overwrite infers a replacement schema
but preserves existing blob v2 fields present in the new data.
"append" and "overwrite". Overwrite adopts explicitly declared blob
layouts, while inferred bytes retain the existing blob v2 fields.
on_bad_vectors: str, default "error"
What to do if any of the vectors are not the same size or contains NaNs.
One of "error", "drop", "fill", "null".
@@ -6563,8 +6575,8 @@ class AsyncTable:
take_query = self.take_offsets(query.take_offsets)
if query.columns:
take_query = take_query.select(query.columns)
if query.use_memwal is not None:
take_query = take_query.use_memwal(query.use_memwal)
if query.use_rte is not None:
take_query = take_query.use_rte(query.use_rte)
if query.with_row_id:
take_query = take_query.with_row_id()
return take_query
@@ -6580,8 +6592,8 @@ class AsyncTable:
async_query = async_query.where(query.filter)
if query.fast_search:
async_query = async_query.fast_search()
if query.use_memwal is not None:
async_query = async_query.use_memwal(query.use_memwal)
if query.use_rte is not None:
async_query = async_query.use_rte(query.use_rte)
if query.with_row_id:
async_query = async_query.with_row_id()
if query.order_by:
@@ -6707,7 +6719,7 @@ class AsyncTable:
when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr,
timeout=merge._timeout,
use_index=merge._use_index,
use_memwal=merge._use_memwal,
use_rte=merge._use_rte,
validate_single_shard=merge._validate_single_shard,
),
)
python/python/tests/test_blob.py
+146
View File
@@ -63,6 +63,25 @@ def _blob_array(name, values):
return pa.ExtensionArray.from_storage(blob_type, storage)
def _blob_layout_data(rows, with_ranges, *, as_extension=True):
field = lancedb.blob("image")
storage_type = field.type.storage_type
if not with_ranges or not as_extension:
storage_type = pa.struct(list(storage_type)[: (4 if with_ranges else 2)])
field = pa.field(
"image",
storage_type,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
images = pa.array([row["image"] for row in rows], type=storage_type)
if isinstance(field.type, pa.ExtensionType):
images = pa.ExtensionArray.from_storage(field.type, images)
return pa.Table.from_arrays(
[pa.array([row["id"] for row in rows], type=pa.int64()), images],
schema=pa.schema([pa.field("id", pa.int64()), field]),
)
def _row_ids_by_id(table):
hits = table.search().with_row_id(True).limit(1000).to_arrow()
assert "_rowid" in hits.column_names
@@ -998,6 +1017,133 @@ def test_merge_insert_writes_python_bytes():
assert blobs.to_pylist() == [b"updated", b"inserted"]
@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize("input_type", ["extension", "struct", "reader"])
@pytest.mark.parametrize("target_with_ranges", [False, True])
async def test_merge_insert_normalizes_blob_layouts(
use_async, input_type, target_with_ranges
):
data = _blob_layout_data(
[{"id": 1, "image": {"data": b"before"}}], target_with_ranges
)
table = lancedb.connect("memory:///").create_table("merge_layouts", data=data)
original_schema = table.schema
updates = _blob_layout_data(
[
{"id": 1, "image": {"data": b"updated"}},
{"id": 2, "image": {"data": b"inserted"}},
{"id": 3, "image": None},
],
not target_with_ranges,
as_extension=input_type != "struct",
)
if input_type == "reader":
updates = updates.to_reader(max_chunksize=1)
writer = table._table if use_async else table
result = (
writer.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(updates)
)
if use_async:
result = await result
assert result.num_updated_rows == 1
assert result.num_inserted_rows == 2
assert table.schema == original_schema
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[i] for i in (1, 2, 3)]).to_pylist() == [
b"updated",
b"inserted",
None,
]
@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize(
"byte_range", [{"position": 0}, {"size": 6}, {"position": 4, "size": 6}]
)
async def test_merge_insert_legacy_blob_rejects_ranges_atomically(
use_async, byte_range
):
data = _blob_layout_data([{"id": 1, "image": {"data": b"before"}}], False)
table = lancedb.connect("memory:///").create_table("reject_ranges", data=data)
original_schema, original_version = table.schema, table.version
updates = _blob_layout_data(
[
{"id": 1, "image": {"data": b"updated"}},
{"id": 2, "image": {"uri": "file:///payload", **byte_range}},
],
True,
).to_reader(max_chunksize=1)
writer = table._table if use_async else table
with pytest.raises(RuntimeError, match="cannot store byte ranges"):
result = (
writer.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(updates)
)
if use_async:
await result
assert table.version == original_version
assert table.schema == original_schema
assert table.count_rows() == 1
assert table.fetch_blobs("image", [_row_ids_by_id(table)[1]]).to_pylist() == [
b"before"
]
@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize("input_type", ["extension", "struct", "reader"])
@pytest.mark.parametrize("source_with_ranges", [False, True])
async def test_add_overwrite_adopts_explicit_blob_layout(
tmp_path, use_async, input_type, source_with_ranges
):
data = _blob_layout_data(
[{"id": 1, "image": {"data": b"before"}}], not source_with_ranges
)
table = lancedb.connect("memory:///").create_table("overwrite_layouts", data=data)
payload = b"0123456789abcdef"
path = tmp_path / "payload.bin"
path.write_bytes(payload)
value = (
{"uri": path.as_uri(), "position": 4, "size": 6}
if source_with_ranges
else {"data": b"updated"}
)
updates = _blob_layout_data(
[{"id": 2, "image": value}, {"id": 3, "image": None}],
source_with_ranges,
as_extension=input_type != "struct",
)
if input_type == "reader":
updates = updates.to_reader(max_chunksize=1)
writer = table._table if use_async else table
result = writer.add(
updates, mode="overwrite", allow_external_blob_outside_bases=True
)
if use_async:
await result
blob_type = table.schema.field("image").type
if isinstance(blob_type, pa.ExtensionType):
blob_type = blob_type.storage_type
assert blob_type.num_fields == (4 if source_with_ranges else 2)
assert table.blob_columns() == ["image"]
assert table.count_rows() == 2
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[2], by_id[3]]).to_pylist() == [
payload[4:10] if source_with_ranges else b"updated",
None,
]
@pytest.mark.parametrize("write", ["merge_insert", "create_table"])
def test_uri_string_blob_write_reaches_external_base_validation(tmp_path, write):
blob_path = tmp_path / "payload.bin"
python/python/tests/test_elastic_dataloader.py
+16
View File
@@ -3259,6 +3259,22 @@ def test_doc_example_checkpoint(lance_table):
)
def test_streaming_dataset_over_table_with_schema_metadata(tmp_path):
"""StreamingDataset builds its permutation at construction; base table schema
metadata (e.g. Hugging Face's `source-column-name-map`) must not break it."""
schema = pa.schema(
[pa.field("id", pa.int64())],
metadata={b"source-column-name-map": b"{}", b"huggingface": b"{}"},
)
table = lancedb.connect(tmp_path).create_table(
"data", pa.table({"id": list(range(NUM_ROWS))}, schema=schema)
)
ds = StreamingDataset(table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS))
# ---------------------------------------------------------------------------
# Remote tables (LanceDB Cloud / Enterprise)
# ---------------------------------------------------------------------------
python/python/tests/test_merge_insert_lsm.py
+50 -56
View File
@@ -1,14 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Tests for the MemWAL LSM ``merge_insert`` dispatch."""
"""Tests for the RTE LSM ``merge_insert`` dispatch."""
from datetime import timedelta
import lancedb
import pyarrow as pa
import pytest
from lancedb._lancedb import MemWalWriteSpec
from lancedb._lancedb import RTEWriteSpec
from lancedb.index import FTS, IvfPq
SCHEMA = pa.schema(
@@ -49,12 +49,12 @@ def _region_reader(rows):
def _bucket_table(tmp_path):
"""A table with ``id`` as the primary key and a single-bucket MemWAL spec."""
"""A table with ``id`` as the primary key and a single-bucket RTE spec."""
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
# num_buckets = 1: every row routes to the single bucket.
table.enable_memwal(MemWalWriteSpec.bucket("id", 1))
table.enable_rte(RTEWriteSpec.bucket("id", 1))
return table
@@ -67,7 +67,7 @@ def test_lsm_merge_insert_bucket(tmp_path):
.when_not_matched_insert_all()
.execute(_reader([3, 4, 5]))
)
# LSM path: rows go to the MemWAL, so only num_rows is populated.
# LSM path: rows go to the RTE, so only num_rows is populated.
assert result.num_rows == 3
assert result.version == 0
assert result.num_inserted_rows == 0
@@ -78,7 +78,7 @@ def test_lsm_merge_insert_unsharded(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.enable_rte(RTEWriteSpec.unsharded())
result = (
table.merge_insert("id")
.when_matched_update_all()
@@ -92,7 +92,7 @@ def test_lsm_merge_insert_identity(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _region_reader([(1, "us"), (2, "us")]))
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.identity("region"))
table.enable_rte(RTEWriteSpec.identity("region"))
# All rows share one identity value, so they route to one shard.
result = (
table.merge_insert([])
@@ -103,31 +103,31 @@ def test_lsm_merge_insert_identity(tmp_path):
assert result.num_rows == 2
def test_lsm_merge_insert_use_memwal_false(tmp_path):
def test_lsm_merge_insert_use_rte_false(tmp_path):
table = _bucket_table(tmp_path) # rows id = 1, 2, 3
# use_memwal(False) opts out: the standard path runs and commits even with a spec.
# use_rte(False) opts out: the standard path runs and commits even with a spec.
result = (
table.merge_insert("id")
.when_not_matched_insert_all()
.use_memwal(False)
.use_rte(False)
.execute(_reader([3, 4, 5]))
)
assert result.num_inserted_rows == 2
assert table.count_rows() == 5
def test_lsm_merge_insert_use_memwal_true_without_spec_errors(tmp_path):
# A table with a primary key but no MemWAL write spec installed.
def test_lsm_merge_insert_use_rte_true_without_spec_errors(tmp_path):
# A table with a primary key but no RTE write spec installed.
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
# use_memwal(True) demands MemWAL routing; without a spec it errors.
with pytest.raises(Exception, match="use_memwal"):
# use_rte(True) demands RTE routing; without a spec it errors.
with pytest.raises(Exception, match="use_rte"):
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.use_memwal(True)
.use_rte(True)
.execute(_reader([3, 4, 5]))
)
@@ -145,7 +145,7 @@ def test_lsm_merge_insert_validate_single_shard_off(tmp_path):
def test_lsm_merge_insert_no_spec_uses_standard_path(tmp_path):
# A table with a primary key but no MemWAL write spec installed.
# A table with a primary key but no RTE write spec installed.
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
@@ -186,7 +186,7 @@ def test_lsm_close_writers(tmp_path):
.when_not_matched_insert_all()
.execute(_reader([7, 8]))
)
table.close_memwal_writers()
table.close_rte_writers()
# The writer reopens lazily on the next merge_insert.
result = (
table.merge_insert([])
@@ -204,14 +204,14 @@ async def test_async_lsm_merge_insert(tmp_path):
)
table = await db.create_table("t", _reader([1, 2, 3]))
await table.set_unenforced_primary_key("id")
await table.enable_memwal(MemWalWriteSpec.bucket("id", 1))
await table.enable_rte(RTEWriteSpec.bucket("id", 1))
builder = (
table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all()
)
result = await builder.execute(_reader([3, 4, 5]))
assert result.num_rows == 3
await table.close_memwal_writers()
await table.close_rte_writers()
def _lsm_upsert(table, ids):
@@ -228,7 +228,7 @@ def test_lsm_read_sees_active_memtable(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3])) # base ids 1,2,3
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.enable_rte(RTEWriteSpec.unsharded())
_lsm_upsert(table, [4, 5]) # active memtable only, not committed to base
@@ -236,8 +236,8 @@ def test_lsm_read_sees_active_memtable(tmp_path):
lsm = table.search().to_arrow()
assert sorted(lsm["id"].to_pylist()) == [1, 2, 3, 4, 5]
# use_memwal(False) bypasses the MemWAL and reads the base table only.
base_only = table.search().use_memwal(False).to_arrow()
# use_rte(False) bypasses the RTE and reads the base table only.
base_only = table.search().use_rte(False).to_arrow()
assert sorted(base_only["id"].to_pylist()) == [1, 2, 3]
@@ -245,7 +245,7 @@ def test_lsm_read_dedup_newest_wins(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3])) # id 2 -> value 1
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.enable_rte(RTEWriteSpec.unsharded())
_lsm_upsert(table, [2, 3, 4]) # ids 2,3,4 -> values 0,1,2
@@ -258,31 +258,31 @@ def test_lsm_read_dedup_newest_wins(tmp_path):
def test_lsm_read_without_spec_reads_base(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id") # no MemWAL write spec
table.set_unenforced_primary_key("id") # no RTE write spec
# No spec: default read and use_memwal(False) both read the base table, no error.
# No spec: default read and use_rte(False) both read the base table, no error.
assert sorted(table.search().to_arrow()["id"].to_pylist()) == [1, 2, 3]
assert sorted(table.search().use_memwal(False).to_arrow()["id"].to_pylist()) == [
assert sorted(table.search().use_rte(False).to_arrow()["id"].to_pylist()) == [
1,
2,
3,
]
def test_lsm_read_unsupported_shape_errors_without_use_memwal_false(tmp_path):
def test_lsm_read_unsupported_shape_errors_without_use_rte_false(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.enable_rte(RTEWriteSpec.unsharded())
_lsm_upsert(table, [4])
# with_row_id is unsupported by the LSM scanner; on a MemWAL table the default
# with_row_id is unsupported by the LSM scanner; on an RTE table the default
# (auto-routed) read hard-errors instead of silently reading a stale base.
with pytest.raises(Exception):
table.search().with_row_id(True).to_arrow()
# use_memwal(False) is the escape hatch: it reads the base table only.
base = table.search().with_row_id(True).use_memwal(False).to_arrow()
# use_rte(False) is the escape hatch: it reads the base table only.
base = table.search().with_row_id(True).use_rte(False).to_arrow()
assert sorted(base["id"].to_pylist()) == [1, 2, 3]
@@ -293,7 +293,7 @@ async def test_async_lsm_read(tmp_path):
)
table = await db.create_table("t", _reader([1, 2, 3]))
await table.set_unenforced_primary_key("id")
await table.enable_memwal(MemWalWriteSpec.unsharded())
await table.enable_rte(RTEWriteSpec.unsharded())
builder = (
table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all()
@@ -331,7 +331,7 @@ def _vector_reader(rows):
def _vector_table(tmp_path):
"""Base table whose vector column is indexed so its rows are visible to the LSM
vector scanner (the base arm uses ``fast_search`` — indexed data only), plus an
unsharded MemWAL spec that maintains that index for the memtable.
unsharded RTE spec that maintains that index for the memtable.
Rows 1,2 are category ``a``, row 3 is ``b``, and 4..60 are filler ``c`` that
give the tiny IVF index enough data to train.
@@ -353,9 +353,7 @@ def _vector_table(tmp_path):
"vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4)
)
index_name = table.list_indices()[0].name
table.enable_memwal(
MemWalWriteSpec.unsharded().with_maintained_indexes([index_name])
)
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name]))
return table
@@ -379,9 +377,9 @@ def test_lsm_read_vector_sees_memtable(tmp_path):
assert {1, 2, 3} <= ids # indexed base rows
assert 1000 in ids # in-flight memtable row
# use_memwal(False) bypasses the MemWAL, so the in-flight row is not visible.
# use_rte(False) bypasses the RTE, so the in-flight row is not visible.
base_ids = set(
table.search(query).use_memwal(False).limit(100).to_arrow()["id"].to_pylist()
table.search(query).use_rte(False).limit(100).to_arrow()["id"].to_pylist()
)
assert {1, 2, 3} <= base_ids
assert 1000 not in base_ids
@@ -453,9 +451,7 @@ def test_lsm_read_fts_sees_memtable(tmp_path):
# Native FTS index (tantivy is not compatible with the LSM memtable index).
table.create_index("text", config=FTS())
index_name = table.list_indices()[0].name
table.enable_memwal(
MemWalWriteSpec.unsharded().with_maintained_indexes([index_name])
)
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name]))
# in-flight doc 4 lands in the memtable's maintained FTS index.
(
@@ -498,8 +494,8 @@ def test_lsm_read_vector_unsupported_knobs_error(tmp_path):
with pytest.raises(Exception, match="use_index"):
table.search(query).bypass_vector_index().to_arrow()
# use_memwal(False) is the escape hatch: the base-only standard path honors them.
base = table.search(query).distance_range(0.0, 100.0).use_memwal(False).to_arrow()
# use_rte(False) is the escape hatch: the base-only standard path honors them.
base = table.search(query).distance_range(0.0, 100.0).use_rte(False).to_arrow()
assert 1000 not in set(base["id"].to_pylist())
@@ -544,7 +540,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path):
table.create_index("text", config=FTS())
# No maintained indexes: the active memtable FTS arm cannot serve un-compacted
# docs, so the search would silently omit them — reject instead.
table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([]))
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([]))
with pytest.raises(Exception, match="maintained"):
table.search("fox", query_type="fts", fts_columns="text").to_arrow()
@@ -553,7 +549,7 @@ def test_lsm_read_time_travel_errors(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.enable_rte(RTEWriteSpec.unsharded())
pinned = table.version
table.add(_reader([4, 5])) # standard add commits a newer version
table.checkout(pinned) # detached head at the historical version
@@ -562,8 +558,8 @@ def test_lsm_read_time_travel_errors(tmp_path):
# historical version is rejected.
with pytest.raises(Exception, match="time-travel"):
table.search().to_arrow()
# use_memwal(False) reads the base table at the pinned version.
base = table.search().use_memwal(False).to_arrow()
# use_rte(False) reads the base table at the pinned version.
base = table.search().use_rte(False).to_arrow()
assert sorted(base["id"].to_pylist()) == [1, 2, 3]
@@ -571,14 +567,14 @@ def test_lsm_read_take_row_ids_errors(tmp_path):
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
table = db.create_table("t", _reader([1, 2, 3]))
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.enable_rte(RTEWriteSpec.unsharded())
_lsm_upsert(table, [4])
# take-by-row-id auto-routes through the LSM scanner, which has no stable _rowid,
# so it hard-errors instead of failing with an opaque column-not-found error.
with pytest.raises(Exception, match="row id"):
table.take_row_ids([0, 1]).to_arrow()
# use_memwal(False) is the escape hatch: it reads the base table.
base = table.take_row_ids([0, 1]).use_memwal(False).to_arrow()
# use_rte(False) is the escape hatch: it reads the base table.
base = table.take_row_ids([0, 1]).use_rte(False).to_arrow()
assert base.num_rows == 2
@@ -588,9 +584,7 @@ def test_lsm_read_fts_postfilter_errors(tmp_path):
table.set_unenforced_primary_key("id")
table.create_index("text", config=FTS())
index_name = table.list_indices()[0].name
table.enable_memwal(
MemWalWriteSpec.unsharded().with_maintained_indexes([index_name])
)
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name]))
# The LSM scanner always prefilters; postfilter on FTS changes result semantics,
# so it hard-errors (previously only the vector arm rejected it).
with pytest.raises(Exception, match="postfilter"):
@@ -608,7 +602,7 @@ def test_lsm_read_fts_multiple_same_type_indexes_errors(tmp_path):
table.set_unenforced_primary_key("id")
table.create_index("text", config=FTS(), name="fts_a")
table.create_index("text", config=FTS(), name="fts_b", replace=False)
table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes(["fts_a"]))
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes(["fts_a"]))
# Two FTS indexes on the column: the base planner's chosen index is ambiguous, so
# the scanner can't pick a catch-up watermark and rejects rather than risk
# dropping rows the actually-used index has not caught up to.
@@ -629,7 +623,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path):
)
# Spec with NO maintained indexes: the base vector index's catch-up is untracked,
# so the scanner rejects rather than risk dropping compacted-but-unindexed rows.
table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([]))
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([]))
with pytest.raises(Exception, match="maintained"):
table.search([1.0] * VECTOR_DIM).to_arrow()
@@ -642,7 +636,7 @@ def test_lsm_read_fts_optimized_index_not_rejected(tmp_path):
table.add(_fts_reader([(i, "lazy fox") for i in range(6, 11)]))
table.optimize() # may split the FTS index into multiple physical segments
name = table.list_indices()[0].name
table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([name]))
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([name]))
# Multiple physical segments of one logical index must not be miscounted as
# multiple indexes and rejected.
ids = set(
python/python/tests/test_permutation.py
+29
View File
@@ -115,6 +115,35 @@ def test_split_random_fixed(mem_db):
assert split_ids.count(split_id) == 25
def test_permutation_over_table_with_schema_metadata(tmp_path):
"""Schema-level metadata on the base table must not break the build.
Hugging Face datasets carry keys like `huggingface` or `source-column-name-map`.
"""
schema = pa.schema(
[pa.field("x", pa.int64())],
# `split_names` collides with a permutation config key; the config must win.
metadata={b"huggingface": b"{}", b"split_names": b'["base"]'},
)
tbl = connect(tmp_path).create_table(
"t", pa.table({"x": list(range(100))}, schema=schema)
)
permutation_tbl = (
permutation_builder(tbl)
.split_random(fixed=2, seed=1, split_names=["train", "test"])
.execute()
)
permutations = Permutations(tbl, permutation_tbl)
assert permutations.split_names == ["train", "test"]
train = Permutation.from_tables(tbl, permutation_tbl, "train")
test = Permutation.from_tables(tbl, permutation_tbl, "test")
assert train.num_rows + test.num_rows == 100
train_x = [row["x"] for batch in train.iter(16) for row in batch]
test_x = [row["x"] for batch in test.iter(16) for row in batch]
assert sorted(train_x + test_x) == list(range(100))
def test_split_random_with_seed(mem_db):
"""Test that seeded random splits are reproducible."""
tbl = mem_db.create_table("test_table", pa.table({"x": range(50), "y": range(50)}))
python/python/tests/test_remote_db.py
+103 -20
View File
@@ -1844,10 +1844,10 @@ def lsm_test_table(lsm_handler):
"""A remote table whose LSM routes are served by ``lsm_handler``.
``lsm_handler(request, route)`` is called for ``/v1/table/test/<route>/``
where route is one of flush_memwal, compact_memwal, get_memwal_stats, and is
where route is one of flush_rte, compact_rte, get_rte_stats, and is
responsible for writing the response.
"""
routes = ("flush_memwal", "compact_memwal", "get_memwal_stats")
routes = ("flush_rte", "compact_rte", "get_rte_stats")
def handler(request):
match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path)
@@ -1879,7 +1879,7 @@ def send_json(request, payload, status=200):
request.wfile.write(json.dumps(payload).encode())
def test_get_memwal_stats_sync():
def test_get_rte_stats_sync():
"""The sync wrapper round-trips the server payload into a dict."""
table_shard = {
"shard_id": "b0",
@@ -1904,29 +1904,29 @@ def test_get_memwal_stats_sync():
seen_bodies = []
def lsm_handler(request, route):
assert route == "get_memwal_stats"
assert route == "get_rte_stats"
seen_bodies.append(read_json_body(request))
send_json(request, {"memwal_stats": {"table_shards": [table_shard]}})
send_json(request, {"rte_stats": {"table_shards": [table_shard]}})
with lsm_test_table(lsm_handler) as table:
assert table.get_memwal_stats() == {"table_shards": [table_shard]}
assert table.get_rte_stats() == {"table_shards": [table_shard]}
# Off by default, and forwarded when asked for.
assert seen_bodies == [{"include_sstable_rows": False}]
table.get_memwal_stats(include_sstable_rows=True)
table.get_rte_stats(include_sstable_rows=True)
assert seen_bodies[-1] == {"include_sstable_rows": True}
def test_get_memwal_stats_sync_returns_none_when_memwal_disabled():
"""A null envelope means the MemWAL write path is not enabled, not an error."""
def test_get_rte_stats_sync_returns_none_when_rte_disabled():
"""A null envelope means the RTE write path is not enabled, not an error."""
def lsm_handler(request, route):
send_json(request, {"memwal_stats": None})
send_json(request, {"rte_stats": None})
with lsm_test_table(lsm_handler) as table:
assert table.get_memwal_stats() is None
assert table.get_rte_stats() is None
def test_flush_and_compact_memwal_sync():
def test_flush_and_compact_rte_sync():
"""Both are one-shot POSTs answered 202 with no body."""
called = []
@@ -1936,12 +1936,12 @@ def test_flush_and_compact_memwal_sync():
request.end_headers()
with lsm_test_table(lsm_handler) as table:
assert table.flush_memwal() is None
assert table.compact_memwal() is None
assert called == ["flush_memwal", "compact_memwal"]
assert table.flush_rte() is None
assert table.compact_rte() is None
assert called == ["flush_rte", "compact_rte"]
def test_checkpoint_memwal_sync():
def test_checkpoint_rte_sync():
"""Freeze, read the watermark, and return once no SSTables remain.
The convergence loop itself is covered in Rust; this pins the sync
@@ -1951,17 +1951,17 @@ def test_checkpoint_memwal_sync():
def lsm_handler(request, route):
called.append(route)
if route == "get_memwal_stats":
if route == "get_rte_stats":
# An empty SSTable tier yields no target watermark, so the loop is done
# after the freeze without ever polling compaction.
send_json(request, {"memwal_stats": {"table_shards": []}})
send_json(request, {"rte_stats": {"table_shards": []}})
else:
request.send_response(202)
request.end_headers()
with lsm_test_table(lsm_handler) as table:
assert table.checkpoint_memwal() is None
assert called == ["flush_memwal", "get_memwal_stats"]
assert table.checkpoint_rte() is None
assert called == ["flush_rte", "get_rte_stats"]
@contextlib.contextmanager
@@ -3254,6 +3254,89 @@ BLOB_DESCRIBE_RESPONSE = {
}
@pytest.mark.asyncio
@pytest.mark.parametrize("use_async", [False, True])
@pytest.mark.parametrize("mode", ["merge", "overwrite"])
@pytest.mark.parametrize("target_with_ranges", [False, True])
async def test_remote_writes_normalize_blob_layouts(
use_async, mode, target_with_ranges
):
description = json.loads(json.dumps(BLOB_DESCRIBE_RESPONSE))
if target_with_ranges:
description["schema"]["fields"][1]["type"]["fields"].extend(
{"name": name, "type": {"type": "uint64"}, "nullable": True}
for name in ("position", "size")
)
received = []
def handler(request):
if request.path == "/v1/table/test/describe/":
response = description
else:
expected = "merge_insert" if mode == "merge" else "insert"
assert urlparse(request.path).path == f"/v1/table/test/{expected}/"
if request.headers.get("Transfer-Encoding") == "chunked":
body = bytearray()
while size := int(request.rfile.readline().split(b";")[0], 16):
body.extend(request.rfile.read(size))
assert request.rfile.read(2) == b"\r\n"
assert request.rfile.read(2) == b"\r\n"
else:
body = request.rfile.read(int(request.headers["Content-Length"]))
received.append(pa.ipc.open_stream(body).read_all())
response = {
"version": 2,
"num_updated_rows": 0,
"num_inserted_rows": 2,
"num_deleted_rows": 0,
}
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", "0.5.0")
request.end_headers()
request.wfile.write(json.dumps(response).encode())
field = lancedb.blob("image")
storage_type = field.type.storage_type
if target_with_ranges:
storage_type = pa.struct(list(storage_type)[:2])
field = pa.field(
"image",
storage_type,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
images = pa.array([{"data": b"payload"}, None], type=storage_type)
if isinstance(field.type, pa.ExtensionType):
images = pa.ExtensionArray.from_storage(field.type, images)
data = pa.Table.from_arrays(
[pa.array([1, 2], type=pa.int64()), images],
schema=pa.schema([pa.field("id", pa.int64()), field]),
)
with mock_lancedb_connection(handler) as db:
remote_table = db.open_table("test")
table = remote_table._table if use_async else remote_table
if mode == "merge":
result = (
table.merge_insert("id").when_not_matched_insert_all().execute(data)
)
else:
result = table.add(data, mode="overwrite")
if use_async:
await result
assert len(received) == 1
images = received[0]["image"].combine_chunks()
if isinstance(images.type, pa.ExtensionType):
images = images.storage
with_ranges = target_with_ranges if mode == "merge" else not target_with_ranges
assert images.type.num_fields == (4 if with_ranges else 2)
expected = {"data": b"payload", "uri": None}
if with_ranges:
expected.update(position=None, size=None)
assert images.to_pylist() == [expected, None]
def blob_query_response_table():
image_field = pa.field(
"image",
python/python/tests/test_memwal_spec.py → python/python/tests/test_rte_spec.py
+60 -60
View File
@@ -1,8 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Tests for installing and clearing an MemWalWriteSpec via
`Table.enable_memwal` / `Table.disable_memwal`.
"""Tests for installing and clearing an RTEWriteSpec via
`Table.enable_rte` / `Table.disable_rte`.
"""
from datetime import timedelta
@@ -10,7 +10,7 @@ from datetime import timedelta
import lancedb
import pyarrow as pa
import pytest
from lancedb._lancedb import MemWalWriteSpec
from lancedb._lancedb import RTEWriteSpec
from lancedb.index import BTree
SCHEMA = pa.schema(
@@ -21,9 +21,9 @@ SCHEMA = pa.schema(
)
def test_memwal_spec_module_metadata():
assert lancedb.MemWalWriteSpec is MemWalWriteSpec
assert MemWalWriteSpec.__module__ == "lancedb._lancedb"
def test_rte_spec_module_metadata():
assert lancedb.RTEWriteSpec is RTEWriteSpec
assert RTEWriteSpec.__module__ == "lancedb._lancedb"
def _batch(ids, vs):
@@ -43,49 +43,49 @@ def _make_table(tmp_path):
return db, table
def test_enable_memwal_validates(tmp_path):
def test_enable_rte_validates(tmp_path):
_db, table = _make_table(tmp_path)
table.set_unenforced_primary_key("id")
# Out-of-range num_buckets.
with pytest.raises(Exception, match="num_buckets"):
table.enable_memwal(MemWalWriteSpec.bucket("id", 0))
table.enable_rte(RTEWriteSpec.bucket("id", 0))
with pytest.raises(Exception, match="num_buckets"):
table.enable_memwal(MemWalWriteSpec.bucket("id", 1025))
table.enable_rte(RTEWriteSpec.bucket("id", 1025))
# Happy path, then a set over the installed spec is refused.
table.enable_memwal(MemWalWriteSpec.bucket("id", 4))
table.enable_rte(RTEWriteSpec.bucket("id", 4))
with pytest.raises(Exception, match="already set"):
table.enable_memwal(MemWalWriteSpec.bucket("id", 8))
table.enable_rte(RTEWriteSpec.bucket("id", 8))
def test_disable_memwal(tmp_path):
def test_disable_rte(tmp_path):
_db, table = _make_table(tmp_path)
table.set_unenforced_primary_key("id")
# unset errors when no spec is set.
with pytest.raises(Exception, match="no MemWAL write spec"):
table.disable_memwal()
with pytest.raises(Exception, match="no RTE write spec"):
table.disable_rte()
# Install a spec, then remove it; afterwards a fresh spec can be set.
table.enable_memwal(MemWalWriteSpec.bucket("id", 4))
table.disable_memwal()
table.enable_rte(RTEWriteSpec.bucket("id", 4))
table.disable_rte()
# A second unset errors — there is no spec left to remove.
with pytest.raises(Exception, match="no MemWAL write spec"):
table.disable_memwal()
table.enable_memwal(MemWalWriteSpec.bucket("id", 8))
with pytest.raises(Exception, match="no RTE write spec"):
table.disable_rte()
table.enable_rte(RTEWriteSpec.bucket("id", 8))
def test_enable_memwal_requires_primary_key(tmp_path):
def test_enable_rte_requires_primary_key(tmp_path):
_db, table = _make_table(tmp_path)
for spec in (
MemWalWriteSpec.unsharded(),
MemWalWriteSpec.bucket("id", 4),
MemWalWriteSpec.identity("id"),
RTEWriteSpec.unsharded(),
RTEWriteSpec.bucket("id", 4),
RTEWriteSpec.identity("id"),
):
with pytest.raises(Exception, match="primary key"):
table.enable_memwal(spec)
assert table.get_memwal_spec() is None
table.enable_rte(spec)
assert table.get_rte_spec() is None
def test_set_unsharded_spec(tmp_path):
@@ -93,12 +93,12 @@ def test_set_unsharded_spec(tmp_path):
# Lance MemWAL still requires a primary key on the dataset; Unsharded
# just skips per-row hashing.
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.unsharded())
table.disable_memwal()
table.enable_rte(RTEWriteSpec.unsharded())
table.disable_rte()
def test_memwal_spec_repr():
s = MemWalWriteSpec.bucket("id", 4)
def test_rte_spec_repr():
s = RTEWriteSpec.bucket("id", 4)
assert s.spec_type == "bucket"
assert s.column == "id"
assert s.num_buckets == 4
@@ -109,20 +109,20 @@ def test_memwal_spec_repr():
assert "id" in repr(s)
assert "4" in repr(s)
u = MemWalWriteSpec.unsharded()
u = RTEWriteSpec.unsharded()
assert u.spec_type == "unsharded"
assert u.column is None
assert u.num_buckets is None
assert "unsharded" in repr(u)
def test_memwal_spec_with_maintained_indexes():
s = MemWalWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"])
def test_rte_spec_with_maintained_indexes():
s = RTEWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"])
assert s.maintained_indexes == ["idx_a", "idx_b"]
@pytest.mark.asyncio
async def test_async_set_disable_memwal(tmp_path):
async def test_async_set_disable_rte(tmp_path):
db = await lancedb.connect_async(
tmp_path, read_consistency_interval=timedelta(seconds=0)
)
@@ -132,11 +132,11 @@ async def test_async_set_disable_memwal(tmp_path):
)
await table.set_unenforced_primary_key("id")
await table.enable_memwal(MemWalWriteSpec.bucket("id", 4))
await table.disable_memwal()
await table.enable_rte(RTEWriteSpec.bucket("id", 4))
await table.disable_rte()
# A second unset errors.
with pytest.raises(Exception, match="no MemWAL write spec"):
await table.disable_memwal()
with pytest.raises(Exception, match="no RTE write spec"):
await table.disable_rte()
def test_set_identity_spec(tmp_path):
@@ -144,12 +144,12 @@ def test_set_identity_spec(tmp_path):
# Identity sharding still requires an unenforced primary key on the
# table; it shards by the raw value of the given column.
table.set_unenforced_primary_key("id")
table.enable_memwal(MemWalWriteSpec.identity("v"))
table.disable_memwal()
table.enable_rte(RTEWriteSpec.identity("v"))
table.disable_rte()
def test_memwal_spec_identity_and_writer_config_defaults():
s = MemWalWriteSpec.identity("v")
def test_rte_spec_identity_and_writer_config_defaults():
s = RTEWriteSpec.identity("v")
assert s.spec_type == "identity"
assert s.column == "v"
assert s.num_buckets is None
@@ -160,12 +160,12 @@ def test_memwal_spec_identity_and_writer_config_defaults():
assert "durable_write" in repr(s)
def test_get_memwal_spec(tmp_path):
def test_get_rte_spec(tmp_path):
_db, table = _make_table(tmp_path)
table.set_unenforced_primary_key("id")
# None when nothing is installed.
assert table.get_memwal_spec() is None
assert table.get_rte_spec() is None
# A real scalar index is needed to name it as a maintained index.
table.create_index("id", config=BTree())
@@ -173,12 +173,12 @@ def test_get_memwal_spec(tmp_path):
# Bucket spec round-trips, including maintained indexes and writer config
# defaults.
table.enable_memwal(
MemWalWriteSpec.bucket("id", 4)
table.enable_rte(
RTEWriteSpec.bucket("id", 4)
.with_maintained_indexes([idx_name])
.with_writer_config_defaults({"durable_write": "false"})
)
spec = table.get_memwal_spec()
spec = table.get_rte_spec()
assert spec is not None
assert spec.spec_type == "bucket"
assert spec.column == "id"
@@ -187,30 +187,30 @@ def test_get_memwal_spec(tmp_path):
assert spec.writer_config_defaults == {"durable_write": "false"}
# After unset, None again.
table.disable_memwal()
assert table.get_memwal_spec() is None
table.disable_rte()
assert table.get_rte_spec() is None
# Identity round-trips (column recovered from the schema). An unnamed
# maintained set reads back as None: what is stored is the intent to
# maintain every index, not the set it resolves to now.
table.enable_memwal(MemWalWriteSpec.identity("id"))
spec = table.get_memwal_spec()
table.enable_rte(RTEWriteSpec.identity("id"))
spec = table.get_rte_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
assert spec.maintained_indexes is None
table.disable_memwal()
table.disable_rte()
# Unsharded round-trips (no routing column). Opting out is distinct from
# the unnamed default.
table.enable_memwal(MemWalWriteSpec.unsharded().with_maintained_indexes([]))
spec = table.get_memwal_spec()
table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([]))
spec = table.get_rte_spec()
assert spec.spec_type == "unsharded"
assert spec.column is None
assert spec.maintained_indexes == []
@pytest.mark.asyncio
async def test_async_get_memwal_spec(tmp_path):
async def test_async_get_rte_spec(tmp_path):
db = await lancedb.connect_async(
tmp_path, read_consistency_interval=timedelta(seconds=0)
)
@@ -219,21 +219,21 @@ async def test_async_get_memwal_spec(tmp_path):
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
)
assert await table.get_memwal_spec() is None
assert await table.get_rte_spec() is None
await table.set_unenforced_primary_key("id")
# A real scalar index is needed to name it as a maintained index.
await table.create_index("id", config=BTree())
idx_name = (await table.list_indices())[0].name
await table.enable_memwal(
MemWalWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
await table.enable_rte(
RTEWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
)
spec = await table.get_memwal_spec()
spec = await table.get_rte_spec()
assert spec is not None
assert spec.spec_type == "bucket"
assert spec.column == "id"
assert spec.num_buckets == 8
assert spec.maintained_indexes == [idx_name]
await table.disable_memwal()
assert await table.get_memwal_spec() is None
await table.disable_rte()
assert await table.get_rte_spec() is None
python/python/tests/test_wal_hybrid.py
+7 -7
View File
@@ -14,7 +14,7 @@ from lancedb.rerankers import RRFReranker
def _wal_table(schema: pa.Schema) -> mock.Mock:
"""A MemWAL-backed table a previous query has already been refused
"""An RTE-backed table a previous query has already been refused
`_rowid` on, so the fusion goes straight to the primary key."""
table = mock.Mock()
table._hybrid_pk_fusion_learned.return_value = True
@@ -51,7 +51,7 @@ def test_wal_hybrid_projects_the_key_instead_of_asking_for_row_ids():
assert builder._pk_fusion.injected == ["id"]
for leg in (builder._vector_query, builder._fts_query):
assert leg._columns == ["text", "id"]
# A MemWAL table rejects with_row_id before it plans.
# An RTE table rejects with_row_id before it plans.
assert leg._with_row_id is not True
@@ -81,15 +81,15 @@ def test_base_table_hybrid_still_joins_on_row_ids():
assert builder._vector_query._columns is None
def test_wal_hybrid_with_use_memwal_false_joins_on_row_ids():
builder = _wal_hybrid().use_memwal(False)
def test_wal_hybrid_with_use_rte_false_joins_on_row_ids():
builder = _wal_hybrid().use_rte(False)
builder._create_query_builders()
assert builder._pk_fusion is None
def test_wal_hybrid_refuses_an_explicit_with_row_id():
builder = _wal_hybrid().with_row_id(True)
with pytest.raises(NotImplementedError, match="use_memwal"):
with pytest.raises(NotImplementedError, match="use_rte"):
builder._create_query_builders()
@@ -141,7 +141,7 @@ def test_wal_hybrid_refuses_row_id_even_with_return_score_all(row_id_first: bool
else:
builder = builder.rerank(reranker).with_row_id(True)
with pytest.raises(NotImplementedError, match="use_memwal"):
with pytest.raises(NotImplementedError, match="use_rte"):
builder._create_query_builders()
@@ -199,7 +199,7 @@ def test_fallback_explains_the_refusal_to_a_caller_who_asked_for_row_ids():
def run():
raise _REFUSAL
with pytest.raises(NotImplementedError, match="use_memwal") as info:
with pytest.raises(NotImplementedError, match="use_rte") as info:
with_pk_fallback(
run, mock.Mock(), fused=lambda: False, caller_requested_row_id=True
)
python/python/tests/utils.py
+3 -3
View File
@@ -73,10 +73,10 @@ class MockPermutationServer:
},
},
)
if path == f"/v1/table/{self.name}/get_memwal_spec/":
if path == f"/v1/table/{self.name}/get_rte_spec/":
self._read_body(request)
# Null spec: this table has no MemWAL write path.
return self._json(request, {"memwal_spec": None})
# Null spec: this table has no RTE write path.
return self._json(request, {"rte_spec": None})
if path == f"/v1/table/{self.name}/count_rows/":
self._read_body(request)
return self._json(request, self.num_rows)
python/src/lib.rs
+3 -3
View File
@@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
use session::Session;
use table::{
AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MemWalWriteSpec, MergeResult,
PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table,
FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MergeResult, PyBlobFile,
RTEWriteSpec, RefreshColumnResult, RefreshMaterializedViewResult, Table,
UpdateFieldMetadataResult, UpdateResult,
};
@@ -104,7 +104,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<UpdateFieldMetadataResult>()?;
m.add_class::<AddResult>()?;
m.add_class::<MergeResult>()?;
m.add_class::<MemWalWriteSpec>()?;
m.add_class::<RTEWriteSpec>()?;
m.add_class::<DeleteResult>()?;
m.add_class::<DropColumnsResult>()?;
m.add_class::<UpdateResult>()?;
python/src/query.rs
+14 -14
View File
@@ -333,7 +333,7 @@ pub struct PyQueryRequest {
pub select_source_columns: Option<HashMap<String, String>>,
pub fast_search: Option<bool>,
pub with_row_id: Option<bool>,
pub use_memwal: Option<bool>,
pub use_rte: Option<bool>,
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub nprobes: Option<usize>,
@@ -366,7 +366,7 @@ impl From<AnyQuery> for PyQueryRequest {
select: PySelect(query_request.select),
fast_search: Some(query_request.fast_search),
with_row_id: Some(query_request.with_row_id),
use_memwal: query_request.use_memwal,
use_rte: query_request.use_rte,
column: None,
query_vector: None,
nprobes: None,
@@ -395,7 +395,7 @@ impl From<AnyQuery> for PyQueryRequest {
select: PySelect(vector_query.base.select),
fast_search: Some(vector_query.base.fast_search),
with_row_id: Some(vector_query.base.with_row_id),
use_memwal: vector_query.base.use_memwal,
use_rte: vector_query.base.use_rte,
column: vector_query.column,
query_vector: Some(PyQueryVectors(vector_query.query_vector)),
nprobes: vector_query.nprobes,
@@ -540,8 +540,8 @@ impl Query {
self.inner = self.inner.clone().fast_search();
}
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
pub fn with_row_id(&mut self) {
@@ -706,8 +706,8 @@ impl TakeQuery {
self.inner = self.inner.clone().with_row_id();
}
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
#[pyo3(signature = ())]
@@ -819,8 +819,8 @@ impl FTSQuery {
self.inner = self.inner.clone().fast_search();
}
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
pub fn with_row_id(&mut self) {
@@ -970,8 +970,8 @@ impl VectorQuery {
self.inner = self.inner.clone().fast_search();
}
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
pub fn use_rte(&mut self, enable: bool) {
self.inner = self.inner.clone().use_rte(enable);
}
pub fn with_row_id(&mut self) {
@@ -1178,9 +1178,9 @@ impl HybridQuery {
self.inner_fts.postfilter();
}
pub fn use_memwal(&mut self, enable: bool) {
self.inner_vec.use_memwal(enable);
self.inner_fts.use_memwal(enable);
pub fn use_rte(&mut self, enable: bool) {
self.inner_vec.use_rte(enable);
self.inner_fts.use_rte(enable);
}
pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> {
python/src/table.rs
+59 -58
View File
@@ -39,13 +39,13 @@ use pyo3::{
mod scannable;
/// Convert `MemWalStats` to a Python dict, preserving the per-table-shard list.
/// Convert `RTEStats` to a Python dict, preserving the per-table-shard list.
///
/// Deliberately not flattened to a table-level summary: a table is N
/// table shards on one node, and the per-shard detail is the reason the
/// endpoint exists — flattening hides the single hot table shard someone opened
/// it to find.
fn memwal_stats_to_py(py: Python<'_>, stats: &lancedb::table::MemWalStats) -> PyResult<Py<PyDict>> {
fn rte_stats_to_py(py: Python<'_>, stats: &lancedb::table::RTEStats) -> PyResult<Py<PyDict>> {
let out = PyDict::new(py);
let table_shards = PyList::empty(py);
for b in &stats.table_shards {
@@ -261,8 +261,8 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
}
}
/// Specification selecting Lance's MemWAL LSM-style write path for
/// `merge_insert`.
/// Specification enabling the realtime engine (RTE), which routes
/// `merge_insert` through Lance's MemWAL LSM-style write path.
///
/// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
/// classmethods, then optionally chain `with_maintained_indexes(...)` and
@@ -270,17 +270,17 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
/// table has, including ones created later.
#[pyclass(module = "lancedb._lancedb", from_py_object)]
#[derive(Clone, Debug)]
pub struct MemWalWriteSpec {
inner: lancedb::table::MemWalWriteSpec,
pub struct RTEWriteSpec {
inner: lancedb::table::RTEWriteSpec,
}
#[pymethods]
impl MemWalWriteSpec {
impl RTEWriteSpec {
/// Hash-bucket sharding by the unenforced primary key column.
#[staticmethod]
pub fn bucket(column: String, num_buckets: u32) -> Self {
Self {
inner: lancedb::table::MemWalWriteSpec::bucket(column, num_buckets),
inner: lancedb::table::RTEWriteSpec::bucket(column, num_buckets),
}
}
@@ -294,20 +294,20 @@ impl MemWalWriteSpec {
#[staticmethod]
pub fn identity(column: String) -> Self {
Self {
inner: lancedb::table::MemWalWriteSpec::identity(column),
inner: lancedb::table::RTEWriteSpec::identity(column),
}
}
/// No sharding — every `merge_insert` call writes to a single
/// MemWAL shard.
/// RTE shard.
#[staticmethod]
pub fn unsharded() -> Self {
Self {
inner: lancedb::table::MemWalWriteSpec::unsharded(),
inner: lancedb::table::RTEWriteSpec::unsharded(),
}
}
/// Set which indexes the MemWAL maintains. `None` (the default) is
/// Set which indexes the RTE maintains. `None` (the default) is
/// every index the table has, including ones created later; a list is
/// verbatim, and an empty list maintains nothing.
#[pyo3(signature = (indexes))]
@@ -327,33 +327,33 @@ impl MemWalWriteSpec {
pub fn __repr__(&self) -> String {
match &self.inner {
lancedb::table::MemWalWriteSpec::Bucket {
lancedb::table::RTEWriteSpec::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => format!(
"MemWalWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
"RTEWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
column,
num_buckets,
fmt_maintained(maintained_indexes),
writer_config_defaults,
),
lancedb::table::MemWalWriteSpec::Identity {
lancedb::table::RTEWriteSpec::Identity {
column,
maintained_indexes,
writer_config_defaults,
} => format!(
"MemWalWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
"RTEWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
column,
fmt_maintained(maintained_indexes),
writer_config_defaults,
),
lancedb::table::MemWalWriteSpec::Unsharded {
lancedb::table::RTEWriteSpec::Unsharded {
maintained_indexes,
writer_config_defaults,
} => format!(
"MemWalWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
"RTEWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
fmt_maintained(maintained_indexes),
writer_config_defaults,
),
@@ -365,9 +365,9 @@ impl MemWalWriteSpec {
#[getter]
pub fn spec_type(&self) -> &'static str {
match &self.inner {
lancedb::table::MemWalWriteSpec::Bucket { .. } => "bucket",
lancedb::table::MemWalWriteSpec::Identity { .. } => "identity",
lancedb::table::MemWalWriteSpec::Unsharded { .. } => "unsharded",
lancedb::table::RTEWriteSpec::Bucket { .. } => "bucket",
lancedb::table::RTEWriteSpec::Identity { .. } => "identity",
lancedb::table::RTEWriteSpec::Unsharded { .. } => "unsharded",
}
}
@@ -375,9 +375,9 @@ impl MemWalWriteSpec {
#[getter]
pub fn column(&self) -> Option<String> {
match &self.inner {
lancedb::table::MemWalWriteSpec::Bucket { column, .. }
| lancedb::table::MemWalWriteSpec::Identity { column, .. } => Some(column.clone()),
lancedb::table::MemWalWriteSpec::Unsharded { .. } => None,
lancedb::table::RTEWriteSpec::Bucket { column, .. }
| lancedb::table::RTEWriteSpec::Identity { column, .. } => Some(column.clone()),
lancedb::table::RTEWriteSpec::Unsharded { .. } => None,
}
}
@@ -385,12 +385,12 @@ impl MemWalWriteSpec {
#[getter]
pub fn num_buckets(&self) -> Option<u32> {
match &self.inner {
lancedb::table::MemWalWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
lancedb::table::RTEWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
_ => None,
}
}
/// Indexes the MemWAL keeps up to date, or `None` for every supported one.
/// Indexes the RTE keeps up to date, or `None` for every supported one.
#[getter]
pub fn maintained_indexes(&self) -> Option<Vec<String>> {
self.inner.maintained_indexes().map(<[String]>::to_vec)
@@ -403,14 +403,14 @@ impl MemWalWriteSpec {
}
}
impl From<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
fn from(spec: MemWalWriteSpec) -> Self {
impl From<RTEWriteSpec> for lancedb::table::RTEWriteSpec {
fn from(spec: RTEWriteSpec) -> Self {
spec.inner
}
}
impl From<lancedb::table::MemWalWriteSpec> for MemWalWriteSpec {
fn from(inner: lancedb::table::MemWalWriteSpec) -> Self {
impl From<lancedb::table::RTEWriteSpec> for RTEWriteSpec {
fn from(inner: lancedb::table::RTEWriteSpec) -> Self {
Self { inner }
}
}
@@ -1636,8 +1636,8 @@ impl Table {
if let Some(use_index) = parameters.use_index {
builder.use_index(use_index);
}
if let Some(use_memwal) = parameters.use_memwal {
builder.use_memwal(use_memwal);
if let Some(use_rte) = parameters.use_rte {
builder.use_rte(use_rte);
}
if let Some(validate_single_shard) = parameters.validate_single_shard {
builder.validate_single_shard(validate_single_shard);
@@ -1662,29 +1662,29 @@ impl Table {
})
}
pub fn enable_memwal<'a>(
pub fn enable_rte<'a>(
self_: PyRef<'a, Self>,
spec: MemWalWriteSpec,
spec: RTEWriteSpec,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.inner_ref()?.clone();
let native_spec = lancedb::table::MemWalWriteSpec::from(spec);
let native_spec = lancedb::table::RTEWriteSpec::from(spec);
future_into_py(self_.py(), async move {
inner.enable_memwal(native_spec).await.infer_error()
inner.enable_rte(native_spec).await.infer_error()
})
}
pub fn disable_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
pub fn disable_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.disable_memwal().await.infer_error()
inner.disable_rte().await.infer_error()
})
}
pub fn get_memwal_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
pub fn get_rte_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let spec = inner.get_memwal_spec().await.infer_error()?;
Ok(spec.map(MemWalWriteSpec::from))
let spec = inner.get_rte_spec().await.infer_error()?;
Ok(spec.map(RTEWriteSpec::from))
})
}
@@ -1703,54 +1703,55 @@ impl Table {
Ok(())
}
/// Converge the table's MemWAL write path into its base table.
/// Converge the table's RTE write path into its base table.
///
/// Best-effort: with writes flowing, new rows may land after the last
/// pass. Errors if the table stops making progress.
pub fn checkpoint_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
pub fn checkpoint_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.checkpoint_memwal().await.infer_error()
inner.checkpoint_rte().await.infer_error()
})
}
/// Freeze every table shard's active memtable into an SSTable.
pub fn flush_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
pub fn flush_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.flush_memwal().await.infer_error()
})
future_into_py(
self_.py(),
async move { inner.flush_rte().await.infer_error() },
)
}
/// Trigger a background SSTable compaction pass per table shard. Returns once the
/// passes are dispatched, not once they finish — watch `get_memwal_stats`.
pub fn compact_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
/// passes are dispatched, not once they finish — watch `get_rte_stats`.
pub fn compact_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.compact_memwal().await.infer_error()
inner.compact_rte().await.infer_error()
})
}
/// Live MemWAL state, or `None` when the MemWAL write path is not enabled.
/// Live RTE state, or `None` when the RTE write path is not enabled.
#[pyo3(signature = (include_sstable_rows=false))]
pub fn get_memwal_stats(
pub fn get_rte_stats(
self_: PyRef<'_, Self>,
include_sstable_rows: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let stats = inner
.get_memwal_stats(include_sstable_rows)
.get_rte_stats(include_sstable_rows)
.await
.infer_error()?;
Python::attach(|py| stats.map(|s| memwal_stats_to_py(py, &s)).transpose())
Python::attach(|py| stats.map(|s| rte_stats_to_py(py, &s)).transpose())
})
}
pub fn close_memwal_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
pub fn close_rte_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.close_memwal_writers().await.infer_error()
inner.close_rte_writers().await.infer_error()
})
}
@@ -2082,7 +2083,7 @@ pub struct MergeInsertParams {
when_not_matched_by_source_condition_expr: Option<PyExpr>,
timeout: Option<std::time::Duration>,
use_index: Option<bool>,
use_memwal: Option<bool>,
use_rte: Option<bool>,
validate_single_shard: Option<bool>,
}
python/vendor/lance-arrow/Cargo.toml
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lance-arrow"
version = "14.0.0-beta.21"
version = "14.0.0-beta.22"
edition = "2024"
authors = ["Lance Devs <dev@lance.org>"]
license = "Apache-2.0"
python/vendor/lance-arrow/README.lancedb.md
Vendored
+2 -2
View File
@@ -1,5 +1,5 @@
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.21`
(commit `8c233c2e6aa6948524f352e5e4cd52bd1364f6ff`), licensed under Apache-2.0.
This directory contains `rust/lance-arrow` from Lance tag `v14.0.0-beta.22`
(commit `72c5b645600bde5d2e0e0b629ed986e2018c62bf`), licensed under Apache-2.0.
LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
until a Lance release includes the fix. The patch is selected in the root
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.41.0-beta.9"
version = "0.41.0-beta.11"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/blob.rs
+58 -5
View File
@@ -244,8 +244,34 @@ impl BlobRangeRequest {
/// Creates an Arrow field for a Lance blob v2 column.
///
/// `Struct<data, uri>` with the `lance.blob.v2` marker. Same layout Lance
/// expects on write.
/// `Struct<data, uri, position, size>` with nullable children and the
/// `lance.blob.v2` marker. Set both range fields to read a byte range from a
/// URI, or leave both null for a whole file. Inline bytes are also supported.
///
/// # Migration
///
/// This helper previously returned only `data` and `uri`. Rust callers building
/// a `StructArray` from this field must now supply four children: append typed
/// null UInt64 arrays for `position` and `size` when no range is needed:
///
/// ```
/// use std::sync::Arc;
/// use arrow_array::{ArrayRef, LargeBinaryArray, StringArray, StructArray, UInt64Array};
/// use arrow_schema::DataType;
/// let field = lancedb::blob("image", true);
/// let DataType::Struct(children) = field.data_type() else { unreachable!() };
/// let image = StructArray::new(children.clone(), vec![
/// Arc::new(LargeBinaryArray::from(vec![Some(b"image".as_slice())])) as ArrayRef,
/// Arc::new(StringArray::from(vec![None::<&str>])),
/// Arc::new(UInt64Array::new_null(1)),
/// Arc::new(UInt64Array::new_null(1)),
/// ], None);
/// ```
///
/// Existing two-field tables remain writable: append and merge-insert fill
/// missing range fields or drop all-null range fields to match the table.
/// Non-null ranges cannot be written to a two-field table. Overwrite replaces
/// the table schema with the input schema rather than normalizing its layout.
///
/// A blob column may be top-level or nested inside a struct or list. Blobs
/// nested inside structs use dotted paths (e.g. `info.blob`) in the read APIs.
@@ -261,9 +287,21 @@ impl BlobRangeRequest {
/// ]);
/// ```
pub fn blob(name: impl AsRef<str>, nullable: bool) -> Field {
lance::blob::blob_field(name.as_ref(), nullable)
blob_with_options(name, nullable, BlobOptions::default())
}
static BLOB_STORAGE_TYPE: std::sync::LazyLock<DataType> = std::sync::LazyLock::new(|| {
DataType::Struct(
vec![
Field::new("data", DataType::LargeBinary, true),
Field::new("uri", DataType::Utf8, true),
Field::new("position", DataType::UInt64, true),
Field::new("size", DataType::UInt64, true),
]
.into(),
)
});
/// Options for creating a blob v2 column field with tier thresholds.
///
/// Re-exported from lance so the thresholds are built exactly the way
@@ -293,6 +331,7 @@ pub use lance::blob::BlobFieldOptions as BlobOptions;
/// ```
pub fn blob_with_options(name: impl AsRef<str>, nullable: bool, options: BlobOptions) -> Field {
lance::blob::blob_field_with_options(name.as_ref(), nullable, options)
.with_data_type(BLOB_STORAGE_TYPE.clone())
}
/// Returns true if `field` is a blob v2 column.
@@ -306,7 +345,7 @@ pub fn is_blob(field: &Field) -> bool {
}
/// Returns true if `field`, or any field nested under it, is a blob v2 column.
fn field_tree_has_blob_v2(field: &Field) -> bool {
pub(crate) fn field_tree_has_blob_v2(field: &Field) -> bool {
if field.is_blob_v2() {
return true;
}
@@ -673,7 +712,21 @@ mod tests {
field.metadata().get(ARROW_EXT_NAME_KEY).map(String::as_str),
Some("lance.blob.v2")
);
assert!(matches!(field.data_type(), DataType::Struct(_)));
let DataType::Struct(children) = field.data_type() else {
panic!("expected struct")
};
assert_eq!(
children
.iter()
.map(|f| (f.name().as_str(), f.data_type(), f.is_nullable()))
.collect::<Vec<_>>(),
vec![
("data", &DataType::LargeBinary, true),
("uri", &DataType::Utf8, true),
("position", &DataType::UInt64, true),
("size", &DataType::UInt64, true),
]
);
}
#[test]
rust/lancedb/src/database/listing.rs
+3 -3
View File
@@ -540,7 +540,7 @@ impl ListingDatabase {
// pairs survive (or none existed in the first place), so an empty
// string here must be treated the same as "no query" — otherwise
// every table URI ends up with a trailing `?`, which makes downstream
// sub-paths (e.g. MemWAL gen paths) re-parse as path=<base table> +
// sub-paths (e.g. RTE gen paths) re-parse as path=<base table> +
// query=<sub-path>, causing Lance to find the base table dataset
// when looking up the sub-path.
let query_string = url.query().filter(|q| !q.is_empty()).map(|s| s.to_string());
@@ -3104,7 +3104,7 @@ mod tests {
///
/// Earlier, `query_pairs_mut().clear()` left the URL with
/// `query=Some("")`, which then propagated as a trailing `?` on
/// every table URI. Sub-path lookups against that URI (e.g. MemWAL
/// every table URI. Sub-path lookups against that URI (e.g. RTE
/// `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parsed as
/// `path=<base table>` + `query=/_mem_wal/...`, causing
/// `Dataset::write` to find the base table dataset and falsely
@@ -3178,7 +3178,7 @@ mod tests {
/// `url::Url::parse` and the `query_pairs_mut()` path) must not
/// append a trailing `?` to per-table URIs when the input URI has
/// no query string. Sub-path lookups against such a URI (e.g.
/// MemWAL `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parse
/// RTE `<table_uri>/_mem_wal/<shard>/<rand>_gen_<n>`) re-parse
/// as `path=<base table>` + `query=/_mem_wal/...`, causing
/// `Dataset::write` to find the base table dataset and falsely
/// report `Dataset already exists`.
rust/lancedb/src/dataloader/permutation/builder.rs
+89 -14
View File
@@ -3,6 +3,7 @@
use std::{collections::HashMap, sync::Arc};
use arrow_array::RecordBatch;
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_execution::{disk_manager::DiskManagerBuilder, runtime_env::RuntimeEnvBuilder};
use datafusion_expr::col;
@@ -220,6 +221,12 @@ impl PermutationBuilder {
Ok(Box::pin(SimpleRecordBatchStream { schema, stream }))
}
/// Replaces the stream's schema metadata with the permutation config.
///
/// The scan of the base table can carry the base table's schema metadata (e.g. Hugging
/// Face's `huggingface` key). That metadata describes columns the permutation table does
/// not have, and its keys could collide with the config keys, so it is dropped rather
/// than merged.
fn add_config_metadata(
data: SendableRecordBatchStream,
metadata: HashMap<String, String>,
@@ -227,7 +234,13 @@ impl PermutationBuilder {
let schema = data.schema().as_ref().clone().with_metadata(metadata);
let schema = Arc::new(schema);
let schema_clone = schema.clone();
let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap());
// `RecordBatch::with_schema` only accepts a superset of the batch's schema, which a
// schema with replaced metadata is not, so rebuild the batch around the new schema.
let stream = data.and_then(move |batch| {
std::future::ready(
RecordBatch::try_new(schema.clone(), batch.columns().to_vec()).map_err(Error::from),
)
});
Ok(Box::pin(SimpleRecordBatchStream {
schema: schema_clone,
stream,
@@ -250,17 +263,17 @@ impl PermutationBuilder {
}
// Unflushed rows have no row id, so a permutation cannot address them.
match self.base_table.base_table().get_memwal_spec().await {
match self.base_table.base_table().get_rte_spec().await {
Ok(Some(_)) => {
return Err(Error::NotSupported {
message: "the data loader does not support tables with a MemWAL write \
message: "the data loader does not support tables with an RTE write \
spec: rows that have not been flushed to the base table \
have no row id, so a permutation cannot reference them"
.to_string(),
});
}
Ok(None) => {}
// No MemWAL write path means no spec.
// No RTE write path means no spec.
Err(Error::NotSupported { .. }) => {}
Err(err) => return Err(err),
}
@@ -489,9 +502,9 @@ mod tests {
)
.unwrap()
}
"/v1/table/remote_base/get_memwal_spec/" => http::Response::builder()
"/v1/table/remote_base/get_rte_spec/" => http::Response::builder()
.status(200)
.body(br#"{"memwal_spec":null}"#.to_vec())
.body(br#"{"rte_spec":null}"#.to_vec())
.unwrap(),
"/v1/table/remote_base/count_rows/" => {
let body = body.unwrap();
@@ -737,12 +750,12 @@ mod tests {
/// permutation cannot reference them. Reading the base table alone would drop
/// them from training without saying so, so the table is refused instead.
#[tokio::test]
async fn test_permutation_rejects_memwal_spec() {
use crate::table::MemWalWriteSpec;
async fn test_permutation_rejects_rte_spec() {
use crate::table::RTEWriteSpec;
use arrow_array::{Int32Array, RecordBatchIterator};
use arrow_schema::{DataType, Field, Schema};
// MemWAL needs a real dataset directory and a non-nullable primary key.
// RTE needs a real dataset directory and a non-nullable primary key.
let temp_dir = tempfile::tempdir().unwrap();
let db = connect(temp_dir.path().to_str().unwrap())
.execute()
@@ -765,15 +778,77 @@ mod tests {
.unwrap();
table.set_unenforced_primary_key(["idx"]).await.unwrap();
table
.enable_memwal(MemWalWriteSpec::unsharded())
.await
.unwrap();
table.enable_rte(RTEWriteSpec::unsharded()).await.unwrap();
let err = PermutationBuilder::new(table).build().await.unwrap_err();
assert!(
err.to_string().contains("MemWAL write spec"),
err.to_string().contains("RTE write spec"),
"expected the pre-check to refuse the table, got: {err}"
);
}
/// Base table schema metadata (e.g. Hugging Face's `huggingface` key) describes the base
/// table's columns, so the permutation table drops it and keeps only its own config.
#[tokio::test]
async fn test_permutation_over_table_with_schema_metadata() {
use arrow_array::{Int64Array, RecordBatchIterator};
use arrow_schema::{DataType, Field, Schema};
let temp_dir = tempfile::tempdir().unwrap();
let db = connect(temp_dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let schema = Arc::new(
Schema::new(vec![Field::new("x", DataType::Int64, false)]).with_metadata(
HashMap::from([
("huggingface".to_string(), "{}".to_string()),
// Collides with a config key; the config value must win.
(SPLIT_NAMES_CONFIG_KEY.to_string(), "not json".to_string()),
]),
),
);
let batch = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(Int64Array::from_iter_values(0..100))],
)
.unwrap();
let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
let table = db.create_table("tbl", reader).execute().await.unwrap();
assert!(
table
.schema()
.await
.unwrap()
.metadata
.contains_key("huggingface")
);
let permutation_table = PermutationBuilder::new(table)
.with_split_strategy(
SplitStrategy::Random {
seed: Some(1),
sizes: SplitSizes::Fixed(2),
clump_size: None,
},
Some(vec!["train".to_string(), "test".to_string()]),
)
.with_shuffle_strategy(ShuffleStrategy::Random {
seed: Some(1),
clump_size: None,
})
.build()
.await
.unwrap();
assert_eq!(permutation_table.count_rows(None).await.unwrap(), 100);
let metadata = permutation_table.schema().await.unwrap().metadata.clone();
assert!(!metadata.contains_key("huggingface"));
assert_eq!(
metadata.get(SPLIT_NAMES_CONFIG_KEY).map(String::as_str),
Some(r#"["train","test"]"#)
);
assert!(metadata.contains_key(BASE_VERSION_CONFIG_KEY));
}
}
rust/lancedb/src/dataloader/permutation/split.rs
+13 -13
View File
@@ -214,9 +214,9 @@ impl Splitter {
}
arrays.push(Arc::new(split_ids));
std::future::ready(Some(Ok(
RecordBatch::try_new(new_schema.clone(), arrays).unwrap()
)))
std::future::ready(Some(
RecordBatch::try_new(new_schema.clone(), arrays).map_err(Error::from),
))
});
Ok(Box::pin(SimpleRecordBatchStream::new(
@@ -289,23 +289,23 @@ impl Splitter {
.collect::<Vec<_>>();
let new_schema_clone = new_schema.clone();
let stream = source.map_ok(move |batch| {
let stream = source.and_then(move |batch| {
let split_ids = Self::hash_split_id(&batch, &thresholds, total_weight);
if split_ids.null_count() > 0 {
let is_valid = split_ids.nulls().unwrap().inner();
let is_valid_mask = BooleanArray::new(is_valid.clone(), None);
let split_ids = arrow::compute::filter(&split_ids, &is_valid_mask).unwrap();
let row_ids = batch.column(row_id_index);
let row_ids = arrow::compute::filter(row_ids.as_ref(), &is_valid_mask).unwrap();
RecordBatch::try_new(new_schema.clone(), vec![row_ids, split_ids]).unwrap()
let batch = if let Some(nulls) = split_ids.nulls().filter(|n| n.null_count() > 0) {
let is_valid_mask = BooleanArray::new(nulls.inner().clone(), None);
arrow::compute::filter(&split_ids, &is_valid_mask).and_then(|split_ids| {
let row_ids = batch.column(row_id_index);
let row_ids = arrow::compute::filter(row_ids.as_ref(), &is_valid_mask)?;
RecordBatch::try_new(new_schema.clone(), vec![row_ids, split_ids])
})
} else {
RecordBatch::try_new(
new_schema.clone(),
vec![batch.column(row_id_index).clone(), Arc::new(split_ids)],
)
.unwrap()
}
};
std::future::ready(batch.map_err(Error::from))
});
Ok(Box::pin(SimpleRecordBatchStream::new(
rust/lancedb/src/lib.rs
+1 -1
View File
@@ -167,7 +167,7 @@
//! # }
//! ```
// The MemWAL read path (`table::query::lsm`) deepens the `create_plan` future's
// The RTE read path (`table::query::lsm`) deepens the `create_plan` future's
// type graph enough to overflow the default trait-recursion limit while evaluating
// auto-traits (`Send`) through the Linux io_uring build's moka cache. Raise it.
#![recursion_limit = "256"]
rust/lancedb/src/materialized_view/refresh.rs
+2 -4
View File
@@ -288,15 +288,13 @@ fn refresh_conflict_filter(dataset: &Dataset) -> Result<KeyExistenceFilter> {
Ok(filter.build())
}
/// Reject MemWAL/MemWAL state because an ordinary dataset scan cannot see its
/// Reject RTE/RTE state because an ordinary dataset scan cannot see its
/// un-compacted tiers.
pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> {
let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty();
if retained || dataset.mem_wal_index_details().await?.is_some() {
return Err(Error::NotSupported {
message: format!(
"{role} '{name}' has a MemWAL write spec or retained un-compacted rows"
),
message: format!("{role} '{name}' has an RTE write spec or retained un-compacted rows"),
});
}
Ok(())
rust/lancedb/src/query.rs
+19 -19
View File
@@ -544,25 +544,25 @@ pub trait QueryBase {
/// This allows ordering query results by one or more columns in either ascending or descending order.
fn order_by(self, ordering: Option<Vec<ColumnOrdering>>) -> Self;
/// Control MemWAL read routing for this query.
/// Control RTE read routing for this query.
///
/// By default (unset), when the table carries a MemWAL write spec (see
/// [`crate::Table::enable_memwal`]), reads are routed through the LSM
/// By default (unset), when the table carries an RTE write spec (see
/// [`crate::Table::enable_rte`]), reads are routed through the LSM
/// scanner so they also return data written via the `merge_insert` LSM path
/// that has not yet been compacted into the base table (active/frozen
/// memtables and SSTables); a table without a spec reads the base
/// table.
///
/// - `use_memwal(true)` forces MemWAL routing and errors if the table has no
/// MemWAL write spec.
/// - `use_memwal(false)` bypasses the MemWAL and reads the base table only,
/// - `use_rte(true)` forces RTE routing and errors if the table has no
/// RTE write spec.
/// - `use_rte(false)` bypasses the RTE and reads the base table only,
/// even when a spec is present.
///
/// Note: the LSM scanner does not support every query shape (e.g. reranking,
/// hybrid search, `order_by`). On a MemWAL table those shapes error unless
/// `use_memwal(false)` is set, because a base-only read would silently
/// exclude un-compacted MemWAL data.
fn use_memwal(self, enable: bool) -> Self;
/// hybrid search, `order_by`). On an RTE table those shapes error unless
/// `use_rte(false)` is set, because a base-only read would silently
/// exclude un-compacted RTE data.
fn use_rte(self, enable: bool) -> Self;
}
pub trait HasQuery {
@@ -634,8 +634,8 @@ impl<T: HasQuery> QueryBase for T {
self
}
fn use_memwal(mut self, enable: bool) -> Self {
self.mut_query().use_memwal = Some(enable);
fn use_rte(mut self, enable: bool) -> Self {
self.mut_query().use_rte = Some(enable);
self
}
}
@@ -898,19 +898,19 @@ pub struct QueryRequest {
/// This allows ordering query results by one or more columns in either ascending or descending order.
pub order_by: Option<Vec<ColumnOrdering>>,
/// Controls MemWAL read routing. When unset (the default), a query against a
/// table that carries a MemWAL write spec (see
/// [`crate::Table::enable_memwal`]) is routed through the LSM scanner so
/// Controls RTE read routing. When unset (the default), a query against a
/// table that carries an RTE write spec (see
/// [`crate::Table::enable_rte`]) is routed through the LSM scanner so
/// it also sees data written via the `merge_insert` LSM path that has not yet
/// been compacted into the base table — the active and frozen in-memory
/// memtables and the SSTables, deduplicated by primary key
/// against the base table (newest generation wins); a table without a spec
/// reads the base table.
///
/// - `Some(true)` forces MemWAL routing and errors if the table has no MemWAL
/// - `Some(true)` forces RTE routing and errors if the table has no RTE
/// write spec.
/// - `Some(false)` reads only the base table, bypassing the MemWAL.
pub use_memwal: Option<bool>,
/// - `Some(false)` reads only the base table, bypassing the RTE.
pub use_rte: Option<bool>,
}
impl Default for QueryRequest {
@@ -930,7 +930,7 @@ impl Default for QueryRequest {
norm: None,
disable_scoring_autoprojection: false,
order_by: None,
use_memwal: None,
use_rte: None,
}
}
}
rust/lancedb/src/query/wal_fusion.rs
+17 -17
View File
@@ -1,14 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables.
//! WAL-PK-FUSION: hybrid search's primary-key fallback for RTE tables.
//!
//! Hybrid search joins its vector and full-text legs on `_rowid`, which a
//! MemWAL table cannot supply: the fresh tier has no stable row id, so the
//! RTE table cannot supply: the fresh tier has no stable row id, so the
//! server rejects `with_row_id` before it plans. When that happens the legs are
//! joined on a surrogate `_rowid` built from the table's primary key instead.
//!
//! This is a stopgap until MemWAL supports `_rowid`, and everything it needs
//! This is a stopgap until RTE supports `_rowid`, and everything it needs
//! lives here so it can be removed in one pass:
//!
//! 1. Delete this file and its `mod wal_fusion;` line in `query.rs`.
@@ -31,44 +31,44 @@ use lance_core::utils::parse::str_is_truthy;
use super::{Select, VectorQuery};
use crate::error::{Error, Result};
/// The part of the MemWAL `with_row_id` refusal that identifies it.
/// The part of the RTE `with_row_id` refusal that identifies it.
///
/// Both layers raise this same sentence verbatim — sophon's
/// `LSM_WITH_ROW_ID_UNSUPPORTED` says so in as many words, and the OSS LSM
/// scanner builds it from the same words — so this substring survives either
/// path and the wrapping each adds. It is a coupling to a message rather than
/// a code, because the refusal arrives as a generic 400; if the server ever
/// rewords it the fallback stops firing and hybrid on a MemWAL table goes back
/// rewords it the fallback stops firing and hybrid on an RTE table goes back
/// to erroring visibly, which is loud rather than silently wrong.
const WAL_ROW_ID_REFUSAL: &str = "does not support with_row_id";
/// Whether `error` is the server declining `_rowid` because the table is
/// MemWAL-backed, as opposed to any other failure the query might hit.
/// RTE-backed, as opposed to any other failure the query might hit.
fn is_wal_row_id_refusal(error: &Error) -> bool {
error.to_string().contains(WAL_ROW_ID_REFUSAL)
}
/// Why a hybrid query on a MemWAL table cannot hand back `_rowid`, raised
/// Why a hybrid query on an RTE table cannot hand back `_rowid`, raised
/// whether the refusal was anticipated or came back from the server.
fn wal_row_id_unsupported() -> Error {
Error::NotSupported {
message: "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has no \
message: "hybrid search on an RTE table cannot return _rowid: the fresh tier has no \
stable row id, and the ids the fusion joins on are synthesized from the \
primary key. Set use_memwal(false) to read the base table only (results will \
exclude un-compacted MemWAL data)"
primary key. Set use_rte(false) to read the base table only (results will \
exclude un-compacted RTE data)"
.to_string(),
}
}
/// Run a hybrid query, joining its legs on the primary key instead of `_rowid`
/// when the table is MemWAL-backed.
/// when the table is RTE-backed.
///
/// `run(None)` must join the legs on `_rowid`; `run(Some(fusion))` must build
/// them with `fusion` instead.
///
/// `_rowid` is asked for first and the primary key used only when the server
/// refuses, rather than paying a round trip up front to find out. Only a
/// MemWAL table refuses, so probing would tax every table to learn something
/// RTE table refuses, so probing would tax every table to learn something
/// almost none of them need — and the refusal has to be handled regardless,
/// since a write spec installed elsewhere can arrive between any two queries.
/// The refusal costs nothing server-side: it is raised before the query is
@@ -80,7 +80,7 @@ where
{
// A previous query on this table was already refused, so skip straight to
// the key.
if query.request.base.use_memwal != Some(false) && query.parent.hybrid_pk_fusion_learned() {
if query.request.base.use_rte != Some(false) && query.parent.hybrid_pk_fusion_learned() {
return run(Some(PkFusion::resolve(query).await?)).await;
}
match run(None).await {
@@ -130,7 +130,7 @@ impl PkFusionMemory {
}
}
/// How the legs of a hybrid query on a MemWAL table are joined: on a surrogate
/// How the legs of a hybrid query on an RTE table are joined: on a surrogate
/// `_rowid` derived client-side from the primary key.
pub(super) struct PkFusion {
columns: Vec<String>,
@@ -152,7 +152,7 @@ impl PkFusion {
Ok(Self { columns, injected })
}
/// Make a leg return the key columns, in place of the `_rowid` a MemWAL
/// Make a leg return the key columns, in place of the `_rowid` an RTE
/// table would refuse.
pub(super) fn prepare_leg(&self, select: &mut Select) {
inject_pk_columns(select, &self.columns);
@@ -266,7 +266,7 @@ fn pk_columns(schema: &arrow_schema::Schema) -> Vec<String> {
/// Replace each batch's join column with a dense surrogate derived from
/// `pk_columns`, in place and in the order given.
///
/// The fusion joins its two legs on [`ROW_ID`], which a MemWAL table cannot
/// The fusion joins its two legs on [`ROW_ID`], which an RTE table cannot
/// supply — the fresh tier has no stable row id. It only ever compares the
/// column for equality, though (`merge_results` dedups, the rerankers group and
/// the score restore looks up), so any injective mapping of the primary key
@@ -280,7 +280,7 @@ fn pk_columns(schema: &arrow_schema::Schema) -> Vec<String> {
fn stamp_surrogate_row_ids(batches: &mut [RecordBatch], pk_columns: &[String]) -> Result<()> {
if pk_columns.is_empty() {
return Err(Error::InvalidInput {
message: "hybrid search on a MemWAL table needs an unenforced primary key to \
message: "hybrid search on an RTE table needs an unenforced primary key to \
deduplicate on, and this table declares none"
.to_string(),
});
rust/lancedb/src/remote/table.rs
+426 -125
View File
File diff suppressed because it is too large Load Diff
rust/lancedb/src/remote/table/blobs.rs
+3 -3
View File
@@ -716,8 +716,8 @@ impl<S: HttpSend> RemoteTable<S> {
)),
select: Select::columns(&[column]),
with_row_id: true,
// Row ids name base-table rows, and the MemWAL scanner has no stable `_rowid`.
use_memwal: Some(false),
// Row ids name base-table rows, and the RTE scanner has no stable `_rowid`.
use_rte: Some(false),
..Default::default()
});
let body = self
@@ -1153,7 +1153,7 @@ mod tests {
.unwrap();
assert_eq!(body["columns"], serde_json::json!(["image"]));
assert_eq!(body["with_row_id"], serde_json::json!(true));
assert_eq!(body["use_memwal"], serde_json::json!(false));
assert_eq!(body["use_rte"], serde_json::json!(false));
let requested = filter_row_ids(body["filter"].as_str().unwrap());
captured.lock().unwrap().push(TakeMockRequest::Query {
row_ids: requested.clone(),
rust/lancedb/src/remote/table/insert.rs
+1 -1
View File
@@ -1497,7 +1497,7 @@ mod tests {
when_not_matched_by_source_delete: false,
when_not_matched_by_source_delete_filt: None,
use_index: true,
use_memwal: None,
use_rte: None,
};
let input: Arc<dyn ExecutionPlan> = Arc::new(ErroringExec::new());

Some files were not shown because too many files have changed in this diff Show More