rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-23 20:22:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-2325-1

Browse Source 
Preserve the compaction option tests and upstream CUDA tests by retaining both Any and MagicMock imports.

Validation: rebuilt the Python extension against Lance v15; 29 optimize/CUDA tests and 7 storage documentation tests passed. Rust formatting and Python source checks passed. Full repository Ruff still reports 49 inherited documentation example errors.
This commit is contained in:
Gatefixer
2026-10-22 17:57:02 +00:00
parent 7ba76d15ff 7f932b8552
commit 465805a853
95 changed files with 11650 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.41.0-beta.10"
current_version = "0.41.0-beta.13"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/workflows/npm-publish.yml
+16 -16
View File
@@ -71,9 +71,9 @@ jobs:
# https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
pre_build: |-
set -e &&
apt-get update &&
apt-get install -y protobuf-compiler pkg-config &&
set -e
apt-get update
apt-get install -y protobuf-compiler pkg-config
# The base image (manylinux2014-cross) sets TARGET_CC to the old
# GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
# so it picks up GCC even though the napi-rs image sets CC=clang.
@@ -84,10 +84,10 @@ jobs:
host: ubuntu-2404-8x-x64
features: fp16kernels
pre_build: |-
set -e &&
sudo apt-get update &&
sudo apt-get install -y protobuf-compiler pkg-config &&
rustup target add x86_64-unknown-linux-musl &&
set -e
sudo apt-get update
sudo apt-get install -y protobuf-compiler pkg-config
rustup target add x86_64-unknown-linux-musl
export EXTRA_ARGS="-x"
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-x64
@@ -103,13 +103,13 @@ jobs:
# linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
linker: /tmp/aarch64-lld-clang
pre_build: |-
set -e &&
apt-get update &&
apt-get install -y protobuf-compiler pkg-config &&
export TARGET_CC=clang TARGET_CXX=clang++ &&
set -e
apt-get update
apt-get install -y protobuf-compiler pkg-config
export TARGET_CC=clang TARGET_CXX=clang++
# The manylinux2014 sysroot has glibc 2.17 headers which lack
# AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
export CFLAGS="$CFLAGS -DAT_HWCAP2=26"
rustup target add aarch64-unknown-linux-gnu
# Not `&&`-chained: in dash, errexit does not fire for a
# non-final command in an `&&` list, so failures were ignored.
@@ -136,10 +136,10 @@ jobs:
lto: thin
codegen_units: 16
pre_build: |-
set -e &&
sudo apt-get update &&
sudo apt-get install -y protobuf-compiler &&
rustup target add aarch64-unknown-linux-musl &&
set -e
sudo apt-get update
sudo apt-get install -y protobuf-compiler
rustup target add aarch64-unknown-linux-musl
export EXTRA_ARGS="-x"
name: build - ${{ matrix.settings.target }}
runs-on: ${{ matrix.settings.host }}
AGENTS.md
+2 -2
View File
@@ -102,8 +102,8 @@ TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `pnpm build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
3. Add typescript method on abstract class `Table` in `nodejs/lancedb/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/lancedb/table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `pnpm run docs` to generate TypeScript documentation.
Cargo.lock
Generated
+120 -59
View File
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4441,7 +4441,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots 1.0.7",
"webpki-roots 1.0.10",
]
[[package]]
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arc-swap",
"arrow",
@@ -5110,6 +5110,7 @@ dependencies = [
"lance-tokenizer",
"log",
"moka",
"num-traits",
"object_store 0.14.2",
"permutation",
"pin-project",
@@ -5134,8 +5135,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5157,7 +5157,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5171,7 +5171,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5180,8 +5180,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrayref",
"crunchy",
@@ -5191,8 +5191,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5229,8 +5229,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow",
"arrow-array",
@@ -5261,8 +5261,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow",
"arrow-array",
@@ -5279,8 +5279,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"proc-macro2",
"quote",
@@ -5289,8 +5289,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5323,8 +5323,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5356,8 +5356,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"datafusion",
"geo-traits",
@@ -5371,8 +5371,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arc-swap",
"arrow",
@@ -5386,6 +5386,7 @@ dependencies = [
"async-recursion",
"async-trait",
"bitvec",
"blake3",
"bytes",
"crossbeam-queue",
"datafusion",
@@ -5420,6 +5421,7 @@ dependencies = [
"lance-tokenizer",
"libsais-rs",
"log",
"nalgebra",
"ndarray",
"num-traits",
"object_store 0.14.2",
@@ -5443,8 +5445,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5467,8 +5469,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow",
"arrow-array",
@@ -5511,8 +5513,8 @@ dependencies = [
[[package]]
name = "lance-jsonb"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"byteorder",
"ethnum",
@@ -5529,8 +5531,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5544,8 +5546,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow",
"async-trait",
@@ -5559,8 +5561,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arc-swap",
"arrow",
@@ -5586,7 +5588,7 @@ dependencies = [
"object_store 0.14.2",
"quick-xml 0.40.1",
"rand 0.9.5",
"reqwest 0.12.28",
"reqwest 0.13.4",
"roaring",
"serde",
"serde_json",
@@ -5614,8 +5616,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5629,8 +5631,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow",
"arrow-array",
@@ -5670,8 +5672,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5684,8 +5686,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "14.0.0-beta.21"
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.21#8c233c2e6aa6948524f352e5e4cd52bd1364f6ff"
version = "15.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5698,7 +5700,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.41.0-beta.10"
version = "0.41.0-beta.13"
dependencies = [
"ahash",
"anyhow",
@@ -5796,7 +5798,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.41.0-beta.10"
version = "0.41.0-beta.13"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5821,7 +5823,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.41.0-beta.10"
version = "0.41.0-beta.13"
dependencies = [
"arc-swap",
"arrow",
@@ -5831,6 +5833,7 @@ dependencies = [
"datafusion-common",
"env_logger",
"futures",
"lance-arrow",
"lance-core",
"lance-io",
"lance-namespace",
@@ -6410,6 +6413,21 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "nalgebra"
version = "0.33.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d43ddcacf343185dfd6de2ee786d9e8b1c2301622afab66b6c73baf9882abfd"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"simba",
"typenum",
]
[[package]]
name = "napi"
version = "3.12.5"
@@ -6649,6 +6667,17 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -8909,7 +8938,7 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams 0.4.2",
"web-sys",
"webpki-roots 1.0.7",
"webpki-roots 1.0.10",
]
[[package]]
@@ -9239,9 +9268,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"aws-lc-rs",
"ring",
@@ -9267,6 +9296,15 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "safetensors"
version = "0.7.0"
@@ -9675,6 +9713,19 @@ dependencies = [
"rand_core 0.6.4",
]
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "simd-adler32"
version = "0.3.9"
@@ -11143,9 +11194,9 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.7"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
dependencies = [
"rustls-pki-types",
]
@@ -11156,14 +11207,14 @@ version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
"webpki-roots 1.0.7",
"webpki-roots 1.0.10",
]
[[package]]
name = "webpki-roots"
version = "1.0.7"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
dependencies = [
"rustls-pki-types",
]
@@ -11181,6 +11232,16 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "winapi"
version = "0.3.9"
Cargo.toml
+19 -14
View File
@@ -13,20 +13,21 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.21", default-features = false, "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.21", "tag" = "v14.0.0-beta.21", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=15.0.0-beta.1", tag = "v15.0.0-beta.1", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=15.0.0-beta.1", path = "python/vendor/lance-arrow" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -79,6 +80,10 @@ tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[patch."https://github.com/lance-format/lance.git"]
# Carry the offset-safe validity fix until it is available in a Lance release.
lance-arrow = { path = "python/vendor/lance-arrow" }
[patch.crates-io]
# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision.
object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" }
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.41.0-beta.10</version>
<version>0.41.0-beta.13</version>
</dependency>
```
docs/src/python/python.md
+16
View File
@@ -379,6 +379,22 @@ listing a storage directory.
## Materialized Views (Synchronous)
On Cloud and Enterprise, creation accepts `auto_refresh_enabled` and
`auto_refresh_interval_seconds`:
```python
view = db.create_materialized_view(
"recent_orders",
"orders",
auto_refresh_enabled=True,
auto_refresh_interval_seconds=300,
)
```
Omit `auto_refresh_interval_seconds` to let the server determine when to refresh.
These options also apply to asynchronous creation and creation jobs. Embedded
connections support manual refresh and reject automatic scheduling requests.
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
docs/web-tests/rs/Cargo.lock
Generated
+2 -2
View File
@@ -5561,9 +5561,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"ring",
"rustls-pki-types",
docs/web/docs.json
+1
View File
@@ -327,6 +327,7 @@
"management/namespace",
"management/table",
"tables/branching",
"tables/realtime-engine",
"management/materialized-view",
"management/view"
]
docs/web/enterprise/authentication.mdx
+282 -30
View File
@@ -3,7 +3,7 @@ title: "Authentication"
sidebarTitle: "Authentication"
description: "Authentication modes for LanceDB Enterprise: API keys and OAuth credentials."
icon: key
keywords: ["authentication", "oauth", "api key", "azure managed identity", "client credentials"]
keywords: ["authentication", "oauth", "api key", "authorization code", "device code", "azure managed identity", "client credentials"]
availability:
oss: unavailable
enterprise: available
@@ -13,9 +13,9 @@ availability:
[Architecture](/architecture).
---
A deployment authenticates every request before
[authorization](/governance/authorization) decides what it may do, against a
`db://` remote table, in one of two ways:
A deployment authenticates each request over a `db://` connection before
[authorization](/governance/authorization) decides what it may do.
Choose one of these methods:
- **API keys**: a long-lived shared secret passed as `api_key` on connect. Works in every SDK.
- **OAuth 2.0**: short-lived bearer tokens obtained from your identity provider, refreshed automatically by the client.
@@ -28,55 +28,91 @@ identities so no secret material lives on the client.
Pass the API key from your Enterprise tenant on `connect`. This works with
both the synchronous and asynchronous Python clients, as well as TypeScript
and Rust.
and Rust. Load the key from an environment variable or secret manager.
```python
<CodeGroup>
```python Python icon="python"
import os
import lancedb
db = lancedb.connect(
uri="db://your-database-uri",
api_key="your-api-key",
api_key=os.environ["LANCEDB_API_KEY"],
region="us-east-1",
host_override="https://your-enterprise-endpoint.com",
)
```
An API key belongs in the environment rather than in source. See
[Database](/management/database#deployment) for the connection options.
```typescript TypeScript icon="square-js"
import * as lancedb from "@lancedb/lancedb";
const apiKey = process.env.LANCEDB_API_KEY;
if (!apiKey) throw new Error("LANCEDB_API_KEY is required");
const db = await lancedb.connect("db://your-database-uri", {
apiKey,
region: "us-east-1",
hostOverride: "https://your-enterprise-endpoint.com",
});
```
```rust Rust icon="rust"
let api_key = std::env::var("LANCEDB_API_KEY")?;
let db = lancedb::connect("db://your-database-uri")
.api_key(&api_key)
.region("us-east-1")
.host_override("https://your-enterprise-endpoint.com")
.execute()
.await?;
```
</CodeGroup>
For Rust, enable the `remote` feature on the `lancedb` dependency to use
API key or OAuth authentication.
See [Database](/management/database#deployment) for the connection options.
## OAuth {#oauth}
The async Python client and the TypeScript client can obtain bearer tokens
The async Python, TypeScript, and Rust clients can obtain bearer tokens
from an OIDC issuer and attach them to every request. Token acquisition,
caching, and refresh are handled inside the client. Your application code
only provides the configuration.
A deployment accepts these tokens only when its administrator has configured
it with your identity provider. Ask them for the issuer URL, client ID and
scopes to use.
it with your identity provider. Ask them for the issuer URL, client ID, scopes,
and any resource or audience required by the provider. For Authorization Code,
register the loopback redirect URI with the provider. For Device Code, enable
the provider's device authorization grant for your application.
<Info>
In Python, OAuth is supported through `lancedb.connect_async`. The synchronous
`connect` entry point continues to use API key authentication for `db://`
URIs. In TypeScript, `lancedb.connect` accepts `oauthConfig` directly.
In Rust, use `.oauth_config(config)` on the connection builder.
</Info>
### Supported flows {#supported-flows}
`OAuthFlowType` selects how the client acquires tokens:
`OAuthFlowType` (Python and TypeScript) or `OAuthFlow` (Rust) selects how the
client acquires tokens:
| Flow | Python value | TypeScript value | When to use |
| --- | --- | --- | --- |
| Client Credentials | `OAuthFlowType.CLIENT_CREDENTIALS` | `OAuthFlowType.ClientCredentials` | Service-to-service / machine-to-machine. Requires a client ID and client secret registered with your identity provider. |
| Azure Managed Identity | `OAuthFlowType.AZURE_MANAGED_IDENTITY` | `OAuthFlowType.AzureManagedIdentity` | Workloads running on Azure compute (VMs, AKS, App Service, Container Apps). Tokens are fetched from the Azure IMDS endpoint, so no client secret is stored on the client. |
| Flow | Python value | TypeScript value | Rust value | When to use |
| --- | --- | --- | --- | --- |
| Client Credentials | `OAuthFlowType.CLIENT_CREDENTIALS` | `OAuthFlowType.ClientCredentials` | `OAuthFlow::ClientCredentials` | Service-to-service / machine-to-machine. Requires a client ID and client secret registered with your identity provider. |
| Authorization Code | `OAuthFlowType.AUTHORIZATION_CODE` | `OAuthFlowType.AuthorizationCode` | `OAuthFlow::AuthorizationCode(AuthorizationCodeOptions::new())` | Interactive user sign-in through a browser and local callback server. Uses S256 PKCE by default. |
| Device Code | `OAuthFlowType.DEVICE_CODE` | `OAuthFlowType.DeviceCode` | `OAuthFlow::DeviceCode` | CLI or headless sign-in. The client displays a verification URL and user code for the user to enter on another device. |
| Azure Managed Identity | `OAuthFlowType.AZURE_MANAGED_IDENTITY` | `OAuthFlowType.AzureManagedIdentity` | `OAuthFlow::AzureManagedIdentity { client_id: None }` | Azure workloads with access to the IMDS endpoint. Tokens are fetched from IMDS, so no client secret is stored on the client. |
### Configure OAuth {#configure-oauth}
Build an OAuth config and pass it on connect. Use `oauth_config` in Python and
`oauthConfig` in TypeScript.
Rust, and `oauthConfig` in TypeScript.
<CodeGroup>
```python Python icon="python"
import os
import lancedb
from lancedb.remote import OAuthConfig, OAuthFlowType
@@ -84,7 +120,7 @@ from lancedb.remote import OAuthConfig, OAuthFlowType
oauth_config = OAuthConfig(
issuer_url="https://login.microsoftonline.com/{tenant}/v2.0",
client_id="your-application-client-id",
client_secret="your-application-client-secret",
client_secret=os.environ["LANCEDB_OAUTH_CLIENT_SECRET"],
scopes=["api://lancedb-api/.default"],
flow=OAuthFlowType.CLIENT_CREDENTIALS,
)
@@ -101,11 +137,14 @@ db = await lancedb.connect_async(
import * as lancedb from "@lancedb/lancedb";
import { OAuthConfig, OAuthFlowType } from "@lancedb/lancedb";
const clientSecret = process.env.LANCEDB_OAUTH_CLIENT_SECRET;
if (!clientSecret) throw new Error("LANCEDB_OAUTH_CLIENT_SECRET is required");
// Client Credentials (service-to-service)
const oauthConfig: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
clientId: "your-application-client-id",
clientSecret: "your-application-client-secret",
clientSecret,
scopes: ["api://lancedb-api/.default"],
flow: OAuthFlowType.ClientCredentials,
};
@@ -116,9 +155,34 @@ const db = await lancedb.connect("db://your-database-uri", {
oauthConfig,
});
```
```rust Rust icon="rust"
use lancedb::remote::{OAuthConfig, OAuthFlow};
// Client Credentials (service-to-service)
let oauth_config = OAuthConfig {
issuer_url: "https://login.microsoftonline.com/{tenant}/v2.0".into(),
client_id: "your-application-client-id".into(),
client_secret: Some(std::env::var("LANCEDB_OAUTH_CLIENT_SECRET")?),
scopes: vec!["api://lancedb-api/.default".into()],
flow: OAuthFlow::ClientCredentials,
client_auth_method: None,
resource: None,
audience: None,
refresh_buffer_secs: None,
token_cache: None,
};
let db = lancedb::connect("db://your-database-uri")
.region("us-east-1")
.host_override("https://your-enterprise-endpoint.com")
.oauth_config(oauth_config)
.execute()
.await?;
```
</CodeGroup>
For workloads running on Azure compute, use a managed identity instead of a
For Azure workloads with access to IMDS, use a managed identity instead of a
client secret:
<CodeGroup>
@@ -130,8 +194,8 @@ oauth_config = OAuthConfig(
client_id="your-application-client-id",
scopes=["api://lancedb-api/.default"],
flow=OAuthFlowType.AZURE_MANAGED_IDENTITY,
# Optional: required only for user-assigned managed identities.
managed_identity_client_id="your-user-assigned-identity-client-id",
# Set only for a user-assigned managed identity:
# managed_identity_client_id="your-user-assigned-identity-client-id",
)
```
@@ -143,31 +207,219 @@ const oauthConfig: OAuthConfig = {
clientId: "your-application-client-id",
scopes: ["api://lancedb-api/.default"],
flow: OAuthFlowType.AzureManagedIdentity,
// Optional: required only for user-assigned managed identities.
managedIdentityClientId: "your-user-assigned-identity-client-id",
// Set only for a user-assigned managed identity:
// managedIdentityClientId: "your-user-assigned-identity-client-id",
};
```
```rust Rust icon="rust"
use lancedb::remote::{OAuthConfig, OAuthFlow};
let oauth_config = OAuthConfig {
issuer_url: "https://login.microsoftonline.com/{tenant}/v2.0".into(),
client_id: "your-application-client-id".into(),
client_secret: None,
scopes: vec!["api://lancedb-api/.default".into()],
flow: OAuthFlow::AzureManagedIdentity {
// Set to Some("your-user-assigned-identity-client-id".into())
// only for a user-assigned managed identity.
client_id: None,
},
client_auth_method: None,
resource: None,
audience: None,
refresh_buffer_secs: None,
token_cache: None,
};
```
</CodeGroup>
`managed_identity_client_id` / `managedIdentityClientId` selects a user-assigned
identity. Omit it for a system-assigned identity. The required `client_id` /
`clientId` field does not select the Azure managed identity.
In Rust, set the `client_id` inside `OAuthFlow::AzureManagedIdentity` to
`Some(...)` for a user-assigned identity, or `None` for a system-assigned
identity. The top-level `OAuthConfig::client_id` is the application client ID.
For interactive sign-in, configure Authorization Code or Device Code. The
authorization URL or device verification instructions are written to standard
error. Authorization Code also tries to open a browser and listens for the
callback at `http://127.0.0.1:8400/callback` by default. Register that redirect
URI with your identity provider. Public clients can omit `client_secret`;
request a refresh token from your provider (often by including
`offline_access`) if you want to reuse the session.
<CodeGroup>
```python Python icon="python"
import lancedb
from lancedb.remote import OAuthConfig, OAuthFlowType, TokenCacheOptions
# Use DEVICE_CODE for a CLI without a local browser or callback server.
oauth_config = OAuthConfig(
issuer_url="https://your-issuer.example.com",
client_id="your-public-client-id",
scopes=["openid", "offline_access", "lancedb.read"],
flow=OAuthFlowType.AUTHORIZATION_CODE,
token_cache=TokenCacheOptions(),
)
db = await lancedb.connect_async(
uri="db://your-database-uri",
region="us-east-1",
host_override="https://your-enterprise-endpoint.com",
oauth_config=oauth_config,
)
```
```typescript TypeScript icon="square-js"
import * as lancedb from "@lancedb/lancedb";
import { OAuthFlowType, type OAuthConfig } from "@lancedb/lancedb";
// Use OAuthFlowType.DeviceCode for a CLI without a local browser or callback server.
const oauthConfig: OAuthConfig = {
issuerUrl: "https://your-issuer.example.com",
clientId: "your-public-client-id",
scopes: ["openid", "offline_access", "lancedb.read"],
flow: OAuthFlowType.AuthorizationCode,
tokenCache: {},
};
const db = await lancedb.connect("db://your-database-uri", {
region: "us-east-1",
hostOverride: "https://your-enterprise-endpoint.com",
oauthConfig,
});
```
```rust Rust icon="rust"
use lancedb::remote::{AuthorizationCodeOptions, OAuthConfig, OAuthFlow, TokenCacheOptions};
// Use OAuthFlow::DeviceCode for a CLI without a local browser or callback server.
let oauth_config = OAuthConfig {
issuer_url: "https://your-issuer.example.com".into(),
client_id: "your-public-client-id".into(),
client_secret: None,
scopes: vec![
"openid".into(),
"offline_access".into(),
"lancedb.read".into(),
],
flow: OAuthFlow::AuthorizationCode(AuthorizationCodeOptions::new()),
client_auth_method: None,
resource: None,
audience: None,
refresh_buffer_secs: None,
token_cache: Some(TokenCacheOptions::new()),
};
let db = lancedb::connect("db://your-database-uri")
.region("us-east-1")
.host_override("https://your-enterprise-endpoint.com")
.oauth_config(oauth_config.clone())
.execute()
.await?;
```
</CodeGroup>
With `DEVICE_CODE` / `DeviceCode` / `OAuthFlow::DeviceCode`, the same
configuration and connection calls apply; only the `flow` value changes.
### Reuse an interactive session {#reuse-an-interactive-session}
By default, tokens stay in process memory. Set `token_cache` / `tokenCache`
for Authorization Code or Device Code to keep refresh tokens across process
restarts. Only refresh tokens are persisted; access tokens and client secrets
are not written to the cache. Set `cache_dir` / `cacheDir` to choose the
directory, or leave it unset to use the platform cache directory. Your
identity provider must issue a refresh token for a later process to sign in
without prompting again.
Use `OAuthSession` with the same config to log in before connecting, inspect
whether a refresh token is cached, or remove the local cached credential:
<CodeGroup>
```python Python icon="python"
from lancedb.remote import OAuthSession
session = OAuthSession(oauth_config)
status = await session.login()
print(status.refreshable)
status = await session.status() # Does not contact the provider.
print(status.refreshable)
logout = await session.logout()
print(logout.removed)
```
```typescript TypeScript icon="square-js"
import { OAuthSession } from "@lancedb/lancedb";
const session = new OAuthSession(oauthConfig);
const status = await session.login();
console.log(status.refreshable);
const current = await session.status(); // Does not contact the provider.
console.log(current.refreshable);
const logout = await session.logout();
console.log(logout.removed);
```
```rust Rust icon="rust"
use lancedb::remote::OAuthSession;
let session = OAuthSession::new(oauth_config)?;
let status = session.login().await?;
println!("{}", status.refreshable);
let status = session.status().await?; // Does not contact the provider.
println!("{}", status.refreshable);
let logout = session.logout().await?;
println!("{}", logout.removed);
```
</CodeGroup>
`login()` runs the interactive flow even when a session is cached. `logout()`
removes the local cache entry; it does not revoke the provider's token or end
a browser sign-in. `OAuthSession` requires a persistent cache and supports
only Authorization Code and Device Code. A cache is unused for Client
Credentials and rejected for Azure Managed Identity.
### Configuration reference {#configuration-reference}
The same configuration is available in both SDKs. Python uses `snake_case`
field names; TypeScript uses `camelCase`.
The same configuration is available in all three SDKs. Python and Rust use
`snake_case` field names; TypeScript uses `camelCase`. In Rust, set optional
`OAuthConfig` fields to `Some(value)` or `None`. Configure `redirect_uri`,
`callback_port`, and `use_pkce` through the builder methods on
`AuthorizationCodeOptions`, and select a managed identity through
`OAuthFlow::AzureManagedIdentity { client_id: ... }`.
| Python field / TypeScript field | Required | Description |
| --- | --- | --- |
| `issuer_url` / `issuerUrl` | Yes | OIDC issuer URL or OAuth authority URL. For Azure, use `https://login.microsoftonline.com/{tenant_id}/v2.0`. |
| `client_id` / `clientId` | Yes | Application / client ID registered with your identity provider. |
| `scopes` | Yes | List of OAuth scopes to request. For Azure managed identity, provide exactly one scope or resource, for example `["api://your-app-id/.default"]`. |
| `flow` | No | Selects the OAuth flow. Defaults to Client Credentials. |
| `client_secret` / `clientSecret` | Conditional | Required for Client Credentials. Redacted from the config's `repr` (Python) and `Debug` output (TypeScript native binding) so it does not leak into logs. |
| `flow` | Yes in Rust; otherwise no | Selects the OAuth flow. Defaults to Client Credentials in Python and TypeScript; Rust requires an explicit `OAuthFlow`. |
| `client_secret` / `clientSecret` | Conditional | Required for Client Credentials; only required for Authorization Code and Device Code when the provider registers a confidential client. Redacted from the config's `repr` (Python) and `Debug` output (Rust and TypeScript native binding). |
| `client_auth_method` / `clientAuthMethod` | No | Token endpoint authentication: `ClientAuthMethod.NONE` / `ClientAuthMethod.None` for a public client, `ClientAuthMethod.CLIENT_SECRET_BASIC` / `ClientAuthMethod.ClientSecretBasic`, or `ClientAuthMethod.CLIENT_SECRET_POST` / `ClientAuthMethod.ClientSecretPost`. Defaults to Basic when a client secret is set and no client authentication otherwise. The secret methods require `client_secret` / `clientSecret`. Omit for Azure Managed Identity. |
| `redirect_uri` / `redirectUri` | No | Authorization Code loopback callback URI. Defaults to `http://127.0.0.1:{callback_port}/callback`; register it with the provider. |
| `callback_port` / `callbackPort` | No | Authorization Code callback server port (default 8400). If `redirect_uri` / `redirectUri` has an explicit port, the two ports must match. |
| `use_pkce` / `usePkce` | No | Enable S256 PKCE for Authorization Code (default `true`). |
| `managed_identity_client_id` / `managedIdentityClientId` | No | Client ID of a user-assigned managed identity. Only used with Azure Managed Identity; omit for system-assigned identities. |
| `resource` | No | RFC 8707 resource indicator, an absolute URI without a fragment. Sent to authorization and token endpoints, including refresh requests. Not supported for Azure Managed Identity. |
| `audience` | No | Provider-specific audience sent to authorization and token endpoints, including refresh requests. Not supported for Azure Managed Identity. |
| `refresh_buffer_secs` / `refreshBufferSecs` | No | How many seconds before token expiry to proactively refresh. Defaults to 300. Keep this well below the token TTL: setting it greater than or equal to the TTL forces a refresh on every request. |
| `token_cache` / `tokenCache` | No | Opt in to a persistent refresh-token cache for Authorization Code or Device Code. Use `TokenCacheOptions()` in Python, `{}` in TypeScript, or `Some(TokenCacheOptions::new())` in Rust. Defaults to an in-memory session. |
`TokenCacheOptions` also accepts `cache_dir` / `cacheDir` to select a cache
directory and `lock_timeout_secs` / `lockTimeoutSecs` to set the wait for a
cross-process refresh lock (default 30 seconds).
In Rust, use the `TokenCacheOptions::cache_dir(...)` and
`TokenCacheOptions::lock_timeout_secs(...)` builder methods. Token endpoint
authentication uses `Some(ClientAuthMethod::None)`,
`Some(ClientAuthMethod::ClientSecretBasic)`, or
`Some(ClientAuthMethod::ClientSecretPost)`; `None` selects the default.
<Warning>
Treat the client secret like any other production credential. Load it from a
secret manager or environment variable rather than committing it to source
control. Both SDKs deliberately omit the secret from their debug/repr output
control. All three SDKs deliberately omit the secret from their debug/repr output
so accidental log lines do not expose the value.
</Warning>
docs/web/feature-engineering/materialized-views.mdx
+1 -1
View File
@@ -52,7 +52,7 @@ In Python SDK 0.40.0, create the source with
above does. Stable row IDs let the view track source rows across updates and
compaction. Set this option when creating the table; it cannot be added later.
A source with a MemWAL write spec or retained, uncompacted write-ahead-log
A source with the [realtime engine](/tables/realtime-engine) enabled or retained, uncompacted write-ahead-log
rows is not accepted by the local materialized-view implementation.
## Refresh behavior {#incremental}
docs/web/feature-engineering/refresh-column.mdx
+48
View File
@@ -57,6 +57,54 @@ That is what makes the call idempotent and safe to retry. However, unlike a
spreadsheet formula, nothing is recomputed until you refresh, so a derived
column is stale between an update and the next refresh.
{/*
Draft for automatic computed-column refresh. Keep the released manual-refresh
instructions above until the server feature is available.
## Automatic refresh {#automatic}
Set scheduling options when declaring a computed column:
```python
table.add_columns(
computed={"doubled": "x * 2"},
auto_refresh_enabled=True,
auto_refresh_interval_seconds=300,
)
```
Both options apply to every computed output declared in that call, including
registered Function outputs. Automatic refresh is enabled by default. When
the interval is omitted, the server determines when to refresh. Local tables
persist only an explicitly configured interval; automatic execution requires a
Cloud or Enterprise server with its separate `column_refresh.enabled` switch
turned on. That server switch is disabled in the base configuration.
The interval is a positive minimum delay after the last successful refresh
of that output. A first refresh is immediately eligible. The scheduler still
checks input freshness and waits for computed ancestors to settle; the timer
does not make unchanged inputs require recomputation. Each Function output's
interval is independent, so refreshing a sibling does not postpone it.
Pacing uses retained live Job Registry history. If completed jobs are archived
before a configured interval expires, the scheduler can lose the last-success
timestamp and refresh earlier. Leave archival disabled, or retain live history
for at least the longest configured column interval. Archival is disabled in
the base configuration; its default retention threshold is seven days.
Change a stored setting through SQL:
```sql
ALTER TABLE my_table ALTER COLUMN doubled SET auto_refresh_enabled = false;
ALTER TABLE my_table ALTER COLUMN doubled SET auto_refresh_interval_seconds = 600;
ALTER TABLE my_table ALTER COLUMN doubled SET auto_refresh_enabled = true;
```
Disabling automatic refresh prevents new automatic jobs. Inflight jobs may
finish, and manual refresh remains available. Re-enabling wakes maintenance
to consider accumulated changes under the interval and dependency rules.
*/}
## Refreshing part of a table {#partial}
<Badge color="red">Enterprise</Badge> In SQL a refresh can be scoped to a
docs/web/indexing/fts-index.mdx
+6 -2
View File
@@ -99,7 +99,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True))
| `stem` | bool | `True` | Apply stemming (`running` → `run`) |
| `remove_stop_words` | bool | `True` | Drop common stop words |
| `ascii_folding` | bool | `True` | Normalize accented characters |
| `custom_stop_words` | list[str] | `None` | Extra stop words to drop in addition to the language defaults. Requires `remove_stop_words=True`. |
| `custom_stop_words` | list[str] | `None` | Explicit stop-word list that replaces the built-in list. Requires `remove_stop_words=True`. |
| `ngram_min_length` | int | `3` | Minimum n-gram length. Applies only when `base_tokenizer="ngram"`. |
| `ngram_max_length` | int | `3` | Maximum n-gram length. Applies only when `base_tokenizer="ngram"`. |
| `prefix_only` | bool | `False` | Index only prefix n-grams rather than all substrings. Applies only when `base_tokenizer="ngram"`. |
@@ -123,7 +123,11 @@ await async_table.create_index("payload.text", config=FTS(with_position=True))
Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `<model-home>/jieba/default/...`. For a table in LanceDB Enterprise, the files must be present where the deployment builds and queries the index, so ask its administrator whether they are installed.
`language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`.
`language` is used by token filters, not by the base tokenizer. Stemming and language-specific stop-word removal support Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. The ICU tokenizer uses a combined multilingual stop-word list. An explicit `custom_stop_words` list replaces the built-in list for either tokenizer.
<Warning title="Rebuild FTS indexes when upgrading to Lance 15">
Lance `15.0.0-beta.1` revises the built-in stop-word lists, including the multilingual list used by ICU. Existing index metadata records tokenizer settings, but does not freeze these built-in lists. Rebuild affected FTS indexes with the same configuration before serving queries with the updated SDK, so stored terms and query tokens use the same lists. Indexes with stop-word removal disabled or an explicit `custom_stop_words` list are unaffected by this change.
</Warning>
### Posting block size {#posting-block-size}
docs/web/maintenance/background-jobs.mdx
+3 -3
View File
@@ -27,13 +27,13 @@ also start cache-warming jobs through the admin API.
## Write-ahead log maintenance {#wal}
For a table with a [write-ahead log](/sql/ddl/table#properties) enabled, two jobs keep the
For a table with the [realtime engine](/tables/realtime-engine) enabled, two jobs keep the
log from growing without bound:
| Job type | What it does |
| --- | --- |
| `sstable_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments. The two are unrelated. |
| `wal_index_catchup` | Extends a base-table index over the rows `sstable_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. |
| `memwal_compaction` | Folds a table shard's SSTables into the base table by primary key. Distinct from `compact`, which merges base-table fragments. The two are unrelated. |
| `memwal_catchup_<type>` | Extends a base-table index over the rows `memwal_compaction` just folded in, and records how far the index has caught up. Nothing else advances that record. The suffix names the index type, for example `memwal_catchup_btree`, `memwal_catchup_fts`, or `memwal_catchup_ivfpq`. |
## Index maintenance {#index}
docs/web/sql/ddl/table.mdx
+50 -9
View File
@@ -51,6 +51,43 @@ The column is committed empty and the expression is stored, not evaluated:
declaring one on a billion-row table costs what declaring one on an empty table
costs. [`REFRESH COLUMN`](/sql/dml/refresh) fills it.
{/*
Draft for automatic computed-column refresh. Enable this section when the
server feature is released.
### Automatic refresh options {#automatic-refresh-options}
```sql
ALTER TABLE t ADD COLUMN doubled AS value * 2
WITH (auto_refresh_enabled = true, auto_refresh_interval_seconds = 300);
ALTER TABLE t ALTER COLUMN doubled SET auto_refresh_enabled = false;
ALTER TABLE t ALTER COLUMN doubled SET auto_refresh_interval_seconds = 600;
ALTER TABLE t ALTER COLUMN doubled SET auto_refresh_enabled = true;
```
Automatic refresh is enabled by default. An explicitly configured interval is
stored in each computed output's field metadata; when omitted, the server
determines when to refresh. The interval must be a positive integer
and is minimum pacing after a successful refresh of that output; unchanged
inputs do not force work. A new column is immediately eligible for its first
refresh. The server's separate automatic-refresh switch must also be enabled.
These options also work beside a Function binding's `version` and `branch`:
```sql
ALTER TABLE t ADD COLUMN (label, score) AS classify(text)
WITH (version = 'fv_1', auto_refresh_enabled = true,
auto_refresh_interval_seconds = 300);
ALTER TABLE t ALTER COLUMN score SET auto_refresh_enabled = false;
```
Declaration settings initialize every output in the group; later SET commands
change one output. Disabling automatic refresh prevents new submissions while
allowing inflight jobs to finish and explicit REFRESH COLUMN to run. Column
SUSPEND/RESUME syntax is not supported.
*/}
{/*
Draft for the Functions release.
@@ -81,12 +118,14 @@ ALTER TABLE t ADD COLUMN (label, score) AS classify(text) WITH (version = 'fv_1'
```sql
ALTER TABLE t ADD COLUMN c AS f(x) FILTER (WHERE keep) WITH (version = 'fv_1')
ALTER TABLE t ADD COLUMN c AS f(x) WITH (num_workers = '2')
REFRESH COLUMN c ON t WITH (num_workers = 2)
```
<Note>
`WITH` on a plain computed column accepts only `version`. Resource options such
as `num_workers` apply to function columns, where there are workers to size.
Declaration `WITH` accepts `auto_refresh_enabled` and
`auto_refresh_interval_seconds`. Function bindings also accept `version` and
`branch`. Worker resource options such as `num_workers` belong on
`REFRESH COLUMN ... WITH (...)`.
</Note>
*/}
@@ -102,18 +141,20 @@ duplicates impossible. It tells the planner what to assume. The column must be
declared `NOT NULL`.
```sql
ALTER TABLE t SET MEMWAL
ALTER TABLE t SET MEMWAL WITH (sharding = 'bucket', column = 'id', buckets = 4)
ALTER TABLE t SET MEMWAL WITH (maintained_indexes = '')
ALTER TABLE t UNSET MEMWAL
ALTER TABLE t SET RTE
ALTER TABLE t SET RTE WITH (sharding = 'bucket', column = 'id', buckets = 4)
ALTER TABLE t SET RTE WITH (maintained_indexes = '')
ALTER TABLE t UNSET RTE
```
A write-ahead log makes small frequent writes cheap by batching them out of the
The [realtime engine](/tables/realtime-engine) puts a write-ahead log in front
of the table, making small frequent writes cheap by batching them out of the
commit path. `sharding` and `buckets` spread the log so concurrent writers do
not contend on one place; `bucket` sharding names its `column`, which must be
the table's unenforced primary key, and inserts into the table then need a
scalar index on that key. `maintained_indexes` chooses which indexes are kept
current against WAL rows. A table with a computed column cannot take a
current against WAL rows; omitted, every supported index is kept current,
including ones created later. A table with a computed column cannot take a
write-ahead log. See
[Background jobs](/maintenance/background-jobs#wal) for how the log gets
folded back into the table.
docs/web/tables/realtime-engine.mdx
+229
View File
@@ -0,0 +1,229 @@
---
title: "Realtime engine (RTE)"
sidebarTitle: "Realtime Engine"
description: "Absorb small, frequent writes cheaply and make them queryable within milliseconds."
icon: "bolt"
keywords: ["rte", "realtime engine", "wal", "memwal", "lsm", "write-ahead log", "streaming writes", "upsert"]
---
<Badge color="red">Enterprise</Badge>
## What is the realtime engine? {#what-is-rte}
Every write to a LanceDB table is normally a commit to the underlying Lance
dataset. That is ideal for bulk loads, but a stream of small upserts becomes a
stream of small commits, and each write is only visible once its commit lands.
The realtime engine (RTE) puts an LSM-style write tier in front of the table.
It turns many small writes into a few large commits, and makes each write
**searchable within milliseconds** of being acknowledged. It is enabled per
table, and nothing about how you query that table changes.
## How it works {#how-it-works}
- **Writes.** A `merge_insert` or `delete` lands in an in-memory **memtable**
and is acknowledged once it is durable in the write-ahead log on object
storage.
- **Flush and compaction.** Memtables freeze into **SSTables**, which background
jobs compact into the base table, catching its indexes up as they go. See
[Background jobs](/maintenance/background-jobs#wal).
- **Reads.** Every query runs against the base table and the fresh tier
(memtables and SSTables) in parallel. Fresh rows supersede base rows with the
same primary key, and the two are merged into one result.
- **Indexes stay fresh.** Vector, full-text, BTree, bitmap, and most other
scalar indexes are maintained on the memtable, so new rows are searchable
through the index as soon as they are written.
## Getting started {#getting-started}
### 1. Create a table with a primary key {#primary-key}
RTE deduplicates writes by primary key. Create the table with a non-nullable
key column, then declare it with `set_unenforced_primary_key`. The key is a
single `int32`, `int64`, string, or binary column, and cannot be changed once
set.
```python
from datetime import timedelta
import lancedb
import numpy as np
import pyarrow as pa
from lancedb import RTEWriteSpec
from lancedb.index import FTS, BTree, IvfPq
db = lancedb.connect("db://my_db", api_key="...", host_override="https://...")
n, dim = 1024, 128
schema = pa.schema([
pa.field("id", pa.int64(), nullable=False),
pa.field("vector", pa.list_(pa.float32(), dim)),
pa.field("text", pa.utf8()),
])
initial_data = pa.table({
"id": pa.array(range(n), pa.int64()),
"vector": pa.FixedSizeListArray.from_arrays(
pa.array(np.random.rand(n * dim).astype("float32")), dim),
"text": pa.array([f"doc {i} alpha bravo" for i in range(n)]),
}, schema=schema)
table = db.create_table("docs", initial_data, schema=schema)
table.set_unenforced_primary_key("id")
```
Or in SQL:
```sql
ALTER TABLE docs SET UNENFORCED PRIMARY KEY (id)
```
### 2. Create indexes {#indexes}
By default, RTE maintains every supported index on the table, including
indexes created after it is enabled. A scalar index on the primary key is
required for partial updates and deletes.
```python
table.create_index("vector", config=IvfPq())
table.create_index("text", config=FTS())
table.create_index("id", config=BTree())
table.wait_for_index(["vector_idx", "text_idx", "id_idx"], timeout=timedelta(minutes=5))
```
### 3. Enable RTE {#enable}
```python
table.enable_rte(RTEWriteSpec.unsharded())
```
Or in SQL:
```sql
ALTER TABLE docs SET RTE
```
### 4. Write and query {#write-and-query}
<Warning>
On an RTE table, an upsert through `merge_insert` is the only way to write
rows. Use the form below exactly: `when_matched_update_all()` with no filter,
plus `when_not_matched_insert_all()`. Other `merge_insert` forms, such as
conditional updates or deleting rows missing from the source, are rejected.
</Warning>
Writes go through `merge_insert` on the primary key, and are visible to queries
immediately:
```python
new_rows = pa.table({
"id": pa.array([2000, 2001], pa.int64()),
"vector": pa.FixedSizeListArray.from_arrays(
pa.array(np.random.rand(2 * dim).astype("float32")), dim),
"text": pa.array(["fresh charlie delta", "fresh echo foxtrot"]),
}, schema=schema)
query_vector = np.random.rand(dim).astype("float32")
table.merge_insert("id") \
.when_matched_update_all() \
.when_not_matched_insert_all() \
.execute(new_rows)
table.search(query_vector).limit(10).to_list()
table.search("charlie").limit(10).to_list()
```
**Partial updates.** Send the primary key plus only the columns you want to
change; omitted columns keep their current values. This requires a scalar
index on the primary key.
```python
patch = pa.table(
{"id": [0, 1], "text": ["doc 0 revised", "doc 1 revised"]},
schema=pa.schema([schema.field("id"), schema.field("text")]),
)
table.merge_insert("id").when_matched_update_all().when_not_matched_insert_all().execute(patch)
```
**Deletes.** The predicate must bind the primary key to literal values. Ranges,
`OR`, and filters on other columns are rejected. Deletes require a scalar index
on the primary key.
```python
table.delete("id IN (2000, 2001)")
```
### Disabling RTE {#disable}
Stop writes and wait for in-flight writes to finish, then flush pending writes
into the base table and disable. `checkpoint_rte()` only drains writes that
existed when it started, so anything written during the checkpoint can be left
behind. `disable_rte()` is rejected while the write-ahead log still holds rows,
so run `checkpoint_rte()` first.
```python
table.checkpoint_rte()
table.disable_rte()
```
## Performance {#performance}
### Writes {#write-performance}
RTE acknowledges a write as soon as it is durable in the write-ahead log, so
ingest is no longer limited by how fast the table can commit.
**Compared with writing directly to the table.** On tables ranging from
hundreds of thousands to tens of millions of rows, RTE delivered 4–12× the
write throughput and 24–52× lower p99 write latency, depending on the mix of
inserts, updates, and deletes.
**Scaling with shards.** An RTE table can be split into
[shards](/sql/ddl/table#properties) that write independently, and a single
table's ingest grows with its shard count. In our testing on AWS, with every
index on the table (vector, full-text, and BTree) kept current, a single table
reached these peak rates:
| Shards | Peak ingest |
| --: | --: |
| 1 | 90 MB/s |
| 2 | 170 MB/s |
| 4 | 340 MB/s |
| 8 | 625 MB/s |
| 16 | 690 MB/s |
| 32 | 885 MB/s |
Adding shards scales a table's ingest nearly linearly.
### Reads {#read-performance}
Reads on an RTE table stay close to those on a fully compacted table, even
with the fresh tier at its fullest before compaction.
The table below compares a 5M-row table with a 128-dimension IVF-PQ index plus
BTree and FTS indexes, with and without RTE. The RTE table holds one
unflushed memtable and three SSTables, each from a 100k-row write wave, half of
which are updates to existing rows. Latencies are p50 in milliseconds.
| Query | Without RTE | With RTE |
| --- | --: | --: |
| Primary-key lookup | 62 | 63 |
| Vector search | 68 | 71 |
| Vector search with prefilter | 90 | 91 |
| Full-text term | 63 | 64 |
| Full-text phrase | 68 | 82 |
| Hybrid search (RRF) | 68 | 83 |
| SQL point lookup | 189 | 187 |
Most queries are within a few milliseconds of the base table. Hybrid and phrase
queries add roughly 15 ms to merge the two tiers.
<Note>
Support for all table operations on RTE tables is coming soon. Time travel,
branching, `count_rows`, row IDs (`with_row_id`), and `TABLESAMPLE` are not yet
available.
</Note>
## Next steps {#next-steps}
- [Merge insert](/ingestion/merge-insert)
- [Background jobs](/maintenance/background-jobs#wal)
- [`ALTER TABLE ... SET RTE`](/sql/ddl/table#properties)
docs/web/training/torch.mdx
+135 -28
View File
@@ -1,7 +1,7 @@
---
title: "PyTorch Integration"
sidebarTitle: "PyTorch"
description: Learn how to use LanceDB with PyTorch for training and inference.
description: Use a LanceDB table as a PyTorch IterableDataset, mapped onto the `torch.utils.data` concepts you already know.
icon: fire
availability:
oss: available
@@ -11,44 +11,151 @@ availability:
tables in the Python SDK.
---
Use LanceDB tables as input to a PyTorch `DataLoader`. Select the columns you
need, choose a batch format, and read data in parallel for training or inference.
`StreamingDataset` is a LanceDB table exposed as a PyTorch
[`IterableDataset`](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset),
so it drops into a stock [`DataLoader`](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader).
It handles per-epoch shuffling, sharding across ranks and workers, prefetching, and exact resume, so you
don't write those yourself. The shuffle is fixed when the dataset is constructed, so each epoch needs a new
`StreamingDataset` built with that epoch's number.
## Quickstart {#quickstart}
The `Table` class in LanceDB implements a contract for a PyTorch
[Dataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.Dataset).
This means you can simply use a LanceDB table in a PyTorch dataloader directly.
```py Python icon=Python
import lancedb, pyarrow as pa, torch
from torch import nn
from lancedb.streaming import StreamingDataset
```py Python icon=Python
import lancedb
import torch
import pyarrow as pa
from lancedb.util import tbl_to_tensor
def to_samples(rb: pa.RecordBatch):
f = lambda c: torch.from_dlpack(rb[c].flatten()).view(len(rb), -1)
rows = zip(f("observation_state"), f("action"))
return [{"state": s, "action": a} for s, a in rows]
mem_db = lancedb.connect("memory://")
table = mem_db.create_table("test_table", pa.table({"a": range(1000)}))
uri = "hf://datasets/lance-format/lerobot-xvla-soft-fold/data"
db = lancedb.connect(uri, storage_options={"hf_enable_resolve_cache": "true"})
table = db.open_table("frames")
train = StreamingDataset(
table, columns=["observation_state", "action"],
transform=to_samples)
# Any LanceDB table can be used as a PyTorch Dataset
dataloader = torch.utils.data.DataLoader(
table, batch_size=1024, shuffle=True, collate_fn=tbl_to_tensor
)
for batch in dataloader:
print(batch)
model = nn.Sequential(nn.Linear(96, 256), nn.ReLU(), nn.Linear(256, 14))
opt = torch.optim.AdamW(model.parameters(), lr=1e-3)
for batch in torch.utils.data.DataLoader(train, batch_size=256):
loss = nn.MSELoss()(model(batch["state"]), batch["action"])
opt.zero_grad()
loss.backward()
opt.step()
```
Although the `Table` class in LanceDB implements the `torch.utils.data.Dataset` interface, you may find that using
a table [Permutation](/training/data-loading) is more flexible.
`StreamingDataset` is a standard PyTorch `IterableDataset`, so it works with everything built for one: a stock
`DataLoader`, its default `collate_fn`, and worker processes. The transform returns plain tensors, so the
model, loss and optimizer are unmodified PyTorch. Here it streams the
[`lerobot-xvla-soft-fold`](https://huggingface.co/datasets/lance-format/lerobot-xvla-soft-fold) dataset
straight from the Hugging Face Hub to train a policy that predicts each `action` from the robot's
`observation_state`.
```py Python icon=Python
from lancedb.permutation import Permutation
## PyTorch concepts in LanceDB {#concept-map}
permutation = Permutation.identity(table)
dataloader = torch.utils.data.DataLoader(permutation)
| In PyTorch you would write | In LanceDB you use |
| --- | --- |
| An `IterableDataset` that reads files in `__iter__` | `StreamingDataset(table)` |
| `DistributedSampler`, or sharding by `get_worker_info()` in `__iter__` | `rank=` and `world_size=`. Each rank and each DataLoader worker reads its own disjoint splits |
| Decoding in `__getitem__` or `collate_fn` | `transform=`, which receives a whole Arrow `RecordBatch` and runs in a thread pool |
| `random_split`, `Subset`, or reading only the fields a model needs | `filter="<SQL predicate>"` and `columns=[...]`. Unselected columns are never read from storage |
| `StatefulDataLoader` | `state_dict()` and `load_state_dict()`, plus `StreamingDataLoader` for workers |
The dataset does not copy your data. At construction it computes a shuffled assignment of row IDs to
`num_splits` fixed splits, then reads rows from the table as it iterates. It reads the table version
that exists at construction, so rows appended later do not appear mid-epoch.
## Optimizations {#optimizations}
The quickstart works as written. These sections cover what to tune as training scales up: the transform that
builds each sample, running across many GPUs, and the output format of a `Table` or `Permutation`.
### Transform {#transform}
A `transform` turns a batch of Arrow rows into the per-sample items your `DataLoader` collates. It receives a
whole `pyarrow.RecordBatch` (`read_batch_size` rows, 64 by default) rather than one sample at a time, so
vectorized code is cheap, and it must return exactly one item per row. The quickstart's transform is the
whole pattern: columns become tensors and each row becomes a dict, which the default `collate_fn` stacks into
a batch. Without a transform, rows arrive as plain Python dicts.
Heavier per-sample work belongs in the same function, such as decoding video. Store each clip in a blob v2
column (`lancedb.blob("video")`). A scan returns small descriptors instead of the video bytes, so reading
other columns never pulls video in. `fetch_blob_files` opens the clips for a batch of rows as file handles,
and torchcodec's `VideoDecoder` decodes just the frames you ask for. `ExtractFrames` samples 8 evenly spaced
frames from each clip:
```py Python icon=Python
import functools
from torchcodec.decoders import VideoDecoder
class ExtractFrames:
def __init__(self, uri: str, name: str, num_frames: int = 8) -> None:
self.uri, self.name, self.num_frames = uri, name, num_frames
@functools.cached_property
def table(self) -> lancedb.table.Table:
return lancedb.connect(self.uri).open_table(self.name)
def __call__(self, rb: pa.RecordBatch) -> list[dict[str, torch.Tensor]]:
samples = []
for blob in self.table.fetch_blob_files("video", rb["_rowid"]):
decoder = VideoDecoder(blob)
idx = torch.linspace(0, len(decoder) - 1, self.num_frames).long()
samples.append({"frames": decoder.get_frames_at(idx.tolist()).data})
return samples
# `clips` is the table "clips" in the database at `uri`, with a blob v2 column "video"
train = StreamingDataset(clips, columns=["_rowid"], transform=ExtractFrames(uri, "clips"))
```
## Output Formats {#output-formats}
Transforms run in a thread pool, one thread per CPU by default (`transform_parallelism`), overlapping with
storage reads and your training step. NumPy, PyArrow and PyTorch release the GIL, so vectorized
work runs in parallel. A pure-Python loop does not. If a profile shows the transform is the bottleneck, move
it into [DataLoader workers](#using-multiple-dataloader-workers).
Real data has bad rows, such as a truncated frame. Raise inside the transform and set
`on_transform_error="skip"` (or `"warn"` to log each failure). LanceDB re-runs the failing batch one row at a
time, drops only the rows that raise, and counts them in `dataset.rows_skipped`. When the bad rows match a SQL
predicate, prefer `filter=`: it runs before the splits are built, so every split keeps the same length.
### Distributed Training {#distributed-training}
`StreamingDataset` takes the place of `DistributedSampler`. Pass the rank and world size, and each rank reads
its own disjoint part of the table, together covering it once per epoch:
```py Python icon=Python
import torch.distributed as dist
dist.init_process_group("nccl")
train = StreamingDataset(
table, columns=["observation_state", "action"], transform=to_samples,
num_splits=48, shuffle_seed=42, epoch=epoch,
rank=dist.get_rank(), world_size=dist.get_world_size())
```
Everything after the dataset is standard distributed PyTorch. Wrap the model in `DistributedDataParallel`
and pass `batch_size` to the `DataLoader` per GPU, so the global batch is `batch_size * world_size`. Nothing
in the data path changes.
Three arguments matter in distributed runs:
- **`num_splits`** is how many fixed streams the table is divided into. It must be divisible by `world_size`,
and by `world_size * num_workers` when you use DataLoader workers. Choose a number with many divisors, such
as 48, and runs on 8, 4 or 2 GPUs see the same data at every step, provided the global batch size is a multiple of
`num_splits`.
- **`shuffle_seed` and `epoch`** must be identical on every rank. Pass a fixed `shuffle_seed` and increase
`epoch` each epoch. `shuffle_seed=None` draws a different random seed in each process, so ranks would read
overlapping rows and miss others.
- **`rank` and `world_size`** come from your launcher, for example `torchrun`.
To resume after a failure, save `train.state_dict()` next to the model. It records progress per split, so the
resumed run continues from the same row even on a different number of GPUs. See
[Parallelism & Elasticity](/training/data-loading#parallelism-&-elasticity) and
[Checkpointing](/training/data-loading#checkpointing) for the batch-size rules and the multi-worker case.
### Output Formats {#output-formats}
By default, a `Table` data loader will emit Arrow data. `collate_fn` is PyTorch's batching hook: PyTorch calls it to
turn the fetched items into one batch. PyTorch's default collate function only knows how to combine tensors, NumPy
@@ -68,7 +175,7 @@ the Arrow data in different ways. The `arrow` and `polars` formats will always
`pandas`, and `torch_col` formats will also avoid data copies in most cases. The `python`, `python_col`, and
`torch` formats will all require at least one full copy of the data and are the slowest options.
### Using the torch column format with a torch data loader {#using-the-torch-column-format-with-a-torch-data-loader}
#### Using the torch column format with a torch data loader {#using-the-torch-column-format-with-a-torch-data-loader}
The `torch_col` format is the most efficient way to convert from Arrow to a `torch.Tensor`. It will convert the
entire Arrow batch to a _column-major_ `torch.Tensor`. In other words, given C columns and R rows, the resulting
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.10</version>
<version>0.41.0-beta.13</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.41.0-beta.10</version>
<version>0.41.0-beta.13</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>14.0.0-beta.21</lance-core.version>
<lance-core.version>15.0.0-beta.1</lance-core.version>
<jackson.version>2.18.11</jackson.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
nodejs/.npmignore
+1
View File
@@ -15,6 +15,7 @@ renovate.json
src
lancedb
examples
scripts
nodejs-artifacts
Cargo.toml
biome.json
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.41.0-beta.10"
version = "0.41.0-beta.13"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/native_symbols.test.ts
+47
View File
@@ -0,0 +1,47 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
const {
checkNativeBinary,
findForbiddenUndefinedSymbols,
} = require("../scripts/check-native-symbols.js");
test("detects the unresolved AVX-512 symbol from broken musl binaries", () => {
const output = [
" U napi_create_function",
" U sum_4bit_dist_table_32bytes_batch_avx512",
" U strlen",
].join("\n");
expect(findForbiddenUndefinedSymbols(output)).toEqual([
"sum_4bit_dist_table_32bytes_batch_avx512",
]);
});
test("accepts native binaries without unresolved internal Lance symbols", () => {
const runNm = jest.fn(() => ({
status: 0,
stdout:
" U napi_create_function\n U strlen\n",
stderr: "",
}));
expect(() => checkNativeBinary("lancedb.node", runNm)).not.toThrow();
expect(runNm).toHaveBeenCalledWith(
"nm",
["-D", "--undefined-only", "lancedb.node"],
{ encoding: "utf8" },
);
});
test("rejects native binaries with the unresolved AVX-512 symbol", () => {
const runNm = jest.fn(() => ({
status: 0,
stdout: " U sum_4bit_dist_table_32bytes_batch_avx512\n",
stderr: "",
}));
expect(() => checkNativeBinary("lancedb.node", runNm)).toThrow(
"lancedb.node contains unresolved internal Lance symbols: sum_4bit_dist_table_32bytes_batch_avx512",
);
});
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+3 -2
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.41.0-beta.10",
"version": "0.41.0-beta.13",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -90,7 +90,8 @@
"postdocs": "node typedoc_post_process.js",
"lint": "biome check . && biome format .",
"lint-fix": "biome check --write . && biome format --write .",
"prepublishOnly": "napi prepublish -t npm",
"check:native-symbols": "node scripts/check-native-symbols.js",
"prepublishOnly": "pnpm check:native-symbols && napi prepublish -t npm",
"test": "jest --verbose",
"integration": "S3_TEST=1 pnpm test",
"universal": "napi universalize",
nodejs/scripts/check-native-symbols.js
+77
View File
@@ -0,0 +1,77 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
const { spawnSync } = require("node:child_process");
const { existsSync } = require("node:fs");
const path = require("node:path");
const FORBIDDEN_UNDEFINED_SYMBOLS = new Set([
"sum_4bit_dist_table_32bytes_batch_avx512",
]);
function findForbiddenUndefinedSymbols(output) {
const found = new Set();
for (const line of output.split(/\r?\n/)) {
const columns = line.trim().split(/\s+/);
const symbol = columns.at(-1)?.split("@")[0];
if (symbol && FORBIDDEN_UNDEFINED_SYMBOLS.has(symbol)) {
found.add(symbol);
}
}
return [...found].sort();
}
function checkNativeBinary(binaryPath, runNm = spawnSync) {
const result = runNm("nm", ["-D", "--undefined-only", binaryPath], {
encoding: "utf8",
});
if (result.error) {
throw new Error(`Unable to inspect ${binaryPath}: ${result.error.message}`);
}
if (result.status !== 0) {
throw new Error(
`Unable to inspect ${binaryPath}: nm exited with status ${result.status}\n${result.stderr}`,
);
}
const forbidden = findForbiddenUndefinedSymbols(result.stdout);
if (forbidden.length > 0) {
throw new Error(
`${binaryPath} contains unresolved internal Lance symbols: ${forbidden.join(
", ",
)}`,
);
}
}
function main() {
const binaryPath = path.resolve(
__dirname,
"..",
"npm",
"linux-x64-musl",
"lancedb.linux-x64-musl.node",
);
if (!existsSync(binaryPath)) {
throw new Error(
`Missing ${binaryPath}; assemble the native artifacts before publishing`,
);
}
checkNativeBinary(binaryPath);
}
if (require.main === module) {
try {
main();
} catch (error) {
console.error(error instanceof Error ? error.message : error);
process.exitCode = 1;
}
}
module.exports = { checkNativeBinary, findForbiddenUndefinedSymbols };
nodejs/src/connection.rs
+31 -1
View File
@@ -673,7 +673,7 @@ impl Connection {
let req = ListNamespacesRequest {
id: namespace_path,
page_token,
limit: limit.map(|l| l as i32),
limit: checked_limit(limit)?,
..Default::default()
};
let resp = self
@@ -787,3 +787,33 @@ impl Connection {
.default_error()
}
}
fn checked_limit(limit: Option<u32>) -> napi::Result<Option<i32>> {
limit
.map(i32::try_from)
.transpose()
.map_err(|_| napi::Error::from_reason("limit must not exceed 2147483647"))
}
#[cfg(test)]
mod tests {
use super::checked_limit;
#[test]
fn checked_limit_rejects_values_above_i32_max() {
// u32 values above i32::MAX wrapped negative through `as i32`,
// sending a negative page size to the server.
assert!(checked_limit(Some(3_000_000_000)).is_err());
assert!(checked_limit(Some(u32::MAX)).is_err());
}
#[test]
fn checked_limit_passes_boundary_and_missing_values() {
assert_eq!(
checked_limit(Some(i32::MAX as u32)).unwrap(),
Some(i32::MAX)
);
assert_eq!(checked_limit(Some(0)).unwrap(), Some(0));
assert_eq!(checked_limit(None).unwrap(), None);
}
}
python/Cargo.toml
+3 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.10"
version = "0.41.0-beta.13"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
@@ -22,6 +22,8 @@ bytes.workspace = true
lancedb.workspace = true
datafusion-common.workspace = true
lance-core.workspace = true
# The workspace path dependency includes the patched crate in Maturin sources.
lance-arrow.workspace = true
lance-namespace.workspace = true
lance-namespace-impls.workspace = true
lance-io.workspace = true
python/python/lancedb/_lancedb.pyi
+25 -5
View File
@@ -270,6 +270,8 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Table: ...
async def create_materialized_view_async(
self,
@@ -279,6 +281,8 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job: ...
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
@@ -501,10 +505,17 @@ class Table:
async def delete(self, filter: Union[str, PyExpr]) -> DeleteResult: ...
async def add_columns(self, columns: list[tuple[str, str]]) -> AddColumnsResult: ...
async def add_computed_columns(
self, columns: list[tuple[str, str]]
self,
columns: list[tuple[str, str]],
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[int] = None,
) -> AddColumnsResult: ...
async def add_function_columns(
self, application_json: str, output_name: Optional[str]
self,
application_json: str,
output_name: Optional[str],
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[int] = None,
) -> AddColumnsResult: ...
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
async def refresh_column_async(self, column: str) -> Job: ...
@@ -851,9 +862,9 @@ class RTEWriteSpec:
@staticmethod
def unsharded() -> "RTEWriteSpec": ...
def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "RTEWriteSpec":
"""Set which indexes the RTE keeps up to date. None resolves every
index on the table at install, failing if one cannot be maintained;
a list is verbatim, empty means none."""
"""Set which indexes the RTE keeps up to date. None is every index on
the table, now and later, skipping any it cannot maintain; a list is
verbatim, empty means none."""
...
def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "RTEWriteSpec":
"""Return a copy of this spec recording the given default
@@ -941,6 +952,15 @@ class AsyncPermutationBuilder:
counts: Optional[List[int]] = None,
fixed: Optional[int] = None,
) -> "AsyncPermutationBuilder": ...
def bootstrap_sample(
self,
num_samples: int,
*,
ratios: Optional[List[float]] = None,
counts: Optional[List[int]] = None,
fixed: Optional[int] = None,
seed: Optional[int] = None,
) -> "AsyncPermutationBuilder": ...
def split_calculated(self, calculation: str) -> "AsyncPermutationBuilder": ...
def shuffle(
self, seed: Optional[int], clump_size: Optional[int]
python/python/lancedb/db.py
+38 -2
View File
@@ -5,6 +5,7 @@
from __future__ import annotations
from abc import abstractmethod
import asyncio
from datetime import timedelta
from pathlib import Path
import sys
@@ -607,6 +608,8 @@ class DBConnection(EnforceOverrides):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
@@ -631,6 +634,13 @@ class DBConnection(EnforceOverrides):
Cap the view at this many rows, in materialization order.
with_no_data: bool, default False
Skip the initial refresh and leave the backing table empty.
auto_refresh_enabled: bool, optional
Enable or disable automatic refresh on Cloud and Enterprise.
When omitted, the server determines whether automatic refresh is enabled.
Embedded databases do not support automatic refresh.
auto_refresh_interval_seconds: float, optional
Positive, finite refresh interval in seconds on Cloud and Enterprise.
When omitted, the server determines when to refresh.
Returns
-------
@@ -649,6 +659,8 @@ class DBConnection(EnforceOverrides):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
"""Submit materialized-view creation and return its job.
@@ -1706,6 +1718,8 @@ class LanceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
@@ -1734,6 +1748,8 @@ class LanceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return MaterializedView(self.open_table(name))
@@ -1748,6 +1764,8 @@ class LanceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
job = LOOP.run(
self._conn.create_materialized_view_async(
@@ -1757,6 +1775,8 @@ class LanceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return Job(job)
@@ -2633,8 +2653,16 @@ class AsyncConnection(object):
if fill_value is None:
fill_value = 0.0
data, schema = sanitize_create_table(
data, schema, metadata, on_bad_vectors, fill_value
# Input preparation may advance a user-provided iterator. Keep that work
# off the background event loop so an iterator can use the synchronous
# LanceDB API without blocking the loop that API needs to make progress.
data, schema = await asyncio.to_thread(
sanitize_create_table,
data,
schema,
metadata,
on_bad_vectors,
fill_value,
)
validate_schema(schema)
@@ -2767,6 +2795,8 @@ class AsyncConnection(object):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncMaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
@@ -2782,6 +2812,8 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
**native_options,
)
return AsyncMaterializedView(AsyncTable(inner))
@@ -2795,6 +2827,8 @@ class AsyncConnection(object):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncJob[None]:
"""Submit materialized-view creation and return its job.
@@ -2810,6 +2844,8 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
**native_options,
)
return AsyncJob(inner)
python/python/lancedb/embeddings/openai.py
+18 -7
View File
@@ -101,24 +101,35 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
valid_indices.append(idx)
# TODO retry, rate limit, token limit
try:
def embed_batch(texts_batch, indices_batch):
kwargs = {
"input": valid_texts,
"input": texts_batch,
"model": self.name,
}
if self.name != "text-embedding-ada-002":
kwargs["dimensions"] = self.dim
rs = self._openai_client.embeddings.create(**kwargs)
valid_embeddings = {
idx: v.embedding for v, idx in zip(rs.data, valid_indices)
}
return {idx: v.embedding for v, idx in zip(rs.data, indices_batch)}
def embed_one_or_log(idx, text):
# Retry a single rejected input
try:
return embed_batch([text], [idx])
except openai.BadRequestError:
logging.exception("Bad request for input at index %d: %r", idx, text)
return {}
try:
valid_embeddings = embed_batch(valid_texts, valid_indices)
except openai.AuthenticationError:
logging.error("Authentication failed: Invalid API key provided")
raise
except openai.BadRequestError:
logging.exception("Bad request: %s", texts)
return [None] * len(texts)
# Retry each input individually
valid_embeddings = {}
for idx, text in zip(valid_indices, valid_texts):
valid_embeddings.update(embed_one_or_log(idx, text))
except Exception:
logging.exception("OpenAI embeddings error")
raise
python/python/lancedb/namespace.py
+16
View File
@@ -665,6 +665,8 @@ class LanceNamespaceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> "MaterializedView":
"""Define a materialized view over a table in the root namespace.
See
@@ -680,6 +682,8 @@ class LanceNamespaceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
).name
)
@@ -695,6 +699,8 @@ class LanceNamespaceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
job = LOOP.run(
self._inner.create_materialized_view_async(
@@ -704,6 +710,8 @@ class LanceNamespaceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return Job(job)
@@ -1288,6 +1296,8 @@ class AsyncLanceNamespaceDBConnection:
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> "AsyncMaterializedView":
"""Define a materialized view over a table in the root namespace."""
view = await self._inner.create_materialized_view(
@@ -1297,6 +1307,8 @@ class AsyncLanceNamespaceDBConnection:
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
# Reopen through the namespace so the view's table carries the
# namespace client and pushdown configuration a bare inner table lacks.
@@ -1311,6 +1323,8 @@ class AsyncLanceNamespaceDBConnection:
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncJob[None]:
"""Submit materialized-view creation and return its job."""
return await self._inner.create_materialized_view_async(
@@ -1320,6 +1334,8 @@ class AsyncLanceNamespaceDBConnection:
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
python/python/lancedb/permutation.py
+44
View File
@@ -169,6 +169,50 @@ class PermutationBuilder:
)
return self
def bootstrap_sample(
self,
num_samples: int,
*,
ratios: Optional[list[float]] = None,
counts: Optional[list[int]] = None,
fixed: Optional[int] = None,
seed: Optional[int] = None,
split_names: Optional[list[str]] = None,
) -> "PermutationBuilder":
"""
Sample rows uniformly **with replacement**, independently per split.
Unlike the other split strategies, the number of output rows is
``num_samples``, not the number of (optionally filtered) input rows: the
same row can appear zero, one, or many times, and ``num_samples`` may be
larger or smaller than the input row count. Useful for bootstrap
resampling, oversampling rare classes, or any training workload that
wants a fixed-size stream of random draws rather than a single
deterministic pass over every row. Combine with :meth:`filter` to
restrict the eligible population.
One of ratios, counts, or fixed must be provided, with the same meaning
as in :meth:`split_random`, except they divide ``num_samples`` across
splits rather than the input row count: if fixed is 4, each split gets
``num_samples // 4`` samples.
Each split draws its own sample using a seed derived independently from
``(seed, split_index)``, so a split's content does not change if the samples are
later redistributed across a different number of splits.
The optional split_names can be provided to name the splits. If not provided,
the splits can only be referenced by their index.
"""
self._async.bootstrap_sample(
num_samples,
ratios=ratios,
counts=counts,
fixed=fixed,
seed=seed,
split_names=split_names,
)
return self
def split_calculated(
self, calculation: str, split_names: Optional[list[str]] = None
) -> "PermutationBuilder":
python/python/lancedb/query.py
+7 -4
View File
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
T = TypeVar("T", bound="LanceModel")
AnalyzePlanDistributedMetrics = Literal["aggregate", "per_worker", "full"]
_SYNC_VECTOR_QUERY_TYPES = (list, tuple, np.ndarray, pa.Array, pa.ChunkedArray)
# Number of rows a hybrid query returns when no limit was set on it. This
# mirrors the default the Rust query builder applies to its sub-queries.
@@ -1024,10 +1025,12 @@ class LanceQueryBuilder(ABC):
fast_search=fast_search,
)
if isinstance(query, list):
if isinstance(query, (list, tuple)):
query = np.array(query, dtype=np.float32)
elif isinstance(query, np.ndarray):
query = query.astype(np.float32)
elif isinstance(query, (pa.Array, pa.ChunkedArray)):
query = np.array(query.to_pylist(), dtype=np.float32)
else:
raise TypeError(f"Unsupported query type: {type(query)}")
@@ -1049,7 +1052,7 @@ class LanceQueryBuilder(ABC):
query = cls._query_to_vector(table, query, vector_column_name)
return query, query_type
elif query_type == "auto":
if isinstance(query, (list, np.ndarray)):
if isinstance(query, _SYNC_VECTOR_QUERY_TYPES):
return query, "vector"
else:
conf = table.embedding_functions.get(vector_column_name)
@@ -1065,7 +1068,7 @@ class LanceQueryBuilder(ABC):
@classmethod
def _query_to_vector(cls, table, query, vector_column_name):
if isinstance(query, (list, np.ndarray)):
if isinstance(query, _SYNC_VECTOR_QUERY_TYPES):
return query
conf = table.embedding_functions.get(vector_column_name)
if conf is not None:
@@ -2438,7 +2441,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
if query is not None and (vector is not None or text is not None):
raise ValueError(
"You can either provide a string query in search() method "
"or set `vector()` and `text()` explicitly for hybrid search."
"or set `vector()` and `text()` explicitly for hybrid search. "
"But not both."
)
python/python/lancedb/remote/db.py
+8
View File
@@ -723,6 +723,8 @@ class RemoteDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
from .table import RemoteTable
@@ -734,6 +736,8 @@ class RemoteDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return MaterializedView(
@@ -755,6 +759,8 @@ class RemoteDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
job = LOOP.run(
self._conn.create_materialized_view_async(
@@ -764,6 +770,8 @@ class RemoteDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return Job(job)
python/python/lancedb/remote/header.py
+1 -1
View File
@@ -141,7 +141,7 @@ class OAuthProvider(HeaderProvider):
# Set expiration if provided
expires_in = token_data.get("expires_in")
if expires_in:
if expires_in is not None:
self._token_expires_at = time.time() + expires_in
else:
# Token doesn't expire or expiration unknown
python/python/lancedb/remote/table.py
+10 -1
View File
@@ -1003,8 +1003,17 @@ class RemoteTable(Table):
| None = None,
*,
computed: Dict[str, str] | None = None,
auto_refresh_enabled: bool | None = None,
auto_refresh_interval_seconds: int | None = None,
) -> AddColumnsResult:
return LOOP.run(self._table.add_columns(transforms, computed=computed))
return LOOP.run(
self._table.add_columns(
transforms,
computed=computed,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
def refresh_column(self, column: str):
return LOOP.run(self._table.refresh_column(column))
python/python/lancedb/rerankers/base.py
+4 -1
View File
@@ -241,6 +241,7 @@ class Reranker(ABC):
This op is slower than just keeping relevance score but can be useful
for debugging.
"""
combined_schema = pa.unify_schemas([vector_results.schema, fts_results.schema])
# add nulls to fts results for _distance
if "_distance" not in fts_results.column_names:
fts_results = fts_results.append_column(
@@ -265,7 +266,9 @@ class Reranker(ABC):
else:
vector_results_dict[key] = value
combined = pa.Table.from_pylist(list(vector_results_dict.values()))
combined = pa.Table.from_pylist(
list(vector_results_dict.values()), schema=combined_schema
)
return combined
def _keep_relevance_score(self, combined_results: pa.Table):
python/python/lancedb/rerankers/voyageai.py
+1 -1
View File
@@ -32,7 +32,7 @@ class VoyageAIReranker(Reranker):
return_score : str, default "relevance"
options are "relevance" or "all". Only "relevance" is supported for now.
api_key : str, default None
The API key to use. If None, will use the OPENAI_API_KEY environment variable.
The API key to use. If None, will use the VOYAGE_API_KEY environment variable.
truncation : Optional[bool], default None
"""
python/python/lancedb/streaming.py
+147 -3
View File
@@ -19,6 +19,7 @@ above.
"""
import ctypes
import hashlib
import heapq
import logging
import os
@@ -39,6 +40,7 @@ from torch.utils.data import DataLoader, IterableDataset, get_worker_info
from .permutation import (
Permutation,
PermutationBuilder,
Transforms,
permutation_builder,
_drop_base_version,
@@ -244,6 +246,20 @@ class StreamingDataset(IterableDataset):
default) rows are shuffled using ``shuffle_seed`` and ``epoch``.
When ``False`` rows are divided into splits sequentially in storage
order, which can be useful for deterministic debugging or evaluation.
Ignored when ``split_strategy`` is provided.
split_strategy:
Escape hatch for any ``PermutationBuilder`` split strategy this class
doesn't otherwise expose a dedicated parameter for -- ``split_hash``,
``split_calculated``, ``bootstrap_sample`` (uniform sampling with
replacement), or any strategy added later. A callable that receives a
fresh ``PermutationBuilder`` for ``table`` and returns a configured one;
it is called and ``.execute()``'d directly, taking over permutation
construction entirely. When provided, ``shuffle``, ``shuffle_seed``,
``epoch``, and ``shuffle_clump_size`` are ignored -- the callable is
solely responsible for how rows are split and ordered. For example,
``split_strategy=lambda b: b.bootstrap_sample(10_000, fixed=num_splits,
seed=42)`` draws 10,000 samples with replacement, divided evenly across
``num_splits`` splits.
shuffle_seed:
Base seed for the random permutation. Combined with ``epoch`` so
each epoch produces a different ordering. Pass ``None`` to generate
@@ -393,6 +409,9 @@ class StreamingDataset(IterableDataset):
*,
num_splits: Optional[int] = None,
shuffle: bool = True,
split_strategy: Optional[
Callable[[PermutationBuilder], PermutationBuilder]
] = None,
shuffle_seed: Optional[int] = 0,
epoch: int = 0,
rank: int = 0,
@@ -583,13 +602,70 @@ class StreamingDataset(IterableDataset):
builder = permutation_builder(table)
if filter is not None:
builder = builder.filter(filter)
if shuffle:
if split_strategy is not None:
# Escape hatch: lets any PermutationBuilder split strategy drive this
# dataset (split_hash, split_calculated, bootstrap_sample, or any future
# strategy) without StreamingDataset needing a bespoke parameter for each
# one. shuffle/shuffle_seed/epoch/shuffle_clump_size are ignored in this
# path -- the caller's split_strategy is solely responsible for ordering.
self._perm_table = split_strategy(builder).execute()
# _rank_splits below assumes split ids are exactly 0..num_splits-1 with no
# gaps (every other path guarantees this by always building with
# fixed=num_splits) -- a split_strategy callable has no such guarantee, and
# a mismatch would otherwise silently drop rows (ids >= num_splits) or leave
# some ranks reading an empty split (missing ids < num_splits).
split_value_counts = (
self._perm_table.to_arrow().column("split_id").value_counts()
)
counts_by_split = dict(
zip(
split_value_counts.field("values").to_pylist(),
split_value_counts.field("counts").to_pylist(),
)
)
expected_splits = set(range(num_splits))
if set(counts_by_split) != expected_splits:
raise ValueError(
f"split_strategy produced split ids {sorted(counts_by_split)}, "
f"but num_splits={num_splits} requires exactly "
f"{{0, ..., {num_splits - 1}}}. Pass num_splits matching the "
"strategy's own split count (e.g. `fixed=num_splits` on "
"bootstrap_sample/split_random/etc.)."
)
# The round-robin iteration below (and in _iter_owned) assumes every
# split has the same row count -- it stops as soon as any one owned
# split is exhausted (see its own comment: "all splits exhaust
# simultaneously"). Every other path guarantees this by always building
# with fixed=num_splits; a split_strategy using ratios/counts/split_hash
# weights has no such guarantee, and unequal sizes would otherwise
# silently drop rows from the larger splits -- or, with world_size > 1
# where each rank owns different splits, end ranks after different row
# counts and desync or hang a distributed run's end-of-epoch collectives.
if len(set(counts_by_split.values())) > 1:
raise ValueError(
f"split_strategy produced unequal split sizes "
f"{counts_by_split} (split id -> row count). Every split must "
"have the same row count. Use `fixed=num_splits` rather than "
"`ratios`/`counts`/weighted `split_hash`, which can produce "
"unequal sizes, or split unevenly-sized groups into separate "
"StreamingDatasets instead."
)
elif shuffle:
perm_seed = shuffle_seed + epoch * _EPOCH_PRIME
self._perm_table = builder.split_random(
fixed=num_splits, seed=perm_seed, clump_size=shuffle_clump_size
).execute()
else:
self._perm_table = builder.split_sequential(fixed=num_splits).execute()
# Only the split_strategy path needs a permutation fingerprint in the
# checkpoint (see `_checkpoint_snapshot`): split_random/split_sequential are
# already fully pinned by shuffle_seed/num_splits, and a fingerprint over raw
# row ids would spuriously reject an otherwise-valid resume after those row
# ids get renumbered by compaction (row ids are not stable by default).
self._uses_split_strategy = split_strategy is not None
# Lazily computed and cached by `_perm_fingerprint` -- `_perm_table` never
# changes after construction, so this is correct to compute once.
self._perm_fingerprint_cache: Optional[str] = None
if self._blocks_per_epoch == "auto":
self._blocks_per_epoch = self._estimate_blocks_per_epoch()
@@ -1406,11 +1482,51 @@ class StreamingDataset(IterableDataset):
)
return state
@property
def _perm_fingerprint(self) -> str:
"""Cheap identity check for the built permutation, computed once and cached
(``_perm_table`` never changes after construction).
Only meaningful -- and only recorded in a checkpoint -- for the
``split_strategy`` path (see ``_checkpoint_snapshot``): ``num_splits``/
``shuffle_seed`` alone cannot detect a `load_state_dict` resuming saved
per-split progress against a *different* permutation than the one that
produced it, since a ``split_strategy`` callable ignores ``shuffle_seed``
entirely -- e.g. a ``bootstrap_sample`` with no seed (or any changed
callable) can rebuild a completely different permutation on the next run
while ``shuffle_seed`` keeps its same default value and the existing check
passes anyway. ``split_random``/``split_sequential`` are already fully
pinned by ``shuffle_seed``/``num_splits`` and must *not* additionally be
fingerprinted: this hashes physical row ids, which are not stable across
compaction by default, so checking it on those paths would reject an
otherwise-valid resume after the table gets compacted between save and
load even though the seeded permutation itself is unchanged.
"""
if self._perm_fingerprint_cache is None:
arrow_tbl = self._perm_table.to_arrow()
hasher = hashlib.sha256()
hasher.update(str(arrow_tbl.num_rows).encode())
for name in ("row_id", "split_id"):
column = arrow_tbl.column(name).combine_chunks()
hasher.update(column.to_numpy(zero_copy_only=False).tobytes())
self._perm_fingerprint_cache = hasher.hexdigest()
return self._perm_fingerprint_cache
def _checkpoint_snapshot(self) -> dict:
# Only recorded for split_strategy: split_random/split_sequential are
# already fully pinned by shuffle_seed/num_splits below, and row ids are not
# stable across compaction by default, so fingerprinting them unconditionally
# would reject an otherwise-valid resume after the table got compacted.
perm_fingerprint = (
{"perm_fingerprint": self._perm_fingerprint}
if (self._uses_split_strategy)
else {}
)
if self._pack_sequences is not None:
return {
"shuffle_seed": self._shuffle_seed,
"num_splits": self._num_splits,
**perm_fingerprint,
"epoch": self._epoch,
"pack_sequences": self._pack_sequences,
"eos_id": self._eos_id,
@@ -1431,6 +1547,7 @@ class StreamingDataset(IterableDataset):
return {
"shuffle_seed": self._shuffle_seed,
"num_splits": self._num_splits,
**perm_fingerprint,
"epoch": self._epoch,
"samples_consumed_per_split": samples,
"positions_consumed_per_split": positions,
@@ -1560,8 +1677,16 @@ class StreamingDataset(IterableDataset):
Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ
from the checkpoint, since a different split structure or shuffle order
makes mid-epoch resumption meaningless. Packed checkpoints
pin ``pack_sequences``, ``eos_id``, ``pad_id``,
makes mid-epoch resumption meaningless. Also raises if the checkpoint
was built with ``split_strategy`` and records a permutation fingerprint
that does not match the current dataset's permutation -- this catches a
changed or non-deterministically-reseeded ``split_strategy`` that
``num_splits``/``shuffle_seed`` alone cannot detect, since ``shuffle_seed``
is unused on that path. Checkpoints from ``split_random``/
``split_sequential`` never carry this fingerprint (row ids are not stable
across compaction, so it would reject an otherwise-valid resume), and
continue to rely on ``shuffle_seed``/``num_splits`` alone as before.
Packed checkpoints pin ``pack_sequences``, ``eos_id``, ``pad_id``,
``blocks_per_epoch``, and ``epoch``.
"""
if state["num_splits"] != self._num_splits:
@@ -1574,6 +1699,19 @@ class StreamingDataset(IterableDataset):
f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, "
f"current dataset has {self._shuffle_seed}"
)
checkpoint_fingerprint = state.get("perm_fingerprint")
if (
checkpoint_fingerprint is not None
and checkpoint_fingerprint != self._perm_fingerprint
):
raise ValueError(
"permutation mismatch: this checkpoint was saved against a "
"different permutation (e.g. a split_strategy that resamples "
"differently between runs, such as bootstrap_sample with no "
"seed, or a changed callable) than the current dataset built. "
"Resuming would apply saved per-split positions to the wrong "
"rows."
)
self._consumer_checkpoint_requires_uniform = False
if "pack_buffers" in state or self._pack_sequences is not None:
@@ -1675,6 +1813,10 @@ class StreamingDataset(IterableDataset):
first = states[0]
packed = "pack_buffers" in first
config_keys = ["shuffle_seed", "num_splits", "epoch"]
if "perm_fingerprint" in first:
# Absent only in checkpoints saved before this key existed; skip rather
# than KeyError so old checkpoints keep merging.
config_keys.append("perm_fingerprint")
if packed:
config_keys.extend(
["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"]
@@ -1684,6 +1826,8 @@ class StreamingDataset(IterableDataset):
if ("pack_buffers" in state) != packed:
raise ValueError("cannot merge packed and unpacked state dicts")
for key in config_keys:
if key not in state:
continue
if state[key] != first[key]:
raise ValueError(
f"{key} mismatch across state dicts: "
python/python/lancedb/table.py
+135 -18
View File
@@ -120,6 +120,33 @@ def _should_push_down_query_table(
return namespace_client is not None and "QueryTable" in pushdown_operations
_CUDA_UNAVAILABLE_MESSAGE = (
"CUDA acceleration is unavailable in this Python environment. Install a "
"CUDA-enabled PyTorch build and verify that the CUDA device is accessible, or "
"omit `accelerator` (or set `accelerator=None`) to create the index on CPU."
)
def _validate_index_accelerator(accelerator: Any) -> None:
if not isinstance(accelerator, str) or accelerator.split(":", 1)[0] != "cuda":
return
try:
import torch
cuda_available = torch.cuda.is_available()
except (ImportError, RuntimeError, AssertionError) as err:
raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
if not cuda_available:
raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE)
try:
torch.cuda.init()
except (RuntimeError, AssertionError) as err:
raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
"""Return a Polars frame unchanged while blocking predicate pushdown."""
return frame
@@ -2653,6 +2680,8 @@ class Table(ABC):
| None = None,
*,
computed: Dict[str, str] | None = None,
auto_refresh_enabled: bool | None = None,
auto_refresh_interval_seconds: int | None = None,
):
"""
Add new columns with defined values.
@@ -2697,6 +2726,15 @@ class Table(ABC):
[`refresh_column_async`][lancedb.table.Table.refresh_column_async].
Cannot be combined with ``transforms``.
auto_refresh_enabled: bool, optional
Enable automatic refresh for each declared computed output. Defaults
to True. Local tables store this option; scheduling requires a server.
auto_refresh_interval_seconds: int, optional
Positive minimum interval after a successful refresh, in seconds.
When omitted, the server determines when to refresh. Applies to every
computed output in this call.
Scheduling options require computed or Function columns.
Returns
-------
AddColumnsResult
@@ -3681,7 +3719,9 @@ class LanceTable(Table):
# Handle accelerator through pylance
if accelerator is not None:
self.to_lance().create_index(
dataset = self.to_lance()
_validate_index_accelerator(accelerator)
dataset.create_index(
column=column,
index_type=index_type,
metric=metric,
@@ -3706,6 +3746,8 @@ class LanceTable(Table):
acc = getattr(config, "accelerator", None)
if acc is not None:
# Dispatch to pylance for GPU acceleration
dataset = self.to_lance()
_validate_index_accelerator(acc)
index_type_map = {
"IvfFlat": "IVF_FLAT",
"IvfSq": "IVF_SQ",
@@ -3717,7 +3759,7 @@ class LanceTable(Table):
cfg_type = type(config).__name__
lance_index_type = index_type_map.get(cfg_type, "IVF_PQ")
self.to_lance().create_index(
dataset.create_index(
column=column,
index_type=lance_index_type,
metric=getattr(config, "distance_type", "l2"),
@@ -4902,8 +4944,17 @@ class LanceTable(Table):
| None = None,
*,
computed: Dict[str, str] | None = None,
auto_refresh_enabled: bool | None = None,
auto_refresh_interval_seconds: int | None = None,
) -> AddColumnsResult:
return LOOP.run(self._table.add_columns(transforms, computed=computed))
return LOOP.run(
self._table.add_columns(
transforms,
computed=computed,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
def refresh_column(self, column: str) -> "RefreshColumnResult":
"""Fill a computed column's unfilled rows and recompute those whose
@@ -5414,6 +5465,12 @@ def _infer_target_schema(
# Use the most common length of the list as the dimensions
dim = _modal_list_size(peeked.column(i))
if dim is None:
# Every list entry is null (e.g. embeddings not computed yet),
# so the dimension is unknowable. Leave the column as a
# variable-length list instead of guessing a fixed size.
continue
# Determine target type based on value type
if pa.types.is_floating(field.type.value_type):
target_type = pa.list_(pa.float32(), dim)
@@ -5453,9 +5510,14 @@ def _infer_target_schema(
return schema, reader
def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> int:
# Use the most common length of the list as the dimensions
return pc.mode(pc.list_value_length(arr))[0].as_py()["mode"]
def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> Optional[int]:
# Use the most common length of the list as the dimensions. pyarrow's
# `mode` skips nulls, so an all-null column yields an empty result; return
# None in that case so the caller can keep a variable-length list.
modes = pc.mode(pc.list_value_length(arr))
if len(modes) == 0:
return None
return modes[0].as_py()["mode"]
def _infer_vector_dim(arr: Union[pa.Array, pa.ChunkedArray]) -> Optional[int]:
@@ -5663,12 +5725,12 @@ class AsyncTable:
via [`set_unenforced_primary_key`]; bucket sharding additionally
requires it to be the single column being bucketed.
By default the RTE maintains every index on the table, resolved
here — a snapshot, so an index created afterwards needs the spec unset
and set again. This fails if one cannot be maintained; name the set
with ``with_maintained_indexes`` to install anyway. That pins an exact
set (a still-building index is rejected, not omitted); ``[]`` maintains
none.
By default the RTE maintains every index on the table, re-read as the
table changes, so an index created later is maintained too; one of a
kind the RTE cannot mirror is skipped. Name the set with
``with_maintained_indexes`` to pin it instead: each name must already
exist and be maintainable (a still-building index is rejected, not
omitted); ``[]`` maintains none.
Parameters
----------
@@ -6292,8 +6354,12 @@ class AsyncTable:
if mode == "overwrite":
# For overwrite, apply the same preprocessing as create_table
# so vector columns are inferred as FixedSizeList.
data, _ = sanitize_create_table(
data, None, on_bad_vectors=on_bad_vectors, fill_value=fill_value
data, _ = await asyncio.to_thread(
sanitize_create_table,
data,
None,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
)
if blob_v2_column_paths(schema):
# Plain bytes cannot be inferred as blob v2. Keep matching blob
@@ -6303,7 +6369,8 @@ class AsyncTable:
elif on_bad_vectors != "error" or (
schema.metadata is not None and b"embedding_functions" in schema.metadata
):
data = _sanitize_data(
data = await asyncio.to_thread(
_sanitize_data,
data,
schema,
metadata=schema.metadata,
@@ -6312,7 +6379,10 @@ class AsyncTable:
allow_subschema=True,
)
_register_optional_converters()
data = to_scannable(data)
# Converting an iterator peeks at its first item. A synchronous query in
# that iterator schedules work on LOOP, so peeking on LOOP's own thread
# would deadlock waiting for itself.
data = await asyncio.to_thread(to_scannable, data)
progress, owns = _normalize_progress(progress)
try:
return await self._inner.add(
@@ -6951,6 +7021,8 @@ class AsyncTable:
| None = None,
*,
computed: dict[str, str] | None = None,
auto_refresh_enabled: bool | None = None,
auto_refresh_interval_seconds: int | None = None,
) -> AddColumnsResult:
"""
Add new columns with defined values.
@@ -6990,12 +7062,46 @@ class AsyncTable:
On LanceDB Cloud and Enterprise the expression is planned by
the server. Cannot be combined with ``transforms``.
auto_refresh_enabled: bool, optional
Enable automatic refresh for each computed output declared by this
call. Defaults to True. Local tables store the setting; automatic
scheduling requires Cloud or Enterprise.
auto_refresh_interval_seconds: int, optional
Positive minimum seconds after a successful refresh before another
automatic refresh is eligible. When omitted, the server determines when
to refresh. Unchanged inputs do
not force recomputation. Applies to every declared computed output.
These options cannot be used with one-time transforms or ordinary
all-null columns.
Examples
--------
>>> await table.add_columns( # doctest: +SKIP
... computed={"total": "price * quantity"},
... auto_refresh_enabled=True,
... auto_refresh_interval_seconds=300,
... )
Returns
-------
AddColumnsResult
version: the new version number of the table after adding columns.
"""
if auto_refresh_enabled is not None and type(auto_refresh_enabled) is not bool:
raise ValueError("auto_refresh_enabled must be a boolean")
if auto_refresh_interval_seconds is not None and (
type(auto_refresh_interval_seconds) is not int
or not 0 < auto_refresh_interval_seconds <= (2**64 - 1) // 1000
):
raise ValueError(
"auto_refresh_interval_seconds must be a positive integer "
"no greater than u64::MAX / 1000"
)
options_supplied = (
auto_refresh_enabled is not None
or auto_refresh_interval_seconds is not None
)
function_application = None
function_output_name = None
if isinstance(transforms, FunctionApplication):
@@ -7019,7 +7125,10 @@ class AsyncTable:
)
function_application._ensure_declarable()
return await self._inner.add_function_columns(
function_application.to_canonical_json(), function_output_name
function_application.to_canonical_json(),
function_output_name,
auto_refresh_enabled,
auto_refresh_interval_seconds,
)
if isinstance(transforms, pa.Field):
@@ -7033,7 +7142,15 @@ class AsyncTable:
raise ValueError(
"add_columns cannot take both transforms and computed columns"
)
return await self._inner.add_computed_columns(list(computed.items()))
return await self._inner.add_computed_columns(
list(computed.items()),
auto_refresh_enabled,
auto_refresh_interval_seconds,
)
if options_supplied:
raise ValueError(
"automatic refresh options require computed or Function columns"
)
if transforms is None:
raise ValueError("add_columns requires transforms or computed columns")
if isinstance(transforms, pa.Schema):
python/python/lancedb/util.py
+18 -8
View File
@@ -34,20 +34,20 @@ def safe_import_adlfs():
adlfs = safe_import_adlfs()
def get_uri_scheme(uri: str) -> str:
def get_uri_scheme(uri: Union[str, pathlib.Path]) -> str:
"""
Get the scheme of a URI. If the URI does not have a scheme, assume it is a file URI.
Parameters
----------
uri : str
uri : str or pathlib.Path
The URI to parse.
Returns
-------
str: The scheme of the URI.
"""
parsed = urlparse(uri)
parsed = urlparse(str(uri))
scheme = parsed.scheme
if not scheme:
scheme = "file"
@@ -61,25 +61,26 @@ def get_uri_scheme(uri: str) -> str:
return scheme
def get_uri_location(uri: str) -> str:
def get_uri_location(uri: Union[str, pathlib.Path]) -> str:
"""
Get the location of a URI. If the parameter is not a url, assumes it is just a path
Parameters
----------
uri : str
uri : str or pathlib.Path
The URI to parse.
Returns
-------
str: Location part of the URL, without scheme
"""
parsed = urlparse(uri)
uri_str = str(uri)
parsed = urlparse(uri_str)
if len(parsed.scheme) == 1:
# Windows drive names are parsed as the scheme
# e.g. "c:\path" -> ParseResult(scheme="c", netloc="", path="/path", ...)
# So we add special handling here for schemes that are a single character
return uri
return uri_str
if not parsed.netloc:
return parsed.path
@@ -309,7 +310,16 @@ def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
if query.ndim == 0:
return None
return query.shape[-1]
if isinstance(query, list) and query:
if isinstance(query, (pa.Array, pa.ChunkedArray)):
if len(query) == 0:
return None
if pa.types.is_fixed_size_list(query.type):
return query.type.list_size
if pa.types.is_list(query.type) or pa.types.is_large_list(query.type):
first = query[0].as_py()
return len(first) if first is not None else None
return len(query)
if isinstance(query, (list, tuple)) and query:
first = query[0]
if isinstance(first, (list, tuple, np.ndarray)):
return len(first)
python/python/tests/test_elastic_dataloader.py
+227
View File
@@ -3297,3 +3297,230 @@ def test_streaming_dataset_over_remote_table():
assert len(server.scans) == 1, "the permutation is built with one row-id scan"
assert server.takes, "rows must be fetched with row-id takes"
assert_server_safe_row_id_requests(server)
# ---------------------------------------------------------------------------
# split_strategy escape hatch
# ---------------------------------------------------------------------------
def test_split_strategy_bootstrap_sample(lance_table):
"""split_strategy lets StreamingDataset use bootstrap_sample directly, producing
more rows than the base table has (sampling with replacement)."""
num_samples = 4 * NUM_ROWS
ds = StreamingDataset(
lance_table,
split_strategy=lambda b: b.bootstrap_sample(
num_samples, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
),
num_splits=NUM_SPLITS,
)
ids = [row["id"] for row in ds]
assert len(ids) == num_samples
# 4x oversampling a small table must produce at least one repeat.
assert len(set(ids)) < len(ids)
assert set(ids).issubset(set(range(NUM_ROWS)))
def test_split_strategy_matches_direct_permutation_builder_use(lance_table):
"""StreamingDataset(split_strategy=...), read single-process (no DataLoader
workers, so this one process owns every split and round-robins across all of
them), must yield exactly what calling the same strategy directly on a
PermutationBuilder would read, interleaved one row per split per cycle.
Reads each split via __getitems__ (explicit indices), not plain iteration:
Permutation's sequential __iter__ returns empty for a permutation with repeated
row_ids (confirmed directly -- len() is correct, __getitems__ returns the right
rows, but `for row in perm` yields nothing). That's a pre-existing limitation of
Permutation's iterator, unrelated to Bootstrap sampling's correctness: it doesn't
affect StreamingDataset or this benchmark, since both only ever read via
explicit-index batched fetches (take_offsets/__getitems__), never the plain
iterator -- flagged separately, not fixed here (out of scope for this change).
"""
from lancedb.permutation import Permutation, permutation_builder
def strategy(builder):
return builder.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=1)
direct_perm_tbl = strategy(permutation_builder(lance_table)).execute()
expected_by_split = []
for split in range(NUM_SPLITS):
perm = Permutation.from_tables(lance_table, direct_perm_tbl, split=split)
rows = perm.__getitems__(list(range(len(perm))))
expected_by_split.append([row["id"] for row in rows])
samples_per_split = len(expected_by_split[0])
expected = [
expected_by_split[split][cycle]
for cycle in range(samples_per_split)
for split in range(NUM_SPLITS)
]
ds = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
actual = [row["id"] for row in ds]
assert actual == expected
def test_split_strategy_is_reproducible_across_instances(lance_table):
def strategy(builder):
return builder.bootstrap_sample(3 * NUM_ROWS, fixed=NUM_SPLITS, seed=99)
ds1 = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
ds2 = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
assert [row["id"] for row in ds1] == [row["id"] for row in ds2]
def test_split_strategy_num_splits_mismatch_raises(lance_table):
"""StreamingDataset(split_strategy=...) must reject a strategy whose split
count doesn't match num_splits. `_rank_splits` assumes split ids are exactly
0..num_splits-1 with no gaps; a mismatch would otherwise silently drop rows
(split ids >= num_splits) or leave some ranks reading an empty split (missing
ids < num_splits)."""
with pytest.raises(ValueError, match="num_splits"):
StreamingDataset(
lance_table,
split_strategy=lambda b: b.bootstrap_sample(
2 * NUM_ROWS, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
),
num_splits=NUM_SPLITS + 1, # the strategy above always builds NUM_SPLITS
)
def test_split_strategy_unequal_split_sizes_raises(lance_table):
"""StreamingDataset(split_strategy=...) must reject a strategy whose splits
all exist (ids 0..num_splits-1) but aren't the same size. The round-robin
iteration (see _iter_owned) assumes every owned split has the same row count
and stops as soon as any one is exhausted -- `ratios`/`counts` (unlike
`fixed`) can produce unequal split sizes, which would otherwise silently
drop rows from the larger splits."""
with pytest.raises(ValueError, match="unequal split sizes"):
StreamingDataset(
lance_table,
split_strategy=lambda b: b.bootstrap_sample(
100, ratios=[0.7, 0.3], seed=SHUFFLE_SEED
),
num_splits=2,
)
def test_split_strategy_permutation_mismatch_on_resume_raises(lance_table):
"""load_state_dict must reject resuming against a different permutation than
the one that produced the checkpoint. num_splits/shuffle_seed alone can't catch
this for split_strategy, since that path ignores shuffle_seed entirely: a
bootstrap_sample reseeded differently resamples completely differently while
shuffle_seed keeps whatever value the caller passed either way, so the
pre-existing checks would pass despite the mismatch."""
def strategy(seed):
return lambda b: b.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=seed)
ds1 = StreamingDataset(
lance_table,
split_strategy=strategy(1),
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
)
it = iter(ds1)
for _ in range(NUM_SPLITS): # one full cycle: every split advances uniformly
next(it)
checkpoint = ds1.state_dict()
ds2 = StreamingDataset(
lance_table,
split_strategy=strategy(2), # different seed -> a different permutation
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED, # unchanged -- irrelevant on this path
)
with pytest.raises(ValueError, match="permutation mismatch"):
ds2.load_state_dict(checkpoint)
def test_split_strategy_resume_with_matching_permutation_succeeds(lance_table):
"""The common case: resuming against the same split_strategy (same seed) must
work exactly like the non-split_strategy checkpoint/resume path, continuing
from exactly the same position with no gaps or repeats."""
def strategy(b):
return b.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=7)
ds1 = StreamingDataset(
lance_table,
split_strategy=strategy,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
)
it = iter(ds1)
for _ in range(NUM_SPLITS):
next(it)
checkpoint = ds1.state_dict()
remaining_original = [row["id"] for row in it]
ds2 = StreamingDataset(
lance_table,
split_strategy=strategy,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
)
ds2.load_state_dict(checkpoint) # must not raise
remaining_resumed = [row["id"] for row in ds2]
assert remaining_original == remaining_resumed
def test_default_checkpoint_omits_perm_fingerprint(lance_table):
"""split_random/split_sequential checkpoints must not carry a
perm_fingerprint -- see test_default_checkpoint_resumes_after_compaction for
why checking one on this path would be wrong, not just unnecessary."""
ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
assert "perm_fingerprint" not in ds.state_dict()
def test_split_strategy_checkpoint_includes_perm_fingerprint(lance_table):
ds = StreamingDataset(
lance_table,
split_strategy=lambda b: b.bootstrap_sample(
2 * NUM_ROWS, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
),
num_splits=NUM_SPLITS,
)
assert "perm_fingerprint" in ds.state_dict()
def test_default_checkpoint_resumes_after_compaction(lance_table):
"""split_random/split_sequential checkpoints must resume normally even after
the table is compacted between save and load. Lance row ids are not stable
across compaction by default (confirmed empirically: a 2-fragment table's
row ids differ before and after `optimize()`), so fingerprinting raw row ids
on this path -- instead of relying on shuffle_seed/num_splits alone, as it
does -- would reject this resume even though the seeded permutation itself
is unchanged."""
lance_table.add(pa.table({"id": [NUM_ROWS]})) # a second fragment to compact
ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
it = iter(ds)
for _ in range(NUM_SPLITS):
next(it)
checkpoint = ds.state_dict()
assert "perm_fingerprint" not in checkpoint
before_compaction = set(
lance_table.to_lance()
.to_table(columns=[], with_row_id=True)["_rowid"]
.to_pylist()
)
lance_table.optimize()
after_compaction = set(
lance_table.to_lance()
.to_table(columns=[], with_row_id=True)["_rowid"]
.to_pylist()
)
assert before_compaction != after_compaction, (
"test setup didn't actually renumber row ids -- fix the fixture, not "
"the assertion below"
)
ds_resumed = StreamingDataset(
lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
)
ds_resumed.load_state_dict(checkpoint) # must not raise
python/python/tests/test_embeddings.py
+45
View File
@@ -596,6 +596,51 @@ def test_openai_no_retry_on_401(mock_sleep):
assert mock_sleep.call_count == 0
def test_openai_bad_request_partial_batch_recovery():
# Regression test for issue #1677
# Fake exception types, avoids requiring the optional openai/httpx deps
class FakeBadRequestError(Exception):
pass
class FakeAuthenticationError(Exception):
pass
fake_openai_module = MagicMock()
fake_openai_module.BadRequestError = FakeBadRequestError
fake_openai_module.AuthenticationError = FakeAuthenticationError
registry = get_registry()
model = registry.get("openai").create(name="text-embedding-ada-002")
def fake_create(input, model=None, **kwargs):
# Full batch always rejected
if len(input) > 1:
raise FakeBadRequestError("bad request")
if input[0] == "bad text":
raise FakeBadRequestError("bad request")
embedding = MagicMock()
embedding.embedding = [0.1, 0.2, 0.3]
result = MagicMock()
result.data = [embedding]
return result
mock_client = MagicMock()
mock_client.embeddings.create.side_effect = fake_create
with (
patch(
"lancedb.embeddings.openai.attempt_import_or_raise",
return_value=fake_openai_module,
),
patch.object(type(model), "_openai_client", new_callable=lambda: mock_client),
):
embeddings = model.generate_embeddings(["good text", "bad text"])
assert embeddings[0] == [0.1, 0.2, 0.3]
assert embeddings[1] is None
def test_ollama_embeddings_pickle():
"""OllamaEmbeddings must pickle even after the cached client is created."""
registry = get_registry()
python/python/tests/test_first_class_function_slice1.py
+20 -3
View File
@@ -398,8 +398,21 @@ class _FunctionDeclarationInner:
def __init__(self):
self.calls = []
async def add_function_columns(self, application_json, output_name):
self.calls.append((json.loads(application_json), output_name))
async def add_function_columns(
self,
application_json,
output_name,
auto_refresh_enabled,
auto_refresh_interval_seconds,
):
self.calls.append(
(
json.loads(application_json),
output_name,
auto_refresh_enabled,
auto_refresh_interval_seconds,
)
)
return "declared"
@@ -420,14 +433,18 @@ async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomica
)
assert result == "declared"
assert inner.calls[-1][1] == "features"
assert inner.calls[-1][2:] == (None, None)
bare = application._copy(update={"columns": {}}).rename(
columns={"normalized_text": "search_text"}
)
result = await table.add_columns(bare)
result = await table.add_columns(
bare, auto_refresh_enabled=False, auto_refresh_interval_seconds=300
)
assert result == "declared"
assert inner.calls[-1][1] is None
assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
assert inner.calls[-1][2:] == (False, 300)
@pytest.mark.asyncio
python/python/tests/test_header_provider.py
+18
View File
@@ -114,6 +114,24 @@ class TestOAuthProvider:
"x-lancedb-credential-type": "oidc",
}
def test_zero_lifetime_token_is_refreshed(self):
"""A token that expires immediately must not be cached indefinitely."""
call_count = 0
def fetcher():
nonlocal call_count
call_count += 1
return {
"access_token": f"token{call_count}",
"expires_in": 0,
}
provider = OAuthProvider(fetcher, refresh_buffer_seconds=0)
assert provider.get_headers() == {"Authorization": "Bearer token1"}
assert provider.get_headers() == {"Authorization": "Bearer token2"}
assert call_count == 2
def test_missing_access_token(self):
"""Test error handling when access_token is missing."""
python/python/tests/test_hybrid_query.py
+12
View File
@@ -536,3 +536,15 @@ def test_normalize_scores():
assert pc.equal(result, expected), (
f"Expected {expected} but got {result} for invert={invert}"
)
def test_hybrid_query_rejects_query_with_vector(sync_table: Table):
# Pins the full message: the space between "hybrid search." and "But" is
# load-bearing (see the _validate_query literal concatenation).
with pytest.raises(ValueError) as excinfo:
sync_table.search("dog", query_type="hybrid").vector([0.5, -0.5]).to_arrow()
assert str(excinfo.value) == (
"You can either provide a string query in search() method "
"or set `vector()` and `text()` explicitly for hybrid search. "
"But not both."
)
python/python/tests/test_index.py
+22
View File
@@ -430,6 +430,28 @@ async def test_create_ivfrq_index(some_table: AsyncTable):
assert indices[0].name == "vector_idx"
@pytest.mark.asyncio
async def test_read_ivfrq_index_created_with_lance(db_async, some_table: AsyncTable):
pytest.importorskip("lance")
dataset = await some_table.to_lance()
dataset.create_index(
"vector",
index_type="IVF_RQ",
metric="cosine",
num_partitions=1,
)
# Open a fresh LanceDB handle so the index is discovered from disk.
table = await db_async.open_table("some_table")
indices = await table.list_indices()
assert len(indices) == 1
assert indices[0].index_type == "IvfRq"
assert indices[0].columns == ["vector"]
results = await table.query().nearest_to(list(range(DIM))).limit(8).to_arrow()
assert len(results) == 8
@pytest.mark.asyncio
async def test_create_hnswpq_index(some_table: AsyncTable):
await some_table.create_index("vector", config=HnswPq(num_partitions=10))
python/python/tests/test_materialized_views.py
+103 -2
View File
@@ -129,8 +129,21 @@ def test_sync_remote_create_uses_public_async_connection():
where=None,
limit=None,
with_no_data=False,
auto_refresh_enabled=None,
auto_refresh_interval_seconds=None,
):
calls.append((name, source, select, where, limit, with_no_data))
calls.append(
(
name,
source,
select,
where,
limit,
with_no_data,
auto_refresh_enabled,
auto_refresh_interval_seconds,
)
)
return StubAsyncMaterializedView()
async def drop_materialized_view(self, name, *, namespace_path=None):
@@ -148,9 +161,11 @@ def test_sync_remote_create_uses_public_async_connection():
where="age >= 18",
limit=10,
with_no_data=True,
auto_refresh_enabled=True,
auto_refresh_interval_seconds=0.5,
)
assert view.name == "adults"
assert calls == [("adults", "people", ["name"], "age >= 18", 10, True)]
assert calls == [("adults", "people", ["name"], "age >= 18", 10, True, True, 0.5)]
db.drop_materialized_view("adults", namespace_path=["analytics"])
assert calls[-1] == ("drop", "adults", ["analytics"])
@@ -550,3 +565,89 @@ def test_local_materialized_view_listing_pages_and_resume(tmp_path):
assert sorted([first, *resumed]) == ["a_view", "z_view"]
assert resumed.page_token() is None
assert list(resumed) == []
@pytest.mark.parametrize(
"options",
[
{},
{"auto_refresh_enabled": True},
{"auto_refresh_enabled": False, "auto_refresh_interval_seconds": 0.5},
],
)
def test_remote_create_forwards_refresh_options_without_interval_default(options):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect("db://dev", api_key="fake", host_override=host)
job = db.create_materialized_view_async("adults", "people", **options)
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/adults/create",
{
"query": 'SELECT * FROM "people"',
**options,
},
)
]
@pytest.mark.asyncio
async def test_async_remote_create_keeps_interval_unspecified():
with mock_remote_materialized_view_create() as (host, requests):
db = await lancedb.connect_async("db://dev", api_key="fake", host_override=host)
job = await db.create_materialized_view_async(
"adults",
"people",
auto_refresh_enabled=True,
)
assert job.id == "mv-create-123"
assert requests[0][1] == {
"query": 'SELECT * FROM "people"',
"auto_refresh_enabled": True,
}
@pytest.mark.parametrize(
"options",
[
{"auto_refresh_enabled": True},
{"auto_refresh_interval_seconds": 300},
],
)
def test_embedded_create_refuses_automatic_refresh_before_creating_view(
tmp_path, options
):
db = make_db(tmp_path)
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
db.create_materialized_view("adults", "people", **options)
assert "adults" not in db.table_names()
@pytest.mark.parametrize("seconds", [0, -1, float("nan"), float("inf")])
def test_remote_create_rejects_invalid_interval_without_request(seconds):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect("db://dev", api_key="fake", host_override=host)
with pytest.raises(ValueError, match="positive, finite"):
db.create_materialized_view_async(
"adults", "people", auto_refresh_interval_seconds=seconds
)
assert requests == []
def test_namespace_forwards_automatic_refresh_request(tmp_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
@pytest.mark.asyncio
async def test_async_namespace_forwards_automatic_refresh_request(tmp_path):
db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
await db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False)
assert view.table.count_rows() == 3
python/python/tests/test_permutation.py
+70
View File
@@ -161,6 +161,76 @@ def test_split_random_with_seed(mem_db):
assert data1["split_id"] == data2["split_id"]
def test_bootstrap_sample_fixed(mem_db):
"""Output row count is num_samples, not the input row count, and splits divide
it evenly."""
tbl = mem_db.create_table("test_table", pa.table({"x": range(20)}))
permutation_tbl = (
permutation_builder(tbl).bootstrap_sample(1000, fixed=4, seed=42).execute()
)
assert permutation_tbl.count_rows() == 1000
data = permutation_tbl.search(None).to_arrow().to_pydict()
split_ids = data["split_id"]
assert set(split_ids) == {0, 1, 2, 3}
for split_id in range(4):
assert split_ids.count(split_id) == 250
# Every sampled row_id must still be one of the 20 real rows.
assert set(data["row_id"]).issubset(set(range(20)))
def test_bootstrap_sample_draws_with_replacement(mem_db):
"""Sampling more rows than exist must produce repeats."""
tbl = mem_db.create_table("test_table", pa.table({"x": range(5)}))
permutation_tbl = (
permutation_builder(tbl).bootstrap_sample(500, fixed=1, seed=1).execute()
)
data = permutation_tbl.search(None).to_arrow().to_pydict()
assert len(data["row_id"]) == 500
# Only 5 distinct rows exist, so with 500 draws every one of them must repeat.
for row_id in range(5):
assert data["row_id"].count(row_id) > 1
def test_bootstrap_sample_with_seed_is_reproducible(mem_db):
tbl = mem_db.create_table("test_table", pa.table({"x": range(50)}))
perm1 = permutation_builder(tbl).bootstrap_sample(200, fixed=2, seed=42).execute()
perm2 = permutation_builder(tbl).bootstrap_sample(200, fixed=2, seed=42).execute()
data1 = perm1.search(None).to_arrow().to_pydict()
data2 = perm2.search(None).to_arrow().to_pydict()
assert data1["row_id"] == data2["row_id"]
assert data1["split_id"] == data2["split_id"]
def test_bootstrap_sample_respects_filter(mem_db):
tbl = mem_db.create_table("test_table", pa.table({"x": range(20)}))
permutation_tbl = (
permutation_builder(tbl)
.filter("x >= 15")
.bootstrap_sample(100, fixed=1, seed=7)
.execute()
)
data = permutation_tbl.search(None).to_arrow().to_pydict()
assert len(data["row_id"]) == 100
assert set(data["row_id"]).issubset(set(range(15, 20)))
def test_bootstrap_sample_requires_exactly_one_size_arg(mem_db):
tbl = mem_db.create_table("test_table", pa.table({"x": range(10)}))
with pytest.raises(ValueError):
permutation_builder(tbl).bootstrap_sample(10).execute()
with pytest.raises(ValueError):
permutation_builder(tbl).bootstrap_sample(
10, ratios=[0.5, 0.5], fixed=2
).execute()
def test_split_hash(mem_db):
"""Test hash-based splitting."""
tbl = mem_db.create_table(
python/python/tests/test_query.py
+22
View File
@@ -833,6 +833,28 @@ def test_with_row_id(table: lancedb.table.Table):
assert rs["_rowid"].to_pylist() == [0, 1]
def test_filter_preserves_deeply_nested_struct_fields(tmp_db):
# Regression test for https://github.com/lancedb/lancedb/issues/2217.
data = [
{
"item": "test",
"sub_obj": {
"second_sub_obj": [
{
"third_sub_obj": {"int_val": 1},
"str_val": "test",
}
]
},
}
]
table = tmp_db.create_table("deeply_nested_struct", data=data)
result = table.search().where("item = 'test'").limit(1).to_arrow()
assert result.to_pylist() == data
def test_blob_v2_query_omits_auto_row_id(tmp_db):
table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_omits_auto_rowid")
python/python/tests/test_rerankers.py
+108
View File
@@ -785,6 +785,114 @@ def test_empty_hybrid_result_reranker():
assert "_rowid" in result.column_names
@pytest.mark.parametrize("return_score", ["relevance", "all"])
@pytest.mark.parametrize("first_row_id", [1, 2**63])
@pytest.mark.parametrize("score_type", [pa.float32(), pa.float64()])
def test_model_reranker_preserves_arrow_schema(
mocker, return_score, first_row_id, score_type
):
common_schema = pa.schema(
[
pa.field("_rowid", pa.uint64()),
pa.field("text", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 2)),
],
metadata={b"source": b"reranker-test"},
)
vector_results = pa.Table.from_pylist(
[
{
"_rowid": first_row_id,
"text": "vector",
"vector": [1, 0],
"_distance": 0.1,
},
{
"_rowid": first_row_id + 1,
"text": "both",
"vector": [0, 1],
"_distance": 0.2,
},
],
schema=common_schema.append(pa.field("_distance", score_type)),
)
fts_results = pa.Table.from_pylist(
[
{
"_rowid": first_row_id + 1,
"text": "both",
"vector": [0, 1],
"_score": 0.9,
},
{
"_rowid": first_row_id + 2,
"text": "fts",
"vector": [1, 1],
"_score": 0.8,
},
],
schema=common_schema.append(pa.field("_score", score_type)),
)
reranker = CohereReranker(return_score=return_score)
reranker._client = mocker.Mock()
reranker._client.rerank.return_value.results = [
mocker.Mock(index=index, relevance_score=score)
for index, score in [(2, 0.9), (1, 0.8), (0, 0.7)]
]
result = reranker.rerank_hybrid("query", vector_results, fts_results)
assert result.select(common_schema.names).schema.equals(
common_schema, check_metadata=True
)
assert result["_rowid"].to_pylist() == [
first_row_id + 2,
first_row_id + 1,
first_row_id,
]
assert result["text"].to_pylist() == ["fts", "both", "vector"]
if return_score == "all":
assert (
result["_distance"]
.combine_chunks()
.equals(pa.array([None, 0.2, 0.1], type=score_type))
)
assert (
result["_score"]
.combine_chunks()
.equals(pa.array([0.8, 0.9, None], type=score_type))
)
@pytest.mark.parametrize("return_score", ["relevance", "all"])
def test_model_reranker_empty_results_preserve_schema(mocker, return_score):
common_schema = pa.schema(
[
("_rowid", pa.uint64()),
("text", pa.string()),
("vector", pa.list_(pa.float32(), 2)),
],
metadata={b"source": b"reranker-test"},
)
vector_results = pa.Table.from_pylist(
[], schema=common_schema.append(pa.field("_distance", pa.float32()))
)
fts_results = pa.Table.from_pylist(
[], schema=common_schema.append(pa.field("_score", pa.float32()))
)
reranker = CohereReranker(return_score=return_score)
reranker._client = mocker.Mock()
result = reranker.rerank_hybrid("query", vector_results, fts_results)
assert len(result) == 0
assert result.select(common_schema.names).schema.equals(
common_schema, check_metadata=True
)
assert result.schema.field("_relevance_score").type == pa.float32()
reranker._client.rerank.assert_not_called()
def test_cross_encoder_reranker_return_all(tmp_path):
pytest.importorskip("sentence_transformers")
reranker = CrossEncoderReranker(return_score="all")
python/python/tests/test_table.py
+180 -1
View File
@@ -15,7 +15,7 @@ from datetime import date, datetime, timedelta
from decimal import Decimal, localcontext
from time import sleep
from typing import Any, List
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import lancedb
from lancedb import table as table_module
@@ -280,6 +280,30 @@ def test_create_table_infers_large_int_vectors(mem_db: DBConnection):
assert vector_column.to_pylist() == [[0.0, 300.0]]
def test_create_table_all_null_vector_column(mem_db: DBConnection):
# A vector column can legitimately arrive with every value null, e.g. when
# the embeddings have not been computed yet. The dimension is unknowable in
# that case, so schema inference must not crash and should leave the column
# as a variable-length list rather than a fixed-size one.
data = pa.table(
{
"id": pa.array([1, 2]),
"vector": pa.array([None, None], type=pa.list_(pa.float32())),
}
)
table = mem_db.create_table("all_null_vector", data=data)
vector_field = table.schema.field("vector")
assert pa.types.is_list(vector_field.type)
assert vector_field.type.value_type == pa.float32()
result = table.to_arrow()
assert result.num_rows == 2
assert result.column("vector").to_pylist() == [None, None]
assert table.count_rows() == 2
@pytest.mark.asyncio
async def test_create_table_async_infers_large_int_vectors(
mem_db_async: AsyncConnection,
@@ -518,6 +542,30 @@ def test_add(mem_db: DBConnection):
_add(table, schema)
def test_add_from_iterator_that_queries_table(mem_db: DBConnection):
source = mem_db.create_table("source", data=pa.table({"id": range(16)}))
target = mem_db.create_table("target", schema=source.schema)
def batches():
for _ in range(5):
yield source.search().limit(10).to_arrow()
target.add(batches())
assert target.count_rows() == 50
def test_create_table_from_iterator_that_queries_table(mem_db: DBConnection):
source = mem_db.create_table("source", data=pa.table({"id": range(16)}))
def batches():
yield source.search().limit(10).to_arrow()
target = mem_db.create_table("target", data=batches())
assert target.count_rows() == 10
def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection):
"""Regression test for https://github.com/lancedb/lancedb/issues/2512."""
schema = pa.schema([pa.field("x", pa.int64())])
@@ -1725,6 +1773,51 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
)
@pytest.mark.parametrize("api", ["legacy", "unified"])
@pytest.mark.parametrize("cuda_failure", ["unavailable", "init_error"])
def test_create_index_cuda_failure_suggests_cpu(
api, cuda_failure, mem_db: DBConnection
):
table = mem_db.create_table(
f"test_cuda_unavailable_{api}",
data=[{"vector": [3.1, 4.1]}, {"vector": [5.9, 26.5]}],
)
torch = MagicMock()
torch.cuda.is_available.return_value = cuda_failure == "init_error"
if cuda_failure == "init_error":
torch.cuda.init.side_effect = RuntimeError("CUDA driver initialization failed")
with (
patch.dict(sys.modules, {"torch": torch}),
patch.object(table, "to_lance") as to_lance,
):
if api == "legacy":
with (
pytest.warns(DeprecationWarning, match="create_index"),
pytest.raises(RuntimeError, match="accelerator=None.*CPU"),
):
table.create_index(
metric="dot",
num_partitions=1,
num_sub_vectors=1,
vector_column_name="vector",
accelerator="cuda",
)
else:
with pytest.raises(RuntimeError, match="accelerator=None.*CPU"):
table.create_index(
"vector",
config=IvfPq(
distance_type="dot",
num_partitions=1,
num_sub_vectors=1,
accelerator="cuda",
),
)
to_lance.return_value.create_index.assert_not_called()
@patch("lancedb.table.AsyncTable.create_index")
def test_create_index_name_and_train_parameters(
mock_create_index, mem_db: DBConnection
@@ -3673,6 +3766,42 @@ def test_multiple_vector_columns(mem_db: DBConnection):
assert result1["text"][0] != result2["text"][0]
@pytest.mark.parametrize(
"query",
[
(0.0, 0.0),
pa.array([0.0, 0.0]),
pa.chunked_array([[0.0], [0.0]]),
],
)
@pytest.mark.parametrize("query_type", ["auto", "vector"])
def test_sync_search_accepts_supported_vector_containers(
mem_db: DBConnection, query, query_type
):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("small", pa.list_(pa.float32(), 2)),
pa.field("large", pa.list_(pa.float32(), 3)),
]
)
table = mem_db.create_table(
"supported_vector_containers",
data=pa.table(
{
"id": [1, 2],
"small": [[0.0, 0.0], [1.0, 1.0]],
"large": [[1.0, 1.0, 1.0], [0.0, 0.0, 0.0]],
},
schema=schema,
),
)
result = table.search(query, query_type=query_type).limit(1).to_arrow()
assert result["id"].to_pylist() == [1]
def test_create_scalar_index(mem_db: DBConnection):
vec_array = pa.array(
[[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], pa.list_(pa.float32(), 2)
@@ -4997,3 +5126,53 @@ async def test_refresh_column_async_job_async_table(tmp_path):
assert result.published_version == 4
assert await job.status() == "finished"
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
@pytest.mark.parametrize("interval", [None, 300])
def test_computed_auto_refresh_options(tmp_path, interval):
db = lancedb.connect(tmp_path)
table = db.create_table("refresh_options", [{"x": 1}])
before = table.version
table.add_columns(
computed={"doubled": "x * 2", "tripled": "x * 3"},
auto_refresh_enabled=False,
auto_refresh_interval_seconds=interval,
)
assert table.version == before + 1
for column in ("doubled", "tripled"):
metadata = table.schema.field(column).metadata
assert metadata[b"computed_column.auto_refresh_enabled"] == b"false"
if interval is None:
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
else:
assert metadata[b"computed_column.auto_refresh_interval_seconds"] == b"300"
table.refresh_column("doubled")
assert table.to_arrow()["doubled"].to_pylist() == [2]
@pytest.mark.parametrize("value", [0, -1, True, 1.5, "300", 2**64])
def test_computed_auto_refresh_rejects_invalid_interval(tmp_path, value):
table = lancedb.connect(tmp_path).create_table("invalid_options", [{"x": 1}])
with pytest.raises(ValueError, match="auto_refresh_interval_seconds"):
table.add_columns(
computed={"doubled": "x * 2"}, auto_refresh_interval_seconds=value
)
assert table.schema.names == ["x"]
def test_auto_refresh_options_require_computed_columns(tmp_path):
table = lancedb.connect(tmp_path).create_table("invalid_kind", [{"x": 1}])
with pytest.raises(ValueError, match="require computed"):
table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True)
@pytest.mark.asyncio
async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
db = await lancedb.connect_async(tmp_path)
table = await db.create_table("server_interval", [{"x": 1}])
before = await table.version()
await table.add_columns(computed={"doubled": "x * 2"})
assert await table.version() == before + 1
metadata = (await table.schema()).field("doubled").metadata
assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
python/python/tests/test_util.py
+17
View File
@@ -30,6 +30,7 @@ import pytest
import lancedb
from lancedb.util import (
flatten_columns,
get_uri_location,
get_uri_scheme,
inf_vector_column_query,
join_uri,
@@ -86,6 +87,22 @@ def test_normalize_uri():
assert parsed_scheme == expected_scheme
def test_get_uri_scheme_and_location_pathlib():
# Test pathlib.Path instances
abs_path = pathlib.Path("/tmp/data")
rel_path = pathlib.Path("relative/data")
assert get_uri_scheme(abs_path) == "file"
assert get_uri_scheme(rel_path) == "file"
assert get_uri_location(abs_path) == str(abs_path)
assert get_uri_location(rel_path) == str(rel_path)
# String URIs still behave identically
assert get_uri_scheme("s3://bucket/data") == "s3"
assert get_uri_location("s3://bucket/data") == "bucket/data"
def test_join_uri_remote():
schemes = ["s3", "az", "gs"]
for scheme in schemes:
python/src/connection.rs
+20 -2
View File
@@ -412,7 +412,8 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
@@ -421,10 +422,18 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(enabled) = auto_refresh_enabled {
builder = builder.auto_refresh_enabled(enabled);
}
if let Some(seconds) = auto_refresh_interval_seconds {
builder = builder.auto_refresh_interval_seconds(seconds);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -440,7 +449,8 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
pub fn create_materialized_view_async(
self_: PyRef<'_, Self>,
name: String,
@@ -449,10 +459,18 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(enabled) = auto_refresh_enabled {
builder = builder.auto_refresh_enabled(enabled);
}
if let Some(seconds) = auto_refresh_interval_seconds {
builder = builder.auto_refresh_interval_seconds(seconds);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
python/src/permutation.rs
+44
View File
@@ -178,6 +178,50 @@ impl PyAsyncPermutationBuilder {
})
}
#[pyo3(signature = (num_samples, *, ratios=None, counts=None, fixed=None, seed=None, split_names=None))]
pub fn bootstrap_sample(
slf: PyRefMut<'_, Self>,
num_samples: u64,
ratios: Option<Vec<f64>>,
counts: Option<Vec<u64>>,
fixed: Option<u64>,
seed: Option<u64>,
split_names: Option<Vec<String>>,
) -> PyResult<Self> {
// Check that exactly one split type is provided
let split_args_count = [ratios.is_some(), counts.is_some(), fixed.is_some()]
.iter()
.filter(|&&x| x)
.count();
if split_args_count != 1 {
return Err(pyo3::exceptions::PyValueError::new_err(
"Exactly one of 'ratios', 'counts', or 'fixed' must be provided",
));
}
let sizes = if let Some(ratios) = ratios {
SplitSizes::Percentages(ratios)
} else if let Some(counts) = counts {
SplitSizes::Counts(counts)
} else if let Some(fixed) = fixed {
SplitSizes::Fixed(fixed)
} else {
unreachable!("One of the split arguments must be provided");
};
slf.modify(|builder| {
builder.with_split_strategy(
SplitStrategy::Bootstrap {
num_samples,
sizes,
seed,
},
split_names,
)
})
}
pub fn split_calculated(
slf: PyRefMut<'_, Self>,
calculation: String,
python/src/table.rs
+27 -2
View File
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use lancedb::table::computed_columns::ComputedColumnOptions;
use std::{
collections::HashMap,
sync::{
@@ -1937,35 +1938,59 @@ impl Table {
})
}
#[pyo3(signature = (columns, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
pub fn add_computed_columns(
self_: PyRef<'_, Self>,
columns: Vec<(String, String)>,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let options = ComputedColumnOptions {
auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
auto_refresh_interval_seconds,
};
options.validate().infer_error()?;
future_into_py(self_.py(), async move {
let mut builder = inner.add_columns();
for (name, expression) in columns {
builder = builder.computed(name, expression);
}
let result = builder.execute().await.infer_error()?;
let result = builder
.computed_options(options)
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
#[pyo3(signature = (application_json, output_name, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
pub fn add_function_columns(
self_: PyRef<'_, Self>,
application_json: String,
output_name: Option<String>,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let application =
lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
let inner = self_.inner_ref()?.clone();
let options = ComputedColumnOptions {
auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
auto_refresh_interval_seconds,
};
options.validate().infer_error()?;
future_into_py(self_.py(), async move {
let builder = match output_name {
Some(name) => inner.add_columns().function_as(name, application),
None => inner.add_columns().function(application),
};
let result = builder.execute().await.infer_error()?;
let result = builder
.computed_options(options)
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
python/uv.lock
Generated
+3 -3
View File
@@ -443,11 +443,11 @@ wheels = [
[[package]]
name = "certifi"
version = "2026.6.17"
version = "2026.10.19"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" }
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" },
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
]
[[package]]
python/vendor/lance-arrow/Cargo.toml
Vendored
+33
View File
@@ -0,0 +1,33 @@
[package]
name = "lance-arrow"
version = "15.0.0-beta.1"
edition = "2024"
authors = ["Lance Devs <dev@lance.org>"]
license = "Apache-2.0"
repository = "https://github.com/lance-format/lance"
readme = "README.md"
description = "Arrow Extension for Lance"
keywords = ["data-format", "data-science", "machine-learning", "apache-arrow", "data-analytics"]
categories = ["database-implementations"]
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies]
arrow-array.workspace = true
arrow-buffer.workspace = true
arrow-data.workspace = true
arrow-ipc = { workspace = true, features = ["zstd"] }
arrow-ord.workspace = true
arrow-schema.workspace = true
arrow-select.workspace = true
bytemuck = { version = "1", default-features = false, features = ["extern_crate_alloc"] }
bytes.workspace = true
futures.workspace = true
half = { workspace = true, features = ["std", "bytemuck"] }
lance-jsonb.workspace = true
num-traits.workspace = true
rand = { workspace = true, features = ["small_rng"] }
[target.'cfg(target_arch = "wasm32")'.dependencies]
getrandom = { version = "0.2", features = ["js"] }
python/vendor/lance-arrow/README.lancedb.md
Vendored
+7
View File
@@ -0,0 +1,7 @@
This directory contains `rust/lance-arrow` from Lance tag `v15.0.0-beta.1`
(commit `46a7cfe1fba5efa1f90deea4c7b14d414b991409`), licensed under Apache-2.0.
LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
until a Lance release includes the fix. The patch is selected in the root
`Cargo.toml`; the rest of Lance stays on the pinned release. An explicit unsafe
block in `src/bfloat16.rs` lets this copy pass LanceDB's Rust 2024 warning gate.
python/vendor/lance-arrow/README.md
Vendored
+6
View File
@@ -0,0 +1,6 @@
# lance-arrow
`lance-arrow` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs)
extensions used by [Lance](https://github.com/lance-format/lance).
**Important Note**: This crate is **not intended for external usage**.
python/vendor/lance-arrow/src/bfloat16.rs
Vendored
+406
View File
@@ -0,0 +1,406 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
//! bfloat16 support for Apache Arrow.
use std::fmt::Formatter;
use std::slice;
use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder};
use arrow_buffer::{Buffer, MutableBuffer};
use arrow_data::ArrayData;
use arrow_schema::{ArrowError, DataType, Field as ArrowField};
use half::bf16;
use crate::{ARROW_EXT_NAME_KEY, FloatArray};
/// The name of the bfloat16 extension in Arrow metadata
pub const BFLOAT16_EXT_NAME: &str = "lance.bfloat16";
/// Check whether the given field is a bfloat16 field
///
/// A field is a bfloat16 field if it has a data type of `FixedSizeBinary(2)` and the metadata
/// contains the bfloat16 extension name.
pub fn is_bfloat16_field(field: &ArrowField) -> bool {
field.data_type() == &DataType::FixedSizeBinary(2)
&& field
.metadata()
.get(ARROW_EXT_NAME_KEY)
.map(|name| name == BFLOAT16_EXT_NAME)
.unwrap_or_default()
}
/// The bfloat16 data type
///
/// This implements the [`ArrowFloatType`](crate::floats::ArrowFloatType) trait for bfloat16 values.
#[derive(Debug)]
pub struct BFloat16Type {}
/// An array of bfloat16 values
///
/// Note that bfloat16 is not the same thing as fp16 which is supported natively by arrow-rs.
#[derive(Clone)]
pub struct BFloat16Array {
inner: FixedSizeBinaryArray,
}
impl std::fmt::Debug for BFloat16Array {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "BFloat16Array\n[\n")?;
from_arrow::print_long_array(&self.inner, f, |array, i, f| {
if array.is_null(i) {
write!(f, "null")
} else {
let binary_values = array.value(i);
let value =
bf16::from_bits(u16::from_le_bytes([binary_values[0], binary_values[1]]));
write!(f, "{:?}", value)
}
})?;
write!(f, "]")
}
}
impl BFloat16Array {
pub fn from_iter_values(iter: impl IntoIterator<Item = bf16>) -> Self {
let values: Vec<bf16> = iter.into_iter().collect();
values.into()
}
pub fn len(&self) -> usize {
self.inner.len()
}
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
pub fn is_null(&self, i: usize) -> bool {
self.inner.is_null(i)
}
pub fn null_count(&self) -> usize {
self.inner.null_count()
}
pub fn iter(&self) -> BFloat16Iter<'_> {
BFloat16Iter {
array: self,
index: 0,
}
}
pub fn value(&self, i: usize) -> bf16 {
assert!(
i < self.len(),
"Trying to access an element at index {} from a BFloat16Array of length {}",
i,
self.len()
);
// Safety:
// `i < self.len()
unsafe { self.value_unchecked(i) }
}
/// # Safety
/// Caller must ensure that `i < self.len()`
pub unsafe fn value_unchecked(&self, i: usize) -> bf16 {
// SAFETY: the caller guarantees that `i` is within this array's bounds.
let binary_value = unsafe { self.inner.value_unchecked(i) };
bf16::from_bits(u16::from_le_bytes([binary_value[0], binary_value[1]]))
}
pub fn into_inner(self) -> FixedSizeBinaryArray {
self.inner
}
}
impl FromIterator<Option<bf16>> for BFloat16Array {
fn from_iter<I: IntoIterator<Item = Option<bf16>>>(iter: I) -> Self {
let mut buffer = MutableBuffer::new(10);
// No null buffer builder :(
let mut nulls = BooleanBufferBuilder::new(10);
let mut len = 0;
for maybe_value in iter {
if let Some(value) = maybe_value {
let bytes = value.to_le_bytes();
buffer.extend(bytes);
} else {
buffer.extend([0u8, 0u8]);
}
nulls.append(maybe_value.is_some());
len += 1;
}
let null_buffer = nulls.finish();
let num_valid = null_buffer.count_set_bits();
let null_buffer = if num_valid == len {
None
} else {
Some(null_buffer.into_inner())
};
let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
.len(len)
.add_buffer(buffer.into())
.null_bit_buffer(null_buffer);
// SAFETY: the value buffer contains exactly `2 * len` bytes (two bytes
// pushed per iteration of the loop above, including the zero-fill for
// null slots), which matches the `FixedSizeBinary(2)` storage layout.
// The null bit buffer, when present, has `len` bits appended above, so
// its length covers the array's logical range.
let array_data = unsafe { array_data.build_unchecked() };
Self {
inner: FixedSizeBinaryArray::from(array_data),
}
}
}
impl FromIterator<bf16> for BFloat16Array {
fn from_iter<I: IntoIterator<Item = bf16>>(iter: I) -> Self {
Self::from_iter_values(iter)
}
}
impl From<Vec<bf16>> for BFloat16Array {
fn from(data: Vec<bf16>) -> Self {
let len = data.len();
// Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives
// `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place —
// no per-element copy or heap alloc. The crate-root `compile_error!`
// pins `target_endian = "little"`, so the resulting bytes match the
// `FixedSizeBinary(2)` on-disk order Lance writes elsewhere.
let raw: Vec<u16> = bytemuck::cast_vec(data);
let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
.len(len)
.add_buffer(Buffer::from_vec(raw));
// SAFETY: the value buffer contains exactly `2 * len` bytes — one
// `u16` per element after the layout-compatible cast — matching the
// `FixedSizeBinary(2)` storage layout. No null buffer is attached, so
// every element is logically valid.
let array_data = unsafe { array_data.build_unchecked() };
Self {
inner: FixedSizeBinaryArray::from(array_data),
}
}
}
impl TryFrom<FixedSizeBinaryArray> for BFloat16Array {
type Error = ArrowError;
fn try_from(value: FixedSizeBinaryArray) -> Result<Self, Self::Error> {
if value.value_length() == 2 {
Ok(Self { inner: value })
} else {
Err(ArrowError::InvalidArgumentError(
"FixedSizeBinaryArray must have a value length of 2".to_string(),
))
}
}
}
impl PartialEq<Self> for BFloat16Array {
fn eq(&self, other: &Self) -> bool {
self.inner.eq(&other.inner)
}
}
pub struct BFloat16Iter<'a> {
array: &'a BFloat16Array,
index: usize,
}
impl<'a> Iterator for BFloat16Iter<'a> {
type Item = Option<bf16>;
fn next(&mut self) -> Option<Self::Item> {
if self.index >= self.array.len() {
return None;
}
let i = self.index;
self.index += 1;
if self.array.is_null(i) {
Some(None)
} else {
Some(Some(self.array.value(i)))
}
}
}
/// Methods that are lifted from arrow-rs temporarily until they are made public.
mod from_arrow {
use arrow_array::Array;
/// Helper function for printing potentially long arrays.
pub(super) fn print_long_array<A, F>(
array: &A,
f: &mut std::fmt::Formatter,
print_item: F,
) -> std::fmt::Result
where
A: Array,
F: Fn(&A, usize, &mut std::fmt::Formatter) -> std::fmt::Result,
{
let head = std::cmp::min(10, array.len());
for i in 0..head {
if array.is_null(i) {
writeln!(f, " null,")?;
} else {
write!(f, " ")?;
print_item(array, i, f)?;
writeln!(f, ",")?;
}
}
if array.len() > 10 {
if array.len() > 20 {
writeln!(f, " ...{} elements...,", array.len() - 20)?;
}
let tail = std::cmp::max(head, array.len() - 10);
for i in tail..array.len() {
if array.is_null(i) {
writeln!(f, " null,")?;
} else {
write!(f, " ")?;
print_item(array, i, f)?;
writeln!(f, ",")?;
}
}
}
Ok(())
}
}
impl FloatArray<BFloat16Type> for FixedSizeBinaryArray {
type FloatType = BFloat16Type;
/// Returns the underlying `bf16` values as a borrowed slice.
///
/// # Preconditions
///
/// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape
/// used by [`BFloat16Array`]). Asserted at entry.
/// - The value buffer must be at least 2-byte aligned. Lance's in-tree
/// constructors always satisfy this: value buffers are built either via
/// `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32
/// bytes) or via `Buffer::from_vec::<u16>` (aligned to `align_of::<u16>()`
/// == 2); both meet `bf16`'s 2-byte requirement. Externally-built
/// `FixedSizeBinaryArray`s arriving via FFI, IPC, or
/// `Buffer::from_custom_allocation` are not required by arrow-rs to be
/// aligned beyond a single byte; passing one to this method violates the
/// precondition. A `debug_assert` below catches such inputs in debug and
/// test builds.
///
/// # Endianness
///
/// `lance-arrow` is gated on `target_endian = "little"` at the crate root,
/// so this method always returns values in the same byte order Lance writes
/// (see [`BFloat16Array::value`] and the [`FromIterator`] impls).
fn as_slice(&self) -> &[bf16] {
assert_eq!(
self.value_length(),
2,
"BFloat16 arrays must use FixedSizeBinary(2) storage"
);
debug_assert_eq!(
(self.value_data().as_ptr() as usize) % std::mem::align_of::<bf16>(),
0,
"BFloat16 value buffer must be at least 2-byte aligned"
);
// SAFETY:
// - The assert above pins `value_size == 2`, so `value_data().len() / 2`
// equals the array's logical element count.
// `FixedSizeBinaryArray::From<ArrayData>` constructs its value buffer
// as `buffers[0].slice_with_length(offset * 2, len * 2)` (arrow-array
// `fixed_size_binary_array.rs`), so `value_data()` already returns
// the offset-adjusted slice. Do not replace `value_data()` with an
// accessor that returns the un-sliced backing buffer.
// - `bf16` is `#[repr(transparent)]` over `u16` (size 2, alignment 2);
// every `u16` bit pattern is a valid `bf16`, so any byte content
// yields a defined value — never UB.
// - Alignment is the caller's responsibility per the precondition
// documented above. The `debug_assert_eq!` immediately preceding this
// block catches violations in debug and test builds only — release
// builds rely on callers honoring the precondition. arrow-rs
// declares `FixedSizeBinary(n)`'s
// `BufferSpec::FixedWidth { alignment: align_of::<u8>() == 1 }`
// (arrow-data `data.rs`), so arrow-rs alone does not guarantee
// 2-byte alignment. Lance's in-tree construction paths build value
// buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant,
// ≥32 bytes) or `Buffer::from_vec::<u16>` (2-byte aligned), both of
// which satisfy `bf16`'s 2-byte requirement.
// - The returned slice borrows from `self`; the underlying ref-counted,
// immutable Arrow buffer cannot be mutated or freed for the slice's
// lifetime.
unsafe {
slice::from_raw_parts(
self.value_data().as_ptr() as *const bf16,
self.value_data().len() / 2,
)
}
}
fn from_values(values: Vec<bf16>) -> Self {
BFloat16Array::from(values).into_inner()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basics() {
let values: Vec<f32> = vec![1.0, 2.0, 3.0];
let values: Vec<bf16> = values.iter().map(|v| bf16::from_f32(*v)).collect();
let array = BFloat16Array::from_iter_values(values.clone());
let array2 = BFloat16Array::from(values.clone());
assert_eq!(array, array2);
assert_eq!(array.len(), 3);
// Pin the raw little-endian bytes emitted by `From<Vec<bf16>>` (rewritten to
// reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order
// regression is caught directly rather than only through Debug formatting.
// bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040.
let inner = array2.clone().into_inner();
let raw_bytes: Vec<u8> = (0..inner.len())
.flat_map(|i| inner.value(i).to_vec())
.collect();
assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]);
let expected_fmt = "BFloat16Array\n[\n 1.0,\n 2.0,\n 3.0,\n]";
assert_eq!(expected_fmt, format!("{:?}", array));
for (expected, value) in values.iter().zip(array.iter()) {
assert_eq!(Some(*expected), value);
}
for (expected, value) in values.as_slice().iter().zip(array2.iter()) {
assert_eq!(Some(*expected), value);
}
let arrow_array = array.into_inner();
assert_eq!(arrow_array.as_slice(), values.as_slice());
}
#[test]
fn test_nulls() {
let values: Vec<Option<bf16>> =
vec![Some(bf16::from_f32(1.0)), None, Some(bf16::from_f32(3.0))];
let array = BFloat16Array::from_iter(values.clone());
assert_eq!(array.len(), 3);
assert_eq!(array.null_count(), 1);
let expected_fmt = "BFloat16Array\n[\n 1.0,\n null,\n 3.0,\n]";
assert_eq!(expected_fmt, format!("{:?}", array));
for (expected, value) in values.iter().zip(array.iter()) {
assert_eq!(*expected, value);
}
}
}
python/vendor/lance-arrow/src/deepcopy.rs
Vendored
+356
View File
@@ -0,0 +1,356 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
use std::sync::Arc;
use arrow_array::{Array, RecordBatch, make_array};
use arrow_buffer::{BooleanBuffer, Buffer, NullBuffer};
use arrow_data::{ArrayData, ArrayDataBuilder, transform::MutableArrayData};
use arrow_schema::DataType;
pub fn deep_copy_buffer(buffer: &Buffer) -> Buffer {
Buffer::from(buffer.as_slice())
}
pub fn deep_copy_nulls(nulls: Option<&NullBuffer>) -> Option<NullBuffer> {
let nulls = nulls?;
let bit_buffer = deep_copy_buffer(nulls.inner().inner());
// SAFETY: `null_count` is taken from the source `NullBuffer`, which already
// upheld `NullBuffer::new_unchecked`'s invariant — the unset-bit count over
// the logical bit slice `[bit_offset, bit_offset + bit_len)`. `NullBuffer::slice`
// adjusts only `BooleanBuffer::bit_offset` / `bit_len` and never byte-advances
// the inner `Buffer`, so `deep_copy_buffer` (which copies the source `Buffer`'s
// `as_slice()` view from byte 0) reproduces the exact bit pattern at the same
// bit offsets; the unset-bit count is therefore preserved. `BooleanBuffer::new`
// panics (does not UB) if `bit_offset + bit_len > 8 * buffer.len()`, and the
// copy has the same length, so that check still passes.
Some(unsafe {
NullBuffer::new_unchecked(
BooleanBuffer::new(bit_buffer, nulls.offset(), nulls.len()),
nulls.null_count(),
)
})
}
pub fn deep_copy_array_data(data: &ArrayData) -> ArrayData {
let data_type = data.data_type().clone();
let len = data.len();
let nulls = deep_copy_nulls(data.nulls());
let offset = data.offset();
let buffers = data
.buffers()
.iter()
.map(deep_copy_buffer)
.collect::<Vec<_>>();
let child_data = data
.child_data()
.iter()
.map(deep_copy_array_data)
.collect::<Vec<_>>();
// SAFETY: `build_unchecked` inherits `ArrayData::new_unchecked`'s contract —
// `(data_type, len, offset, nulls, buffers, child_data)` must form a valid
// Arrow array. This call reproduces `data` structurally: `data_type`, `len`,
// and `offset` are forwarded unchanged; each buffer is replaced by a byte-
// identical copy of its offset-applied `as_slice()` view (the output buffer
// is `MutableBuffer`-allocated, at least as aligned as the source); `nulls`
// is deep-copied with the same bit offset/length and unset-bit count (see
// `deep_copy_nulls`); `child_data` is recursively cloned with the same
// guarantee. Every value-level invariant the source upheld — UTF-8 validity,
// monotonic offsets, in-bounds dictionary indices, run-end monotonicity,
// struct child-length matching — therefore transfers to the copy. If the
// source `ArrayData` was itself constructed via `new_unchecked` with an
// invalid payload, this function faithfully reproduces that invalidity.
unsafe {
ArrayDataBuilder::new(data_type)
.len(len)
.nulls(nulls)
.offset(offset)
.buffers(buffers)
.child_data(child_data)
.build_unchecked()
}
}
pub fn deep_copy_array(array: &dyn Array) -> Arc<dyn Array> {
let data = array.to_data();
let data = deep_copy_array_data(&data);
make_array(data)
}
pub fn deep_copy_batch(batch: &RecordBatch) -> crate::Result<RecordBatch> {
let arrays = batch
.columns()
.iter()
.map(|array| deep_copy_array(array))
.collect::<Vec<_>>();
RecordBatch::try_new(batch.schema(), arrays)
}
/// Deep copy array data, extracting only the sliced portion using MutableArrayData
/// This is the most efficient and correct way to copy just the sliced data
pub fn deep_copy_array_data_sliced(data: &ArrayData) -> ArrayData {
// Use MutableArrayData to efficiently copy just the slice
let mut mutable = MutableArrayData::new(vec![data], false, data.len());
// Which index this takes depends on the layout, because arrow's extenders
// disagree about who applies `ArrayData::offset()`. The bit-packed one adds
// it to the raw values buffer itself, so an offset-applied index there
// reads from twice the offset and runs off the end of the buffer. Every
// other layout either reads through an already-offset-applied view or
// forwards the index to children carrying their own offsets, and takes the
// absolute index it has always been given.
let start = match data.data_type() {
DataType::Boolean => 0,
_ => data.offset(),
};
mutable.extend(0, start, start + data.len());
// Freeze into immutable ArrayData
mutable.freeze()
}
/// Deep copy an array, extracting only the sliced portion using MutableArrayData
pub fn deep_copy_array_sliced(array: &dyn Array) -> Arc<dyn Array> {
let data = array.to_data();
let data = deep_copy_array_data_sliced(&data);
make_array(data)
}
/// Deep copy a RecordBatch, extracting only the sliced portion using MutableArrayData
pub fn deep_copy_batch_sliced(batch: &RecordBatch) -> crate::Result<RecordBatch> {
let arrays = batch
.columns()
.iter()
.map(|array| deep_copy_array_sliced(array))
.collect::<Vec<_>>();
RecordBatch::try_new(batch.schema(), arrays)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch, StringArray};
use arrow_data::ArrayDataBuilder;
use arrow_schema::{DataType, Field, Schema};
#[test]
fn raw_sliced_fixed_size_list_data_keeps_its_child_offset() {
// `ArrayData::slice` records the offset on the parent and leaves the
// child whole -- a shape `FixedSizeListArray::slice` never produces,
// but a legal one this public helper can be handed directly. Arrow's
// extender forwards `start * size` to that unsliced child and adds
// nothing, so the index it gets has to be the absolute one; this is
// here to stop the boolean fix from being generalised over it.
let child = Int32Array::from(vec![10, 11, 20, 21]).to_data();
let field = Arc::new(Field::new_list_field(DataType::Int32, false));
let data = ArrayDataBuilder::new(DataType::FixedSizeList(field, 2))
.len(2)
.add_child_data(child)
.build()
.unwrap();
let sliced = data.slice(1, 1);
let copied = super::deep_copy_array_data_sliced(&sliced);
let copied_child = Int32Array::from(copied.child_data()[0].clone());
assert_eq!(copied_child.values().as_ref(), &[20, 21]);
}
#[test]
fn raw_sliced_struct_of_fixed_size_list_reaches_the_right_values() {
// Slicing raw struct data pushes the window into the struct's children,
// and a fixed-size-list child takes it as its own parent offset with
// its values left whole. Two levels of offset, and the absolute index
// has to remain right through both of them.
let values = Int32Array::from(vec![10, 11, 20, 21, 30, 31]).to_data();
let item = Arc::new(Field::new_list_field(DataType::Int32, false));
let list = ArrayDataBuilder::new(DataType::FixedSizeList(item, 2))
.len(3)
.add_child_data(values)
.build()
.unwrap();
let field = Arc::new(Field::new("l", list.data_type().clone(), false));
let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into()))
.len(3)
.add_child_data(list)
.build()
.unwrap();
let sliced = data.slice(2, 1);
let copied = super::deep_copy_array_data_sliced(&sliced);
let copied_values = Int32Array::from(copied.child_data()[0].child_data()[0].clone());
assert_eq!(copied_values.values().as_ref(), &[30, 31]);
}
#[test]
fn raw_parent_offset_struct_keeps_the_selected_child_row() {
// A struct whose children are whole and whose window is the parent
// offset: arrow's struct extender forwards the index to those children
// untouched, so it has to be the absolute one.
let field = Arc::new(Field::new("a", DataType::Int32, false));
let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into()))
.len(1)
.offset(1)
.add_child_data(Int32Array::from(vec![10, 20]).to_data())
.build()
.unwrap();
let copied = super::deep_copy_array_data_sliced(&data);
let copied_child = Int32Array::from(copied.child_data()[0].clone());
assert_eq!(copied_child.values().as_ref(), &[20]);
}
#[test]
fn sliced_boolean_deep_copy_reads_from_the_slice() {
// A boolean slice keeps its `ArrayData::offset()` -- the buffer cannot
// advance by a fraction of a byte -- so a copy that adds the offset on
// top of what `MutableArrayData` already applies reads past the end of
// the values buffer and panics.
let array = BooleanArray::from(vec![true, false, true, false, true, false, true, false]);
for (offset, len) in [(0usize, 8usize), (1, 7), (3, 5), (7, 1)] {
let sliced = array.slice(offset, len);
let copied = super::deep_copy_array_sliced(&sliced);
let copied = copied.as_any().downcast_ref::<BooleanArray>().unwrap();
let expected: Vec<bool> = (0..len).map(|i| sliced.value(i)).collect();
let actual: Vec<bool> = (0..len).map(|i| copied.value(i)).collect();
assert_eq!(actual, expected, "offset={offset} len={len}");
}
}
#[test]
fn sliced_boolean_deep_copy_keeps_its_nulls() {
let array = BooleanArray::from(vec![
Some(true),
None,
Some(false),
Some(true),
None,
Some(false),
]);
let sliced = array.slice(1, 4);
let copied = super::deep_copy_array_sliced(&sliced);
let copied = copied.as_any().downcast_ref::<BooleanArray>().unwrap();
let expected: Vec<Option<bool>> = (0..sliced.len())
.map(|i| (!sliced.is_null(i)).then(|| sliced.value(i)))
.collect();
let actual: Vec<Option<bool>> = (0..copied.len())
.map(|i| (!copied.is_null(i)).then(|| copied.value(i)))
.collect();
assert_eq!(actual, expected);
}
#[test]
fn test_deep_copy_sliced_array_with_nulls() {
let array = Arc::new(Int32Array::from(vec![
Some(1),
None,
Some(3),
None,
Some(5),
]));
let sliced_array = array.slice(1, 3);
let copied_array = super::deep_copy_array(&sliced_array);
assert_eq!(sliced_array.len(), copied_array.len());
assert_eq!(sliced_array.nulls(), copied_array.nulls());
}
#[test]
fn test_deep_copy_array_data_sliced() {
let array = Int32Array::from((0..1000).collect::<Vec<i32>>());
let sliced = array.slice(100, 10);
let sliced_data = sliced.to_data();
let copied_data = super::deep_copy_array_data_sliced(&sliced_data);
assert_eq!(copied_data.len(), 10);
assert_eq!(copied_data.offset(), 0);
// Verify data correctness
let copied_array = Int32Array::from(copied_data);
for i in 0..10 {
assert_eq!(copied_array.value(i), 100 + i as i32);
}
}
#[test]
fn test_deep_copy_array_sliced() {
let array = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
let sliced = array.slice(1, 3);
let copied = super::deep_copy_array_sliced(&sliced);
assert_eq!(copied.len(), 3);
let copied_int = copied.as_any().downcast_ref::<Int32Array>().unwrap();
assert_eq!(copied_int.value(0), 2);
assert_eq!(copied_int.value(1), 3);
assert_eq!(copied_int.value(2), 4);
}
#[test]
fn test_deep_copy_batch_sliced() {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("name", DataType::Utf8, false),
]));
let id_array = Arc::new(Int32Array::from((0..100).collect::<Vec<i32>>()));
let name_array = Arc::new(StringArray::from(
(0..100)
.map(|i| format!("name_{}", i))
.collect::<Vec<String>>(),
));
let batch = RecordBatch::try_new(
schema,
vec![id_array as Arc<dyn Array>, name_array as Arc<dyn Array>],
)
.unwrap();
let sliced = batch.slice(10, 5);
let copied = super::deep_copy_batch_sliced(&sliced).unwrap();
assert_eq!(copied.num_rows(), 5);
assert_eq!(copied.num_columns(), 2);
// Verify data correctness
let id_col = copied
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.unwrap();
let name_col = copied
.column(1)
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
for i in 0..5 {
assert_eq!(id_col.value(i), 10 + i as i32);
assert_eq!(name_col.value(i), format!("name_{}", 10 + i));
}
}
#[test]
fn test_deep_copy_array_sliced_with_nulls() {
let array = Arc::new(Int32Array::from(vec![
Some(1),
None,
Some(3),
None,
Some(5),
]));
let sliced = array.slice(1, 3); // [None, Some(3), None]
let copied = super::deep_copy_array_sliced(&sliced);
assert_eq!(copied.len(), 3);
assert_eq!(copied.null_count(), 2); // Two nulls in the slice
let copied_int = copied.as_any().downcast_ref::<Int32Array>().unwrap();
assert!(!copied_int.is_valid(0)); // None
assert!(copied_int.is_valid(1)); // Some(3)
assert!(!copied_int.is_valid(2)); // None
assert_eq!(copied_int.value(1), 3);
}
}
python/vendor/lance-arrow/src/floats.rs
Vendored
+292
View File
@@ -0,0 +1,292 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
//! Floats Array
use std::fmt::{Debug, Display};
use std::iter::Sum;
use std::sync::Arc;
use std::{
fmt::Formatter,
ops::{AddAssign, DivAssign},
};
use arrow_array::{
Array, FixedSizeBinaryArray, Float16Array, Float32Array, Float64Array,
types::{Float16Type, Float32Type, Float64Type},
};
use arrow_schema::{DataType, Field};
use half::{bf16, f16};
use num_traits::{AsPrimitive, Bounded, Float, FromPrimitive};
use super::bfloat16::{BFloat16Array, BFloat16Type};
use crate::Result;
use crate::bfloat16::is_bfloat16_field;
/// Float data type.
///
/// This helps differentiate between the different float types,
/// because bf16 is not officially supported [DataType] in arrow-rs.
#[derive(Debug)]
pub enum FloatType {
BFloat16,
Float16,
Float32,
Float64,
}
impl std::fmt::Display for FloatType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Self::BFloat16 => write!(f, "bfloat16"),
Self::Float16 => write!(f, "float16"),
Self::Float32 => write!(f, "float32"),
Self::Float64 => write!(f, "float64"),
}
}
}
/// Try to convert a [DataType] to a [FloatType]. To support bfloat16, always
/// prefer using the `TryFrom<&Field>` implementation.
impl TryFrom<&DataType> for FloatType {
type Error = crate::ArrowError;
fn try_from(value: &DataType) -> Result<Self> {
match *value {
DataType::Float16 => Ok(Self::Float16),
DataType::Float32 => Ok(Self::Float32),
DataType::Float64 => Ok(Self::Float64),
_ => Err(crate::ArrowError::InvalidArgumentError(format!(
"{:?} is not a floating type",
value
))),
}
}
}
impl TryFrom<&Field> for FloatType {
type Error = crate::ArrowError;
fn try_from(field: &Field) -> Result<Self> {
match field.data_type() {
DataType::FixedSizeBinary(2) if is_bfloat16_field(field) => Ok(Self::BFloat16),
_ => Self::try_from(field.data_type()),
}
}
}
/// Trait for float types used in Lance indexes
///
/// This mimics the utilities provided by [`arrow_array::ArrowPrimitiveType`]
/// but applies to all float types (including bfloat16)
pub trait ArrowFloatType: Debug {
type Native: FromPrimitive
+ FloatToArrayType<ArrowType = Self>
+ AsPrimitive<f32>
+ Debug
+ Display;
const FLOAT_TYPE: FloatType;
const MIN: Self::Native;
const MAX: Self::Native;
/// Arrow Float Array Type.
type ArrayType: FloatArray<Self>;
/// Returns empty array of this type.
fn empty_array() -> Self::ArrayType {
<Self::ArrayType as FloatArray<Self>>::from_values(Vec::new())
}
}
/// Trait to be implemented by native types that have a corresponding [`ArrowFloatType`]
/// implementation.
///
/// This helps define what operations are supported by native floats and also helps convert
/// from a native type back to the corresponding Arrow float type.
pub trait FloatToArrayType:
Float
+ Bounded
+ Sum
+ AddAssign<Self>
+ AsPrimitive<f64>
+ AsPrimitive<f32>
+ DivAssign
+ Send
+ Sync
+ Copy
{
/// The corresponding [`ArrowFloatType`] implementation for this native type
type ArrowType: ArrowFloatType<Native = Self>;
}
impl FloatToArrayType for bf16 {
type ArrowType = BFloat16Type;
}
impl FloatToArrayType for f16 {
type ArrowType = Float16Type;
}
impl FloatToArrayType for f32 {
type ArrowType = Float32Type;
}
impl FloatToArrayType for f64 {
type ArrowType = Float64Type;
}
impl ArrowFloatType for BFloat16Type {
type Native = bf16;
const FLOAT_TYPE: FloatType = FloatType::BFloat16;
const MIN: Self::Native = bf16::MIN;
const MAX: Self::Native = bf16::MAX;
type ArrayType = FixedSizeBinaryArray;
}
impl ArrowFloatType for Float16Type {
type Native = f16;
const FLOAT_TYPE: FloatType = FloatType::Float16;
const MIN: Self::Native = f16::MIN;
const MAX: Self::Native = f16::MAX;
type ArrayType = Float16Array;
}
impl ArrowFloatType for Float32Type {
type Native = f32;
const FLOAT_TYPE: FloatType = FloatType::Float32;
const MIN: Self::Native = f32::MIN;
const MAX: Self::Native = f32::MAX;
type ArrayType = Float32Array;
}
impl ArrowFloatType for Float64Type {
type Native = f64;
const FLOAT_TYPE: FloatType = FloatType::Float64;
const MIN: Self::Native = f64::MIN;
const MAX: Self::Native = f64::MAX;
type ArrayType = Float64Array;
}
/// [FloatArray] is a trait that is implemented by all float type arrays
///
/// This is similar to [`arrow_array::PrimitiveArray`] but applies to all float types (including bfloat16)
/// and is implemented as a trait and not a struct
pub trait FloatArray<T: ArrowFloatType + ?Sized>: Array + Clone + 'static {
type FloatType: ArrowFloatType;
/// Returns a reference to the underlying data as a slice.
///
/// # Panics
///
/// Implementations may panic if the array's storage shape does not match
/// the expected element layout. In particular, the `bf16` impl panics if
/// `value_length() != 2` (the `FixedSizeBinary(2)` shape required by
/// `BFloat16Array`).
///
/// # Preconditions
///
/// Implementations may impose additional invariants on the underlying
/// buffer. The `bf16` impl requires the value buffer to be at least
/// 2-byte aligned — satisfied automatically by every in-tree Lance
/// constructor, but external callers passing externally-built arrays
/// (FFI, IPC, `Buffer::from_custom_allocation`) must ensure alignment.
/// See the impl's docstring for details.
fn as_slice(&self) -> &[T::Native];
/// Construct an array from a vector of values.
fn from_values(values: Vec<T::Native>) -> Self;
/// Construct an array from an iterator of values.
fn from_iter_values(values: impl IntoIterator<Item = T::Native>) -> Self
where
Self: Sized,
{
Self::from_values(values.into_iter().collect())
}
}
impl FloatArray<Float16Type> for Float16Array {
type FloatType = Float16Type;
fn as_slice(&self) -> &[<Float16Type as ArrowFloatType>::Native] {
self.values()
}
fn from_values(values: Vec<<Float16Type as ArrowFloatType>::Native>) -> Self {
Self::from(values)
}
}
impl FloatArray<Float32Type> for Float32Array {
type FloatType = Float32Type;
fn as_slice(&self) -> &[<Float32Type as ArrowFloatType>::Native] {
self.values()
}
fn from_values(values: Vec<<Float32Type as ArrowFloatType>::Native>) -> Self {
Self::from(values)
}
}
impl FloatArray<Float64Type> for Float64Array {
type FloatType = Float64Type;
fn as_slice(&self) -> &[<Float64Type as ArrowFloatType>::Native] {
self.values()
}
fn from_values(values: Vec<<Float64Type as ArrowFloatType>::Native>) -> Self {
Self::from(values)
}
}
/// Convert a float32 array to another float array
///
/// This is used during queries as query vectors are always provided as float32 arrays
/// and need to be converted to the appropriate float type for the index.
pub fn coerce_float_vector(input: &Float32Array, float_type: FloatType) -> Result<Arc<dyn Array>> {
match float_type {
FloatType::BFloat16 => Ok(Arc::new(
BFloat16Array::from_iter_values(input.values().iter().map(|v| bf16::from_f32(*v)))
.into_inner(),
)),
FloatType::Float16 => Ok(Arc::new(Float16Array::from_iter_values(
input.values().iter().map(|v| f16::from_f32(*v)),
))),
FloatType::Float32 => Ok(Arc::new(input.clone())),
FloatType::Float64 => Ok(Arc::new(Float64Array::from_iter_values(
input.values().iter().map(|v| *v as f64),
))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_coerce_float_vector_bfloat16() {
let input = Float32Array::from(vec![1.0f32, 2.0, 3.0]);
let array = coerce_float_vector(&input, FloatType::BFloat16).unwrap();
assert_eq!(array.data_type(), &DataType::FixedSizeBinary(2));
let fixed = array
.as_any()
.downcast_ref::<FixedSizeBinaryArray>()
.unwrap();
let expected: Vec<bf16> = input.values().iter().map(|v| bf16::from_f32(*v)).collect();
assert_eq!(fixed.as_slice(), expected.as_slice());
}
}
python/vendor/lance-arrow/src/ipc.rs
Vendored
+621
View File
@@ -0,0 +1,621 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
//! Zero-copy Arrow IPC stream read/write utilities.
//!
//! Provides helpers for serializing and deserializing [`RecordBatch`]es as
//! self-delimiting Arrow IPC streams using synchronous [`Read`]/[`Write`] I/O.
//!
//! These are designed for embedding IPC streams inside larger binary formats
//! (e.g. a cache entry that contains multiple IPC sections). Each stream is
//! self-delimiting (schema + batches + EOS marker) and can be read back
//! independently.
//!
//! # Zero-copy reads
//!
//! [`read_ipc_stream`] and [`read_ipc_stream_single`] take `&Bytes` and use
//! [`Bytes::slice`] to produce each message buffer. Because `Bytes::slice`
//! increments a reference count rather than copying, the resulting
//! [`Buffer`]s — and the array data decoded from them by [`FileDecoder`] —
//! are all backed by the same allocation as the input.
use std::io::{Read, Write};
use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_buffer::Buffer;
use arrow_ipc::convert::fb_to_schema;
use arrow_ipc::reader::FileDecoder;
use arrow_ipc::root_as_message;
use arrow_ipc::writer::StreamWriter;
use arrow_schema::ArrowError;
use bytes::Bytes;
// ---------------------------------------------------------------------------
// Length-prefixed byte utilities
// ---------------------------------------------------------------------------
/// Write `data` prefixed by its length as a little-endian `u64`.
///
/// Paired with [`read_len_prefixed_bytes`].
pub fn write_len_prefixed_bytes(writer: &mut dyn Write, data: &[u8]) -> Result<(), ArrowError> {
writer
.write_all(&(data.len() as u64).to_le_bytes())
.map_err(|e| ArrowError::IoError(e.to_string(), e))?;
writer
.write_all(data)
.map_err(|e| ArrowError::IoError(e.to_string(), e))
}
/// Read a byte slice written by [`write_len_prefixed_bytes`].
///
/// Reads an 8-byte little-endian length then exactly that many bytes.
pub fn read_len_prefixed_bytes(reader: &mut dyn Read) -> Result<Vec<u8>, ArrowError> {
let mut len_buf = [0u8; 8];
reader
.read_exact(&mut len_buf)
.map_err(|e| ArrowError::IoError(e.to_string(), e))?;
let len = u64::from_le_bytes(len_buf) as usize;
let mut buf = vec![0u8; len];
reader
.read_exact(&mut buf)
.map_err(|e| ArrowError::IoError(e.to_string(), e))?;
Ok(buf)
}
// ---------------------------------------------------------------------------
// IPC stream utilities
// ---------------------------------------------------------------------------
// 4-byte continuation marker used by modern Arrow IPC streams.
const IPC_CONTINUATION: [u8; 4] = [0xff; 4];
/// Write `batch` as a single-batch Arrow IPC stream to `writer`.
pub fn write_ipc_stream(batch: &RecordBatch, writer: &mut dyn Write) -> Result<(), ArrowError> {
let mut sw = StreamWriter::try_new(&mut *writer, batch.schema_ref())?;
sw.write(batch)?;
sw.finish()
}
/// Write all batches from `iter` as a single Arrow IPC stream to `writer`.
///
/// `iter` must yield at least one batch; the schema is inferred from the first
/// batch. Returns `ArrowError::InvalidArgumentError` if the iterator is empty.
/// If you need to write an empty stream (schema only, no rows), construct a
/// `StreamWriter` directly.
pub fn write_ipc_stream_batches<I>(iter: I, writer: &mut dyn Write) -> Result<(), ArrowError>
where
I: IntoIterator<Item = RecordBatch>,
{
let mut iter = iter.into_iter();
let first = iter
.next()
.ok_or_else(|| ArrowError::InvalidArgumentError("no batches to serialize".into()))?;
let mut sw = StreamWriter::try_new(&mut *writer, first.schema_ref())?;
sw.write(&first)?;
for batch in iter {
sw.write(&batch)?;
}
sw.finish()
}
/// Read one complete Arrow IPC stream message from `data` as a zero-copy [`Buffer`].
///
/// Parses the first message starting at byte 0 of `data`. Returns `None` on
/// EOS (size field == 0) or empty input. The returned [`Buffer`] is backed by
/// `data`'s allocation — no bytes are copied.
///
/// The caller should advance its position by `buf.len()` after each call.
fn read_one_ipc_message(data: &Bytes) -> Result<Option<Buffer>, ArrowError> {
let bytes = data.as_ref();
if bytes.is_empty() {
return Ok(None);
}
if bytes.len() < 4 {
return Err(ArrowError::IoError(
"IPC: truncated header".into(),
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC header"),
));
}
let has_continuation = bytes[..4] == IPC_CONTINUATION;
let (size_bytes, prefix_len): ([u8; 4], usize) = if has_continuation {
if bytes.len() < 8 {
return Err(ArrowError::IoError(
"IPC: truncated header after continuation".into(),
std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"truncated after continuation",
),
));
}
(bytes[4..8].try_into().unwrap(), 8)
} else {
(bytes[..4].try_into().unwrap(), 4)
};
let meta_size = u32::from_le_bytes(size_bytes) as usize;
if meta_size == 0 {
return Ok(None); // EOS
}
let meta_end = prefix_len + meta_size;
if bytes.len() < meta_end {
return Err(ArrowError::IoError(
"IPC: truncated metadata".into(),
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC metadata"),
));
}
let msg = root_as_message(&bytes[prefix_len..meta_end])
.map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
let body_len = msg.bodyLength() as usize;
let total = meta_end + body_len;
if bytes.len() < total {
return Err(ArrowError::IoError(
"IPC: truncated body".into(),
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC body"),
));
}
// Zero-copy: Bytes::slice shares the backing allocation; Buffer::from
// wraps it without copying.
Ok(Some(Buffer::from(data.slice(0..total))))
}
/// Read a length-prefixed byte slice at `offset` in `data`, advancing `offset`.
///
/// Reads an 8-byte little-endian length, then slices exactly that many bytes
/// from `data`. The returned [`Bytes`] is zero-copy (shares `data`'s allocation).
pub fn read_len_prefixed_bytes_at(data: &Bytes, offset: &mut usize) -> Result<Bytes, ArrowError> {
let bytes = data.as_ref();
let len_end = offset
.checked_add(8)
.filter(|&e| e <= bytes.len())
.ok_or_else(|| {
ArrowError::IoError(
"length-prefixed bytes: truncated length field".into(),
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated length"),
)
})?;
let len = u64::from_le_bytes(bytes[*offset..len_end].try_into().unwrap()) as usize;
*offset = len_end;
let data_end = offset
.checked_add(len)
.filter(|&e| e <= bytes.len())
.ok_or_else(|| {
ArrowError::IoError(
"length-prefixed bytes: truncated data".into(),
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated data"),
)
})?;
let result = data.slice(*offset..data_end);
*offset = data_end;
Ok(result)
}
/// Read all [`RecordBatch`]es from one Arrow IPC stream starting at `offset`,
/// advancing `offset` past the stream (including the EOS marker).
///
/// Zero-copy: array buffers borrow from `data`'s allocation.
pub fn read_ipc_stream_at(
data: &Bytes,
offset: &mut usize,
) -> Result<Vec<RecordBatch>, ArrowError> {
let batches = read_ipc_stream(&data.slice(*offset..))?;
// Recompute how many bytes were consumed by re-parsing message sizes.
// We can't get this from read_ipc_stream directly, so we re-walk the
// message headers (metadata only, no body re-read) to sum up lengths.
let slice = &data.as_ref()[*offset..];
let mut consumed = 0usize;
loop {
let rem = &slice[consumed..];
if rem.is_empty() {
break;
}
let has_cont = rem.len() >= 4 && rem[..4] == IPC_CONTINUATION;
let (size_bytes, prefix_len): ([u8; 4], usize) = if has_cont {
if rem.len() < 8 {
break;
}
(rem[4..8].try_into().unwrap(), 8)
} else {
if rem.len() < 4 {
break;
}
(rem[..4].try_into().unwrap(), 4)
};
let meta_size = u32::from_le_bytes(size_bytes) as usize;
if meta_size == 0 {
// EOS — consume it and stop.
consumed += prefix_len;
break;
}
let meta_end = prefix_len + meta_size;
if rem.len() < meta_end {
break;
}
let msg = root_as_message(&rem[prefix_len..meta_end])
.map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
let body_len = msg.bodyLength() as usize;
consumed += meta_end + body_len;
}
*offset += consumed;
Ok(batches)
}
/// Read exactly one [`RecordBatch`] from one Arrow IPC stream starting at `offset`,
/// advancing `offset` past the stream (including the EOS marker).
///
/// Zero-copy: array buffers borrow from `data`'s allocation.
pub fn read_ipc_stream_single_at(
data: &Bytes,
offset: &mut usize,
) -> Result<RecordBatch, ArrowError> {
let mut batches = read_ipc_stream_at(data, offset)?;
match batches.len() {
1 => Ok(batches.remove(0)),
n => Err(ArrowError::ParseError(format!(
"expected exactly 1 IPC record batch, got {n}"
))),
}
}
/// Extract the prefix length and metadata size from a raw IPC message buffer.
///
/// Modern IPC streams have an 8-byte prefix `[continuation: 4][size: 4]`.
/// Legacy streams have a 4-byte prefix `[size: 4]`. Returns `(prefix_len, meta_size)`.
fn parse_ipc_message_prefix(buf: &Buffer) -> Result<(usize, usize), ArrowError> {
let has_continuation = buf.len() >= 4 && buf[..4] == IPC_CONTINUATION;
if has_continuation {
if buf.len() < 8 {
return Err(ArrowError::ParseError(
"IPC message buffer too short".into(),
));
}
let meta_size = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize;
Ok((8, meta_size))
} else {
if buf.len() < 4 {
return Err(ArrowError::ParseError(
"IPC message buffer too short".into(),
));
}
let meta_size = u32::from_le_bytes(buf[..4].try_into().unwrap()) as usize;
Ok((4, meta_size))
}
}
/// Read all [`RecordBatch`]es from one Arrow IPC stream.
///
/// Zero-copy: each batch's array data buffers are borrowed from the input
/// message buffer(s) and not copied during decoding.
///
/// Uses [`FileDecoder`] directly (rather than `StreamDecoder`) to avoid a
/// known edge case where `StreamDecoder` does not produce a batch for messages
/// with a zero-length body when the message exactly fills the decode buffer.
pub fn read_ipc_stream(data: &Bytes) -> Result<Vec<RecordBatch>, ArrowError> {
let mut offset = 0usize;
let schema_buf = read_one_ipc_message(&data.slice(offset..))?.ok_or_else(|| {
ArrowError::ParseError("IPC stream: expected schema message, got EOS".into())
})?;
offset += schema_buf.len();
let (prefix_len, meta_size) = parse_ipc_message_prefix(&schema_buf)?;
let schema_msg = root_as_message(&schema_buf[prefix_len..prefix_len + meta_size])
.map_err(|e| ArrowError::ParseError(format!("IPC schema parse error: {e}")))?;
let schema = Arc::new(fb_to_schema(schema_msg.header_as_schema().ok_or_else(
|| ArrowError::ParseError("IPC stream: first message is not a schema".into()),
)?));
let mut decoder = FileDecoder::new(schema, schema_msg.version());
let mut batches = Vec::new();
loop {
let Some(buf) = read_one_ipc_message(&data.slice(offset..))? else {
break;
};
offset += buf.len();
let (prefix_len, meta_size) = parse_ipc_message_prefix(&buf)?;
let msg = root_as_message(&buf[prefix_len..prefix_len + meta_size])
.map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
let body_len = msg.bodyLength() as usize;
// Block offset = 0 since the buffer starts at the message boundary.
// metaDataLength = prefix_len + meta_size (prefix + flatbuf + padding).
let block = arrow_ipc::Block::new(0, (prefix_len + meta_size) as i32, body_len as i64);
match msg.header_type() {
arrow_ipc::MessageHeader::RecordBatch => {
if let Some(batch) = decoder.read_record_batch(&block, &buf)? {
batches.push(batch);
}
}
arrow_ipc::MessageHeader::DictionaryBatch => {
decoder.read_dictionary(&block, &buf)?;
}
_ => break,
}
}
Ok(batches)
}
/// Read exactly one [`RecordBatch`] from one Arrow IPC stream.
pub fn read_ipc_stream_single(data: &Bytes) -> Result<RecordBatch, ArrowError> {
let mut batches = read_ipc_stream(data)?;
match batches.len() {
1 => Ok(batches.remove(0)),
n => Err(ArrowError::ParseError(format!(
"expected exactly 1 IPC record batch, got {n}"
))),
}
}
// ---------------------------------------------------------------------------
// Aligned IPC sections
// ---------------------------------------------------------------------------
/// Byte alignment that each IPC section's stream start is padded to.
///
/// When several IPC streams are concatenated into one larger blob (e.g. a
/// cache entry), a section that starts at an arbitrary offset would leave its
/// array data misaligned. [`FileDecoder`] with `require_alignment = false`
/// then silently copies each buffer into a freshly aligned allocation on
/// every read, defeating zero-copy. Padding each section start to a 64-byte
/// boundary keeps the decoded buffers borrowed directly from the input.
pub const IPC_SECTION_ALIGNMENT: usize = 64;
/// Number of zero-padding bytes needed to advance `pos` to the next
/// [`IPC_SECTION_ALIGNMENT`] boundary.
fn section_padding(pos: usize) -> usize {
(IPC_SECTION_ALIGNMENT - (pos % IPC_SECTION_ALIGNMENT)) % IPC_SECTION_ALIGNMENT
}
/// A [`Write`] adapter that counts the bytes written through it.
struct CountingWriter<'a> {
inner: &'a mut dyn Write,
count: usize,
}
impl Write for CountingWriter<'_> {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let n = self.inner.write(buf)?;
self.count += n;
Ok(n)
}
fn flush(&mut self) -> std::io::Result<()> {
self.inner.flush()
}
}
/// Write zero padding so the next byte lands on an [`IPC_SECTION_ALIGNMENT`]
/// boundary, advancing `pos` past it.
fn write_section_padding(writer: &mut dyn Write, pos: &mut usize) -> Result<(), ArrowError> {
let pad = section_padding(*pos);
if pad > 0 {
const ZEROS: [u8; IPC_SECTION_ALIGNMENT] = [0u8; IPC_SECTION_ALIGNMENT];
writer
.write_all(&ZEROS[..pad])
.map_err(|e| ArrowError::IoError(e.to_string(), e))?;
*pos += pad;
}
Ok(())
}
/// Write `batch` as a 64-byte-aligned single-batch Arrow IPC section.
///
/// `pos` is the absolute byte offset of `writer` within the enclosing blob.
/// Zero padding is written first so the IPC stream begins on an
/// [`IPC_SECTION_ALIGNMENT`] boundary, then the stream itself. `pos` is
/// advanced past both the padding and the stream so the caller can write
/// further aligned sections.
///
/// Paired with [`read_ipc_section_at`]. For the decoded buffers to be borrowed
/// zero-copy, the blob must ultimately be read back from a buffer whose base
/// address is at least 64-byte aligned.
pub fn write_ipc_section(
writer: &mut dyn Write,
pos: &mut usize,
batch: &RecordBatch,
) -> Result<(), ArrowError> {
write_section_padding(writer, pos)?;
let mut counting = CountingWriter {
inner: writer,
count: 0,
};
write_ipc_stream(batch, &mut counting)?;
*pos += counting.count;
Ok(())
}
/// Read a single [`RecordBatch`] from an aligned IPC section at `offset`.
///
/// Skips the alignment padding written by [`write_ipc_section`], then reads
/// the stream, advancing `offset` past the section (padding + stream + EOS).
///
/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
/// address is at least 64-byte aligned (see [`write_ipc_section`]).
pub fn read_ipc_section_at(data: &Bytes, offset: &mut usize) -> Result<RecordBatch, ArrowError> {
*offset += section_padding(*offset);
read_ipc_stream_single_at(data, offset)
}
/// Write `batches` as a single 64-byte-aligned multi-batch Arrow IPC section.
///
/// Like [`write_ipc_section`] but emits every batch from `iter` into one IPC
/// stream (schema + N batches + EOS). `iter` must yield at least one batch.
/// Paired with [`read_ipc_section_batches_at`].
pub fn write_ipc_section_batches<I>(
writer: &mut dyn Write,
pos: &mut usize,
iter: I,
) -> Result<(), ArrowError>
where
I: IntoIterator<Item = RecordBatch>,
{
write_section_padding(writer, pos)?;
let mut counting = CountingWriter {
inner: writer,
count: 0,
};
write_ipc_stream_batches(iter, &mut counting)?;
*pos += counting.count;
Ok(())
}
/// Read all [`RecordBatch`]es from an aligned multi-batch IPC section at
/// `offset`, advancing `offset` past the section (padding + stream + EOS).
///
/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
/// address is at least 64-byte aligned (see [`write_ipc_section_batches`]).
pub fn read_ipc_section_batches_at(
data: &Bytes,
offset: &mut usize,
) -> Result<Vec<RecordBatch>, ArrowError> {
*offset += section_padding(*offset);
read_ipc_stream_at(data, offset)
}
#[cfg(test)]
mod tests {
use arrow_array::{ArrayRef, record_batch};
use super::*;
#[test]
fn test_ipc_roundtrip() {
let batch1 = record_batch!(
("int", Int32, [1, 2, 3]),
("str", Utf8, ["foo", "bar", "baz"])
)
.unwrap();
let batch2 = record_batch!(("int", Int32, [4, 5]), ("str", Utf8, ["qux", "quux"])).unwrap();
let batches = vec![batch1.clone(), batch2.clone()];
let mut buf = Vec::new();
write_ipc_stream_batches(batches, &mut buf).unwrap();
let data = Bytes::from(buf);
let batches = read_ipc_stream(&data).unwrap();
assert_eq!(batches.len(), 2);
assert_eq!(batches[0], batch1);
assert_eq!(batches[1], batch2);
let data_base = data.as_ptr() as usize;
let data_end = data_base + data.len();
let assert_col_zero_copy = |array: &ArrayRef| {
for buffer in array.to_data().buffers() {
let ptr = buffer.as_ptr() as usize;
assert!(
ptr >= data_base && ptr < data_end,
"buffer at {ptr:#x} is not backed by the input Bytes allocation \
[{data_base:#x}..{data_end:#x})"
);
}
};
for batch in &batches {
assert_eq!(batch.schema(), batch1.schema());
assert_col_zero_copy(batch.column(0));
assert_col_zero_copy(batch.column(1));
}
}
/// Allocate a [`Bytes`] whose base address is 64-byte aligned, modelling a
/// backend that reads cache entries into an aligned buffer. A plain
/// `Bytes::from(vec)` only guarantees the allocator's alignment for `u8`.
fn aligned_bytes(payload: &[u8]) -> Bytes {
let mut v = vec![0u8; payload.len() + IPC_SECTION_ALIGNMENT];
let pad = section_padding(v.as_ptr() as usize);
v[pad..pad + payload.len()].copy_from_slice(payload);
Bytes::from(v).slice(pad..pad + payload.len())
}
#[test]
fn test_aligned_ipc_sections_are_zero_copy() {
// A LargeBinary column exercises the i64-offset buffer whose 8-byte
// alignment requirement triggers a realigning memcpy when misaligned.
let blocks = arrow_array::LargeBinaryArray::from_vec(vec![&b"hello"[..], b"world"]);
let section_a = RecordBatch::try_from_iter([("a", Arc::new(blocks) as ArrayRef)]).unwrap();
let section_b = record_batch!(("b", Int64, [10i64, 20, 30, 40, 50])).unwrap();
let mut buf = Vec::new();
// Arbitrary, deliberately non-64-aligned preamble so the first section
// must be padded rather than landing at offset 0 by luck.
buf.extend_from_slice(&[0xABu8; 7]);
let mut pos = buf.len();
// The first section's stream begins after padding the 7-byte preamble
// up to the next 64-byte boundary.
assert_eq!(7 + section_padding(7), IPC_SECTION_ALIGNMENT);
write_ipc_section(&mut buf, &mut pos, &section_a).unwrap();
write_ipc_section(&mut buf, &mut pos, &section_b).unwrap();
let data = aligned_bytes(&buf);
assert_eq!(
section_padding(data.as_ptr() as usize),
0,
"base not aligned"
);
let mut offset = 7;
let read_a = read_ipc_section_at(&data, &mut offset).unwrap();
let read_b = read_ipc_section_at(&data, &mut offset).unwrap();
assert_eq!(read_a, section_a);
assert_eq!(read_b, section_b);
let data_base = data.as_ptr() as usize;
let data_end = data_base + data.len();
for batch in [&read_a, &read_b] {
for buffer in batch.column(0).to_data().buffers() {
let ptr = buffer.as_ptr() as usize;
assert!(
ptr >= data_base && ptr < data_end,
"section buffer at {ptr:#x} was realigned out of the input \
[{data_base:#x}..{data_end:#x}) — misaligned section",
);
}
}
}
#[test]
fn test_aligned_multi_batch_section_roundtrip_zero_copy() {
// A multi-batch section (e.g. IVF SQ storage chunks) must round-trip
// every batch and decode the first batch's buffers zero-copy.
let b1 = record_batch!(("v", Int64, [1i64, 2, 3])).unwrap();
let b2 = record_batch!(("v", Int64, [4i64, 5])).unwrap();
let b3 = record_batch!(("v", Int64, [6i64])).unwrap();
let mut buf = vec![0xCDu8; 5];
let mut pos = buf.len();
write_ipc_section_batches(&mut buf, &mut pos, [b1.clone(), b2.clone(), b3.clone()])
.unwrap();
let data = aligned_bytes(&buf);
let mut offset = 5;
let read = read_ipc_section_batches_at(&data, &mut offset).unwrap();
assert_eq!(read, vec![b1, b2, b3]);
assert_eq!(offset, buf.len(), "offset should land at section end");
let data_base = data.as_ptr() as usize;
let data_end = data_base + data.len();
for buffer in read[0].column(0).to_data().buffers() {
let ptr = buffer.as_ptr() as usize;
assert!(
ptr >= data_base && ptr < data_end,
"first batch buffer at {ptr:#x} was realigned out of the input",
);
}
}
}
python/vendor/lance-arrow/src/json.rs
Vendored
+1538
View File
File diff suppressed because it is too large Load Diff
python/vendor/lance-arrow/src/lib.rs
Vendored
+3102
View File
File diff suppressed because it is too large Load Diff
python/vendor/lance-arrow/src/list.rs
Vendored
+177
View File
@@ -0,0 +1,177 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
use std::sync::Arc;
use arrow_array::{Array, BooleanArray, GenericListArray, OffsetSizeTrait};
use arrow_buffer::{BooleanBufferBuilder, OffsetBuffer, ScalarBuffer};
use arrow_schema::Field;
pub trait ListArrayExt {
/// Filters out masked null items from the list array
///
/// It is legal for a list array to have a null entry with a non-zero length. The
/// values inside the entry are "garbage" and should be ignored. This function
/// filters the values array to remove the garbage values.
///
/// The output list will always have zero-length nulls.
fn filter_garbage_nulls(&self) -> Self;
/// Returns a copy of the list's values array that has been sliced to size
///
/// It is legal for a list array's offsets to not start with zero. It's also legal
/// for a list array's offsets to not extend to the entire values array. This function
/// behaves similarly to `values()` except it slices the array so that it starts at
/// the first list offset and ends at the last list offset.
fn trimmed_values(&self) -> Arc<dyn Array>;
/// The offset type of the underlying list array.
type Offset: OffsetSizeTrait;
/// Returns offsets shifted so the first offset is zero, matching
/// [`Self::trimmed_values`].
///
/// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the
/// original values buffer, so combining them with trimmed values produces
/// offsets that exceed the values length. Use this together with
/// `trimmed_values` when constructing a new list array.
fn trimmed_offsets(&self) -> OffsetBuffer<Self::Offset>;
}
impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize> {
fn filter_garbage_nulls(&self) -> Self {
if self.is_empty() {
return self.clone();
}
let Some(validity) = self.nulls().cloned() else {
return self.clone();
};
let mut should_keep = BooleanBufferBuilder::new(self.values().len());
// Handle case where offsets do not start at 0
let preamble_len = self.offsets().first().unwrap().to_usize().unwrap();
should_keep.append_n(preamble_len, false);
let mut new_offsets: Vec<OffsetSize> = Vec::with_capacity(self.len() + 1);
new_offsets.push(OffsetSize::zero());
let mut cur_len = OffsetSize::zero();
for (offset, is_valid) in self.offsets().windows(2).zip(validity.iter()) {
let len = offset[1] - offset[0];
if is_valid {
cur_len += len;
should_keep.append_n(len.to_usize().unwrap(), true);
new_offsets.push(cur_len);
} else {
should_keep.append_n(len.to_usize().unwrap(), false);
new_offsets.push(cur_len);
}
}
// Offsets may not reference entire values buffer
let trailer = self.values().len() - should_keep.len();
should_keep.append_n(trailer, false);
let should_keep = should_keep.finish();
let should_keep = BooleanArray::new(should_keep, None);
let new_values = arrow_select::filter::filter(self.values(), &should_keep).unwrap();
let new_offsets = ScalarBuffer::from(new_offsets);
let new_offsets = OffsetBuffer::new(new_offsets);
Self::new(
Arc::new(Field::new(
"item",
self.value_type(),
self.values().is_nullable(),
)),
new_offsets,
new_values,
Some(validity),
)
}
fn trimmed_values(&self) -> Arc<dyn Array> {
let first_value = self
.offsets()
.first()
.map(|v| v.to_usize().unwrap())
.unwrap_or(0);
let last_value = self
.offsets()
.last()
.map(|v| v.to_usize().unwrap())
.unwrap_or(0);
self.values().slice(first_value, last_value - first_value)
}
type Offset = OffsetSize;
fn trimmed_offsets(&self) -> OffsetBuffer<OffsetSize> {
let offsets = self.offsets();
let Some(&first) = offsets.first() else {
return offsets.clone();
};
if first == OffsetSize::zero() {
return offsets.clone();
}
let shifted: Vec<OffsetSize> = offsets.iter().map(|&o| o - first).collect();
OffsetBuffer::new(ScalarBuffer::from(shifted))
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_array::{ListArray, UInt64Array};
use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer};
use arrow_schema::{DataType, Field};
use super::ListArrayExt;
#[test]
fn test_filter_garbage_nulls() {
let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
let offsets = ScalarBuffer::<i32>::from(vec![2, 5, 8, 9]);
let offsets = OffsetBuffer::new(offsets);
let list_validity = NullBuffer::new(BooleanBuffer::from(vec![true, false, true]));
let list_arr = ListArray::new(
Arc::new(Field::new("item", DataType::UInt64, true)),
offsets,
Arc::new(items),
Some(list_validity.clone()),
);
let filtered = list_arr.filter_garbage_nulls();
let expected_items = UInt64Array::from(vec![2, 3, 4, 8]);
let offsets = ScalarBuffer::<i32>::from(vec![0, 3, 3, 4]);
let expected = ListArray::new(
Arc::new(Field::new("item", DataType::UInt64, false)),
OffsetBuffer::new(offsets),
Arc::new(expected_items),
Some(list_validity),
);
assert_eq!(filtered, expected);
}
#[test]
fn test_trim_values() {
let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
let offsets = ScalarBuffer::<i32>::from(vec![2, 5, 6, 8, 9]);
let offsets = OffsetBuffer::new(offsets);
let list_arr = ListArray::new(
Arc::new(Field::new("item", DataType::UInt64, true)),
offsets,
Arc::new(items),
None,
);
let list_arr = list_arr.slice(1, 2);
let trimmed = list_arr.trimmed_values();
let expected_items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
let expected_items = expected_items.slice(5, 3);
assert_eq!(trimmed.as_ref(), &expected_items);
}
}
python/vendor/lance-arrow/src/memory.rs
Vendored
+92
View File
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
use std::collections::HashSet;
use arrow_array::{Array, RecordBatch};
use arrow_data::ArrayData;
/// Counts memory used by buffers of Arrow arrays and RecordBatches.
///
/// This is meant to capture how much memory is being used by the Arrow data
/// structures as they are. It does not represent the memory used if the data
/// were to be serialized and then deserialized. In particular:
///
/// * This does not double count memory used by buffers shared by multiple
/// arrays or batches. Round-tripped data may use more memory because of this.
/// * This counts the **total** size of the buffers, even if the array is a slice.
/// Round-tripped data may use less memory because of this.
#[derive(Default)]
pub struct MemoryAccumulator {
seen: HashSet<usize>,
total: usize,
}
impl MemoryAccumulator {
pub fn record_array(&mut self, array: &dyn Array) {
let data = array.to_data();
self.record_array_data(&data);
}
fn record_array_data(&mut self, data: &ArrayData) {
for buffer in data.buffers() {
let ptr = buffer.as_ptr();
if self.seen.insert(ptr as usize) {
self.total += buffer.capacity();
}
}
if let Some(nulls) = data.nulls() {
let null_buf = nulls.inner().inner();
let ptr = null_buf.as_ptr();
if self.seen.insert(ptr as usize) {
self.total += null_buf.capacity();
}
}
for child in data.child_data() {
self.record_array_data(child);
}
}
pub fn record_batch(&mut self, batch: &RecordBatch) {
for array in batch.columns() {
self.record_array(array);
}
}
pub fn total(&self) -> usize {
self.total
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_array::Int32Array;
use arrow_schema::{DataType, Field, Schema};
use super::*;
#[test]
fn test_memory_accumulator() {
let batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
)
.unwrap();
let slice = batch.slice(1, 2);
let mut acc = MemoryAccumulator::default();
// Should record whole buffer, not just slice
acc.record_batch(&slice);
assert_eq!(acc.total(), 3 * std::mem::size_of::<i32>());
// Should not double count
acc.record_batch(&slice);
assert_eq!(acc.total(), 3 * std::mem::size_of::<i32>());
}
}
python/vendor/lance-arrow/src/scalar.rs
Vendored
+302
View File
@@ -0,0 +1,302 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
use arrow_array::{ArrayRef, UInt64Array, make_array};
use arrow_buffer::Buffer;
use arrow_data::ArrayDataBuilder;
use arrow_schema::{ArrowError, DataType};
use arrow_select::take::take;
use crate::DataTypeExt;
type Result<T> = std::result::Result<T, ArrowError>;
pub const INLINE_VALUE_MAX_BYTES: usize = 32;
pub fn extract_scalar_value(array: &ArrayRef, idx: usize) -> Result<ArrayRef> {
if idx >= array.len() {
return Err(ArrowError::InvalidArgumentError(
"Scalar index out of bounds".to_string(),
));
}
take(array.as_ref(), &UInt64Array::from(vec![idx as u64]), None)
}
fn read_u32(buf: &[u8], offset: &mut usize) -> Result<u32> {
if *offset + 4 > buf.len() {
return Err(ArrowError::InvalidArgumentError(
"Invalid scalar value buffer: unexpected EOF".to_string(),
));
}
let bytes = [
buf[*offset],
buf[*offset + 1],
buf[*offset + 2],
buf[*offset + 3],
];
*offset += 4;
Ok(u32::from_le_bytes(bytes))
}
fn read_bytes<'a>(buf: &'a [u8], offset: &mut usize, len: usize) -> Result<&'a [u8]> {
if *offset + len > buf.len() {
return Err(ArrowError::InvalidArgumentError(
"Invalid scalar value buffer: unexpected EOF".to_string(),
));
}
let slice = &buf[*offset..*offset + len];
*offset += len;
Ok(slice)
}
fn write_u32(out: &mut Vec<u8>, v: u32) {
out.extend_from_slice(&v.to_le_bytes());
}
fn write_bytes(out: &mut Vec<u8>, bytes: &[u8]) {
out.extend_from_slice(bytes);
}
pub fn encode_scalar_value_buffer(scalar: &ArrayRef) -> Result<Vec<u8>> {
if scalar.len() != 1 || scalar.null_count() != 0 {
return Err(ArrowError::InvalidArgumentError(
"Scalar value buffer must be a single non-null value".to_string(),
));
}
let data = scalar.to_data();
if data.offset() != 0 {
return Err(ArrowError::InvalidArgumentError(
"Scalar value buffer must have offset=0".to_string(),
));
}
if !data.child_data().is_empty() {
return Err(ArrowError::InvalidArgumentError(
"Scalar value buffer does not support nested types".to_string(),
));
}
// Minimal format (RFC): store the Arrow value buffers for a length-1 array.
// Null bitmap and child data are intentionally not supported here.
//
// | u32 num_buffers |
// | u32 buffer_0_len | ... | u32 buffer_{n-1}_len |
// | buffer_0 bytes | ... | buffer_{n-1} bytes |
let mut out = Vec::with_capacity(128);
let buffers = data.buffers();
write_u32(&mut out, buffers.len() as u32);
for b in buffers {
write_u32(&mut out, b.len() as u32);
}
for b in buffers {
write_bytes(&mut out, b.as_slice());
}
Ok(out)
}
pub fn decode_scalar_from_value_buffer(
data_type: &DataType,
value_buffer: &[u8],
) -> Result<ArrayRef> {
if matches!(
data_type,
DataType::Struct(_) | DataType::FixedSizeList(_, _)
) {
return Err(ArrowError::InvalidArgumentError(format!(
"Scalar value buffer does not support nested data type {:?}",
data_type
)));
}
let mut offset = 0;
let num_buffers = read_u32(value_buffer, &mut offset)? as usize;
let buffer_lens = (0..num_buffers)
.map(|_| read_u32(value_buffer, &mut offset).map(|l| l as usize))
.collect::<Result<Vec<_>>>()?;
let mut buffers = Vec::with_capacity(num_buffers);
for len in buffer_lens {
let bytes = read_bytes(value_buffer, &mut offset, len)?;
buffers.push(Buffer::from_vec(bytes.to_vec()));
}
if offset != value_buffer.len() {
return Err(ArrowError::InvalidArgumentError(
"Invalid scalar value buffer: trailing bytes".to_string(),
));
}
let mut builder = ArrayDataBuilder::new(data_type.clone())
.len(1)
.null_count(0);
for b in buffers {
builder = builder.add_buffer(b);
}
Ok(make_array(builder.build()?))
}
pub fn decode_scalar_from_inline_value(
data_type: &DataType,
inline_value: &[u8],
) -> Result<ArrayRef> {
// I expect our input to be safe here, but I added some debug_assert_eq statements just in case.
// If they are triggered, we may need to change them to return actual errors.
//
// Boolean values are bit-packed in Arrow and therefore are not "fixed-stride" in bytes.
// As a result, `byte_width_opt()` returns `None` for `DataType::Boolean`, even though a
// length-1 scalar can be represented inline using a single byte (matching `try_inline_value`).
if matches!(data_type, DataType::Boolean) {
debug_assert_eq!(
inline_value.len(),
1,
"Invalid boolean inline scalar length (expected 1 byte, got {})",
inline_value.len()
);
} else if let Some(byte_width) = data_type.byte_width_opt() {
debug_assert_eq!(
inline_value.len(),
byte_width,
"Inline constant length mismatch for {:?}: expected {} bytes but got {}",
data_type,
byte_width,
inline_value.len()
);
}
let data = ArrayDataBuilder::new(data_type.clone())
.len(1)
.null_count(0)
.add_buffer(Buffer::from_vec(inline_value.to_vec()))
.build()?;
Ok(make_array(data))
}
pub fn try_inline_value(scalar: &ArrayRef) -> Option<Vec<u8>> {
if scalar.null_count() != 0 || scalar.len() != 1 {
return None;
}
let data = scalar.to_data();
if !data.child_data().is_empty() {
return None;
}
if data.buffers().len() != 1 {
return None;
}
let bytes = data.buffers()[0].as_slice();
if bytes.len() > INLINE_VALUE_MAX_BYTES {
return None;
}
Some(bytes.to_vec())
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_array::{
BooleanArray, DictionaryArray, FixedSizeBinaryArray, Int8Array, Int32Array, StringArray,
cast::AsArray, types::Int8Type,
};
use super::*;
#[test]
fn test_extract_scalar_value() {
let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
let scalar = extract_scalar_value(&array, 2).unwrap();
assert_eq!(scalar.len(), 1);
assert_eq!(
scalar
.as_primitive::<arrow_array::types::Int32Type>()
.value(0),
3
);
}
#[test]
fn test_extract_scalar_value_from_full_dictionary() {
let values = Arc::new(StringArray::from(
(0..=i8::MAX)
.map(|value| format!("value-{value}"))
.collect::<Vec<_>>(),
));
let keys = Int8Array::from((0..=i8::MAX).collect::<Vec<_>>());
let array: ArrayRef = Arc::new(DictionaryArray::<Int8Type>::new(keys, values));
let scalar = extract_scalar_value(&array, i8::MAX as usize).unwrap();
let scalar = scalar.as_dictionary::<Int8Type>();
assert_eq!(scalar.len(), 1);
assert_eq!(scalar.key(0), Some(i8::MAX as usize));
assert_eq!(scalar.values().len(), i8::MAX as usize + 1);
}
#[test]
fn test_scalar_value_buffer_utf8_round_trip() {
let scalar: ArrayRef = Arc::new(StringArray::from(vec!["hello"]));
let buf = encode_scalar_value_buffer(&scalar).unwrap();
let decoded = decode_scalar_from_value_buffer(&DataType::Utf8, &buf).unwrap();
assert_eq!(decoded.len(), 1);
assert_eq!(decoded.null_count(), 0);
assert_eq!(decoded.as_string::<i32>().value(0), "hello");
}
#[test]
fn test_scalar_value_buffer_fixed_size_binary_round_trip() {
let val = vec![0xABu8; 33];
let scalar: ArrayRef = Arc::new(
FixedSizeBinaryArray::try_from_sparse_iter_with_size(
std::iter::once(Some(val.as_slice())),
33,
)
.unwrap(),
);
let buf = encode_scalar_value_buffer(&scalar).unwrap();
let decoded =
decode_scalar_from_value_buffer(&DataType::FixedSizeBinary(33), &buf).unwrap();
assert_eq!(decoded.len(), 1);
assert_eq!(decoded.as_fixed_size_binary().value(0), val.as_slice());
}
#[test]
fn test_inline_value_boolean_round_trip() {
let scalar: ArrayRef = Arc::new(BooleanArray::from_iter([Some(true)]));
let inline = try_inline_value(&scalar).unwrap();
let decoded = decode_scalar_from_inline_value(&DataType::Boolean, &inline).unwrap();
assert_eq!(decoded.len(), 1);
assert_eq!(decoded.null_count(), 0);
assert!(decoded.as_boolean().value(0));
}
#[test]
fn test_scalar_value_buffer_rejects_nested_type() {
let field = Arc::new(arrow_schema::Field::new("item", DataType::Int32, false));
let list: ArrayRef = Arc::new(arrow_array::FixedSizeListArray::new(
field,
2,
Arc::new(Int32Array::from(vec![1, 2])),
None,
));
let scalar = list.slice(0, 1);
assert!(encode_scalar_value_buffer(&scalar).is_err());
}
#[test]
fn test_decode_scalar_from_value_buffer_rejects_nested_type() {
let buf = Vec::<u8>::new();
let res =
decode_scalar_from_value_buffer(&DataType::Struct(arrow_schema::Fields::empty()), &buf);
assert!(res.is_err());
}
#[test]
fn test_decode_scalar_from_value_buffer_trailing_bytes() {
// num_buffers = 0, plus an extra byte
let mut bytes = Vec::new();
bytes.extend_from_slice(&0u32.to_le_bytes());
bytes.push(1);
let res = decode_scalar_from_value_buffer(&DataType::Int32, &bytes);
assert!(res.is_err());
}
}
python/vendor/lance-arrow/src/schema.rs
Vendored
+208
View File
@@ -0,0 +1,208 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
//! Extension to arrow schema
use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema};
use crate::{ARROW_EXT_NAME_KEY, BLOB_META_KEY, BLOB_V2_EXT_NAME};
pub enum Indentation {
OneLine,
MultiLine(u8),
}
impl Indentation {
fn value(&self) -> String {
match self {
Self::OneLine => "".to_string(),
Self::MultiLine(spaces) => " ".repeat(*spaces as usize),
}
}
fn deepen(&self) -> Self {
match self {
Self::OneLine => Self::OneLine,
Self::MultiLine(spaces) => Self::MultiLine(spaces + 2),
}
}
}
/// Extends the functionality of [arrow_schema::Field].
pub trait FieldExt {
/// Create a compact string representation of the field
///
/// This is intended for display purposes and not for serialization
fn to_compact_string(&self, indent: Indentation) -> String;
/// Check if the field is marked as a packed struct
fn is_packed_struct(&self) -> bool;
/// Check if the field is marked as a blob
fn is_blob(&self) -> bool;
/// Check if the field is marked as a blob
fn is_blob_v2(&self) -> bool;
}
impl FieldExt for Field {
fn to_compact_string(&self, indent: Indentation) -> String {
let mut result = format!("{}: ", self.name().clone());
match self.data_type() {
DataType::Struct(fields) => {
result += "{";
result += &indent.value();
for (field_idx, field) in fields.iter().enumerate() {
result += field.to_compact_string(indent.deepen()).as_str();
if field_idx < fields.len() - 1 {
result += ",";
}
result += indent.value().as_str();
}
result += "}";
}
DataType::List(field)
| DataType::LargeList(field)
| DataType::ListView(field)
| DataType::LargeListView(field) => {
result += "[";
result += field.to_compact_string(indent.deepen()).as_str();
result += "]";
}
DataType::FixedSizeList(child, dimension) => {
result += &format!(
"[{}; {}]",
child.to_compact_string(indent.deepen()),
dimension
);
}
DataType::Dictionary(key_type, value_type) => {
result += &value_type.to_string();
result += "@";
result += &key_type.to_string();
}
_ => {
result += &self.data_type().to_string();
}
}
if self.is_nullable() {
result += "?";
}
result
}
// Check if field has metadata `packed` set to true, this check is case insensitive.
fn is_packed_struct(&self) -> bool {
let field_metadata = self.metadata();
const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
PACKED_KEYS.iter().any(|key| {
field_metadata
.get(*key)
.map(|value| value.eq_ignore_ascii_case("true"))
.unwrap_or(false)
})
}
fn is_blob(&self) -> bool {
let field_metadata = self.metadata();
field_metadata.get(BLOB_META_KEY).is_some()
|| field_metadata
.get(ARROW_EXT_NAME_KEY)
.map(|value| value == BLOB_V2_EXT_NAME)
.unwrap_or(false)
}
fn is_blob_v2(&self) -> bool {
let field_metadata = self.metadata();
field_metadata
.get(ARROW_EXT_NAME_KEY)
.map(|value| value == BLOB_V2_EXT_NAME)
.unwrap_or(false)
}
}
/// Extends the functionality of [arrow_schema::Schema].
pub trait SchemaExt {
/// Create a new [`Schema`] with one extra field.
fn try_with_column(&self, field: Field) -> std::result::Result<Schema, ArrowError>;
fn try_with_column_at(
&self,
index: usize,
field: Field,
) -> std::result::Result<Schema, ArrowError>;
fn field_names(&self) -> Vec<&String>;
fn without_column(&self, column_name: &str) -> Schema;
/// Create a compact string representation of the schema
///
/// This is intended for display purposes and not for serialization
fn to_compact_string(&self, indent: Indentation) -> String;
}
impl SchemaExt for Schema {
fn try_with_column(&self, field: Field) -> std::result::Result<Schema, ArrowError> {
if self.column_with_name(field.name()).is_some() {
return Err(ArrowError::SchemaError(format!(
"Can not append column {} on schema: {:?}",
field.name(),
self
)));
};
let mut fields: Vec<FieldRef> = self.fields().iter().cloned().collect();
fields.push(FieldRef::new(field));
Ok(Self::new_with_metadata(fields, self.metadata.clone()))
}
fn try_with_column_at(
&self,
index: usize,
field: Field,
) -> std::result::Result<Schema, ArrowError> {
if self.column_with_name(field.name()).is_some() {
return Err(ArrowError::SchemaError(format!(
"Failed to modify schema: Inserting column {} would create a duplicate column in schema: {:?}",
field.name(),
self
)));
};
let mut fields: Vec<FieldRef> = self.fields().iter().cloned().collect();
fields.insert(index, FieldRef::new(field));
Ok(Self::new_with_metadata(fields, self.metadata.clone()))
}
/// Project the schema to remove the given column.
///
/// This only works on top-level fields right now. If a field does not exist,
/// the schema will be returned as is.
fn without_column(&self, column_name: &str) -> Schema {
let fields: Vec<FieldRef> = self
.fields()
.iter()
.filter(|f| f.name() != column_name)
.cloned()
.collect();
Self::new_with_metadata(fields, self.metadata.clone())
}
fn field_names(&self) -> Vec<&String> {
self.fields().iter().map(|f| f.name()).collect()
}
fn to_compact_string(&self, indent: Indentation) -> String {
let mut result = "{".to_string();
result += &indent.value();
for (field_idx, field) in self.fields.iter().enumerate() {
result += field.to_compact_string(indent.deepen()).as_str();
if field_idx < self.fields.len() - 1 {
result += ",";
}
result += indent.value().as_str();
}
result += "}";
result
}
}
python/vendor/lance-arrow/src/stream.rs
Vendored
+554
View File
@@ -0,0 +1,554 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
//! Utilities for working with streams of [`RecordBatch`].
use arrow_array::RecordBatch;
use arrow_schema::{ArrowError, SchemaRef};
use futures::stream::{self, Stream, StreamExt};
use std::pin::Pin;
use crate::deepcopy::deep_copy_batch_sliced;
/// Rechunks a stream of [`RecordBatch`] so that each output batch has
/// approximately `target_bytes` of array data.
///
/// Small input batches are accumulated (by concatenation) until at least
/// `min_bytes` of data has been collected. If the resulting batch exceeds
/// `max_bytes`, it is sliced into roughly equal pieces of ~`max_bytes`
/// (assuming uniform row sizes).
pub fn rechunk_stream_by_size<S, E>(
input: S,
input_schema: SchemaRef,
min_bytes: usize,
max_bytes: usize,
) -> impl Stream<Item = Result<RecordBatch, E>>
where
S: Stream<Item = Result<RecordBatch, E>>,
E: From<ArrowError>,
{
rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, false)
}
/// Like [`rechunk_stream_by_size`] but deep-copies slices so that
/// `get_array_memory_size` reflects the true size of each output batch.
///
/// After a normal `RecordBatch::slice`, the backing buffers are shared with
/// the original batch, so `get_array_memory_size` still reports the full
/// parent size. This variant deep-copies every slice produced during the
/// splitting phase, which allows the stream to detect and re-split slices
/// that still exceed `max_bytes` (e.g. because a single row is much larger
/// than average).
///
/// The deep copy is a last resort and potentially expensive for large
/// batches. However, it is only performed when a batch actually needs to be
/// sliced — batches that are already within the target range pass through at
/// zero cost. Use this only when the hard cap on `max_bytes` is a
/// correctness requirement, not merely a performance hint.
pub fn rechunk_stream_by_size_deep_copy<S, E>(
input: S,
input_schema: SchemaRef,
min_bytes: usize,
max_bytes: usize,
) -> impl Stream<Item = Result<RecordBatch, E>>
where
S: Stream<Item = Result<RecordBatch, E>>,
E: From<ArrowError>,
{
rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, true)
}
fn rechunk_stream_by_size_inner<S, E>(
input: S,
input_schema: SchemaRef,
min_bytes: usize,
max_bytes: usize,
deep_copy: bool,
) -> impl Stream<Item = Result<RecordBatch, E>>
where
S: Stream<Item = Result<RecordBatch, E>>,
E: From<ArrowError>,
{
stream::try_unfold(
RechunkState {
input: Box::pin(input),
accumulated: Vec::new(),
acc_bytes: 0,
done: false,
input_schema,
min_bytes,
max_bytes,
deep_copy,
},
|mut state| async move {
if state.done && state.accumulated.is_empty() {
return Ok(None);
}
// Pull batches until we reach the byte target or exhaust input.
// Always pull at least one batch so that min_bytes=0 works.
while !state.done && (state.accumulated.is_empty() || state.acc_bytes < state.min_bytes)
{
match state.input.next().await {
Some(Ok(batch)) => {
state.acc_bytes += batch.get_array_memory_size();
state.accumulated.push(batch);
}
Some(Err(e)) => return Err(e),
None => {
state.done = true;
}
}
}
if state.accumulated.is_empty() {
return Ok(None);
}
// Fast path: if the first accumulated batch already meets the
// byte threshold, deliver it directly instead of concatenating
// everything together (which would just get sliced back apart).
if state.accumulated.len() > 1
&& state.accumulated[0].get_array_memory_size() >= state.min_bytes
{
let b = state.accumulated.remove(0);
state.acc_bytes -= b.get_array_memory_size();
return Ok(Some((b, state)));
}
let batch = if state.accumulated.len() == 1 {
state.accumulated.pop().unwrap()
} else {
let b =
arrow_select::concat::concat_batches(&state.input_schema, &state.accumulated)
.map_err(E::from)?;
state.accumulated.clear();
b
};
state.acc_bytes = 0;
// Slice the batch into ~max_bytes pieces assuming uniform row sizes.
let mut slices =
slice_batch(batch, state.max_bytes, state.deep_copy).map_err(E::from)?;
if slices.len() == 1 {
Ok(Some((slices.pop().unwrap(), state)))
} else {
let first = slices.remove(0);
// Stash leftover slices for subsequent iterations.
for a in &slices {
state.acc_bytes += a.get_array_memory_size();
}
state.accumulated = slices;
Ok(Some((first, state)))
}
},
)
}
/// Slice a batch into pieces of at most `max_bytes`.
///
/// When `deep_copy` is false, slices share buffers with the original batch
/// and `get_array_memory_size` will still report the parent buffer size.
/// This is fine when the caller only needs approximate sizing.
///
/// When `deep_copy` is true, each slice is deep-copied so that
/// `get_array_memory_size` reflects the true size. If a deep-copied slice
/// still exceeds `max_bytes` (due to non-uniform row sizes), it is
/// recursively split until every piece is within budget or contains only a
/// single row.
fn slice_batch(
batch: RecordBatch,
max_bytes: usize,
deep_copy: bool,
) -> Result<Vec<RecordBatch>, ArrowError> {
let batch_bytes = batch.get_array_memory_size();
let num_rows = batch.num_rows();
if batch_bytes <= max_bytes {
return Ok(vec![batch]);
}
if num_rows <= 1 {
// A single row cannot be split further, but the size just measured may
// belong to its source buffer rather than to the row:
// `get_array_memory_size` reports whole buffers, and a slice shares
// them. Returning it uncopied hands the caller that inflated figure --
// a 100 KB row inside a 200 MB buffer measures as 200 MB, and a caller
// budgeting on it concludes the row cannot fit anywhere.
if deep_copy {
return Ok(vec![deep_copy_batch_sliced(&batch)?]);
}
return Ok(vec![batch]);
}
let rows_per_chunk = (max_bytes as u64 * num_rows as u64 / batch_bytes as u64).max(1) as usize;
let mut result = Vec::new();
let mut offset = 0;
while offset < num_rows {
let len = rows_per_chunk.min(num_rows - offset);
let slice = batch.slice(offset, len);
if deep_copy {
let copied = deep_copy_batch_sliced(&slice)?;
// Recurse: the deep-copied slice has accurate sizes, so if it
// still exceeds max_bytes we can split further.
result.extend(slice_batch(copied, max_bytes, true)?);
} else {
result.push(slice);
}
offset += len;
}
Ok(result)
}
/// Internal state for [`rechunk_stream`].
///
/// Kept as a named struct so the `try_unfold` closure stays readable.
struct RechunkState<S> {
input: Pin<Box<S>>,
accumulated: Vec<RecordBatch>,
acc_bytes: usize,
done: bool,
input_schema: SchemaRef,
min_bytes: usize,
max_bytes: usize,
deep_copy: bool,
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use arrow_array::Int32Array;
use arrow_schema::{DataType, Field, Schema};
use futures::executor::block_on;
/// A single row that shares a large source buffer must report its own
/// size, not the buffer's. Callers budget on this figure, and an inflated
/// one makes a small row look impossible to place.
#[test]
fn a_single_row_slice_reports_its_own_size() {
use arrow_array::LargeStringArray;
let row = "x".repeat(100 * 1024);
let values: Vec<&str> = (0..2000).map(|_| row.as_str()).collect();
let schema = Arc::new(Schema::new(vec![Field::new(
"text",
DataType::LargeUtf8,
false,
)]));
let batch =
RecordBatch::try_new(schema, vec![Arc::new(LargeStringArray::from(values))]).unwrap();
let whole = batch.get_array_memory_size();
let one_row = batch.slice(7, 1);
assert_eq!(
one_row.get_array_memory_size(),
whole,
"a slice shares its source buffer, which is the reason this test exists"
);
// Budget far below the row so the split recurses to the single-row floor.
let chunks = slice_batch(one_row, 1024, true).unwrap();
assert_eq!(chunks.len(), 1);
let measured = chunks[0].get_array_memory_size();
assert_eq!(chunks[0].num_rows(), 1);
assert!(
measured < whole / 100,
"single row still measured as {measured} bytes against a {whole} byte source"
);
}
fn make_batch(num_rows: usize) -> RecordBatch {
let schema = test_schema();
let values: Vec<i32> = (0..num_rows as i32).collect();
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(values))]).unwrap()
}
fn test_schema() -> SchemaRef {
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]))
}
fn collect_rechunked(
batches: Vec<RecordBatch>,
min_bytes: usize,
max_bytes: usize,
) -> Vec<RecordBatch> {
let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>));
let rechunked = rechunk_stream_by_size(input, test_schema(), min_bytes, max_bytes);
block_on(rechunked.collect::<Vec<_>>())
.into_iter()
.map(|r| r.unwrap())
.collect()
}
fn total_rows(batches: &[RecordBatch]) -> usize {
batches.iter().map(|b| b.num_rows()).sum()
}
#[test]
fn test_empty_stream() {
let result = collect_rechunked(vec![], 100, 200);
assert!(result.is_empty());
}
#[test]
fn test_single_batch_passthrough() {
let batch = make_batch(100);
let bytes = batch.get_array_memory_size();
// Batch is between min and max — should pass through as-is.
let result = collect_rechunked(vec![batch], bytes / 2, bytes * 2);
assert_eq!(result.len(), 1);
assert_eq!(result[0].num_rows(), 100);
}
#[test]
fn test_small_batches_concatenated() {
let one_batch_bytes = make_batch(10).get_array_memory_size();
let batches: Vec<_> = (0..8).map(|_| make_batch(10)).collect();
// min = 5 batches worth, max = 10 batches worth.
let result = collect_rechunked(batches, one_batch_bytes * 5, one_batch_bytes * 10);
assert_eq!(total_rows(&result), 80);
// Should have been concatenated into fewer batches than the 8 inputs.
assert!(
result.len() < 8,
"expected fewer output batches, got {}",
result.len()
);
}
#[test]
fn test_large_batch_sliced() {
let batch = make_batch(1000);
let bytes = batch.get_array_memory_size();
let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
assert_eq!(total_rows(&result), 1000);
assert!(
result.len() >= 4,
"expected at least 4 slices, got {}",
result.len()
);
}
#[test]
fn test_sliced_leftovers_are_not_recombined() {
// Key test for the fast-path optimisation. When a large batch is
// sliced, leftover slices should be delivered one-at-a-time without
// being concatenated back together. We verify this by checking that
// every output buffer pointer falls inside the original batch's
// allocation (i.e. they are all zero-copy slices, not fresh copies).
let batch = make_batch(1000);
let bytes = batch.get_array_memory_size();
let orig_data = batch.column(0).to_data();
let orig_buf = &orig_data.buffers()[0];
let orig_start = orig_buf.as_ptr() as usize;
let orig_end = orig_start + orig_buf.len();
let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
assert_eq!(total_rows(&result), 1000);
assert!(result.len() >= 4);
for (i, b) in result.iter().enumerate() {
let ptr = b.column(0).to_data().buffers()[0].as_ptr() as usize;
assert!(
ptr >= orig_start && ptr < orig_end,
"slice {i} buffer at {ptr:#x} is outside the original allocation \
[{orig_start:#x}, {orig_end:#x}) — it was re-concatenated"
);
}
}
#[test]
fn test_flush_remainder_on_stream_end() {
// Data below min_bytes should still be flushed when the stream ends.
let batch = make_batch(10);
let bytes = batch.get_array_memory_size();
let result = collect_rechunked(vec![batch], bytes * 100, bytes * 200);
assert_eq!(result.len(), 1);
assert_eq!(result[0].num_rows(), 10);
}
#[test]
fn test_large_then_small_batches() {
// After a large batch is fully drained, subsequent small batches
// should be accumulated normally.
let large = make_batch(1000);
let small_bytes = make_batch(10).get_array_memory_size();
let batches = vec![
large,
make_batch(10),
make_batch(10),
make_batch(10),
make_batch(10),
make_batch(10),
];
let result = collect_rechunked(batches, small_bytes * 3, small_bytes * 100);
assert_eq!(total_rows(&result), 1050);
// The large batch should appear (possibly sliced) followed by
// concatenated small batches, so we should have fewer output batches
// than the 6 inputs.
assert!(result.len() < 6);
}
#[test]
fn test_row_preservation_across_slicing() {
// Verify that every input row appears exactly once in the output
// and in the correct order after slicing.
let batch = make_batch(237); // odd count to exercise remainder slice
let bytes = batch.get_array_memory_size();
let result = collect_rechunked(vec![batch], bytes / 8, bytes / 5);
assert_eq!(total_rows(&result), 237);
let values: Vec<i32> = result
.iter()
.flat_map(|b| {
b.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.unwrap()
.values()
.iter()
.copied()
})
.collect();
let expected: Vec<i32> = (0..237).collect();
assert_eq!(values, expected);
}
#[test]
fn test_min_bytes_zero_still_yields_all_rows() {
// When min_bytes=0, the stream should still yield every batch.
// This is the "chop only, don't coalesce" use case.
let batches: Vec<_> = (0..5).map(|_| make_batch(100)).collect();
let batch_bytes = batches[0].get_array_memory_size();
let result = collect_rechunked(batches, 0, batch_bytes * 2);
assert_eq!(total_rows(&result), 500);
}
#[test]
fn test_min_bytes_zero_slices_oversized() {
// min_bytes=0 with a small max_bytes should still slice large batches.
let batch = make_batch(1000);
let bytes = batch.get_array_memory_size();
let result = collect_rechunked(vec![batch], 0, bytes / 4);
assert_eq!(total_rows(&result), 1000);
assert!(
result.len() >= 4,
"expected at least 4 slices, got {}",
result.len()
);
}
/// Build a batch with one variable-length string column.
/// Every row is `small_size` bytes except the row at index `big_row_idx`
/// which is `big_size` bytes.
fn make_variable_batch(
num_rows: usize,
small_size: usize,
big_row_idx: usize,
big_size: usize,
) -> RecordBatch {
let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
let values: Vec<String> = (0..num_rows)
.map(|i| {
if i == big_row_idx {
"X".repeat(big_size)
} else {
"x".repeat(small_size)
}
})
.collect();
let array = arrow_array::StringArray::from(values);
RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap()
}
fn variable_schema() -> SchemaRef {
Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]))
}
fn collect_rechunked_variable(
batches: Vec<RecordBatch>,
min_bytes: usize,
max_bytes: usize,
) -> Vec<RecordBatch> {
let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>));
let rechunked =
rechunk_stream_by_size_deep_copy(input, variable_schema(), min_bytes, max_bytes);
block_on(rechunked.collect::<Vec<_>>())
.into_iter()
.map(|r| r.unwrap())
.collect()
}
#[test]
fn test_oversized_row_at_end() {
// 100 rows: 99 small (64 bytes each) + 1 large (100KiB) at the end.
let batch = make_variable_batch(100, 64, 99, 100 * 1024);
let max_bytes = 64 * 1024;
let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
assert_eq!(total_rows(&result), 100);
for (i, b) in result.iter().enumerate() {
let size = b.get_array_memory_size();
assert!(
size <= max_bytes || b.num_rows() == 1,
"batch {i} has {size} bytes (max {max_bytes}) and {} rows",
b.num_rows()
);
}
}
#[test]
fn test_oversized_row_at_start() {
// 100 rows: 1 large (100KiB) at the start + 99 small (64 bytes each).
let batch = make_variable_batch(100, 64, 0, 100 * 1024);
let max_bytes = 64 * 1024;
let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
assert_eq!(total_rows(&result), 100);
for (i, b) in result.iter().enumerate() {
let size = b.get_array_memory_size();
assert!(
size <= max_bytes || b.num_rows() == 1,
"batch {i} has {size} bytes (max {max_bytes}) and {} rows",
b.num_rows()
);
}
}
#[test]
fn test_oversized_row_in_middle() {
// 100 rows: 1 large (100KiB) in the middle + 99 small (64 bytes each).
let batch = make_variable_batch(100, 64, 50, 100 * 1024);
let max_bytes = 64 * 1024;
let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
assert_eq!(total_rows(&result), 100);
for (i, b) in result.iter().enumerate() {
let size = b.get_array_memory_size();
assert!(
size <= max_bytes || b.num_rows() == 1,
"batch {i} has {size} bytes (max {max_bytes}) and {} rows",
b.num_rows()
);
}
}
#[test]
fn test_error_propagation() {
let input = stream::iter(vec![
Ok(make_batch(10)),
Err(ArrowError::ComputeError("boom".into())),
Ok(make_batch(10)),
]);
let rechunked = rechunk_stream_by_size(input, test_schema(), 1, usize::MAX);
let results: Vec<Result<RecordBatch, ArrowError>> = block_on(rechunked.collect());
assert!(results.iter().any(|r| r.is_err()));
}
}
python/vendor/lance-arrow/src/struct.rs
Vendored
+227
View File
@@ -0,0 +1,227 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// SPDX-FileCopyrightText: Copyright The Lance Authors
//! Extension to arrow struct arrays
use arrow_array::{Array, StructArray, cast::AsArray, make_array};
use arrow_buffer::NullBuffer;
use arrow_data::{ArrayData, ArrayDataBuilder};
use arrow_schema::{ArrowError, DataType};
pub trait StructArrayExt {
/// Removes the offset / length of the struct array by pushing it into the children
///
/// In arrow-rs when slice is called it recursively slices the children.
/// In arrow-cpp when slice is called it just sets the offset/length of
/// the struct array and leaves the children as-is
///
/// Both are legal approaches (╥﹏╥)
///
/// This method helps reduce complexity by folding into the arrow-rs approach
fn normalize_slicing(&self) -> Result<Self, ArrowError>
where
Self: Sized;
/// Structs are allowed to mask valid items. For example, a struct array might be:
///
/// [ {"items": [1, 2, 3]}, NULL, {"items": NULL}]
///
/// However, the underlying items array might be: [[1, 2, 3], [4, 5], NULL]
///
/// The [4, 5] list is masked out because the struct array is null.
///
/// The struct validity would be [true, false, true] and the list validity would be [true, true, false]
///
/// This method pushes nulls down into all children. In the above example the list validity would become
/// [true, false, false].
///
/// This method is not recursive. If a child is a struct array it will not push that child's nulls down.
///
/// This method does not remove garbage lists. It only updates the validity so a future call to
/// [crate::list::ListArrayExt::filter_garbage_nulls] will remove the garbage lists (without
/// this pushdown it would not)
fn pushdown_nulls(&self) -> Result<Self, ArrowError>
where
Self: Sized;
}
fn normalized_struct_array_data(data: ArrayData) -> Result<ArrayData, ArrowError> {
let parent_offset = data.offset();
let parent_len = data.len();
let modified_children = data
.child_data()
.iter()
.map(|d| {
let d = normalized_struct_array_data(d.clone())?;
let offset = d.offset();
let len = d.len();
if len < parent_len + parent_offset {
return Err(ArrowError::InvalidArgumentError(format!(
"Child array {} has length {} which is less than the parent length {} plus the parent offset {}",
d.data_type(),
len,
parent_len,
parent_offset
)));
}
let new_offset = offset + parent_offset;
d.into_builder().offset(new_offset)
.len(parent_len)
.build()
})
.collect::<Result<Vec<_>, _>>()?;
ArrayDataBuilder::new(data.data_type().clone())
.len(parent_len)
.offset(0)
.buffers(data.buffers().to_vec())
.child_data(modified_children)
.build()
}
impl StructArrayExt for StructArray {
fn normalize_slicing(&self) -> Result<Self, ArrowError>
where
Self: Sized,
{
if self.offset() == 0 && self.columns().iter().all(|c| c.len() == self.len()) {
return Ok(self.clone());
}
let data = normalized_struct_array_data(self.to_data())?;
Ok(Self::from(data))
}
fn pushdown_nulls(&self) -> Result<Self, ArrowError>
where
Self: Sized,
{
let Some(validity) = self.nulls() else {
return Ok(self.clone());
};
let data = self.to_data();
let children = data
.child_data()
.iter()
.map(|c| {
if c.data_type() == &DataType::Null {
// A Null array is already null in every slot, and arrow-rs rejects
// a validity buffer on it.
Ok(c.clone())
} else if let Some(child_validity) = c.nulls() {
let new_validity = child_validity.inner() & validity.inner();
c.clone()
.into_builder()
.nulls(Some(NullBuffer::from(new_validity)))
.build()
} else {
Ok(c.clone()
.into_builder()
.nulls(Some(validity.clone()))
.build()?)
}
})
.collect::<Result<Vec<_>, _>>()?;
let arr = make_array(data.into_builder().child_data(children).build()?);
Ok(arr.as_struct().clone())
}
}
#[cfg(test)]
mod tests {
use arrow_array::{
Array, Int32Array, NullArray, StructArray, cast::AsArray, make_array, types::Int32Type,
};
use arrow_buffer::NullBuffer;
use arrow_schema::{DataType, Field, Fields};
use std::sync::Arc;
use crate::r#struct::StructArrayExt;
#[test]
fn test_normalize_slicing_no_offset() {
let x = Int32Array::from(vec![1, 2, 3]);
let y = Int32Array::from(vec![4, 5, 6]);
let struct_array = StructArray::new(
Fields::from(vec![
Field::new("x", DataType::Int32, true),
Field::new("y", DataType::Int32, true),
]),
vec![Arc::new(x), Arc::new(y)],
None,
);
let normalized = struct_array.normalize_slicing().unwrap();
assert_eq!(normalized, struct_array);
}
#[test]
fn test_arrow_rs_slicing() {
let x = Int32Array::from(vec![1, 2, 3, 4]);
let y = Int32Array::from(vec![5, 6, 7, 8]);
let struct_array = StructArray::new(
Fields::from(vec![
Field::new("x", DataType::Int32, true),
Field::new("y", DataType::Int32, true),
]),
vec![Arc::new(x), Arc::new(y)],
None,
);
// Slicing with arrow-rs propagates the slicing to the children so there should
// be no change needed to the struct array
let sliced = struct_array.slice(1, 2);
let normalized = sliced.normalize_slicing().unwrap();
assert_eq!(normalized, sliced);
}
#[test]
fn test_arrow_cpp_slicing() {
let x = Int32Array::from(vec![1, 2, 3, 4]);
let y = Int32Array::from(vec![5, 6, 7, 8]);
let struct_array = StructArray::new(
Fields::from(vec![
Field::new("x", DataType::Int32, true),
Field::new("y", DataType::Int32, true),
]),
vec![Arc::new(x), Arc::new(y)],
None,
);
let data = struct_array.to_data();
let sliced = data.into_builder().offset(1).len(2).build().unwrap();
let sliced = make_array(sliced);
let normalized = sliced.as_struct().clone().normalize_slicing().unwrap();
assert_eq!(normalized, struct_array.slice(1, 2));
}
#[test]
fn test_pushdown_nulls_with_null_typed_child() {
// `pa.Table.from_pylist([{"s": {"a": 1, "b": None}}, {"s": None}])` infers a
// struct with a Null-typed child. Arrow forbids a validity buffer on a Null
// array, so the struct's nulls must not be pushed into it.
let struct_array = StructArray::new(
Fields::from(vec![
Field::new("a", DataType::Int32, true),
Field::new("b", DataType::Null, true),
]),
vec![
Arc::new(Int32Array::from(vec![1, 2, 3])),
Arc::new(NullArray::new(3)),
],
Some(NullBuffer::from(vec![true, false, true])),
);
let pushed = struct_array.pushdown_nulls().unwrap();
assert_eq!(pushed.nulls(), struct_array.nulls());
assert_eq!(
pushed.column(0).as_primitive::<Int32Type>(),
&Int32Array::from(vec![Some(1), None, Some(3)])
);
assert_eq!(pushed.column(1).data_type(), &DataType::Null);
assert_eq!(pushed.column(1).logical_null_count(), 3);
}
}
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.41.0-beta.10"
version = "0.41.0-beta.13"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/database/listing.rs
+94 -26
View File
@@ -750,26 +750,39 @@ impl ListingDatabase {
uri.push_str(&format!("?{}", query_string));
}
let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params)).await?;
// One name that cannot be dropped must not strand the names after it: a caller
// dropping every table would otherwise be left with a partly dropped database that
// the same call cannot finish. Every name is attempted and the first failure is
// still what the call reports, so a caller matching on the error sees what it saw
// before.
let mut first_error = None;
for name in names {
let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
let full_path = self.base_path.clone().join(dir_name.clone());
commit_handler.delete(&full_path).await?;
let dropped: Result<()> = async {
commit_handler.delete(&full_path).await?;
self.object_store
.remove_dir_all(full_path.clone())
.await
.map_err(|err| match err {
// this error is not lance::Error::DatasetNotFound, as the method
// `remove_dir_all` may be used to remove something not be a dataset
lance::Error::NotFound { .. } => Error::TableNotFound {
name: name.clone(),
source: Box::new(err),
},
_ => Error::from(err),
})?;
self.object_store
.remove_dir_all(full_path.clone())
.await
.map_err(|err| match err {
// this error is not lance::Error::DatasetNotFound, as the method
// `remove_dir_all` may be used to remove something not be a dataset
lance::Error::NotFound { .. } => Error::TableNotFound {
name: name.clone(),
source: Box::new(err),
},
_ => Error::from(err),
})
}
.await;
if let Err(err) = dropped {
first_error.get_or_insert(err);
}
}
Ok(())
first_error.map_or(Ok(()), Err)
}
/// Inherit storage options from the connection into the target map
@@ -930,20 +943,18 @@ impl Database for ListingDatabase {
if !request.namespace_path.is_empty() {
return self.namespace_database().table_names(request).await;
}
let mut f = self
// Only child directories can be tables, the same rule `list_tables` lists by: a
// loose object named like a table is not one, and a name handed back for it is a
// name the caller can neither open nor drop.
let dir_suffix = format!(".{LANCE_EXTENSION}");
let listing = self
.object_store
.read_dir(self.base_path.clone())
.await?
.list_with_delimiter(Some(&self.base_path))
.await?;
let mut f = listing
.common_prefixes
.iter()
.map(Path::new)
.filter(|path| {
let is_lance = path
.extension()
.and_then(|e| e.to_str())
.map(|e| e == LANCE_EXTENSION);
is_lance.unwrap_or(false)
})
.filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
.filter_map(|location| table_name(location, &dir_suffix))
.collect::<Vec<String>>();
f.sort();
if let Some(start_after) = request.start_after {
@@ -1697,6 +1708,63 @@ mod tests {
.unwrap();
assert_eq!(page.tables, vec!["real"]);
// The deprecated name listing answers the same question and so follows the same
// rule: a name it hands back has to be one a caller can open and drop.
#[allow(deprecated)]
let names = db.table_names(TableNamesRequest::default()).await.unwrap();
assert_eq!(names, vec!["real"]);
}
/// Dropping the database drops its tables and leaves everything else alone. A loose
/// `*.lance` object under the database prefix is not a table, so it is neither dropped
/// nor allowed to strand the tables that are.
#[tokio::test]
async fn test_drop_all_tables_leaves_loose_lance_files() {
let (tempdir, db) = setup_database().await;
create_tables(&db, &["a", "b"]).await;
// Sorts between the two tables, so a drop that stops at the first failure would
// leave `b` behind.
let loose = tempdir.path().join("aaa-loose.lance");
std::fs::write(&loose, b"not a table").unwrap();
db.drop_all_tables(&[]).await.unwrap();
#[allow(deprecated)]
let names = db.table_names(TableNamesRequest::default()).await.unwrap();
assert!(names.is_empty(), "a table survived the drop: {names:?}");
assert!(!tempdir.path().join("a.lance").exists());
assert!(!tempdir.path().join("b.lance").exists());
assert!(
loose.exists(),
"a loose object was dropped as if it were a table"
);
}
/// A name that cannot be dropped does not strand the names after it: every name is
/// attempted, and the failure is still what the call reports.
#[tokio::test]
async fn test_drop_tables_attempts_every_name() {
let (tempdir, db) = setup_database().await;
create_tables(&db, &["a", "b"]).await;
let result = db
.drop_tables(vec![
"a".to_string(),
"missing".to_string(),
"b".to_string(),
])
.await;
assert!(
matches!(result, Err(Error::TableNotFound { .. })),
"dropping a table that is not there did not report it: {result:?}"
);
assert!(!tempdir.path().join("a.lance").exists());
assert!(
!tempdir.path().join("b.lance").exists(),
"the table after the failing name was left behind"
);
}
#[tokio::test]
rust/lancedb/src/dataloader/permutation/builder.rs
+5 -1
View File
@@ -316,6 +316,10 @@ impl PermutationBuilder {
} else {
self.sort_by_column(rows, ROW_ID).await?
};
// Bootstrap's output row count (num_samples) differs from its input row count
// (num_rows); every other strategy's output matches its input. The shuffler
// must be told the former, not assume it always equals the latter.
let output_rows = splitter.output_row_count(num_rows);
let split_data = splitter.apply(rows, num_rows).await?;
// Shuffle data if requested
@@ -328,7 +332,7 @@ impl PermutationBuilder {
temp_dir: self.config.temp_dir.clone(),
max_rows_per_file: 10 * 1024 * 1024,
});
shuffler.shuffle(split_data, num_rows).await?
shuffler.shuffle(split_data, output_rows).await?
}
};
rust/lancedb/src/dataloader/permutation/split.rs
+363 -2
View File
@@ -11,6 +11,7 @@ use arrow_schema::{DataType, Field, Schema};
use datafusion_common::hash_utils::{RandomState, create_hashes};
use futures::{StreamExt, TryStreamExt};
use lance_arrow::SchemaExt;
use rand::Rng;
use lance_core::ROW_ID;
@@ -19,7 +20,7 @@ use crate::{
arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
dataloader::{
permutation::shuffle::{Shuffler, ShufflerConfig},
permutation::util::TemporaryDirectory,
permutation::util::{TemporaryDirectory, non_crypto_rng},
},
query::{Query, QueryBase, Select},
};
@@ -76,6 +77,59 @@ pub enum SplitStrategy {
///
/// If this strategy is used then the counts/percentages in the SplitSizes are ignored.
Calculated { calculation: String },
/// Rows are sampled uniformly **with replacement** from the base table (combine with
/// [`PermutationBuilder::with_filter`] to restrict the eligible population), independently
/// per split. Named after "the bootstrap" -- the statistical technique of estimating a
/// dataset's properties by resampling it with replacement -- the same sense `bootstrap=True`
/// has in scikit-learn's bagging classifiers.
///
/// Every other strategy partitions the table: each row goes to exactly one split, exactly
/// once. `Bootstrap` does not partition -- it resamples. The number of output rows is
/// `num_samples`, not the number of input rows: the same row can appear zero, one, or many
/// times, and `num_samples` may be larger or smaller than the (filtered) input row count.
/// Typical uses:
///
/// - **Bagging / ensemble training** (e.g. random forests): each split is an independent
/// bootstrap resample of the training set.
/// - **Oversampling a rare class**: filter to the minority class, then sample it up to
/// whatever count balances it against the majority, instead of exhausting its small set
/// once per epoch.
/// - **Fixed-size training steps independent of dataset size**: `num_samples` is set
/// directly rather than being implicitly `len(dataset)`.
/// - **Weighted sampling without a weight column**: filter to "eligible" rows (e.g. a
/// windowed-sampling workload filtering to valid window-start positions), then bootstrap
/// the filtered set -- an item with more eligible rows is naturally drawn proportionally
/// more often, with no explicit per-row weight needed.
///
/// `sizes` divides `num_samples` across splits (reusing [`SplitSizes`], so `Fixed(n)` means
/// "n splits, each getting `num_samples / n` samples", the same convention `Random` and
/// `Sequential` use). Each split draws its own sample using a seed derived independently
/// from `(seed, split_index)`, so a given split's content does not change if `num_samples`
/// is redistributed across a different number of splits -- the same elastic-determinism
/// property documented for the data loader's other strategies.
///
/// Example: a 5-row base table (`row_id` 0..4), `Bootstrap { num_samples: 8, sizes:
/// Fixed(2), seed: Some(42) }` produced (an actual run, not a hypothetical one):
///
/// ```text
/// row_id split_id
/// 0 0
/// 2 0
/// 4 0
/// 4 0 <- row 4 repeated within split 0
/// 4 1 <- row 4 also drawn in split 1 (independent draw)
/// 4 1
/// 1 1
/// 2 1
/// ```
///
/// 8 output rows from a 5-row table, row 4 drawn 4 times total, row 3 never drawn --
/// exactly the with-replacement behavior the name promises, not an even round-robin.
Bootstrap {
num_samples: u64,
sizes: SplitSizes,
seed: Option<u64>,
},
}
impl SplitStrategy {
@@ -107,6 +161,25 @@ impl SplitStrategy {
}
Self::Sequential { sizes } => sizes.validate(num_rows),
Self::Calculated { .. } => Ok(()),
Self::Bootstrap {
num_samples, sizes, ..
} => {
if num_rows == 0 {
return Err(Error::InvalidInput {
message: "Bootstrap strategy requires at least one eligible row to \
sample from (check the filter, if one is set)"
.to_string(),
});
}
if *num_samples == 0 {
return Err(Error::InvalidInput {
message: "Bootstrap strategy requires num_samples > 0".to_string(),
});
}
// Validated against num_samples, not num_rows: sampling is with replacement,
// so num_samples may exceed the number of eligible rows.
sizes.validate(*num_samples)
}
}
}
}
@@ -314,6 +387,95 @@ impl Splitter {
)))
}
/// Materializes the full (filtered) row id stream -- random access is required since
/// with-replacement sampling can draw any position, including ones already drawn -- then
/// independently samples each split's share with its own derived RNG seed.
async fn apply_bootstrap(
&self,
source: SendableRecordBatchStream,
num_samples: u64,
sizes: &SplitSizes,
seed: Option<u64>,
) -> Result<SendableRecordBatchStream> {
let schema = source.schema();
let row_id_field = schema.field(0).clone();
let batches: Vec<RecordBatch> = source.try_collect().await?;
let row_id_arrays = batches
.iter()
.map(|b| b.column(0).as_ref())
.collect::<Vec<_>>();
let row_ids = arrow::compute::concat(&row_id_arrays)?;
// Sample against however many rows this scan actually returned, not the
// separately computed `num_rows` count checked by `validate` -- that count and
// this scan are two different queries, not pinned to one snapshot, so a
// concurrent append or delete between them could otherwise push
// `random_range` out of `row_ids`'s real bounds.
let num_eligible_rows = row_ids.len() as u64;
if num_eligible_rows == 0 {
return Err(Error::InvalidInput {
message: "Bootstrap strategy found no eligible rows to sample from in \
this scan (the table or its filter may have changed since \
validation)"
.to_string(),
});
}
let split_sample_counts = sizes.to_counts(num_samples);
// `try_reserve_exact` (fallible) rather than `with_capacity` (infallible --
// aborts the whole process via `handle_alloc_error` on failure, which no
// caller can catch) so an unreasonably large `num_samples` becomes a normal,
// catchable error instead of a crash. This bounds the failure mode, not peak
// memory itself: the output is still one fully materialized RecordBatch, not
// streamed in chunks.
let too_large = || Error::InvalidInput {
message: format!(
"Bootstrap strategy's num_samples ({num_samples}) is too large to \
allocate in memory"
),
};
let mut position_indices = Vec::<u64>::new();
position_indices
.try_reserve_exact(num_samples as usize)
.map_err(|_| too_large())?;
let mut split_ids = Vec::<u64>::new();
split_ids
.try_reserve_exact(num_samples as usize)
.map_err(|_| too_large())?;
for (split_idx, &count) in split_sample_counts.iter().enumerate() {
// Derive an independent seed per split (rather than one RNG shared across all
// splits) so a split's sample doesn't depend on how many other splits there are.
let split_seed = seed.map(|s| {
s.wrapping_add(
(split_idx as u64)
.wrapping_mul(0x9E37_79B9_7F4A_7C15)
.wrapping_add(1),
)
});
let mut rng = non_crypto_rng(&split_seed);
for _ in 0..count {
position_indices.push(rng.random_range(0..num_eligible_rows));
split_ids.push(split_idx as u64);
}
}
let indices = UInt64Array::from(position_indices);
let sampled_row_ids = arrow::compute::take(row_ids.as_ref(), &indices, None)?;
let split_id_array: Arc<dyn Array> = Arc::new(UInt64Array::from(split_ids));
let new_schema = Arc::new(Schema::new(vec![
row_id_field,
Field::new(SPLIT_ID_COLUMN, DataType::UInt64, false),
]));
let result_batch =
RecordBatch::try_new(new_schema.clone(), vec![sampled_row_ids, split_id_array])?;
Ok(Box::pin(SimpleRecordBatchStream::new(
futures::stream::once(std::future::ready(Ok(result_batch))),
new_schema,
)))
}
pub async fn apply(
&self,
source: SendableRecordBatchStream,
@@ -357,6 +519,14 @@ impl Splitter {
self.apply_hash(source, split_weights, *discard_weight)
.await
}
SplitStrategy::Bootstrap {
num_samples,
sizes,
seed,
} => {
self.apply_bootstrap(source, *num_samples, sizes, *seed)
.await
}
}
}
@@ -377,7 +547,13 @@ impl Splitter {
pub fn orders_by_split_id(&self) -> bool {
match &self.strategy {
SplitStrategy::Hash { .. } | SplitStrategy::Calculated { .. } => true,
// Bootstrap emits samples grouped split-by-split already (see apply_bootstrap),
// rather than relying on the shared pipeline's sort-by-split-id step: within-split
// draw order is meaningful to callers (e.g. a warmup/measured split within one
// split's stream), and sort stability for ties isn't documented behavior to lean on.
SplitStrategy::Hash { .. }
| SplitStrategy::Calculated { .. }
| SplitStrategy::Bootstrap { .. } => true,
SplitStrategy::NoSplit
| SplitStrategy::Sequential { .. }
// It may be strange but for random we shuffle and then assign splits so the result is
@@ -385,6 +561,26 @@ impl Splitter {
| SplitStrategy::Random { .. } => false,
}
}
/// The row count `apply` produces from `num_rows` input rows. Equal to `num_rows`
/// for every strategy except `Bootstrap`, whose output size is `num_samples`
/// regardless of how many (filtered) input rows it draws from. Callers that need
/// to validate or pre-size `apply`'s output (e.g. the shuffler) must use this
/// instead of assuming the input and output row counts match.
///
/// For `Bootstrap`, this is `sizes.to_counts(num_samples)`'s sum, not
/// `num_samples` itself: `to_counts` can emit less than its input (e.g.
/// `Fixed(n)` when `n` does not divide `num_samples` evenly, or explicit
/// `counts`/`ratios` that do not use every sample) and `apply_bootstrap` only
/// ever emits exactly what `to_counts` returns.
pub fn output_row_count(&self, num_rows: u64) -> u64 {
match &self.strategy {
SplitStrategy::Bootstrap {
num_samples, sizes, ..
} => sizes.to_counts(*num_samples).iter().sum(),
_ => num_rows,
}
}
}
/// Split configuration - either percentages or absolute counts
@@ -807,4 +1003,169 @@ mod tests {
assert_eq!(num_in_split_0, 12); // 24%
assert_eq!(num_in_split_1, 30); // 60%
}
#[tokio::test]
async fn test_bootstrap_split_schema_and_counts() {
let splitter = Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Bootstrap {
num_samples: 1000,
sizes: SplitSizes::Fixed(4),
seed: Some(42),
},
);
let split_batches = splitter
.apply(test_data(), 50)
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let schema = split_batches[0].schema();
let split_batch = concat_batches(&schema, &split_batches).unwrap();
// Schema stays exactly [row_id, split_id], same as every other strategy.
let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
assert_eq!(field_names, vec!["id", SPLIT_ID_COLUMN]);
// Output row count is num_samples, not num_rows (50) -- sampling is with replacement.
assert_eq!(split_batch.num_rows(), 1000);
let split_ids = split_batch.column(1).as_primitive::<UInt64Type>().values();
for split_id in [0u64, 1, 2, 3] {
assert_eq!(split_ids.iter().filter(|v| **v == split_id).count(), 250);
}
// With 1000 draws from only 50 distinct rows, some row id must repeat.
let row_ids = split_batch.column(0).as_primitive::<Int32Type>().values();
let mut seen = std::collections::HashSet::new();
assert!(!row_ids.iter().all(|v| seen.insert(*v)));
}
#[tokio::test]
async fn test_bootstrap_split_seed_is_reproducible() {
let make_splitter = || {
Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Bootstrap {
num_samples: 200,
sizes: SplitSizes::Fixed(2),
seed: Some(7),
},
)
};
let first = make_splitter()
.apply(test_data(), 50)
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let second = make_splitter()
.apply(test_data(), 50)
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let schema = first[0].schema();
let first = concat_batches(&schema, &first).unwrap();
let second = concat_batches(&schema, &second).unwrap();
assert_eq!(first.column(0), second.column(0));
assert_eq!(first.column(1), second.column(1));
}
#[tokio::test]
async fn test_bootstrap_split_respects_filter() {
// Bootstrap is driven through a filter the same way every other strategy is -- the
// base table rows reaching Splitter::apply are already filtered by the time they get
// here, so this exercises the same num_rows=0 guard a fully-filtered-out population
// would hit.
let splitter = Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Bootstrap {
num_samples: 10,
sizes: SplitSizes::Fixed(1),
seed: Some(1),
},
);
let Err(err) = splitter.apply(test_data(), 0).await else {
panic!("expected Bootstrap over zero eligible rows to fail");
};
assert!(err.to_string().contains("at least one eligible row"));
}
fn empty_data() -> SendableRecordBatchStream {
lance_datagen::gen_batch()
.with_seed(Seed::from(42))
.col(ID_COLUMN, lance_datagen::array::step::<Int32Type>())
.into_ldb_stream(RowCount::from(0), BatchCount::from(1))
}
#[test]
fn test_output_row_count() {
// Bootstrap's output row count is independent of the input size -- every
// other strategy's output matches its input. The shuffler relies on this
// (see builder.rs) instead of assuming input and output row counts match.
let bootstrap = Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Bootstrap {
num_samples: 1000,
sizes: SplitSizes::Fixed(4),
seed: Some(42),
},
);
assert_eq!(bootstrap.output_row_count(50), 1000);
// Not num_samples itself: `Fixed(3)` doesn't divide 1000 evenly, so
// `to_counts` emits 333 * 3 = 999, one short of num_samples --
// `apply_bootstrap` only ever emits exactly what `to_counts` returns.
let bootstrap_uneven = Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Bootstrap {
num_samples: 1000,
sizes: SplitSizes::Fixed(3),
seed: Some(42),
},
);
assert_eq!(bootstrap_uneven.output_row_count(50), 999);
let sequential = Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Sequential {
sizes: SplitSizes::Fixed(2),
},
);
assert_eq!(sequential.output_row_count(50), 50);
}
#[tokio::test]
async fn test_bootstrap_rejects_empty_scan_cleanly() {
// The eligibility count `validate` checks and this scan are two separate
// queries against the table, not pinned to one snapshot -- a concurrent
// delete could shrink the real scan to zero rows after `validate` already
// passed on a nonzero count. Calling `apply_bootstrap` directly (bypassing
// `apply`'s `validate`) reproduces exactly that: a scan that returns zero
// rows regardless of what count was validated upstream. It must return a
// clean `Err`, not panic on an out-of-range `take` or an empty-range
// `random_range`.
let splitter = Splitter::new(
TemporaryDirectory::None,
SplitStrategy::Bootstrap {
num_samples: 10,
sizes: SplitSizes::Fixed(1),
seed: Some(1),
},
);
let Err(err) = splitter
.apply_bootstrap(empty_data(), 10, &SplitSizes::Fixed(1), Some(1))
.await
else {
panic!("expected Bootstrap over an empty scan to fail cleanly, not panic");
};
assert!(err.to_string().contains("no eligible rows"));
}
}
rust/lancedb/src/index.rs
+30
View File
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use chrono::{DateTime, Utc};
use lance::index::scalar::IndexDetails;
use scalar::FtsIndexBuilder;
use serde::Deserialize;
use serde_with::skip_serializing_none;
@@ -382,10 +383,38 @@ pub enum IndexType {
// FTS
#[serde(alias = "INVERTED", alias = "Inverted")]
FTS,
/// A registered scalar index plugin's display name, preserved verbatim.
///
/// For example, `Custom("Sketch".into()).to_string()` returns `"Sketch"`.
Custom(String),
/// Catch-all for index types not recognized by this version of LanceDB.
Unknown,
}
impl IndexType {
pub(crate) fn from_description(
description: &dyn lance_index::traits::IndexDescription,
) -> Self {
let index_type = description.index_type().parse().unwrap_or(Self::Unknown);
if index_type != Self::Unknown {
return index_type;
}
let Some(metadata) = description.segments().first() else {
return Self::Unknown;
};
if lance_index::is_system_index(metadata) {
return Self::Unknown;
}
let Some(details) = metadata.index_details.as_ref() else {
return Self::Unknown;
};
IndexDetails(details.clone())
.get_plugin()
.map(|plugin| Self::Custom(plugin.name().to_string()))
.unwrap_or(Self::Unknown)
}
}
impl std::fmt::Display for IndexType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
@@ -405,6 +434,7 @@ impl std::fmt::Display for IndexType {
Self::BloomFilter => write!(f, "BLOOM_FILTER"),
Self::RTree => write!(f, "RTREE"),
Self::FTS => write!(f, "FTS"),
Self::Custom(name) => f.write_str(name),
Self::Unknown => write!(f, "UNKNOWN"),
}
}
rust/lancedb/src/index/scalar.rs
+11
View File
@@ -65,6 +65,17 @@ pub struct FmIndexBuilder {}
/// A ZoneMap index stores min/max summaries for ranges of rows and can
/// accelerate range predicates by pruning zones that cannot match.
///
/// ZoneMap indexes use per-fragment write seeds by default for strings, binary
/// data, and supported types wider than eight bytes. Creating a seed-enabled
/// index activates a table capability immediately, before any version 1 seed
/// is written. A mutation retaining a legacy seed-enabled index also activates
/// it. Older clients cannot read or write the resulting snapshots, so upgrade
/// all readers and writers before creating such an index or mutating a table
/// that already has one. The capability
/// remains set in subsequent snapshots, including after dropping the index or
/// restoring an older version; dropping the index does not restore access for
/// older clients. Opening a table alone does not activate the capability.
///
/// ```
/// use lancedb::{
/// index::{scalar::ZoneMapIndexBuilder, Index},
rust/lancedb/src/job.rs
+71 -9
View File
@@ -101,21 +101,19 @@ impl TerminalResult {
fn decode<T: DeserializeOwned>(self) -> Result<T> {
let value = self.value.ok_or_else(|| match &self.request_id {
Some(request_id) => Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
},
Some(request_id) => remote_job_error(
request_id.clone(),
"successful typed job response did not contain a result".to_string(),
),
None => Error::Runtime {
message: "successful typed job did not contain a result".to_string(),
},
})?;
serde_json::from_value(value).map_err(|error| match self.request_id {
Some(request_id) => Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
Some(request_id) => remote_job_error(
request_id,
status_code: None,
},
format!("failed to parse typed job result: {error}"),
),
None => Error::Runtime {
message: format!("failed to parse typed job result: {error}"),
},
@@ -123,6 +121,28 @@ impl TerminalResult {
}
}
/// Build the error for a failed remote typed-job result.
///
/// The `Error::Http` variant is gated behind the `remote` feature, so without
/// it we fall back to `Error::Runtime`. A `request_id` is only ever `Some`
/// for remote jobs, making the fallback unreachable in practice without the
/// feature.
#[cfg(feature = "remote")]
fn remote_job_error(request_id: String, source: String) -> Error {
Error::Http {
source: source.into(),
request_id,
status_code: None,
}
}
#[cfg(not(feature = "remote"))]
fn remote_job_error(request_id: String, source: String) -> Error {
Error::Runtime {
message: format!("{source} (request_id={request_id})"),
}
}
type ResultDecoder<T> = Arc<dyn Fn(TerminalResult) -> Result<T> + Send + Sync>;
enum JobInner<T> {
@@ -634,4 +654,46 @@ mod tests {
assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. })));
assert_eq!(job.status().await.unwrap(), "cancelled");
}
#[test]
fn decode_missing_result_uses_feature_appropriate_error() {
// A `request_id` is only `Some` for remote jobs. The error for a
// missing result must use `Error::Http` with the `remote` feature and
// fall back to `Error::Runtime` without it (where `Error::Http`
// does not exist). See https://github.com/lancedb/lancedb/issues/4142.
let result = TerminalResult {
value: None,
request_id: Some("req-1".to_string()),
};
let err = result.decode::<Value>().unwrap_err();
#[cfg(feature = "remote")]
assert!(
matches!(err, Error::Http { .. }),
"expected Error::Http, got {err:?}"
);
#[cfg(not(feature = "remote"))]
assert!(
matches!(err, Error::Runtime { .. }),
"expected Error::Runtime, got {err:?}"
);
}
#[test]
fn decode_unparsable_result_uses_feature_appropriate_error() {
let result = TerminalResult {
value: Some(Value::String("not a u64".to_string())),
request_id: Some("req-1".to_string()),
};
let err = result.decode::<u64>().unwrap_err();
#[cfg(feature = "remote")]
assert!(
matches!(err, Error::Http { .. }),
"expected Error::Http, got {err:?}"
);
#[cfg(not(feature = "remote"))]
assert!(
matches!(err, Error::Runtime { .. }),
"expected Error::Runtime, got {err:?}"
);
}
}
rust/lancedb/src/materialized_view.rs
+43 -1
View File
@@ -280,7 +280,7 @@ pub struct MaterializedViewInfo {
/// The backend-independent request used to create a remote materialized view.
#[doc(hidden)]
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq)]
pub struct CreateMaterializedViewRequest {
/// Name of the new view.
pub name: String,
@@ -290,6 +290,10 @@ pub struct CreateMaterializedViewRequest {
pub query: String,
/// Whether to skip the initial population job.
pub with_no_data: bool,
/// Whether the server should refresh the view automatically.
pub auto_refresh_enabled: Option<bool>,
/// Refresh interval in seconds. When absent, the server determines when to refresh.
pub auto_refresh_interval_seconds: Option<f64>,
}
/// Prefix of the internal columns holding source columns a computed column
@@ -1570,6 +1574,8 @@ pub struct CreateMaterializedViewBuilder {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
}
fn source_sql_identifier(
@@ -1603,6 +1609,8 @@ impl CreateMaterializedViewBuilder {
filter: None,
limit: None,
with_no_data: false,
auto_refresh_enabled: None,
auto_refresh_interval_seconds: None,
}
}
@@ -1651,6 +1659,32 @@ impl CreateMaterializedViewBuilder {
self
}
/// Enable or disable automatic refresh on Cloud and Enterprise.
/// When omitted, the server determines whether automatic refresh is enabled.
/// Embedded databases do not support automatic refresh.
pub fn auto_refresh_enabled(mut self, enabled: bool) -> Self {
self.auto_refresh_enabled = Some(enabled);
self
}
/// Set a positive, finite refresh interval in seconds on Cloud and Enterprise.
/// When omitted, the server determines when to refresh.
///
/// ```no_run
/// # #![recursion_limit = "256"]
/// # async fn example(db: &lancedb::Connection) -> lancedb::Result<()> {
/// db.create_materialized_view("recent_orders", "orders")
/// .auto_refresh_enabled(true)
/// .auto_refresh_interval_seconds(300.0)
/// .execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn auto_refresh_interval_seconds(mut self, seconds: f64) -> Self {
self.auto_refresh_interval_seconds = Some(seconds);
self
}
fn query(&self) -> Result<String> {
fn quote(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
@@ -1693,6 +1727,8 @@ impl CreateMaterializedViewBuilder {
namespace_path: self.namespace.clone(),
query: self.query()?,
with_no_data: self.with_no_data,
auto_refresh_enabled: self.auto_refresh_enabled,
auto_refresh_interval_seconds: self.auto_refresh_interval_seconds,
})
.await;
}
@@ -1719,6 +1755,12 @@ impl CreateMaterializedViewBuilder {
}
async fn execute_native(self) -> Result<MaterializedView> {
if self.auto_refresh_enabled == Some(true) || self.auto_refresh_interval_seconds.is_some() {
return Err(Error::NotSupported {
message: "automatic materialized-view refresh requires Cloud or Enterprise"
.to_string(),
});
}
let source = self
.connection
.open_table(&self.source)
rust/lancedb/src/remote/db.rs
+112 -28
View File
@@ -51,10 +51,6 @@ use super::table::RemoteTable;
use super::util::parse_server_version;
use super::{ARROW_STREAM_CONTENT_TYPE, extract_job_id};
fn quote_sql_identifier(identifier: &str) -> String {
format!("\"{}\"", identifier.replace('"', "\"\""))
}
// Request structure for the remote clone table API
#[derive(serde::Serialize)]
struct RemoteCloneTableRequest {
@@ -67,6 +63,15 @@ struct RemoteCloneTableRequest {
is_shallow: Option<bool>,
}
#[derive(serde::Serialize)]
struct RemoteCreateMaterializedViewRequest {
query: String,
#[serde(skip_serializing_if = "Option::is_none")]
auto_refresh_enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
auto_refresh_interval_seconds: Option<f64>,
}
// the versions of the server that we support
// for any new feature that we need to change the SDK behavior, we should bump the server version,
// and add a feature flag as method of `ServerVersion` here.
@@ -877,16 +882,43 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
.to_string(),
});
}
let client = self.sql_client.clone().ok_or_else(|| Error::NotSupported {
message: "SQL is unavailable for this remote database client".to_string(),
if let Some(seconds) = request.auto_refresh_interval_seconds
&& (!seconds.is_finite() || seconds <= 0.0)
{
return Err(Error::InvalidInput {
message: "auto_refresh_interval_seconds must be positive, finite seconds".into(),
});
}
let identifier = build_table_identifier(&request.name, &request.namespace_path)?;
let body = RemoteCreateMaterializedViewRequest {
query: request.query,
auto_refresh_enabled: request.auto_refresh_enabled,
auto_refresh_interval_seconds: request.auto_refresh_interval_seconds,
};
let request = self
.client
.post(&format!("/v1/materialized_view/{identifier}/create"))
.json(&body);
let (request_id, response) = self.client.send(request).await?;
let response = self.client.check_response(&request_id, response).await?;
let status = response.status();
let body = response.text().await.err_to_http(request_id.clone())?;
if status != StatusCode::ACCEPTED {
return Err(Error::Http {
source: "materialized-view creation must return 202 Accepted".into(),
request_id,
status_code: Some(status),
});
}
let job_id = extract_job_id(&body).ok_or_else(|| Error::Http {
source: "materialized-view create response did not contain a valid job_id".into(),
request_id,
status_code: Some(status),
})?;
let namespace = request.namespace_path;
let statement = format!(
"CREATE MATERIALIZED VIEW {} AS {}",
quote_sql_identifier(&request.name),
request.query
);
Ok(client.submit_as_job(statement, namespace, || async { Ok(()) }))
Ok(Job::new(Box::new(RemoteJob::new(
self.client.clone(),
job_id,
))))
}
async fn drop_materialized_view_async(
@@ -1964,21 +1996,73 @@ mod tests {
}
#[tokio::test]
async fn test_create_materialized_view_requires_sql_client() {
let db = super::RemoteDatabase::new_mock(|request| -> http::Response<String> {
panic!("unexpected REST request: {}", request.url().path())
});
let error = db
.create_materialized_view_async(CreateMaterializedViewRequest {
name: "adults".into(),
namespace_path: vec!["analytics".into()],
query: "SELECT age AS \"age\" FROM \"raw\".\"people\" WHERE age >= 18 LIMIT 10"
.into(),
with_no_data: false,
})
.await
.unwrap_err();
assert!(error.to_string().contains("SQL is unavailable"));
async fn test_create_materialized_view_uses_rest_and_preserves_optional_settings() {
for (enabled, interval) in [(None, None), (Some(true), None), (Some(false), Some(0.5))] {
let db = super::RemoteDatabase::new_mock(move |request| {
assert_eq!(request.method(), "POST");
assert_eq!(
request.url().path(),
"/v1/materialized_view/analytics$adults/create"
);
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(body["query"], "SELECT * FROM people");
assert_eq!(
body.get("auto_refresh_enabled"),
enabled.map(serde_json::Value::from).as_ref()
);
assert_eq!(
body.get("auto_refresh_interval_seconds"),
interval.map(serde_json::Value::from).as_ref()
);
http::Response::builder()
.status(202)
.body(serde_json::json!({"job_id": "j1-mv-create"}).to_string())
.unwrap()
});
let job = db
.create_materialized_view_async(CreateMaterializedViewRequest {
name: "adults".into(),
namespace_path: vec!["analytics".into()],
query: "SELECT * FROM people".into(),
with_no_data: false,
auto_refresh_enabled: enabled,
auto_refresh_interval_seconds: interval,
})
.await
.unwrap();
assert_eq!(job.id(), Some("j1-mv-create"));
}
}
#[tokio::test]
async fn test_create_materialized_view_rejects_invalid_acceptance() {
for (status, body) in [
(200, r#"{"job_id":"j1"}"#),
(202, "{}"),
(202, r#"{"job_id":""}"#),
] {
let db = super::RemoteDatabase::new_mock(move |_| {
http::Response::builder().status(status).body(body).unwrap()
});
let error = db
.create_materialized_view_async(CreateMaterializedViewRequest {
name: "adults".into(),
namespace_path: vec![],
query: "SELECT * FROM people".into(),
with_no_data: false,
auto_refresh_enabled: None,
auto_refresh_interval_seconds: None,
})
.await
.err()
.unwrap();
assert!(error.to_string().contains(if status == 200 {
"202 Accepted"
} else {
"job_id"
}));
}
}
#[tokio::test]
rust/lancedb/src/remote/table.rs
+57 -29
View File
@@ -3497,7 +3497,11 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
Ok(result)
}
async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
async fn add_computed_columns(
&self,
columns: &[(String, String)],
options: crate::table::computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
self.check_mutable().await?;
crate::table::computed_columns::ensure_not_function_bound(
self.schema().await?.as_ref(),
@@ -3516,7 +3520,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
},
)
.collect::<Vec<_>>();
let mut body = serde_json::json!({ "new_columns": entries });
let mut body = serde_json::json!({ "new_columns": entries, "auto_refresh_enabled": options.auto_refresh_enabled });
if let Some(interval) = options.auto_refresh_interval_seconds {
body["auto_refresh_interval_seconds"] = serde_json::json!(interval);
}
self.apply_branch_body(&mut body);
let request = self
.client
@@ -3550,6 +3557,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
&self,
application: &crate::function::FunctionApplication,
output_name: Option<&str>,
options: crate::table::computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
self.check_mutable().await?;
let schema = self.schema().await?;
@@ -3570,6 +3578,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
.collect::<Vec<_>>();
let mut body = serde_json::json!({
"new_columns": new_columns,
"auto_refresh_enabled": options.auto_refresh_enabled,
"function": {
"application": plan.application,
"binding_metadata_version": plan.binding_metadata_version,
@@ -3579,6 +3588,9 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
"outputs": plan.outputs,
},
});
if let Some(interval) = options.auto_refresh_interval_seconds {
body["auto_refresh_interval_seconds"] = serde_json::json!(interval);
}
self.apply_branch_body(&mut body);
let request = self
.client
@@ -9263,31 +9275,44 @@ mod tests {
/// client never types the expression itself.
#[tokio::test]
async fn test_add_computed_columns_sends_the_expression() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
"/v1/table/my_table/describe/" => simple_describe_response(),
"/v1/table/my_table/add_columns/" => {
assert_eq!(request.method(), "POST");
let body = request.body().unwrap().as_bytes().unwrap();
let value: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(
value["new_columns"],
serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
);
http::Response::builder()
.status(200)
.body(r#"{"version": 7}"#.to_string())
.unwrap()
}
path => panic!("Unexpected path: {path}"),
});
for interval in [None, Some(300)] {
let table =
Table::new_with_handler("my_table", move |request| match request.url().path() {
"/v1/table/my_table/describe/" => simple_describe_response(),
"/v1/table/my_table/add_columns/" => {
assert_eq!(request.method(), "POST");
let body = request.body().unwrap().as_bytes().unwrap();
let value: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(
value["new_columns"],
serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
);
assert_eq!(value["auto_refresh_enabled"], false);
if let Some(interval) = interval {
assert_eq!(value["auto_refresh_interval_seconds"], interval);
} else {
assert!(value.get("auto_refresh_interval_seconds").is_none());
}
http::Response::builder()
.status(200)
.body(r#"{"version": 7}"#.to_string())
.unwrap()
}
path => panic!("Unexpected path: {path}"),
});
let result = table
.add_columns()
.computed("doubled", "x * 2")
.execute()
.await
.unwrap();
assert_eq!(result.version, 7);
let result = table
.add_columns()
.computed("doubled", "x * 2")
.computed_options(crate::table::computed_columns::ComputedColumnOptions {
auto_refresh_enabled: false,
auto_refresh_interval_seconds: interval,
})
.execute()
.await
.unwrap();
assert_eq!(result.version, 7);
}
}
#[tokio::test]
@@ -9305,10 +9330,11 @@ mod tests {
request.body().unwrap().as_bytes().unwrap(),
)
.unwrap();
let expected: serde_json::Value = serde_json::from_str(include_str!(
let mut expected: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json"
))
.unwrap();
expected["auto_refresh_enabled"] = serde_json::json!(true);
assert_eq!(actual, expected);
http::Response::builder()
.status(200)
@@ -9504,10 +9530,11 @@ mod tests {
request.body().unwrap().as_bytes().unwrap(),
)
.unwrap();
let expected: serde_json::Value = serde_json::from_str(include_str!(
let mut expected: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json"
))
.unwrap();
expected["auto_refresh_enabled"] = serde_json::json!(true);
assert_eq!(actual, expected);
http::Response::builder()
.status(200)
@@ -9550,10 +9577,11 @@ mod tests {
"/v1/table/my_table/add_columns/" => {
let actual: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
let expected: serde_json::Value = serde_json::from_str(include_str!(
let mut expected: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json"
))
.unwrap();
expected["auto_refresh_enabled"] = serde_json::json!(true);
assert_eq!(actual, expected);
http::Response::builder()
.status(200)
rust/lancedb/src/table.rs
+10 -10
View File
@@ -816,6 +816,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
async fn add_computed_columns(
&self,
_columns: &[(String, String)],
_options: computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
Err(Error::NotSupported {
message: "computed columns are not supported on this table type".into(),
@@ -826,6 +827,7 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
&self,
_application: &crate::function::FunctionApplication,
_output_name: Option<&str>,
_options: computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
Err(Error::NotSupported {
message: "Function columns are supported only on LanceDB Cloud and Enterprise".into(),
@@ -3770,8 +3772,12 @@ impl BaseTable for NativeTable {
Ok(result)
}
async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
let result = schema_evolution::execute_declare(self, columns).await?;
async fn add_computed_columns(
&self,
columns: &[(String, String)],
options: computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
let result = schema_evolution::execute_declare(self, columns, options).await?;
self.bump_freshness();
Ok(result)
}
@@ -3817,10 +3823,7 @@ impl BaseTable for NativeTable {
let mut indices: Vec<IndexConfig> = descriptions
.iter()
.filter_map(|idx_desc| {
let index_type: crate::index::IndexType = idx_desc
.index_type()
.parse()
.unwrap_or(crate::index::IndexType::Unknown);
let index_type = crate::index::IndexType::from_description(idx_desc.as_ref());
if index_type == crate::index::IndexType::Unknown {
// Internal or future index types that this version doesn't recognize
// (e.g. Lance's internal FragReuseIndex) are silently excluded from
@@ -3956,10 +3959,7 @@ impl BaseTable for NativeTable {
return Ok(None);
};
let index_type: crate::index::IndexType = description
.index_type()
.parse()
.unwrap_or(crate::index::IndexType::Unknown);
let index_type = crate::index::IndexType::from_description(description.as_ref());
let is_vector = matches!(
index_type,
rust/lancedb/src/table/add_columns.rs
+126 -2
View File
@@ -8,6 +8,7 @@ use std::sync::Arc;
use lance::dataset::NewColumnTransform;
use super::BaseTable;
use super::computed_columns::ComputedColumnOptions;
use super::schema_evolution::AddColumnsResult;
use crate::function::FunctionApplication;
use crate::{Error, Result};
@@ -19,6 +20,7 @@ pub struct AddColumnsBuilder {
computed: Vec<(String, String)>,
function: Option<(FunctionApplication, Option<String>)>,
read_columns: Option<Vec<String>>,
options: Option<ComputedColumnOptions>,
}
impl std::fmt::Debug for AddColumnsBuilder {
@@ -41,6 +43,7 @@ impl AddColumnsBuilder {
computed: Vec::new(),
function: None,
read_columns: None,
options: None,
}
}
@@ -139,6 +142,26 @@ impl AddColumnsBuilder {
self
}
/// Configure automatic refresh for every output declared in this call.
/// Settings are stored with the declaration. They require computed or
/// Function columns; local tables persist them without starting a scheduler.
///
/// ```
/// # use lancedb::{Table, table::computed_columns::ComputedColumnOptions};
/// # async fn declare(table: &Table) -> lancedb::Result<()> {
/// table.add_columns().computed("doubled", "x * 2")
/// .computed_options(ComputedColumnOptions {
/// auto_refresh_enabled: true,
/// auto_refresh_interval_seconds: Some(300),
/// }).execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn computed_options(mut self, options: ComputedColumnOptions) -> Self {
self.options = Some(options);
self
}
/// Add the columns.
pub async fn execute(self) -> Result<AddColumnsResult> {
let Self {
@@ -147,7 +170,18 @@ impl AddColumnsBuilder {
computed,
function,
read_columns,
options,
} = self;
if let Some(options) = options {
options.validate()?;
if computed.is_empty() && function.is_none() {
return Err(Error::InvalidInput {
message: "automatic refresh options require computed or Function columns"
.into(),
});
}
}
let options = options.unwrap_or_default();
let declaration_count = usize::from(!computed.is_empty()) + usize::from(function.is_some());
if transform.is_some() && declaration_count != 0 || declaration_count > 1 {
@@ -179,7 +213,7 @@ impl AddColumnsBuilder {
.into(),
});
}
parent.add_computed_columns(&computed).await
parent.add_computed_columns(&computed, options).await
}
(None, true, Some((application, output_name))) => {
if read_columns.is_some() {
@@ -189,7 +223,7 @@ impl AddColumnsBuilder {
});
}
parent
.add_function_columns(&application, output_name.as_deref())
.add_function_columns(&application, output_name.as_deref(), options)
.await
}
_ => unreachable!("mixed add_columns modes were rejected above"),
@@ -201,6 +235,9 @@ impl AddColumnsBuilder {
mod tests {
use std::sync::Arc;
use super::ComputedColumnOptions;
use crate::table::computed_columns::AUTO_REFRESH_INTERVAL_META_KEY;
use arrow_array::{Int32Array, RecordBatch, record_batch};
use arrow_schema::{DataType, Field, Schema};
use lance::dataset::{BatchUDF, NewColumnTransform};
@@ -214,6 +251,93 @@ mod tests {
conn.create_table(name, batch).execute().await.unwrap()
}
#[tokio::test]
async fn computed_options_commit_with_declaration() {
let table = table_with_two_columns("options").await;
let version = table.version().await.unwrap();
let options = ComputedColumnOptions {
auto_refresh_enabled: false,
auto_refresh_interval_seconds: Some(300),
};
let result = table
.add_columns()
.computed("doubled", "x * 2")
.computed_options(options)
.execute()
.await
.unwrap();
assert_eq!(result.version, version + 1);
let schema = table.schema().await.unwrap();
assert_eq!(
ComputedColumnOptions::from_metadata(
schema.field_with_name("doubled").unwrap().metadata()
)
.unwrap(),
options
);
for interval in [0, u64::MAX] {
let invalid = ComputedColumnOptions {
auto_refresh_interval_seconds: Some(interval),
..options
};
assert!(
table
.add_columns()
.computed("bad", "x")
.computed_options(invalid)
.execute()
.await
.is_err()
);
}
assert!(
table
.add_columns()
.transform(NewColumnTransform::SqlExpressions(vec![(
"once".into(),
"x".into()
)]))
.computed_options(options)
.execute()
.await
.is_err()
);
assert!(
table
.schema()
.await
.unwrap()
.field_with_name("bad")
.is_err()
);
}
#[tokio::test]
async fn omitted_refresh_interval_stays_unset() {
let table = table_with_two_columns("server_interval").await;
let version = table.version().await.unwrap();
let result = table
.add_columns()
.computed("doubled", "x * 2")
.execute()
.await
.unwrap();
assert_eq!(result.version, version + 1);
let schema = table.schema().await.unwrap();
let metadata = schema.field_with_name("doubled").unwrap().metadata();
assert!(!metadata.contains_key(AUTO_REFRESH_INTERVAL_META_KEY));
assert_eq!(
ComputedColumnOptions::from_metadata(metadata).unwrap(),
ComputedColumnOptions::default()
);
assert!(
serde_json::to_value(ComputedColumnOptions::default())
.unwrap()
.get("auto_refresh_interval_seconds")
.is_none()
);
}
#[tokio::test]
async fn test_requires_a_transform() {
let table = table_with_two_columns("no_transform").await;
rust/lancedb/src/table/computed_columns.rs
+100
View File
@@ -55,6 +55,106 @@ pub const EXPRESSION_META_KEY: &str = "computed_column.expression";
/// Field metadata key holding the column's inputs, as a JSON array of names.
pub const INPUTS_META_KEY: &str = "computed_column.inputs";
/// Whether the server may automatically refresh this output.
pub const AUTO_REFRESH_ENABLED_META_KEY: &str = "computed_column.auto_refresh_enabled";
/// Minimum interval between successful automatic refreshes, in seconds.
pub const AUTO_REFRESH_INTERVAL_META_KEY: &str = "computed_column.auto_refresh_interval_seconds";
/// Scheduling settings persisted on each computed output. Local tables store
/// these settings; automatic scheduling requires a Cloud or Enterprise server.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct ComputedColumnOptions {
/// Permit automatic refresh. Manual refresh is always available.
pub auto_refresh_enabled: bool,
/// Minimum seconds after a successful refresh before another is eligible.
/// When omitted, the server determines when to refresh.
#[serde(skip_serializing_if = "Option::is_none")]
pub auto_refresh_interval_seconds: Option<u64>,
}
impl Default for ComputedColumnOptions {
fn default() -> Self {
Self {
auto_refresh_enabled: true,
auto_refresh_interval_seconds: None,
}
}
}
impl ComputedColumnOptions {
/// Reject intervals that cannot be represented by the server scheduler.
pub fn validate(&self) -> Result<()> {
if self
.auto_refresh_interval_seconds
.is_some_and(|interval| interval == 0 || interval.checked_mul(1000).is_none())
{
return Err(Error::InvalidInput {
message: "auto_refresh_interval_seconds must be a positive integer no greater than u64::MAX / 1000".into(),
});
}
Ok(())
}
/// Read scheduling settings, using defaults for omitted properties.
pub fn from_metadata(metadata: &HashMap<String, String>) -> Result<Self> {
let mut options = Self::default();
if let Some(value) = metadata.get(AUTO_REFRESH_ENABLED_META_KEY) {
options.auto_refresh_enabled = value.parse().map_err(|_| Error::InvalidInput {
message: "auto_refresh_enabled must be true or false".into(),
})?;
}
if let Some(value) = metadata.get(AUTO_REFRESH_INTERVAL_META_KEY) {
options.auto_refresh_interval_seconds =
Some(value.parse().map_err(|_| Error::InvalidInput {
message: "auto_refresh_interval_seconds must be a positive integer".into(),
})?);
}
options.validate()?;
Ok(options)
}
/// Attach validated settings to newly declared computed output fields.
pub fn apply_to_schema(&self, schema: &ArrowSchema) -> Result<ArrowSchema> {
self.validate()?;
let fields = schema
.fields()
.iter()
.map(|field| {
if computed_column_from_field(field).is_none()
|| field
.metadata()
.get(FUNCTION_OUTPUT_ORDINAL_META_KEY)
.is_some_and(|ordinal| {
ordinal == &FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL.to_string()
})
{
return field.clone();
}
let mut metadata = field.metadata().clone();
metadata.extend(self.metadata());
Arc::new(field.as_ref().clone().with_metadata(metadata))
})
.collect::<Vec<_>>();
Ok(ArrowSchema::new_with_metadata(
fields,
schema.metadata().clone(),
))
}
/// Field metadata values shared by native and server declaration paths.
pub fn metadata(&self) -> HashMap<String, String> {
let mut metadata = HashMap::from([(
AUTO_REFRESH_ENABLED_META_KEY.into(),
self.auto_refresh_enabled.to_string(),
)]);
if let Some(interval) = self.auto_refresh_interval_seconds {
metadata.insert(AUTO_REFRESH_INTERVAL_META_KEY.into(), interval.to_string());
}
metadata
}
}
/// Field metadata key holding the Function binding identity.
pub const FUNCTION_BINDING_ID_META_KEY: &str = "computed_column.function.binding_id";
rust/lancedb/src/table/query.rs
+55 -1
View File
@@ -671,6 +671,12 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result<NsQueryTableRequest> {
});
}
let k = match q.limit {
Some(limit) => limit as i32,
None if q.full_text_search.is_some() => DEFAULT_TOP_K as i32,
None => i32::MAX,
};
let filter = q.filter.as_ref().map(filter_to_sql).transpose()?;
let columns = match &q.select {
@@ -723,7 +729,7 @@ fn convert_to_namespace_query(query: &AnyQuery) -> Result<NsQueryTableRequest> {
Ok(NsQueryTableRequest {
id: None, // Will be set by caller
vector,
k: q.limit.unwrap_or(10) as i32,
k,
filter,
columns,
prefilter: Some(q.prefilter),
@@ -1073,6 +1079,54 @@ mod tests {
assert!(ns_request.vector.single_vector.as_ref().unwrap().is_empty());
}
#[test]
fn test_convert_to_namespace_query_plain_query_without_limit() {
let any_query = AnyQuery::Query(QueryRequest::default());
let ns_request = convert_to_namespace_query(&any_query).unwrap();
assert_eq!(ns_request.k, i32::MAX);
}
#[test]
fn test_convert_to_namespace_query_plain_query_with_explicit_limit() {
let any_query = AnyQuery::Query(QueryRequest {
limit: Some(10),
..Default::default()
});
let ns_request = convert_to_namespace_query(&any_query).unwrap();
assert_eq!(ns_request.k, 10);
}
#[test]
fn test_convert_to_namespace_query_vector_without_limit() {
let query_vector = Arc::new(Float32Array::from(vec![1.0, 2.0, 3.0, 4.0]));
let any_query = AnyQuery::VectorQuery(VectorQueryRequest {
query_vector: vec![query_vector as ArrayRef],
..Default::default()
});
let ns_request = convert_to_namespace_query(&any_query).unwrap();
assert_eq!(ns_request.k, 10);
}
#[test]
fn test_convert_to_namespace_query_fts_without_limit() {
let any_query = AnyQuery::Query(QueryRequest {
full_text_search: Some(lance_index::scalar::FullTextSearchQuery::new(
"hello world".to_string(),
)),
..Default::default()
});
let ns_request = convert_to_namespace_query(&any_query).unwrap();
assert_eq!(ns_request.k, 10);
}
#[tokio::test]
async fn test_execute_query_local_routing() {
use crate::connect;
rust/lancedb/src/table/schema_evolution.rs
+6
View File
@@ -13,6 +13,7 @@ use lance::dataset::transaction::{Operation, Transaction, UpdateMap, UpdateMapEn
use lance::dataset::{ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use super::computed_columns;
use super::{BaseTable, NativeTable};
@@ -128,6 +129,7 @@ pub(crate) async fn execute_add_columns(
pub(crate) async fn execute_declare(
table: &NativeTable,
columns: &[(String, String)],
options: computed_columns::ComputedColumnOptions,
) -> Result<AddColumnsResult> {
use lance::dataset::mem_wal::DatasetMemWalExt;
@@ -157,6 +159,10 @@ pub(crate) async fn execute_declare(
});
}
let transform = computed_columns::declare(table.schema().await?, columns)?;
let NewColumnTransform::AllNulls(schema) = transform else {
unreachable!("declarations are all-null")
};
let transform = NewColumnTransform::AllNulls(Arc::new(options.apply_to_schema(&schema)?));
commit_add_columns(table, transform, None).await
}
rust/lancedb/tests/nullable_struct_merge.rs
+64
View File
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch, StructArray};
use arrow_buffer::NullBuffer;
use arrow_schema::{DataType, Field, Schema};
use lance_arrow::RecordBatchExt;
#[test]
fn merge_nullable_struct_with_sliced_boolean_child() {
let left_struct = StructArray::new(
vec![Field::new("id", DataType::Int32, false)].into(),
vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef],
None,
);
// BooleanArray retains its offset when sliced. The merged child bitmap must
// cover that offset as well as the four visible rows.
let boolean_child: ArrayRef = Arc::new(BooleanArray::from_iter((0..1030).map(|index| {
if index == 1027 {
None
} else {
Some(index % 2 == 1)
}
})));
let boolean_child = boolean_child.slice(1025, 4);
assert_eq!(boolean_child.offset(), 1025);
let right_struct = StructArray::new(
vec![Field::new("flag", DataType::Boolean, true)].into(),
vec![boolean_child],
Some(NullBuffer::from(vec![true, false, true, true])),
);
let left_batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new(
"nested",
left_struct.data_type().clone(),
true,
)])),
vec![Arc::new(left_struct)],
)
.unwrap();
let right_batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new(
"nested",
right_struct.data_type().clone(),
true,
)])),
vec![Arc::new(right_struct)],
)
.unwrap();
let merged = left_batch.merge(&right_batch).unwrap();
let nested = merged.column_by_name("nested").unwrap();
let nested = nested.as_any().downcast_ref::<StructArray>().unwrap();
let flag = nested.column_by_name("flag").unwrap();
let flag = flag.as_any().downcast_ref::<BooleanArray>().unwrap();
assert_eq!(
flag.iter().collect::<Vec<_>>(),
vec![Some(true), None, None, Some(false)]
);
}
rust/lancedb/tests/registered_scalar_plugin.rs
+66
View File
@@ -0,0 +1,66 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use arrow_array::{RecordBatchIterator, record_batch};
use lance::{Dataset, index::DatasetIndexExt};
use lance_index::scalar::ScalarIndexParams;
use lancedb::index::IndexType;
// MinHashLSH is a registered scalar plugin without a LanceDB IndexType variant.
// It exercises the same metadata-only fallback as a downstream plugin.
#[tokio::test]
async fn list_registered_scalar_plugin_with_its_type_name() {
let directory = tempfile::tempdir().unwrap();
let batch = record_batch!((
"value",
Utf8,
["alpha beta", "beta gamma", "gamma delta", "delta alpha"]
))
.unwrap();
let mut dataset = Dataset::write(
RecordBatchIterator::new([Ok(batch.clone())], batch.schema()),
directory.path().join("test.lance").to_str().unwrap(),
None,
)
.await
.unwrap();
dataset
.create_index_builder(
&["value"],
lance_index::IndexType::Scalar,
&ScalarIndexParams::new("minhashlsh".into()),
)
.name("profile".into())
.train(false)
.await
.unwrap();
let connection = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = connection.open_table("test").execute().await.unwrap();
let indices = table.list_indices().await.unwrap();
assert_eq!(indices.len(), 1);
assert_eq!(indices[0].name, "profile");
assert_eq!(
indices[0].index_type,
IndexType::Custom("MinHashLsh".into())
);
assert_eq!(indices[0].index_type.to_string(), "MinHashLsh");
assert_eq!(indices[0].columns, ["value"]);
assert!(
indices[0]
.type_url
.as_deref()
.unwrap()
.ends_with("MinHashLshIndexDetails")
);
let statistics = table.index_stats("profile").await.unwrap().unwrap();
assert_eq!(
statistics.index_type,
IndexType::Custom("MinHashLsh".into())
);
assert_eq!(statistics.num_indexed_rows, 0);
assert_eq!(statistics.num_unindexed_rows, 4);
}