rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge main into blob concurrency repair

Browse Source 
Preserve descriptor-based blob sizing and version pinning together with shared request permits and finite cursor ranges. Adapt probe-only regression tests to the upstream probe helper.
This commit is contained in:
Gatefixer
2026-10-03 01:44:09 +00:00
parent 28c65514c2 b0c0d8ee92
commit c0c9423f5b
76 changed files with 4730 additions and 2018 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.11"
current_version = "0.40.0-beta.12"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/workflows/nodejs.yml
+65
View File
@@ -164,6 +164,71 @@ jobs:
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
Cargo.lock
Generated
+103 -101
View File
@@ -141,7 +141,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -181,9 +181,9 @@ dependencies = [
[[package]]
name = "arc-swap"
version = "1.9.1"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
dependencies = [
"rustversion",
]
@@ -1110,9 +1110,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.6.3"
version = "1.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9"
checksum = "0b791f3ac597193fe1d08b82366986eb1f5bc31f2ac6c194c0855276116c76cd"
dependencies = [
"base64-simd",
"bytes",
@@ -1819,7 +1819,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3087,7 +3087,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3315,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4538,7 +4538,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2 0.6.3",
"system-configuration",
"tokio",
"tower-service",
@@ -4880,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arc-swap",
"arrow",
@@ -5193,7 +5193,7 @@ dependencies = [
"lance-tokenizer",
"log",
"moka",
"object_store 0.14.1",
"object_store 0.14.2",
"permutation",
"pin-project",
"prost",
@@ -5217,8 +5217,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5240,7 +5240,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5254,7 +5254,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5263,8 +5263,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrayref",
"crunchy",
@@ -5274,8 +5274,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5294,7 +5294,7 @@ dependencies = [
"log",
"moka",
"num_cpus",
"object_store 0.14.1",
"object_store 0.14.2",
"pin-project",
"prost",
"quick_cache",
@@ -5312,8 +5312,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5344,8 +5344,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5362,8 +5362,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"proc-macro2",
"quote",
@@ -5372,8 +5372,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5406,8 +5406,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5428,7 +5428,7 @@ dependencies = [
"lance-io",
"log",
"num-traits",
"object_store 0.14.1",
"object_store 0.14.2",
"prost",
"prost-build",
"prost-types",
@@ -5439,8 +5439,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"datafusion",
"geo-traits",
@@ -5454,8 +5454,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arc-swap",
"arrow",
@@ -5482,6 +5482,7 @@ dependencies = [
"geoarrow-array",
"geoarrow-schema",
"half",
"hashbrown 0.16.1",
"hex",
"itertools 0.14.0",
"jieba-rs",
@@ -5504,7 +5505,7 @@ dependencies = [
"log",
"ndarray",
"num-traits",
"object_store 0.14.1",
"object_store 0.14.2",
"prost",
"prost-build",
"prost-types",
@@ -5525,8 +5526,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5549,8 +5550,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5570,7 +5571,7 @@ dependencies = [
"log",
"metrics",
"moka",
"object_store 0.14.1",
"object_store 0.14.2",
"object_store_opendal",
"opendal",
"opendal-service-goosefs",
@@ -5592,8 +5593,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5607,8 +5608,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"async-trait",
@@ -5622,9 +5623,10 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arc-swap",
"arrow",
"arrow-ipc",
"arrow-schema",
@@ -5645,7 +5647,7 @@ dependencies = [
"lance-namespace",
"lance-table",
"log",
"object_store 0.14.1",
"object_store 0.14.2",
"quick-xml 0.40.1",
"rand 0.9.5",
"reqwest 0.12.28",
@@ -5662,9 +5664,9 @@ dependencies = [
[[package]]
name = "lance-namespace-reqwest-client"
version = "0.12.0"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
checksum = "828f291764feda98898452bea4ce3756b73e5d5c7c67b62edda27456b7cc72c1"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -5676,8 +5678,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5691,8 +5693,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5713,7 +5715,7 @@ dependencies = [
"lance-io",
"lance-select",
"log",
"object_store 0.14.1",
"object_store 0.14.2",
"prost",
"prost-build",
"prost-types",
@@ -5732,8 +5734,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5746,8 +5748,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "13.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5760,7 +5762,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
dependencies = [
"ahash",
"anyhow",
@@ -5824,7 +5826,7 @@ dependencies = [
"moka",
"num-traits",
"oauth2",
"object_store 0.14.1",
"object_store 0.14.2",
"pin-project",
"polars",
"polars-arrow",
@@ -5857,7 +5859,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5882,7 +5884,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
dependencies = [
"arc-swap",
"arrow",
@@ -6473,9 +6475,9 @@ dependencies = [
[[package]]
name = "napi"
version = "3.12.2"
version = "3.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
checksum = "f0c007d4a8ead952a81887661d41fd56b7e7a70d3d4d921f445fb37a8f6efa1e"
dependencies = [
"bitflags 2.11.1",
"chrono",
@@ -6493,15 +6495,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.2"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "860e7c40864f95cfb83cde99f9ebadd88ef3d9bdccd7dd2cee0cc96a2dd4ffa7"
checksum = "0941983559cd62bfa7f04646a16428967a847365adeef27711e4e3927046bbed"
[[package]]
name = "napi-derive"
version = "3.6.5"
version = "3.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350057056a30368aa76c11a0d406b0aa61321710be2c36656ab4f6efe0b785a2"
checksum = "6b1d232c24dc6754979b27efdf2047a5849efc7c92c1245de87e6bedbb9f5c3c"
dependencies = [
"convert_case",
"ctor",
@@ -6513,9 +6515,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.3"
version = "6.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c1c87a71568f3fe5c736b10878ff55064b250bb4ea4b48c8055713e07b9e463"
checksum = "fd6856fe1686867da90ed0dc292d7c0c87a695eb7c578b913360f33619ee01e3"
dependencies = [
"convert_case",
"proc-macro2",
@@ -6526,9 +6528,9 @@ dependencies = [
[[package]]
name = "napi-sys"
version = "3.3.0"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85fbf1fa9f1babfe396d74bbbf52b3643770243e8f5b0b46715d4caf7f0dfc9a"
checksum = "e22a4f25c16a5c5411d987cd6fbd48313522ae1789ade2d6dd3efdc6d40a0fc8"
dependencies = [
"libloading",
]
@@ -6635,7 +6637,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6764,7 +6766,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.21.7",
"base64 0.22.1",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -6881,13 +6883,13 @@ dependencies = [
[[package]]
name = "object_store"
version = "0.14.1"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"
checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94"
dependencies = [
"async-trait",
"aws-lc-rs",
"base64 0.22.1",
"base64 0.23.1",
"bytes",
"chrono",
"crc-fast",
@@ -6933,7 +6935,7 @@ dependencies = [
"bytes",
"chrono",
"futures",
"object_store 0.14.1",
"object_store 0.14.2",
"opendal",
"pin-project",
"tokio",
@@ -8138,8 +8140,8 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.4.1",
"itertools 0.11.0",
"heck 0.5.0",
"itertools 0.14.0",
"log",
"multimap",
"petgraph",
@@ -8158,7 +8160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.11.0",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -8380,7 +8382,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls 0.23.45",
"socket2 0.5.10",
"socket2 0.6.3",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -8418,7 +8420,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.10",
"socket2 0.6.3",
"tracing",
"windows-sys 0.60.2",
]
@@ -9218,7 +9220,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -9289,7 +9291,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -9580,9 +9582,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.23.0"
version = "3.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4"
checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f"
dependencies = [
"base64 0.23.1",
"bs58",
@@ -9601,9 +9603,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.23.0"
version = "3.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc"
checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49"
dependencies = [
"darling 0.24.1",
"proc-macro2",
@@ -9836,7 +9838,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9848,7 +9850,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9871,7 +9873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -9990,7 +9992,7 @@ dependencies = [
"cfg-if 1.0.4",
"libc",
"psm",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -10282,7 +10284,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11303,7 +11305,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.11</version>
<version>0.40.0-beta.12</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+1 -1
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/BlobFile.md
+101 -5
View File
@@ -10,16 +10,52 @@ A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fet
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(): Promise<Buffer>
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor to the end and advances the cursor.
Reads from the cursor and advances the cursor.
A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does
not move the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
@@ -30,7 +66,10 @@ not move the cursor.
### readRange()
```ts
readRange(start, end): Promise<Buffer>
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
@@ -43,12 +82,55 @@ Fails when `end` is past the blob size. Does not move the cursor.
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
@@ -60,3 +142,17 @@ Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Query.md
+1 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/QueryBase.md
+1 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/Table.md
+13 -3
View File
@@ -516,7 +516,10 @@ Drop an index from the table.
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]>
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
@@ -531,6 +534,8 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
@@ -540,7 +545,10 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
### fetchBlobs()
```ts
abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]>
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
@@ -559,6 +567,8 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
@@ -1371,7 +1381,7 @@ repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+1 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/classes/VectorQuery.md
+1 -1
View File
@@ -673,7 +673,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
docs/src/js/globals.md
+3
View File
@@ -158,8 +158,11 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.11</version>
<version>0.40.0-beta.12</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.11</version>
<version>0.40.0-beta.12</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>13.0.0-beta.14</lance-core.version>
<lance-core.version>13.1.0-beta.1</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
publish = false
license.workspace = true
description.workspace = true
nodejs/NODEJS_THIRD_PARTY_LICENSES.md
+8
View File
@@ -275,6 +275,7 @@
[@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
@@ -306,6 +307,9 @@
[apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0
[argparse@1.0.10](https://github.com/nodeca/argparse) - MIT
[argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0
[array-back@3.1.0](https://github.com/75lb/array-back) - MIT
@@ -348,6 +352,7 @@
[color@4.2.3](https://github.com/Qix-/color) - MIT
[combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT
[command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT
[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT
[command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT
[concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT
[convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT
@@ -398,12 +403,14 @@
[file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT
[fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT
[find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT
[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT
[find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT
[find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT
[flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT
[flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0
[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0
[flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC
[follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT
[foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC
@@ -644,6 +651,7 @@
[uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT
[undici-types@5.26.5](https://github.com/nodejs/undici) - MIT
[undici-types@6.19.8](https://github.com/nodejs/undici) - MIT
[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT
[update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT
[uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause
[uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT
nodejs/__test__/arrow.test.ts
+1415 -1456
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/arrow_versions.ts
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// Every Apache Arrow release the package accepts as a peer dependency, as
// separately installed copies so the tests can feed objects built by one
// Arrow version into a LanceDB running on another. When a new Arrow major
// is released, add the alias to `package.json` and extend this list; the
// test files pick the change up from here.
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as arrow19 from "apache-arrow-19";
import * as arrow20 from "apache-arrow-20";
import * as arrow21 from "apache-arrow-21";
export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 };
export type ApacheArrow =
| typeof arrow15
| typeof arrow16
| typeof arrow17
| typeof arrow18
| typeof arrow19
| typeof arrow20
| typeof arrow21;
export const arrowVersions: ApacheArrow[] = [
arrow15,
arrow16,
arrow17,
arrow18,
arrow19,
arrow20,
arrow21,
];
/** The newest supported Arrow release, for tests that only need one. */
export const latestArrow = arrow21;
nodejs/__test__/registry.test.ts
+5 -8
View File
@@ -3,13 +3,10 @@
import * as apiArrow from "apache-arrow";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as tmp from "tmp";
import { arrowVersions } from "./arrow_versions";
import { connect } from "../lancedb";
import {
EmbeddingFunction,
@@ -18,7 +15,7 @@ import {
} from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
describe.each(arrowVersions)("LanceSchema", (arrow) => {
test("should preserve input order", async () => {
const schema = LanceSchema({
id: new arrow.Int32(),
@@ -29,7 +26,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
});
});
describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
describe.each(arrowVersions)("Registry", (arrow) => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
@@ -240,7 +237,7 @@ describe("Registry.setVar", () => {
return data.map(() => [1, 2, 3]);
}
embeddingDataType() {
return new arrow18.Float32() as apiArrow.Float;
return new apiArrow.Float32();
}
protected getSensitiveKeys() {
return ["secretKey"];
nodejs/__test__/sanitize.test.ts
+113 -1
View File
@@ -2,7 +2,13 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as arrow from "../lancedb/arrow";
import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
import {
sanitizeField,
sanitizeMap,
sanitizeSchema,
sanitizeType,
} from "../lancedb/sanitize";
import { arrow21 } from "./arrow_versions";
describe("sanitize", function () {
describe("sanitizeType function", function () {
@@ -59,6 +65,8 @@ describe("sanitize", function () {
["durationmicrosecond", new arrow.DurationMicrosecond()],
["durationmillisecond", new arrow.DurationMillisecond()],
["durationsecond", new arrow.DurationSecond()],
["largebinary", new arrow.LargeBinary()],
["largeutf8", new arrow.LargeUtf8()],
] as const;
it.each(allTypeNameTestCases)(
@@ -89,6 +97,110 @@ describe("sanitize", function () {
"Unrecognized type name in schema: invalid_type",
);
});
// These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the
// rest), so the objects come from a second copy of the newest release. The
// library itself may be running on an older Arrow (see the CI matrix), in
// which case sanitizing them has to fail with a clear message instead.
const hostArrow = arrow as unknown as Record<string, unknown>;
const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function";
const whenSupported = hostSupportsNewerTypes ? describe : describe.skip;
const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe;
const newerTypeTestCases = [
["Utf8View", () => new arrow21.Utf8View()],
["BinaryView", () => new arrow21.BinaryView()],
["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()],
[
"LargeList",
() =>
new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())),
],
] as const;
whenSupported("types added in Arrow 20 and 21", function () {
it.each(newerTypeTestCases)(
"should sanitize a %s type from another copy of Arrow",
function (name, makeForeign) {
const foreign = makeForeign();
const result = sanitizeType(foreign);
// Interval subclasses share `Type.Interval`, so the sanitized type
// is the generic `Interval` carrying the same unit, as for the
// other units.
const expected = name === "IntervalMonthDayNano" ? "Interval" : name;
expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown);
expect(result.typeId).toBe(foreign.typeId);
expect(result.toString()).toBe(foreign.toString());
},
);
it.each([
["binaryview", "BinaryView"],
["utf8view", "Utf8View"],
["intervalmonthdaynano", "Interval"],
])('should map type name "%s" to %s', function (name, expected) {
expect(sanitizeType(name)).toBeInstanceOf(
hostArrow[expected] as () => unknown,
);
});
it("should sanitize the child field of a LargeList", function () {
const foreign = new arrow21.LargeList(
new arrow21.Field("item", new arrow21.Int32(), false),
);
const result = sanitizeType(foreign) as arrow.List;
expect(result.children[0]).toBeInstanceOf(arrow.Field);
expect(result.children[0].name).toBe("item");
expect(result.children[0].nullable).toBe(false);
const child = result.children[0].type;
expect(arrow.DataType.isInt(child)).toBe(true);
expect((child as arrow.Int).bitWidth).toBe(32);
expect((child as arrow.Int).isSigned).toBe(true);
});
});
whenUnsupported("types newer than the installed Arrow", function () {
it.each(
newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"),
)(
"should explain that %s needs a newer apache-arrow",
function (name, makeForeign) {
expect(() => sanitizeType(makeForeign())).toThrow(
`The Arrow type ${name} requires apache-arrow`,
);
},
);
it("should sanitize IntervalMonthDayNano to a generic Interval", () => {
// The subclass shares `Type.Interval`, which every supported Arrow
// release has; only the unit is new.
const result = sanitizeType(new arrow21.IntervalMonthDayNano());
expect(result).toBeInstanceOf(arrow.Interval);
expect((result as arrow.Interval).unit).toBe(
arrow21.IntervalUnit.MONTH_DAY_NANO,
);
});
});
it("should rebuild a schema from a second copy of the same Arrow version", function () {
// Arrow 21 makes `instanceof Schema` succeed across copies of the library
// but not `instanceof <DataType>`; the schema must still be rebuilt.
const foreign = new arrow21.Schema([
new arrow21.Field("x", new arrow21.Float32(), true),
]);
// Only Arrow 21+ overrides `Symbol.hasInstance` this way.
const crossCopyInstanceof = Object.getOwnPropertySymbols(
arrow.Schema,
).includes(Symbol.hasInstance);
expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof);
const result = sanitizeSchema(foreign);
expect(result).not.toBe(foreign);
const type = result.fields[0].type;
expect(type).toBeInstanceOf(arrow.DataType);
expect(arrow.DataType.isFloat(type)).toBe(true);
expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE);
expect(() => arrow.makeBuilder({ type })).not.toThrow();
});
});
describe("sanitizeField function", function () {
nodejs/__test__/table.test.ts
+313 -12
View File
@@ -2,15 +2,11 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "fs";
import { getEventListeners } from "node:events";
import { pathToFileURL } from "node:url";
import * as path from "path";
import * as tmp from "tmp";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
AutoQuery,
Connection,
@@ -55,19 +51,21 @@ import {
instanceOfFullTextQuery,
} from "../lancedb/query";
import { LocalTable } from "../lancedb/table";
import {
type ApacheArrow,
arrow18,
arrowVersions,
latestArrow,
} from "./arrow_versions";
describe.each([arrow15, arrow16, arrow17, arrow18])(
describe.each(arrowVersions)(
"Given a table",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
let tmpDir: tmp.DirResult;
let table: Table;
const schema:
| import("apache-arrow-15").Schema
| import("apache-arrow-16").Schema
| import("apache-arrow-17").Schema
| import("apache-arrow-18").Schema = new arrow.Schema([
const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
new arrow.Field("id", new arrow.Float64(), true),
]);
@@ -92,6 +90,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await expect(table.countRows()).resolves.toBe(3);
});
it("creates and adds Arrow tables with large string and binary columns", async () => {
const data = new arrow.Table({
text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()),
bytes: arrow.vectorFromArray(
[Buffer.from("one"), Buffer.from("two")],
new arrow.LargeBinary(),
),
});
const conn = await connect(tmpDir.name);
const largeTable = await conn.createTable("large_columns", data);
await largeTable.add(data);
const rows = await largeTable.query().toArray();
expect(rows.map((row) => row.text)).toEqual([
"alpha",
"beta",
"alpha",
"beta",
]);
expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([
"one",
"two",
"one",
"two",
]);
const fields = (await largeTable.schema()).fields;
expect(fields.map((field) => field.type.typeId)).toEqual([
arrow.Type.LargeUtf8,
arrow.Type.LargeBinary,
]);
});
it("should support a foreign Float64 vector schema end to end", async () => {
const conn = await connect(tmpDir.name);
const schema = new arrow.Schema([
@@ -709,6 +739,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
// https://github.com/lancedb/lancedb/issues/1963
// These types were added in Arrow 21, so they are built with a second copy of
// that release rather than the per-version matrix above, and reading them back
// needs the library's own Arrow to know them too.
const hostArrowSupportsViewTypes =
typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
(hostArrowSupportsViewTypes ? it : it.skip)(
"creates and adds tables with LargeList, Utf8View and BinaryView columns",
async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const makeData = () =>
new latestArrow.Table({
id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
tags: latestArrow.vectorFromArray(
[["a", "b"], ["c"]],
new latestArrow.LargeList(
new latestArrow.Field("item", new latestArrow.Utf8(), true),
),
),
text: latestArrow.vectorFromArray(
["short", "a string that is longer than the twelve inline bytes"],
new latestArrow.Utf8View(),
),
bytes: latestArrow.vectorFromArray(
[new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
new latestArrow.BinaryView(),
),
});
const table = await db.createTable("views", makeData());
await table.add(makeData());
const schema = await table.schema();
// Take the enum from the Arrow 21 copy: the host Arrow may predate
// `Type.LargeList`, and this file still has to type-check there.
expect(schema.fields.map((f) => f.type.typeId)).toEqual([
latestArrow.Type.Int,
latestArrow.Type.LargeList,
// Lance stores the view types as their offset-based equivalents.
latestArrow.Type.Utf8,
latestArrow.Type.Binary,
]);
const rows = await table.query().toArrow();
expect(rows.numRows).toBe(4);
const row = rows.get(1)!.toJSON();
expect(row.id).toBe(2);
expect(row.tags.toJSON()).toEqual(["c"]);
expect(row.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
// Arrow JS 21 serializes a *sliced* view column with a truncated views
// buffer; the library rebuilds view columns as Utf8/Binary so this works.
const sliced = makeData().slice(1, 2);
expect(sliced.getChild("text")!.data[0].offset).toBe(1);
const slicedTable = await db.createTable("views_sliced", sliced);
await slicedTable.add(sliced);
const slicedRows = await slicedTable.query().toArrow();
expect(slicedRows.numRows).toBe(2);
for (const i of [0, 1]) {
const r = slicedRows.get(i)!.toJSON();
expect(r.id).toBe(2);
expect(r.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(r.bytes)).toEqual(Array(20).fill(7));
}
// Regression: a field with an integer-like name. JavaScript enumerates
// such keys first, so a key-ordered rebuild would pair "1" with the text
// column's values. The batches are built explicitly to pin field order.
const numericSchema = new latestArrow.Schema([
new latestArrow.Field("text", new latestArrow.Utf8View(), true),
new latestArrow.Field("1", new latestArrow.Utf8View(), true),
new latestArrow.Field("n", new latestArrow.Int32(), true),
]);
const column = (
values: unknown[],
type: InstanceType<typeof latestArrow.DataType>,
) => latestArrow.vectorFromArray(values, type).data[0];
const numericBatch = new latestArrow.RecordBatch(
numericSchema,
latestArrow.makeData({
type: new latestArrow.Struct(numericSchema.fields),
length: 3,
nullCount: 0,
children: [
column(["t0", "t1", "t2"], new latestArrow.Utf8View()),
column(["one0", "one1", "one2"], new latestArrow.Utf8View()),
column([0, 1, 2], new latestArrow.Int32()),
],
}),
);
const numericData = new latestArrow.Table(numericSchema, [numericBatch]);
const numericTable = await db.createTable("views_numeric", numericData);
await numericTable.add(numericData.slice(1));
const numericRows = (await numericTable.query().toArrow()).toArray();
expect(numericRows.map((r) => r.toJSON())).toEqual([
{ text: "t0", "1": "one0", n: 0 },
{ text: "t1", "1": "one1", n: 1 },
{ text: "t2", "1": "one2", n: 2 },
{ text: "t1", "1": "one1", n: 1 },
{ text: "t2", "1": "one2", n: 2 },
]);
tmpDir.removeCallback();
},
);
it("should query documents with LangChain PDF metadata", async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
try {
@@ -2601,6 +2739,27 @@ describe("when dealing with blob columns", () => {
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
});
it("adds a LargeBinary Arrow column to a blob table", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
const table = await db.createTable(
"blobs",
[{ id: 1n, image: Buffer.from("alpha") }],
{ schema },
);
const payload = Buffer.from("beta");
const data = new arrow18.Table({
id: arrow18.vectorFromArray([2n], new arrow18.Int64()),
image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()),
});
await table.add(data);
const rows = await table.query().withRowId().toArray();
const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint;
const [actual] = await table.fetchBlobs("image", [rowId]);
expect(actual).toEqual(payload);
});
it("reads a half-open range", async () => {
const { table, rowIds } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
@@ -2625,6 +2784,148 @@ describe("when dealing with blob columns", () => {
).rejects.toThrow(/exceeds blob size/);
});
it("reads at most maxBytes and advances the cursor", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect((await handle!.read(2n)).toString()).toBe("al");
expect(await handle!.tell()).toBe(2n);
expect((await handle!.read(2n)).toString()).toBe("ph");
expect((await handle!.read(10n)).toString()).toBe("a");
expect(await handle!.tell()).toBe(5n);
expect(await handle!.read(1n)).toEqual(Buffer.alloc(0));
});
it("seeks the cursor", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await handle!.seek(3n);
expect(await handle!.tell()).toBe(3n);
expect((await handle!.read()).toString()).toBe("ha");
await handle!.seek(0n);
expect(await handle!.read()).toEqual(alpha);
});
it("rejects a negative seek position", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await expect(handle!.seek(-1n)).rejects.toThrow(/cannot be negative/);
});
it("readRanges returns one buffer per range in order", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const buffers = await handle!.readRanges([
{ start: 3n, end: 5n },
{ start: 0n, end: 2n },
{ start: 3n, end: 5n },
]);
expect(buffers.map((b) => b.toString())).toEqual(["ha", "al", "ha"]);
expect(await handle!.tell()).toBe(0n);
});
it("fails when a readRanges end is past the blob size", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await expect(
handle!.readRanges([
{ start: 0n, end: 1n },
{ start: 0n, end: BigInt(alpha.length + 1) },
]),
).rejects.toThrow(/exceeds blob size/);
});
it("fails reads after close and closes once", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect(handle!.isClosed()).toBe(false);
await handle!.close();
expect(handle!.isClosed()).toBe(true);
await expect(handle!.close()).resolves.toBeUndefined();
await expect(handle!.read()).rejects.toThrow(/already closed/);
await expect(handle!.read(1n)).rejects.toThrow(/already closed/);
await expect(handle!.readRange(0n, 1n)).rejects.toThrow(/already closed/);
await expect(handle!.readRanges([{ start: 0n, end: 1n }])).rejects.toThrow(
/already closed/,
);
await expect(handle!.seek(0n)).rejects.toThrow(/already closed/);
await expect(handle!.tell()).rejects.toThrow(/already closed/);
expect(handle!.size()).toBe(5n);
});
it("rejects reads with an already aborted signal", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const signal = AbortSignal.abort();
const abortError = { name: "AbortError" };
await expect(handle!.read(undefined, { signal })).rejects.toMatchObject(
abortError,
);
await expect(handle!.read(2n, { signal })).rejects.toMatchObject(
abortError,
);
await expect(handle!.readRange(0n, 1n, { signal })).rejects.toMatchObject(
abortError,
);
await expect(
handle!.readRanges([{ start: 0n, end: 1n }], { signal }),
).rejects.toMatchObject(abortError);
await expect(
table.fetchBlobs("image", rowIds, { signal }),
).rejects.toMatchObject(abortError);
await expect(
table.fetchBlobFiles("image", rowIds, { signal }),
).rejects.toMatchObject(abortError);
expect(await handle!.read()).toEqual(alpha);
});
it("rejects with the abort reason", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const reason = new Error("caller gave up");
await expect(
handle!.readRange(0n, 1n, { signal: AbortSignal.abort(reason) }),
).rejects.toBe(reason);
});
it("reads with a signal and removes its abort listener", async () => {
const { table, rowIds, alpha, beta } = await openBlobTable();
const controller = new AbortController();
const { signal } = controller;
const [handle] = await table.fetchBlobFiles("image", rowIds, { signal });
expect((await handle!.readRange(0n, 2n, { signal })).toString()).toBe("al");
expect(await table.fetchBlobs("image", rowIds, { signal })).toEqual([
alpha,
beta,
null,
]);
expect(getEventListeners(signal, "abort")).toHaveLength(0);
controller.abort();
expect(await handle!.read()).toEqual(alpha);
});
it("cancels an in-flight read", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("payload", { inlineSizeThreshold: 0, dedicatedSizeThreshold: 1 }),
]);
const payload = Buffer.alloc(32 * 1024 * 1024, 7);
const table = await db.createTable("large_blobs", [{ id: 1n, payload }], {
schema,
});
const [row] = await table.query().withRowId().toArray();
const [handle] = await table.fetchBlobFiles("payload", [
row._rowid as bigint,
]);
const controller = new AbortController();
const pending = handle!.readRange(0n, handle!.size(), {
signal: controller.signal,
});
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4));
});
it("rejects fetchBlobs on a non-blob column", async () => {
const { table, rowIds } = await openBlobTable();
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
@@ -3007,7 +3308,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
});
describe.each([arrow15, arrow16, arrow17, arrow18])(
describe.each(arrowVersions)(
"when optimizing a dataset",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
nodejs/lancedb/arrow.ts
+83 -4
View File
@@ -66,11 +66,20 @@ export type SchemaLike =
export type FieldLike =
| Field
| {
type: string;
type: string | DataTypeLike;
name: string;
nullable: boolean;
metadata?: Map<string, string>;
};
/**
* A `DataType` from any copy or version of apache-arrow.
*
* Arrow 21 brands its classes with `unique symbol` properties, so a type
* object from a second copy of the library no longer satisfies the `DataType`
* type of this one even though it is structurally identical. Inputs that only
* need to be sanitized accept this looser shape instead.
*/
export type DataTypeLike = DataType | { readonly typeId: number };
/**
* Create an Arrow field backed by LanceDB's JSON extension type.
@@ -1059,6 +1068,71 @@ export async function fromRecordsToStreamBuffer(
return Buffer.from(await writer.toUint8Array());
}
// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist
// in Arrow 21+, and this module compiles against every supported release.
const UTF8_VIEW_TYPE_ID = 24;
const BINARY_VIEW_TYPE_ID = 23;
/**
* Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`.
*/
function materializeViewData(child: ArrowData, type: DataType): ArrowData {
const builder = makeBuilder({ type, nullValues: [null] });
for (const value of new Vector([child])) {
builder.append(value);
}
return builder.finish().flush();
}
/**
* Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
*
* Lance stores the view types as their offset-based equivalents anyway, so
* nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC
* writer emits a truncated views buffer for a *sliced* view array, which the
* Rust reader rejects with "Need at least N bytes in buffers[0]".
*
* The record batches are rebuilt positionally rather than through a
* `Record<string, Vector>`: JavaScript enumerates integer-like keys first, so
* a field named e.g. `"1"` would otherwise be paired with the wrong column.
*
* Tables without view columns are returned as-is.
*/
function materializeViewColumns(table: ArrowTable): ArrowTable {
const replacements = new Map<number, DataType>();
table.schema.fields.forEach((field, i) => {
if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
replacements.set(i, new Utf8());
} else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
replacements.set(i, new Binary());
}
});
if (replacements.size === 0) {
return table;
}
const fields = table.schema.fields.map((field, i) => {
const type = replacements.get(i);
return type === undefined
? field
: new Field(field.name, type, field.nullable, field.metadata);
});
const schema = new Schema(fields, table.schema.metadata);
const batches = table.batches.map((batch) => {
const children = batch.data.children.map((child, i) => {
const type = replacements.get(i);
return type === undefined ? child : materializeViewData(child, type);
});
const data = makeData({
type: new Struct(fields),
length: batch.numRows,
nullCount: 0,
children,
});
return new RecordBatch(schema, data);
});
return new ArrowTable(schema, batches);
}
/**
* Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
*
@@ -1075,7 +1149,9 @@ export async function fromTableToBuffer(
if (schema !== undefined && schema !== null) {
schema = sanitizeSchema(schema);
}
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
const tableWithEmbeddings = materializeViewColumns(
await applyEmbeddings(table, embeddings, schema),
);
validateBlobSchema(tableWithEmbeddings.schema);
const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
@@ -1146,7 +1222,8 @@ export async function fromRecordBatchToBuffer(
export async function fromRecordBatchToStreamBuffer(
batch: RecordBatch,
): Promise<Buffer> {
const writer = RecordBatchStreamWriter.writeAll([batch]);
const table = materializeViewColumns(new ArrowTable([batch]));
const writer = RecordBatchStreamWriter.writeAll(table);
return Buffer.from(await writer.toUint8Array());
}
@@ -1163,7 +1240,9 @@ export async function fromTableToStreamBuffer(
embeddings?: EmbeddingFunctionConfig,
schema?: SchemaLike,
): Promise<Buffer> {
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
const tableWithEmbeddings = materializeViewColumns(
await applyEmbeddings(table, embeddings, schema),
);
const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
}
nodejs/lancedb/blob.ts
+95 -7
View File
@@ -141,13 +141,16 @@ export class BlobFile {
}
/**
* Reads from the cursor to the end and advances the cursor.
* Reads from the cursor and advances the cursor.
*
* A second call returns an empty buffer. {@link BlobFile.readRange} does
* not move the cursor.
* Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
* otherwise. Returns an empty buffer at the end of the blob.
* {@link BlobFile.readRange} does not move the cursor.
*/
read(): Promise<Buffer> {
return this.inner.read();
read(maxBytes?: bigint, options?: BlobReadOptions): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.read(maxBytes, signal),
);
}
/**
@@ -155,11 +158,96 @@ export class BlobFile {
*
* Fails when `end` is past the blob size. Does not move the cursor.
*/
readRange(start: bigint, end: bigint): Promise<Buffer> {
return this.inner.readRange(start, end);
readRange(
start: bigint,
end: bigint,
options?: BlobReadOptions,
): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRange(start, end, signal),
);
}
/**
* Reads several half-open byte ranges. Returns one buffer per range, in
* the order given.
*
* Fails when any `end` is past the blob size. Does not move the cursor.
*/
readRanges(
ranges: BlobRange[],
options?: BlobReadOptions,
): Promise<Buffer[]> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRanges(ranges, signal),
);
}
/** Moves the cursor to `position`, in bytes from the start of the blob. */
seek(position: bigint): Promise<void> {
return this.inner.seek(position);
}
/** Returns the cursor position in bytes. */
tell(): Promise<bigint> {
return this.inner.tell();
}
/**
* Releases the handle. Reads after `close()` fail. Calling it again does
* nothing.
*/
close(): Promise<void> {
return this.inner.close();
}
/** Returns true after {@link BlobFile.close}. */
isClosed(): boolean {
return this.inner.isClosed();
}
}
/** Options for blob reads. */
export type BlobReadOptions = {
/**
* Cancels the read. The call rejects with `signal.reason`, and the native
* read stops, including in-flight requests to a remote table.
*/
signal?: AbortSignal;
};
/** @ignore */
export async function runWithSignal<T>(
signal: AbortSignal | undefined,
run: (signal: AbortSignal | undefined) => Promise<T>,
): Promise<T> {
if (signal === undefined) {
return run(undefined);
}
signal.throwIfAborted();
// napi-rs replaces `onabort` on the signal it receives, so pass a private
// signal and forward the caller's abort to it.
const controller = new AbortController();
const onAbort = () => controller.abort();
signal.addEventListener("abort", onAbort, { once: true });
try {
return await run(controller.signal);
} catch (err) {
if (signal.aborted) {
throw signal.reason;
}
throw err;
} finally {
signal.removeEventListener("abort", onAbort);
}
}
/** A half-open byte range `[start, end)` for {@link BlobFile.readRanges}. */
export type BlobRange = {
start: bigint;
end: bigint;
};
export function coerceBlobValue(value: unknown): BlobInput | null {
if (value == null) {
return null;
nodejs/lancedb/index.ts
+2 -1
View File
@@ -84,7 +84,7 @@ export {
} from "./arrow";
export { blob, isBlobField, BlobFile } from "./blob";
export type { BlobOptions } from "./blob";
export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob";
export {
Connection,
@@ -195,6 +195,7 @@ export * as rerankers from "./rerankers";
export {
SchemaLike,
TableLike,
DataTypeLike,
FieldLike,
RecordBatchLike,
DataLike,
nodejs/lancedb/sanitize.ts
+137 -15
View File
@@ -9,6 +9,7 @@
// comes from the exact same library instance. This is not always the case
// and so we must sanitize the input to ensure that it is compatible.
import * as arrowModule from "apache-arrow";
import { BufferType, Data, Vector } from "apache-arrow";
import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
import {
@@ -43,6 +44,8 @@ import {
Interval,
IntervalDayTime,
IntervalYearMonth,
LargeBinary,
LargeUtf8,
List,
Map_,
Null,
@@ -74,6 +77,66 @@ import {
Utf8,
} from "./arrow";
/**
* Whether `value` was created by *this* copy of apache-arrow's `ctor`.
*
* Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
* `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
* built by a different copy of the library. Those are exactly the objects
* this module has to rebuild: the `DataType` instances nested inside them are
* deliberately *not* given that treatment upstream, so Arrow's own builders and
* writers reject them. Walk the real prototype chain instead of trusting
* `instanceof`.
*/
function isOwnInstance<T>(
value: unknown,
// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
ctor: abstract new (...args: any[]) => T,
): value is T {
return (
typeof value === "object" &&
value !== null &&
Object.prototype.isPrototypeOf.call(ctor.prototype, value)
);
}
/**
* Types added to Arrow JS after release 18, keyed by the `Type` enum value
* they use. The library compiles and runs against every Arrow release in the
* peer range, so these are looked up by name at runtime instead of imported:
* an older Arrow simply does not have the class.
*/
type NewerTypeName =
| "IntervalMonthDayNano"
| "LargeList"
| "BinaryView"
| "Utf8View";
const NEWER_TYPES: ReadonlyArray<{
name: NewerTypeName;
typeId: number;
since: number;
}> = [
{ name: "IntervalMonthDayNano", typeId: -31, since: 20 },
{ name: "LargeList", typeId: 21, since: 21 },
{ name: "BinaryView", typeId: 23, since: 21 },
{ name: "Utf8View", typeId: 24, since: 21 },
];
// biome-ignore lint/suspicious/noExplicitAny: constructor arity varies by type
type DataTypeConstructor = new (...args: any[]) => DataType;
function newerTypeConstructor(name: NewerTypeName): DataTypeConstructor {
const ctor = (arrowModule as unknown as Record<string, unknown>)[name];
if (typeof ctor !== "function") {
const since = NEWER_TYPES.find((t) => t.name === name)?.since;
throw new Error(
`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
"but the installed apache-arrow does not provide it",
);
}
return ctor as DataTypeConstructor;
}
type SanitizationContext = {
types: WeakMap<object, DataType>;
vectors: WeakMap<object, Vector>;
@@ -210,19 +273,37 @@ export function sanitizeList(typeLike: object) {
return sanitizeListWithContext(typeLike, createSanitizationContext());
}
function sanitizeListChildWithContext(
typeLike: object,
context: SanitizationContext,
typeName: string,
): Field {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
`Expected a ${typeName} type to have an array-like \`children\` property`,
);
}
if (typeLike.children.length !== 1) {
throw Error(`Expected a ${typeName} type to have exactly one child`);
}
return sanitizeFieldWithContext(typeLike.children[0], context);
}
function sanitizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a List type to have an array-like `children` property",
);
}
if (typeLike.children.length !== 1) {
throw Error("Expected a List type to have exactly one child");
}
return new List(sanitizeFieldWithContext(typeLike.children[0], context));
return new List(sanitizeListChildWithContext(typeLike, context, "List"));
}
function sanitizeLargeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
const LargeList = newerTypeConstructor("LargeList");
return new LargeList(
sanitizeListChildWithContext(typeLike, context, "LargeList"),
);
}
export function sanitizeStruct(typeLike: object) {
@@ -449,11 +530,30 @@ function sanitizeTypeWithContext(
return type;
}
function sanitizeNewerTypeById(
typeLike: object,
typeId: number,
context: SanitizationContext,
): DataType | undefined {
const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
if (entry === undefined) {
return undefined;
}
if (entry.name === "LargeList") {
return sanitizeLargeListWithContext(typeLike, context);
}
return new (newerTypeConstructor(entry.name))();
}
function sanitizeTypeById(
typeLike: object,
typeId: Type,
context: SanitizationContext,
): DataType {
const newer = sanitizeNewerTypeById(typeLike, typeId, context);
if (newer !== undefined) {
return newer;
}
switch (typeId) {
case Type.NONE:
throw Error("Received a Type with a typeId of NONE");
@@ -465,8 +565,12 @@ function sanitizeTypeById(
return sanitizeFloat(typeLike);
case Type.Binary:
return new Binary();
case Type.LargeBinary:
return new LargeBinary();
case Type.Utf8:
return new Utf8();
case Type.LargeUtf8:
return new LargeUtf8();
case Type.Bool:
return new Bool();
case Type.Decimal:
@@ -566,7 +670,7 @@ function sanitizeFieldWithContext(
fieldLike: unknown,
context: SanitizationContext,
): Field {
if (fieldLike instanceof Field) {
if (isOwnInstance(fieldLike, Field)) {
return fieldLike;
}
if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -620,7 +724,7 @@ function sanitizeSchemaWithContext(
schemaLike: SchemaLike,
context: SanitizationContext,
): Schema {
if (schemaLike instanceof Schema) {
if (isOwnInstance(schemaLike, Schema)) {
return schemaLike;
}
if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -647,7 +751,7 @@ function sanitizeSchemaWithContext(
}
export function sanitizeTable(tableLike: TableLike): Table {
if (tableLike instanceof Table) {
if (isOwnInstance(tableLike, Table)) {
return tableLike;
}
if (typeof tableLike !== "object" || tableLike === null) {
@@ -675,7 +779,7 @@ function sanitizeRecordBatch(
batchLike: RecordBatchLike,
context: SanitizationContext,
): RecordBatch {
if (batchLike instanceof RecordBatch) {
if (isOwnInstance(batchLike, RecordBatch)) {
return batchLike;
}
if (typeof batchLike !== "object" || batchLike === null) {
@@ -704,11 +808,17 @@ type DictionaryDataLike = DataLike & {
dictionary?: DictionaryVectorLike;
};
// Arrow 21 added `variadicBuffers` to `Data` for the Utf8View / BinaryView
// types. Older Arrow versions do not have the property, so it is optional.
type VariadicDataLike = DataLike & {
variadicBuffers?: ReadonlyArray<Uint8Array>;
};
function sanitizeData(
dataLike: DataLike,
context: SanitizationContext,
): Data<DataType> {
if (dataLike instanceof Data) {
if (isOwnInstance(dataLike, Data)) {
return dataLike;
}
const cachedData = context.data.get(dataLike);
@@ -726,7 +836,12 @@ function sanitizeData(
context.vectors.set(dictionaryLike, dictionary);
}
}
const data = new Data(
// Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
// releases ignore the extra argument but their typings do not declare it.
const DataCtor = Data as unknown as new (
...args: unknown[]
) => Data<DataType>;
const data = new DataCtor(
sanitizeTypeWithContext(dataLike.type, context),
dataLike.offset,
dataLike.length,
@@ -739,6 +854,7 @@ function sanitizeData(
},
dataLike.children.map((child) => sanitizeData(child, context)),
dictionary,
(dataLike as VariadicDataLike).variadicBuffers,
);
context.data.set(dataLike, data);
return data;
@@ -747,7 +863,11 @@ function sanitizeData(
const constructorsByTypeName = {
null: () => new Null(),
binary: () => new Binary(),
largebinary: () => new LargeBinary(),
binaryview: () => new (newerTypeConstructor("BinaryView"))(),
utf8: () => new Utf8(),
largeutf8: () => new LargeUtf8(),
utf8view: () => new (newerTypeConstructor("Utf8View"))(),
bool: () => new Bool(),
int8: () => new Int8(),
int16: () => new Int16(),
@@ -768,6 +888,8 @@ const constructorsByTypeName = {
timesecond: () => new TimeSecond(),
intervaldaytime: () => new IntervalDayTime(),
intervalyearmonth: () => new IntervalYearMonth(),
intervalmonthdaynano: () =>
new (newerTypeConstructor("IntervalMonthDayNano"))(),
durationnanosecond: () => new DurationNanosecond(),
durationmicrosecond: () => new DurationMicrosecond(),
durationmillisecond: () => new DurationMillisecond(),
nodejs/lancedb/table.ts
+10 -5
View File
@@ -17,7 +17,7 @@ import {
tableFromIPC,
} from "./arrow";
import { BlobFile } from "./blob";
import { BlobFile, BlobReadOptions, runWithSignal } from "./blob";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { IndexOptions } from "./indices";
import { Job } from "./job";
@@ -545,6 +545,7 @@ export abstract class Table {
abstract fetchBlobs(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(Buffer | null)[]>;
/**
@@ -557,6 +558,7 @@ export abstract class Table {
abstract fetchBlobFiles(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(BlobFile | null)[]>;
/**
@@ -1248,8 +1250,11 @@ export class LocalTable extends Table {
async fetchBlobs(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(Buffer | null)[]> {
const values = await this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds));
const values = await runWithSignal(options?.signal, (signal) =>
this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds), signal),
);
// N-API Option maps missing values to undefined. Collapse those to null.
return values.map((value) => value ?? null);
}
@@ -1257,10 +1262,10 @@ export class LocalTable extends Table {
async fetchBlobFiles(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(BlobFile | null)[]> {
const files = await this.inner.fetchBlobFiles(
column,
rowIdsToBigInts(rowIds),
const files = await runWithSignal(options?.signal, (signal) =>
this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal),
);
// N-API Option maps missing values to undefined. Collapse those to null.
return files.map((file) =>
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+6 -2
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -49,10 +49,14 @@
"@types/jest": "^29.1.2",
"@types/node": "22.7.4",
"@types/tmp": "^0.2.6",
"apache-arrow": "21.2.0",
"apache-arrow-15": "npm:apache-arrow@15.0.0",
"apache-arrow-16": "npm:apache-arrow@16.0.0",
"apache-arrow-17": "npm:apache-arrow@17.0.0",
"apache-arrow-18": "npm:apache-arrow@18.0.0",
"apache-arrow-19": "npm:apache-arrow@19.0.1",
"apache-arrow-20": "npm:apache-arrow@20.0.0",
"apache-arrow-21": "npm:apache-arrow@21.2.0",
"eslint": "^8.57.0",
"jest": "^29.7.0",
"shx": "^0.3.4",
@@ -102,7 +106,7 @@
},
"peerDependencies": {
"@types/node": ">=22",
"apache-arrow": ">=15.0.0 <=18.1.0"
"apache-arrow": ">=15.0.0 <22.0.0"
},
"peerDependenciesMeta": {
"@types/node": {
nodejs/pnpm-lock.yaml
Generated
+91 -7
View File
@@ -14,9 +14,6 @@ importers:
'@opentelemetry/api':
specifier: ^1.9.0
version: 1.9.1
apache-arrow:
specifier: '>=15.0.0 <=18.1.0'
version: 18.1.0
reflect-metadata:
specifier: ^0.2.2
version: 0.2.2
@@ -54,6 +51,9 @@ importers:
'@types/tmp':
specifier: ^0.2.6
version: 0.2.6
apache-arrow:
specifier: 21.2.0
version: 21.2.0
apache-arrow-15:
specifier: npm:apache-arrow@15.0.0
version: apache-arrow@15.0.0
@@ -66,6 +66,15 @@ importers:
apache-arrow-18:
specifier: npm:apache-arrow@18.0.0
version: apache-arrow@18.0.0
apache-arrow-19:
specifier: npm:apache-arrow@19.0.1
version: apache-arrow@19.0.1
apache-arrow-20:
specifier: npm:apache-arrow@20.0.0
version: apache-arrow@20.0.0
apache-arrow-21:
specifier: npm:apache-arrow@21.2.0
version: apache-arrow@21.2.0
eslint:
specifier: ^8.57.0
version: 8.57.1
@@ -1703,6 +1712,9 @@ packages:
'@types/node@22.7.4':
resolution: {integrity: sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==}
'@types/node@25.9.8':
resolution: {integrity: sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==}
'@types/stack-utils@2.0.3':
resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==}
@@ -1840,8 +1852,16 @@ packages:
resolution: {integrity: sha512-gFlPaqN9osetbB83zC29AbbZqGiCuFH1vyyPseJ+B7SIbfBtESV62mMT/CkiIt77W6ykC/nTWFzTXFs0Uldg4g==}
hasBin: true
apache-arrow@18.1.0:
resolution: {integrity: sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==}
apache-arrow@19.0.1:
resolution: {integrity: sha512-APmMLzS4qbTivLrPdQXexGM4JRr+0g62QDaobzEvip/FdQIrv2qLy0mD5Qdmw4buydtVJgbFeKR8f59I6PPGDg==}
hasBin: true
apache-arrow@20.0.0:
resolution: {integrity: sha512-JUeK0jFRUd7rbmrhhzR3O2KXjLaZ4YYYFOptyUfxOsMIoZCPi6bZR58gVi/xi3HTBMPseXm9PXyQ2V916930pA==}
hasBin: true
apache-arrow@21.2.0:
resolution: {integrity: sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==}
hasBin: true
argparse@1.0.10:
@@ -2033,6 +2053,15 @@ packages:
resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==}
engines: {node: '>=4.0.0'}
command-line-args@6.0.2:
resolution: {integrity: sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==}
engines: {node: '>=12.20'}
peerDependencies:
'@75lb/nature': latest
peerDependenciesMeta:
'@75lb/nature':
optional: true
command-line-usage@7.0.4:
resolution: {integrity: sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==}
engines: {node: '>=12.20.0'}
@@ -2281,6 +2310,15 @@ packages:
resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==}
engines: {node: '>=4.0.0'}
find-replace@5.0.2:
resolution: {integrity: sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==}
engines: {node: '>=14'}
peerDependencies:
'@75lb/nature': latest
peerDependenciesMeta:
'@75lb/nature':
optional: true
find-up@4.1.0:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
@@ -2302,6 +2340,9 @@ packages:
flatbuffers@24.12.23:
resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==}
flatbuffers@25.9.23:
resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}
flatted@3.4.2:
resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==}
@@ -3345,6 +3386,9 @@ packages:
undici-types@6.21.0:
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
undici-types@7.24.6:
resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==}
unist-util-is@6.0.1:
resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==}
@@ -5505,6 +5549,10 @@ snapshots:
dependencies:
undici-types: 6.19.8
'@types/node@25.9.8':
dependencies:
undici-types: 7.24.6
'@types/stack-utils@2.0.3': {}
'@types/tmp@0.2.6': {}
@@ -5694,17 +5742,40 @@ snapshots:
json-bignum: 0.0.3
tslib: 2.8.1
apache-arrow@18.1.0:
apache-arrow@19.0.1:
dependencies:
'@swc/helpers': 0.5.21
'@types/command-line-args': 5.2.3
'@types/command-line-usage': 5.0.4
'@types/node': 20.19.41
command-line-args: 5.2.1
command-line-args: 6.0.2
command-line-usage: 7.0.4
flatbuffers: 24.12.23
json-bignum: 0.0.3
tslib: 2.8.1
transitivePeerDependencies:
- '@75lb/nature'
apache-arrow@20.0.0:
dependencies:
'@swc/helpers': 0.5.21
'@types/command-line-args': 5.2.3
'@types/command-line-usage': 5.0.4
'@types/node': 20.19.41
command-line-args: 6.0.2
command-line-usage: 7.0.4
flatbuffers: 25.9.23
json-bignum: 0.0.3
tslib: 2.8.1
transitivePeerDependencies:
- '@75lb/nature'
apache-arrow@21.2.0:
dependencies:
'@types/node': 25.9.8
flatbuffers: 25.9.23
json-with-bigint: 3.5.8
tslib: 2.8.1
argparse@1.0.10:
dependencies:
@@ -5906,6 +5977,13 @@ snapshots:
lodash.camelcase: 4.3.0
typical: 4.0.0
command-line-args@6.0.2:
dependencies:
array-back: 6.2.3
find-replace: 5.0.2
lodash.camelcase: 4.3.0
typical: 7.3.0
command-line-usage@7.0.4:
dependencies:
array-back: 6.2.3
@@ -6178,6 +6256,8 @@ snapshots:
dependencies:
array-back: 3.1.0
find-replace@5.0.2: {}
find-up@4.1.0:
dependencies:
locate-path: 5.0.0
@@ -6201,6 +6281,8 @@ snapshots:
flatbuffers@24.12.23: {}
flatbuffers@25.9.23: {}
flatted@3.4.2: {}
follow-redirects@1.16.0: {}
@@ -7405,6 +7487,8 @@ snapshots:
undici-types@6.21.0: {}
undici-types@7.24.6: {}
unist-util-is@6.0.1:
dependencies:
'@types/unist': 3.0.3
nodejs/src/blob.rs
+133 -10
View File
@@ -1,25 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::future::Future;
use std::ops::Range;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use arrow_array::{Array, LargeBinaryArray};
use futures::future::abortable;
use lancedb::blob::BlobFile as LanceBlobFile;
use napi::Env;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::error::convert_error;
#[napi(object)]
pub struct BlobRange {
pub start: BigInt,
pub end: BigInt,
}
#[napi]
pub struct BlobFile {
inner: Arc<LanceBlobFile>,
closed: AtomicBool,
}
impl BlobFile {
pub(crate) fn new(inner: LanceBlobFile) -> Self {
Self {
inner: Arc::new(inner),
closed: AtomicBool::new(false),
}
}
fn ensure_open(&self) -> napi::Result<()> {
if self.closed.load(Ordering::Acquire) {
Err(napi::Error::from_reason("blob file is already closed"))
} else {
Ok(())
}
}
}
@@ -32,21 +52,124 @@ impl BlobFile {
}
#[napi]
pub async fn read(&self) -> napi::Result<Buffer> {
let bytes = self.inner.read().await.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
pub fn read<'env>(
&self,
env: &'env Env,
max_bytes: Option<BigInt>,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Buffer>> {
self.ensure_open()?;
let max_bytes = max_bytes
.map(|max_bytes| {
usize::try_from(parse_u64(max_bytes, "maxBytes")?)
.map_err(|_| napi::Error::from_reason("maxBytes is too large"))
})
.transpose()?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let bytes = match max_bytes {
None => inner.read().await,
Some(max_bytes) => inner.read_up_to(max_bytes).await,
}
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
})
}
#[napi]
pub async fn read_range(&self, start: BigInt, end: BigInt) -> napi::Result<Buffer> {
pub fn read_range<'env>(
&self,
env: &'env Env,
start: BigInt,
end: BigInt,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Buffer>> {
self.ensure_open()?;
let range = bigint_range(start, end)?;
let bytes = self
.inner
.read_range(range)
.await
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let bytes = inner
.read_range(range)
.await
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
})
}
#[napi]
pub fn read_ranges<'env>(
&self,
env: &'env Env,
ranges: Vec<BlobRange>,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Buffer>>> {
self.ensure_open()?;
let ranges = ranges
.into_iter()
.map(|range| bigint_range(range.start, range.end))
.collect::<napi::Result<Vec<_>>>()?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let buffers = inner
.read_ranges(&ranges)
.await
.map_err(|err| convert_error(&err))?;
Ok(buffers
.iter()
.map(|bytes| Buffer::from(bytes.as_ref()))
.collect())
})
}
#[napi]
pub async fn seek(&self, position: BigInt) -> napi::Result<()> {
self.ensure_open()?;
let position = parse_u64(position, "position")?;
self.inner
.seek(position)
.await
.map_err(|err| convert_error(&err))
}
#[napi]
pub async fn tell(&self) -> napi::Result<BigInt> {
self.ensure_open()?;
let position = self.inner.tell().await.map_err(|err| convert_error(&err))?;
Ok(BigInt::from(position))
}
#[napi]
pub async fn close(&self) -> napi::Result<()> {
if self.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
self.inner.close().await.map_err(|err| convert_error(&err))
}
#[napi]
pub fn is_closed(&self) -> bool {
self.closed.load(Ordering::Acquire)
}
}
/// Runs `fut` on the napi runtime. Aborting `signal` drops the future, which
/// cancels in-flight reads.
pub fn spawn_abortable<'env, T>(
env: &'env Env,
signal: Option<AbortSignal>,
fut: impl Future<Output = napi::Result<T>> + Send + 'static,
) -> napi::Result<PromiseRaw<'env, T>>
where
T: ToNapiValue + Send + 'static,
{
let (fut, handle) = abortable(fut);
if let Some(signal) = signal {
signal.on_abort(move || handle.abort());
}
env.spawn_future(async move {
fut.await
.unwrap_or_else(|_| Err(napi::Error::from_reason("blob read aborted")))
})
}
fn bigint_range(start: BigInt, end: BigInt) -> napi::Result<Range<u64>> {
nodejs/src/query.rs
+1
View File
@@ -658,6 +658,7 @@ impl JsFullTextQuery {
FtsQuery::Phrase(_) => "phrase".to_string(),
FtsQuery::Boost(_) => "boost".to_string(),
FtsQuery::MultiMatch(_) => "multi_match".to_string(),
FtsQuery::CombinedFields(_) => "combined_fields".to_string(),
FtsQuery::Boolean(_) => "boolean".to_string(),
}
}
nodejs/src/table.rs
+29 -20
View File
@@ -11,11 +11,12 @@ use lancedb::table::{
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
};
use napi::Env;
use napi::bindgen_prelude::*;
use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode};
use napi_derive::napi;
use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids};
use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids, spawn_abortable};
use crate::error::NapiErrorExt;
use crate::index::Index;
use crate::merge::NativeMergeInsertBuilder;
@@ -340,36 +341,44 @@ impl Table {
}
#[napi(catch_unwind)]
pub async fn fetch_blobs(
pub fn fetch_blobs<'env>(
&self,
env: &'env Env,
column: String,
row_ids: Vec<BigInt>,
) -> napi::Result<Vec<Option<Buffer>>> {
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Option<Buffer>>>> {
let row_ids = parse_row_ids(row_ids)?;
let array = self
.inner_ref()?
.fetch_blobs(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(copy_blob_buffers(array))
let table = self.inner_ref()?.clone();
spawn_abortable(env, signal, async move {
let array = table
.fetch_blobs(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(copy_blob_buffers(array))
})
}
#[napi(catch_unwind)]
pub async fn fetch_blob_files(
pub fn fetch_blob_files<'env>(
&self,
env: &'env Env,
column: String,
row_ids: Vec<BigInt>,
) -> napi::Result<Vec<Option<BlobFile>>> {
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Option<BlobFile>>>> {
let row_ids = parse_row_ids(row_ids)?;
let files = self
.inner_ref()?
.fetch_blob_files(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(files
.into_iter()
.map(|file| file.map(BlobFile::new))
.collect())
let table = self.inner_ref()?.clone();
spawn_abortable(env, signal, async move {
let files = table
.fetch_blob_files(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(files
.into_iter()
.map(|file| file.map(BlobFile::new))
.collect())
})
}
#[napi(catch_unwind)]
nodejs/tsconfig.json
+4 -1
View File
@@ -10,7 +10,10 @@
"resolveJsonModule": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"moduleResolution": "Node"
"moduleResolution": "Node",
// flatbuffers 25 (via apache-arrow 21) ships declarations that need
// TypeScript 5.7+ generic typed arrays; skip checking third-party .d.ts.
"skipLibCheck": true
},
"exclude": ["./dist/*", "./examples/*"],
"typedocOptions": {
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/_lancedb.pyi
+13 -5
View File
@@ -149,12 +149,20 @@ class Connection(object):
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
async def open_job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def create_function_async(
self, request_json: str, namespace_path: Optional[List[str]] = None
) -> Job: ...
async def get_function(
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> str: ...
async def list_functions(
self, namespace_path: Optional[List[str]] = None
) -> List[str]: ...
async def drop_function(
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> bool: ...
async def drop_function_async(
self, name: str, version: str
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> Tuple[bool, Job]: ...
async def create_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
python/python/lancedb/db.py
+123 -31
View File
@@ -784,6 +784,7 @@ class DBConnection(EnforceOverrides):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Build and register a scalar Python UDF, then return its version.
@@ -800,6 +801,9 @@ class DBConnection(EnforceOverrides):
Function needs, each naming a Secret and the environment variable
its value arrives in. The Function's source is unchanged by this;
it reads the variable the way it already did.
namespace_path : list of str, optional
The namespace holding the Function. None or an empty list is the
root namespace.
Examples
--------
@@ -815,13 +819,16 @@ class DBConnection(EnforceOverrides):
)
```
"""
return self.create_function_async(definition, secrets=secrets).wait()
return self.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
).wait()
def create_function_async(
self,
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
"""Submit a scalar Python UDF for building and registration.
@@ -833,17 +840,26 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def get_function(self, name: str, *, version: str) -> FunctionVersion:
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
"""List every published immutable Function version in a namespace.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Functions in child namespaces are not included. Results are ordered by
Function name then version. Local connections raise
``NotImplementedError``.
Examples
--------
@@ -857,7 +873,13 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
"""Drop a Function name and the object it was bound to.
The requested version must exist in the currently named object. Returns
@@ -870,7 +892,13 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
"""Drop a Function name and return its cleanup job.
The name is unbound before this returns; the object's content may still
@@ -1808,25 +1836,62 @@ class LanceDBConnection(DBConnection):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
)
)
return Job(job)
@override
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
return LOOP.run(
self._conn.get_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions(namespace_path=namespace_path))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
return LOOP.run(
self._conn.drop_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
dropped, job = LOOP.run(
self._conn.drop_function_async(
name, version=version, namespace_path=namespace_path
)
)
return dropped, Job(job)
@override
@@ -2787,6 +2852,7 @@ class AsyncConnection(object):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> AsyncJob[FunctionVersion]:
"""Submit a scalar Python UDF for building and registration.
@@ -2794,36 +2860,60 @@ class AsyncConnection(object):
artifact. Waiting on the job returns the immutable Function version.
``secrets`` is a sequence of
[EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and
the environment variable its value arrives in. Local connections raise
``NotImplementedError``.
the environment variable its value arrives in. ``namespace_path`` is
the namespace the Function is registered in; None or an empty list is
the root. Local connections raise ``NotImplementedError``.
"""
if not isinstance(definition, UdfDefinition):
raise TypeError("create_function_async requires a @udf definition")
request = definition.bind_secrets(secrets)
inner = await self._inner.create_function_async(request.to_canonical_json())
inner = await self._inner.create_function_async(
request.to_canonical_json(), namespace_path
)
return _typed_job(inner, FunctionVersion.from_json)
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
async def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
return FunctionVersion.from_json(
await self._inner.get_function(name, version, namespace_path)
)
async def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
async def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
"""List every published immutable Function version in a namespace.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Functions in child namespaces are not included. Results are ordered by
Function name then version. Local connections raise
``NotImplementedError``.
"""
return [
FunctionVersion.from_json(value)
for value in await self._inner.list_functions()
for value in await self._inner.list_functions(namespace_path)
]
async def drop_function(self, name: str, *, version: str) -> bool:
async def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
"""Drop a Function name and the object it was bound to."""
return await self._inner.drop_function(name, version)
return await self._inner.drop_function(name, version, namespace_path)
async def drop_function_async(
self, name: str, *, version: str
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, AsyncJob]:
"""Drop a Function name and return its cleanup job.
@@ -2831,7 +2921,9 @@ class AsyncConnection(object):
be being deleted. Await :meth:`AsyncJob.wait` to wait for that to
finish.
"""
dropped, job = await self._inner.drop_function_async(name, version)
dropped, job = await self._inner.drop_function_async(
name, version, namespace_path
)
return dropped, AsyncJob(job)
async def create_secret(
python/python/lancedb/embeddings/base.py
+5 -1
View File
@@ -182,7 +182,11 @@ class EmbeddingFunction(BaseModel, ABC):
def VectorField(self, **kwargs):
"""
Creates a pydantic Field that can automatically annotate
the target vector column for this embedding function
the target vector column for this embedding function.
The field can be annotated as ``list[float]`` for compatibility with
static type checkers. LanceDB will infer the fixed vector dimension from
this embedding function.
"""
return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
python/python/lancedb/functions.py
+10 -1
View File
@@ -356,9 +356,16 @@ _ObjectVersion = Annotated[
class FunctionVersion(_RemoteValue):
"""A pinned object revision, independent of its executable image digest."""
"""A pinned object revision, independent of its executable image digest.
``name`` and ``namespace_path`` are the Function's parts, as a
[SecretReference][lancedb.functions.SecretReference] carries a Secret's;
they are what [get_function][lancedb.db.DBConnection.get_function] takes.
An empty ``namespace_path`` is the root namespace.
"""
name: str
namespace_path: tuple[str, ...] = ()
object_id: str
location: str
version: _ObjectVersion
@@ -444,6 +451,7 @@ class FunctionVersion(_RemoteValue):
return FunctionApplication(
function=FunctionVersionRef(
name=self.name,
namespace_path=self.namespace_path,
object_id=self.object_id,
location=self.location,
version=self.version,
@@ -502,6 +510,7 @@ class FunctionRegistrationRequest(_RemoteValue):
class FunctionVersionRef(_OpenRemoteValue):
name: str
namespace_path: tuple[str, ...] = ()
object_id: str
location: str
version: _ObjectVersion
python/python/lancedb/pydantic.py
+39
View File
@@ -88,6 +88,17 @@ def Vector(
... pa.field("url", pa.utf8(), False),
... pa.field("embeddings", pa.list_(pa.float32(), 768))
... ])
Notes
-----
``Vector`` creates a type dynamically, so calls such as ``Vector(768)`` are
not valid static type annotations. For an embedding field, use the standard
``list[float]`` annotation when running mypy; ``VectorField`` supplies the
fixed dimension to LanceDB::
class MyModel(LanceModel):
text: str = embeddings.SourceField()
vector: list[float] = embeddings.VectorField()
"""
# TODO: make a public parameterized type.
@@ -313,6 +324,10 @@ def _unwrap_optional_annotation(annotation: Any) -> Any | None:
def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
"""Convert a Pydantic FieldInfo to Arrow DataType"""
embedding_vector_type = _embedding_vector_to_arrow_type(field)
if embedding_vector_type is not None:
return embedding_vector_type
unwrapped = _unwrap_optional_annotation(field.annotation)
if unwrapped is not None:
return _pydantic_type_to_arrow_type(unwrapped, field)
@@ -326,8 +341,32 @@ def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
return _pydantic_type_to_arrow_type(field.annotation, field)
def _embedding_vector_to_arrow_type(field: FieldInfo) -> pa.DataType | None:
"""Infer a fixed-size vector type from ``VectorField`` metadata."""
if not _is_embedding_vector_annotation(field):
return None
function = get_extras(field, "vector_column_for")
return pa.list_(pa.float32(), function.ndims())
def _is_embedding_vector_annotation(field: FieldInfo) -> bool:
if get_extras(field, "vector_column_for") is None:
return False
annotation = _unwrap_optional_annotation(field.annotation)
if annotation is None:
annotation = field.annotation
origin = getattr(annotation, "__origin__", None)
args = getattr(annotation, "__args__", ())
return origin is list and args == (float,)
def is_nullable(field: FieldInfo) -> bool:
"""Check if a Pydantic FieldInfo is nullable."""
if _is_embedding_vector_annotation(field):
return True
if _unwrap_optional_annotation(field.annotation) is not None:
return True
if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
python/python/lancedb/remote/db.py
+46 -9
View File
@@ -863,25 +863,62 @@ class RemoteDBConnection(DBConnection):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
)
)
return Job(job)
@override
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
return LOOP.run(
self._conn.get_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions(namespace_path=namespace_path))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
return LOOP.run(
self._conn.drop_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
dropped, job = LOOP.run(
self._conn.drop_function_async(
name, version=version, namespace_path=namespace_path
)
)
return dropped, Job(job)
@override
python/python/tests/test_blob.py
+1 -1
View File
@@ -1506,7 +1506,7 @@ def test_merge_insert_mixed_python_blobs_validates_external_uri(tmp_path, as_pan
{"id": 2, "image": blob_path.as_uri()},
]
with pytest.raises(ValueError, match="allow_external_blob_outside_bases"):
with pytest.raises(ValueError, match="outside registered external bases"):
(
table.merge_insert("id")
.when_matched_update_all()
python/python/tests/test_first_class_function_slice1.py
+23
View File
@@ -162,6 +162,29 @@ def test_function_version_binds_named_columns_as_one_immutable_application():
] == [("text", "column", "documents.body")]
def test_function_version_names_its_namespace_as_a_separate_field():
"""A Function is named by its parts, as a Secret is; a root one states no
namespace."""
value = job_result("remote_function_job.json")
root = FunctionVersion.from_json(json.dumps(value))
assert root.namespace_path == ()
assert "namespace_path" not in json.loads(root.to_canonical_json())
nested = FunctionVersion(**{**value, "namespace_path": ["analytics", "features"]})
assert nested.name == "embed"
assert nested.namespace_path == ("analytics", "features")
assert json.loads(nested.to_canonical_json())["namespace_path"] == [
"analytics",
"features",
]
application = nested(text=col("documents.body"))
assert application.function.name == "embed"
assert application.function.namespace_path == ("analytics", "features")
assert json.loads(application.to_canonical_json())["function"][
"namespace_path"
] == ["analytics", "features"]
def test_function_version_binding_validates_names_and_direct_columns():
version = FunctionVersion.from_json(
json.dumps(job_result("remote_function_job.json"))
python/python/tests/test_first_class_function_slice2.py
+114 -4
View File
@@ -1539,8 +1539,10 @@ def _mock_remote_function_catalog():
else (None, None)
)
if function_action[1] == "create":
*namespace_path, name = function_action[0].split("$")
state["version"] = {
"name": "normalize_score",
"name": name,
**({"namespace_path": namespace_path} if namespace_path else {}),
"version": FUNCTION_VERSION,
"object_id": "fixture",
"location": "memory:///fixture",
@@ -1565,10 +1567,18 @@ def _mock_remote_function_catalog():
"job_state": "DONE",
"result": state["version"],
}
elif self.path == "/v1/function/normalize_score/describe":
# A namespaced Function's id is its namespace path and name joined
# by `$`, so the name is the last segment of it.
elif (
function_action[1] == "describe"
and function_action[0].split("$")[-1] == "normalize_score"
):
assert body == {"version": FUNCTION_VERSION}
response = state["version"]
elif self.path == "/v1/function/normalize_score/drop":
elif (
function_action[1] == "drop"
and function_action[0].split("$")[-1] == "normalize_score"
):
assert body == {"version": FUNCTION_VERSION}
response = {"dropped": True}
elif secret_action[1] in ("create", "alter"):
@@ -1602,7 +1612,8 @@ def _mock_remote_function_catalog():
assert query["page_token"] == "next"
self._write_response(200, {"secrets": [{"name": "hf-prod"}]})
return
if url.path != "/v1/namespace/$/function/list":
parts = url.path.strip("/").split("/")
if parts[:2] != ["v1", "namespace"] or parts[3:] != ["function", "list"]:
self._write_response(404, {"error": "not found"})
return
assert query["include_definition"] == "true"
@@ -1840,3 +1851,102 @@ async def test_async_remote_drop_function_sends_exact_version():
{"version": FUNCTION_VERSION},
)
]
def test_remote_function_verbs_address_a_namespace_in_the_path():
"""A Function's namespace is part of the identifier each route addresses;
no body carries it, so a namespaced request differs only in its path."""
namespace = ["analytics", "features"]
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
created = db.create_function(normalize_score, namespace_path=namespace)
# The returned parts are what the other verbs take, so a created
# Function reopens from its own fields.
assert created.name == "normalize_score"
assert created.namespace_path == tuple(namespace)
reopened = db.get_function(
created.name,
version=created.version,
namespace_path=list(created.namespace_path),
)
assert reopened == created
assert reopened(value=lancedb.col("score")).function.namespace_path == tuple(
namespace
)
db.list_functions(namespace_path=namespace)
assert (
db.drop_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
is True
)
dropped, _ = db.drop_function_async(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
assert dropped is True
assert [path for path, _ in state["requests"]] == [
"/v1/function/analytics$features$normalize_score/create",
"/v1/jobs/describe",
"/v1/function/analytics$features$normalize_score/describe",
"/v1/namespace/analytics$features/function/list",
"/v1/namespace/analytics$features/function/list",
"/v1/function/analytics$features$normalize_score/drop",
"/v1/function/analytics$features$normalize_score/drop",
]
@pytest.mark.asyncio
async def test_async_remote_function_verbs_address_a_namespace_in_the_path():
namespace = ["analytics", "features"]
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
registration = await db.create_function_async(
normalize_score, namespace_path=namespace
)
await registration.wait()
await db.get_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
await db.list_functions(namespace_path=namespace)
assert (
await db.drop_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
is True
)
assert [path for path, _ in state["requests"]] == [
"/v1/function/analytics$features$normalize_score/create",
"/v1/jobs/describe",
"/v1/function/analytics$features$normalize_score/describe",
"/v1/namespace/analytics$features/function/list",
"/v1/namespace/analytics$features/function/list",
"/v1/function/analytics$features$normalize_score/drop",
]
def test_a_function_namespace_path_must_be_a_list_of_segments():
"""A bare string is refused rather than read as a sequence of one-character
segments, and nothing reaches the service."""
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
with pytest.raises(TypeError):
db.list_functions(namespace_path="analytics")
assert state["requests"] == []
python/python/tests/test_lsm_write_spec.py
+7 -7
View File
@@ -52,9 +52,9 @@ def test_set_lsm_write_spec_validates(tmp_path):
with pytest.raises(Exception, match="num_buckets"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025))
# Happy path then mutation rejected.
# Happy path, then a set over the installed spec is refused.
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
with pytest.raises(Exception, match="mutation"):
with pytest.raises(Exception, match="already set"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
@@ -176,18 +176,18 @@ def test_get_lsm_write_spec(tmp_path):
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema). Leaving the
# maintained set to be inferred picks up the index on the table, so the
# spec reads back naming it rather than as "infer".
# Identity round-trips (column recovered from the schema). An unnamed
# maintained set reads back as None: what is stored is the intent to
# maintain every index, not the set it resolves to now.
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
assert spec.maintained_indexes == [idx_name]
assert spec.maintained_indexes is None
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column). Opting out is distinct from
# the inferred default.
# the unnamed default.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
python/python/tests/test_pydantic.py
+20
View File
@@ -9,6 +9,7 @@ from typing import List, Optional, Tuple
import pyarrow as pa
import pydantic
import pytest
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.pydantic import (
LanceModel,
MultiVector,
@@ -425,6 +426,25 @@ def test_bare_vector_raises_clear_error():
exec("class TestModel(LanceModel):\n vector: Vector", namespace)
def test_embedding_vector_list_annotation():
embedding = MockTextEmbeddingFunction.create()
class StaticTypingModel(LanceModel):
text: str = embedding.SourceField()
vector: list[float] = embedding.VectorField()
schema = pydantic_to_schema(StaticTypingModel)
assert schema == pa.schema(
[
pa.field("text", pa.utf8(), False),
pa.field("vector", pa.list_(pa.float32(), embedding.ndims()), True),
]
)
model = StaticTypingModel(text="hello", vector=[0.0] * embedding.ndims())
assert model.vector == [0.0] * embedding.ndims()
def test_fixed_size_list_field():
class TestModel(pydantic.BaseModel):
vec: Vector(16)
python/python/tests/test_remote_db.py
+28 -2
View File
@@ -2396,6 +2396,27 @@ def blob_query_response_table():
)
def blob_descriptor_take_table():
"""What `fetch_blob_files` reads for rows 10, 20, 30, and 40."""
queried = blob_query_response_table()
image_field = queried.schema.field("image")
images = pa.StructArray.from_arrays(
[
pa.array([0, 0, 0, 0], type=pa.uint8()),
pa.array([0, 0, 0, 0], type=pa.uint64()),
pa.array([5, 0, 5, 0], type=pa.uint64()),
pa.array([0, 0, 0, 0], type=pa.uint32()),
pa.array(["", "", "", ""], type=pa.string()),
],
fields=image_field.type,
mask=pa.array([False, True, False, False]),
)
return pa.Table.from_arrays(
[images, pa.array([10, 20, 30, 40], type=pa.uint64())],
schema=pa.schema([image_field, pa.field("_rowid", pa.uint64())]),
)
@contextlib.contextmanager
def blob_remote_table(*, server_version=Version("0.5.0")):
def handler(request):
@@ -2434,9 +2455,14 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length", 0))
body = json.loads(request.rfile.read(content_len))
assert body["columns"] == ["id", "image"]
assert body["with_row_id"] is True
response_table = blob_query_response_table()
if body["columns"] == ["image"]:
# fetch_blob_files sizes its handles from a descriptor take.
assert body["filter"].startswith("_rowid IN")
response_table = blob_descriptor_take_table()
else:
assert body["columns"] == ["id", "image"]
response_table = blob_query_response_table()
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
python/src/connection.rs
+26 -6
View File
@@ -754,31 +754,37 @@ impl Connection {
})
}
#[pyo3(signature = (request_json, namespace_path=None))]
pub fn create_function_async(
self_: PyRef<'_, Self>,
request_json: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
.infer_error()?;
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.create_function_async(request)
.create_function_async(request, &namespace_path)
.await
.infer_error()
.map(crate::job::Job::new_typed)
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn get_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.get_function(name, version)
.get_function(name, version, &namespace_path)
.await
.infer_error()?
.to_canonical_json()
@@ -786,11 +792,16 @@ impl Connection {
})
}
pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
#[pyo3(signature = (namespace_path=None))]
pub fn list_functions(
self_: PyRef<'_, Self>,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.list_functions()
.list_functions(&namespace_path)
.await
.infer_error()?
.into_iter()
@@ -799,26 +810,35 @@ impl Connection {
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
inner
.drop_function(name, version, &namespace_path)
.await
.infer_error()
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn drop_function_async(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.drop_function_async(name, version)
.drop_function_async(name, version, &namespace_path)
.await
.infer_error()
.map(|(dropped, job)| (dropped, crate::job::Job::new(job)))
python/src/query.rs
+3
View File
@@ -180,6 +180,9 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
.expect("Failed to import namespace");
match self.0 {
FtsQuery::CombinedFields(_) => Err(PyValueError::new_err(
"Combined-fields FTS queries are not yet supported in Python",
)),
FtsQuery::Match(query) => {
let kwargs = PyDict::new(py);
kwargs.set_item("boost", query.boost)?;
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/connection.rs
+20 -9
View File
@@ -587,7 +587,8 @@ impl Connection {
)
}
/// Build and register a Python callable as an immutable Function version.
/// Build and register a Python callable as an immutable Function version
/// in `namespace_path`, which is empty for the root namespace.
///
/// The server-side job builds the OCI image, then registers the completed
/// artifact. Waiting on the returned typed job yields the durable
@@ -597,8 +598,11 @@ impl Connection {
pub async fn create_function_async(
&self,
request: crate::function::FunctionRegistrationRequest,
namespace_path: &[String],
) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
self.internal.create_function_async(request).await
self.internal
.create_function_async(request, namespace_path)
.await
}
/// Look up one exact immutable Function version in the remote catalog.
@@ -610,13 +614,15 @@ impl Connection {
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
namespace_path: &[String],
) -> Result<crate::function::FunctionVersion> {
self.internal
.get_function(name.as_ref(), version.as_ref())
.get_function(name.as_ref(), version.as_ref(), namespace_path)
.await
}
/// List every published immutable Function version in the remote catalog.
/// List every published immutable Function version in `namespace_path` of
/// the remote catalog. Functions in child namespaces are not included.
///
/// Results are ordered by Function name then version. The client walks all
/// server pages before returning. Local databases return
@@ -628,14 +634,17 @@ impl Connection {
/// # async fn list_functions(
/// # connection: &lancedb::Connection,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// for function in connection.list_functions().await? {
/// for function in connection.list_functions(&[]).await? {
/// println!("{} {}", function.name(), function.version());
/// }
/// # Ok(())
/// # }
/// ```
pub async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
self.internal.list_functions().await
pub async fn list_functions(
&self,
namespace_path: &[String],
) -> Result<Vec<crate::function::FunctionVersion>> {
self.internal.list_functions(namespace_path).await
}
/// Remove the current Function name binding, retaining the object history.
@@ -647,9 +656,10 @@ impl Connection {
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
namespace_path: &[String],
) -> Result<bool> {
self.internal
.drop_function(name.as_ref(), version.as_ref())
.drop_function(name.as_ref(), version.as_ref(), namespace_path)
.await
}
@@ -663,9 +673,10 @@ impl Connection {
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
namespace_path: &[String],
) -> Result<(bool, crate::job::Job)> {
self.internal
.drop_function_async(name.as_ref(), version.as_ref())
.drop_function_async(name.as_ref(), version.as_ref(), namespace_path)
.await
}
rust/lancedb/src/database.rs
+18 -5
View File
@@ -333,10 +333,12 @@ pub trait Database:
///
/// See [`CloneTableRequest`] for detailed documentation and examples.
async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>>;
/// Submit a Function creation job that builds an image and registers it.
/// Submit a Function creation job that builds an image and registers it
/// in `namespace_path`.
async fn create_function_async(
&self,
_request: crate::function::FunctionRegistrationRequest,
_namespace_path: &[String],
) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
function_catalog_not_supported()
}
@@ -412,15 +414,25 @@ pub trait Database:
&self,
_name: &str,
_version: &str,
_namespace_path: &[String],
) -> Result<crate::function::FunctionVersion> {
function_catalog_not_supported()
}
/// List every published immutable Function version in the remote catalog.
async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
/// List every published immutable Function version in one namespace of the
/// remote catalog.
async fn list_functions(
&self,
_namespace_path: &[String],
) -> Result<Vec<crate::function::FunctionVersion>> {
function_catalog_not_supported()
}
/// Remove the current Function name binding, retaining the object history.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
async fn drop_function(
&self,
_name: &str,
_version: &str,
_namespace_path: &[String],
) -> Result<bool> {
function_catalog_not_supported()
}
/// Start dropping a Function and return a handle to the cleanup job.
@@ -431,8 +443,9 @@ pub trait Database:
&self,
name: &str,
version: &str,
namespace_path: &[String],
) -> Result<(bool, crate::job::Job)> {
let dropped = self.drop_function(name, version).await?;
let dropped = self.drop_function(name, version, namespace_path).await?;
Ok((dropped, crate::job::Job::new_done()))
}
/// Create a named Secret in this database. Fails if the name is taken, so
rust/lancedb/src/function.rs
+71 -12
View File
@@ -85,22 +85,25 @@ fn has_unknown_keys(value: &Value, allowed: &[&str]) -> bool {
.is_some_and(|object| object.keys().any(|key| !allowed.contains(&key.as_str())))
}
/// Every field a [`FunctionVersionRef`] document may carry. An application or
/// binding with any other key comes from a newer contract.
pub(crate) const FUNCTION_VERSION_REF_FIELDS: &[&str] = &[
"name",
"namespace_path",
"object_id",
"location",
"version",
"manifest_digest",
];
fn application_has_unknown_nested_fields(value: &Value) -> bool {
let Some(application) = value.as_object() else {
return false;
};
if application.get("function").is_some_and(|value| {
has_unknown_keys(
value,
&[
"name",
"object_id",
"location",
"version",
"manifest_digest",
],
)
}) {
if application
.get("function")
.is_some_and(|value| has_unknown_keys(value, FUNCTION_VERSION_REF_FIELDS))
{
return true;
}
if application
@@ -431,9 +434,17 @@ fn deserialize_object_version<'de, D: Deserializer<'de>>(
}
/// One immutable Function object revision and its executable artifact.
///
/// The Function is named by its parts, `name` and `namespace_path`, the way a
/// [`SecretReference`](crate::secrets::SecretReference) names a Secret.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersion {
name: String,
/// The namespace holding the Function. Empty is the root, and is absent
/// from the wire, as a [`SecretReference`](crate::secrets::SecretReference)
/// omits it.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
namespace_path: Vec<String>,
object_id: String,
location: String,
#[serde(deserialize_with = "deserialize_object_version")]
@@ -463,6 +474,7 @@ impl FunctionVersion {
pub fn reference(&self) -> FunctionVersionRef {
FunctionVersionRef {
name: self.name.clone(),
namespace_path: self.namespace_path.clone(),
object_id: self.object_id.clone(),
location: self.location.clone(),
version: self.version.clone(),
@@ -472,6 +484,9 @@ impl FunctionVersion {
pub fn name(&self) -> &str {
&self.name
}
pub fn namespace_path(&self) -> &[String] {
&self.namespace_path
}
pub fn version(&self) -> &str {
&self.version
}
@@ -556,6 +571,10 @@ impl_json!(FunctionRegistrationRequest);
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersionRef {
pub name: String,
/// The namespace holding the Function; empty is the root and is absent
/// from the wire.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub namespace_path: Vec<String>,
pub object_id: String,
pub location: String,
#[serde(deserialize_with = "deserialize_object_version")]
@@ -954,6 +973,46 @@ mod tests {
);
}
/// A Function is named by its parts, and a root one states no namespace.
#[test]
fn function_version_carries_its_namespace_as_a_separate_field() {
const VERSION: &str = include_str!(
"../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json"
);
let root = FunctionVersion::from_json(VERSION).unwrap();
assert!(root.namespace_path().is_empty());
assert_eq!(root.to_canonical_json().unwrap(), VERSION.trim_end());
let mut document: Value = serde_json::from_str(VERSION).unwrap();
document["namespace_path"] = serde_json::json!(["analytics", "features"]);
let nested = FunctionVersion::from_json(&document.to_string()).unwrap();
assert_eq!(nested.name(), "embed");
assert_eq!(nested.namespace_path(), ["analytics", "features"]);
let reference = nested.reference();
assert_eq!(reference.name, "embed");
assert_eq!(reference.namespace_path, ["analytics", "features"]);
let reference: Value = serde_json::to_value(&reference).unwrap();
assert_eq!(
reference["namespace_path"],
serde_json::json!(["analytics", "features"])
);
let root_reference = serde_json::to_value(root.reference()).unwrap();
assert!(root_reference.get("namespace_path").is_none());
// An application carrying the namespace is not from a newer contract.
let application = serde_json::json!({
"function": reference,
"inputs": [],
"output": serde_json::to_value(nested.signature().output.clone()).unwrap(),
});
let application = FunctionApplication::from_json(&application.to_string()).unwrap();
assert!(!application.has_unknown_fields());
assert_eq!(
application.function().namespace_path,
["analytics", "features"]
);
}
/// Arrays are ordered by the caller, so canonicalization must leave them
/// alone -- sorting them would change what a signature means.
#[test]
rust/lancedb/src/remote/db.rs
+117 -19
View File
@@ -979,8 +979,9 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
async fn create_function_async(
&self,
request: FunctionRegistrationRequest,
namespace_path: &[String],
) -> Result<Job<FunctionVersion>> {
let function_id = build_object_identifier("Function name", &request.name, &[])?;
let function_id = build_object_identifier("Function name", &request.name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/function/{function_id}/create"))
@@ -1005,8 +1006,13 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
))))
}
async fn get_function(&self, name: &str, version: &str) -> Result<FunctionVersion> {
let function_id = build_object_identifier("Function name", name, &[])?;
async fn get_function(
&self,
name: &str,
version: &str,
namespace_path: &[String],
) -> Result<FunctionVersion> {
let function_id = build_object_identifier("Function name", name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/function/{function_id}/describe"))
@@ -1018,8 +1024,8 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
response.json().await.err_to_http(request_id)
}
async fn list_functions(&self) -> Result<Vec<FunctionVersion>> {
let namespace_id = build_namespace_identifier(&[])?;
async fn list_functions(&self, namespace_path: &[String]) -> Result<Vec<FunctionVersion>> {
let namespace_id = build_namespace_identifier(namespace_path)?;
let path = format!("/v1/namespace/{namespace_id}/function/list");
let mut functions = Vec::new();
let mut page_token: Option<String> = None;
@@ -1059,12 +1065,25 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
Ok(functions)
}
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
Ok(self.drop_function_async(name, version).await?.0)
async fn drop_function(
&self,
name: &str,
version: &str,
namespace_path: &[String],
) -> Result<bool> {
Ok(self
.drop_function_async(name, version, namespace_path)
.await?
.0)
}
async fn drop_function_async(&self, name: &str, version: &str) -> Result<(bool, Job)> {
let function_id = build_object_identifier("Function name", name, &[])?;
async fn drop_function_async(
&self,
name: &str,
version: &str,
namespace_path: &[String],
) -> Result<(bool, Job)> {
let function_id = build_object_identifier("Function name", name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/function/{function_id}/drop"))
@@ -2004,7 +2023,7 @@ mod tests {
.body(serde_json::json!({"dropped": true, "job_id": "j1-fn-drop"}).to_string())
.unwrap()
});
let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap();
let (dropped, job) = db.drop_function_async("embed", "1", &[]).await.unwrap();
assert!(dropped);
assert_eq!(job.id(), Some("j1-fn-drop"));
}
@@ -2019,7 +2038,7 @@ mod tests {
.body(serde_json::json!({"dropped": false}).to_string())
.unwrap()
});
let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap();
let (dropped, job) = db.drop_function_async("embed", "1", &[]).await.unwrap();
assert!(!dropped);
assert_eq!(job.id(), None);
assert_eq!(job.status().await.unwrap(), "finished");
@@ -2036,7 +2055,11 @@ mod tests {
let db = super::RemoteDatabase::new_mock(move |_| {
http::Response::builder().status(202).body(body).unwrap()
});
let error = db.drop_function_async("embed", "1").await.err().unwrap();
let error = db
.drop_function_async("embed", "1", &[])
.await
.err()
.unwrap();
assert!(error.to_string().contains("valid job_id"));
}
}
@@ -3905,7 +3928,7 @@ mod tests {
http::Response::builder().status(200).body("{}").unwrap()
});
let error = conn
.drop_function(name, "fv_1")
.drop_function(name, "fv_1", &[])
.await
.expect_err("a dot-only Function name must be refused");
assert!(
@@ -4307,7 +4330,7 @@ mod tests {
path => panic!("unexpected path: {path}"),
});
let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
let job = conn.create_function_async(request).await.unwrap();
let job = conn.create_function_async(request, &[]).await.unwrap();
assert_eq!(job.id(), Some("job-function-1"));
let version = job.wait().await.unwrap();
assert_eq!(version.name(), "embed");
@@ -4327,7 +4350,7 @@ mod tests {
assert_eq!(body, serde_json::json!({"version": "1"}));
http::Response::builder().status(200).body(VERSION).unwrap()
});
let version = conn.get_function("embed", "1").await.unwrap();
let version = conn.get_function("embed", "1", &[]).await.unwrap();
assert_eq!(version.name(), "embed");
assert_eq!(version.version(), "1");
}
@@ -4370,7 +4393,7 @@ mod tests {
}
}
});
let functions = conn.list_functions().await.unwrap();
let functions = conn.list_functions(&[]).await.unwrap();
assert_eq!(functions.len(), 1);
assert_eq!(functions[0].name(), "embed");
assert_eq!(functions[0].version(), "1");
@@ -4393,7 +4416,7 @@ mod tests {
.unwrap()
});
let functions = conn.list_functions().await.unwrap();
let functions = conn.list_functions(&[]).await.unwrap();
assert!(functions.is_empty());
assert_eq!(requests.load(Ordering::SeqCst), 1);
}
@@ -4434,7 +4457,7 @@ mod tests {
.unwrap()
});
let error = conn.list_functions().await.unwrap_err();
let error = conn.list_functions(&[]).await.unwrap_err();
assert!(
matches!(
&error,
@@ -4461,7 +4484,82 @@ mod tests {
.body(r#"{"dropped":false}"#)
.unwrap()
});
assert!(!conn.drop_function("embed", "1").await.unwrap());
assert!(!conn.drop_function("embed", "1", &[]).await.unwrap());
}
/// A Function's namespace is addressed in the path the way a Secret's is:
/// every route takes the joined identifier and no body carries the
/// namespace, so a namespaced request differs from a root one only in its
/// path.
#[tokio::test]
async fn test_a_function_namespace_path_is_addressed_in_the_path() {
const REQUEST: &str = include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
);
const VERSION: &str = include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json"
);
let namespace = ["analytics".to_string(), "features".to_string()];
let mut expected_create: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
expected_create.as_object_mut().unwrap().remove("name");
let conn = Connection::new_with_handler(move |request| {
let body = request
.body()
.and_then(|body| body.as_bytes())
.map(|bytes| serde_json::from_slice::<serde_json::Value>(bytes).unwrap());
match request.url().path() {
"/v1/function/analytics$features$normalize_score/create" => {
assert_eq!(body.unwrap(), expected_create);
http::Response::builder()
.status(202)
.body(r#"{"job_id":"job-function-1"}"#.to_string())
.unwrap()
}
"/v1/function/analytics$features$embed/describe" => {
assert_eq!(body.unwrap(), serde_json::json!({"version": "1"}));
http::Response::builder()
.status(200)
.body(VERSION.to_string())
.unwrap()
}
"/v1/function/analytics$features$embed/drop" => {
assert_eq!(body.unwrap(), serde_json::json!({"version": "1"}));
http::Response::builder()
.status(200)
.body(r#"{"dropped":true}"#.to_string())
.unwrap()
}
// Listing is namespace-scoped, so the namespace is the whole
// identifier.
"/v1/namespace/analytics$features/function/list" => http::Response::builder()
.status(200)
.body(r#"{"functions":[]}"#.to_string())
.unwrap(),
path => panic!("unexpected path: {path}"),
}
});
let request = crate::function::FunctionRegistrationRequest::from_json(REQUEST).unwrap();
let job = conn
.create_function_async(request, &namespace)
.await
.unwrap();
assert_eq!(job.id(), Some("job-function-1"));
conn.get_function("embed", "1", &namespace).await.unwrap();
assert!(conn.list_functions(&namespace).await.unwrap().is_empty());
assert!(conn.drop_function("embed", "1", &namespace).await.unwrap());
}
/// Each namespace segment is checked before a request is built, so an
/// empty segment cannot collapse the identifier onto the parent namespace.
#[tokio::test]
async fn test_an_unaddressable_function_namespace_segment_is_refused() {
let conn = Connection::new_with_handler(|request| -> http::Response<String> {
panic!("reached the transport: {}", request.url().path())
});
let namespace = ["analytics".to_string(), String::new()];
assert!(conn.get_function("embed", "1", &namespace).await.is_err());
assert!(conn.list_functions(&namespace).await.is_err());
assert!(conn.drop_function("embed", "1", &namespace).await.is_err());
}
#[tokio::test]
rust/lancedb/src/remote/table.rs
+49
View File
@@ -96,6 +96,8 @@ const SCHEMA_SELECTOR_CHANGED: &str = "table selector changed while fetching sch
fn fts_query_requires_document_granularity_support(query: &FtsQuery) -> bool {
match query {
// Combined-fields queries do not expose a document granularity option.
FtsQuery::CombinedFields(_) => false,
FtsQuery::Match(query) => query
.document_granularity
.is_some_and(|granularity| granularity.is_list_element()),
@@ -6339,6 +6341,53 @@ mod tests {
.unwrap();
}
#[tokio::test]
async fn test_query_combined_fields_uses_structured_fts() {
use lance_index::scalar::inverted::query::CombinedFieldsQuery;
let table =
Table::new_with_handler_version("my_table", semver::Version::new(0, 3, 0), |request| {
let body = request.body().unwrap().as_bytes().unwrap();
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(
body["full_text_query"]["query"],
serde_json::json!({
"combined_fields": {
"query": "hello world",
"columns": ["title", "text"],
"boost": [1.0, 1.0],
"operator": "Or"
}
})
);
let data = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![1]))],
)
.unwrap();
http::Response::builder()
.status(200)
.header(CONTENT_TYPE, ARROW_FILE_CONTENT_TYPE)
.body(write_ipc_file(&data))
.unwrap()
});
table
.query()
.full_text_search(FullTextSearchQuery::new_query(
CombinedFieldsQuery::try_new(
"hello world".into(),
vec!["title".into(), "text".into()],
)
.unwrap()
.into(),
))
.execute()
.await
.unwrap();
}
#[tokio::test]
async fn test_query_row_document_granularity_uses_structured_fts() {
let table =
rust/lancedb/src/remote/table/blobs.rs
+624 -70
View File
@@ -3,23 +3,28 @@
//! Cloud blob column listing, whole-byte fetch, and seekable HTTP byte-range handles.
use std::collections::HashMap;
use std::ops::Range;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use arrow_array::{Array, LargeBinaryArray};
use arrow_array::{Array, LargeBinaryArray, StructArray, UInt8Array, UInt64Array};
use arrow_schema::DataType;
use bytes::Bytes;
use datafusion_expr::{col, lit};
use futures::{StreamExt, TryStreamExt};
use lance::dataset::ROW_ID;
use lance_core::datatypes::BlobKind;
use reqwest::{Response, StatusCode, header};
use tokio::sync::{Mutex, OwnedSemaphorePermit};
use crate::Error;
use crate::blob::BlobFile;
use crate::error::Result;
use crate::query::{QueryFilter, QueryRequest, Select};
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
use crate::table::BaseTable;
use crate::table::{AnyQuery, BaseTable};
use super::{
FreshnessHeaders, FreshnessState, ReadSnapshot, RemoteTable, VERSION_HEADER,
@@ -142,7 +147,7 @@ struct RemoteBlobState {
/// Seekable Cloud blob handle over HTTP Range.
///
/// Nonempty handles read the table version returned by their size probe.
/// Handles read the table version used to fetch their descriptor or probe their size.
#[derive(Debug)]
pub struct RemoteBlobFile {
requester: Arc<dyn BlobRangeRequester>,
@@ -503,7 +508,11 @@ impl<S: HttpSend> RemoteTable<S> {
/// Open seekable handles for `row_ids`.
///
/// Each non-null row is probed once to determine its size.
/// Descriptors for the requested rows come from one `_rowid` take query per
/// [`MAX_FETCH_BLOBS_ROW_IDS`] unique row ids, all read at one exact table
/// version. Each handle is sized from its descriptor and reads that
/// version. Only external blobs whose descriptor records no size are
/// probed, because their length is the whole object's.
pub(super) async fn fetch_blob_files_impl(
&self,
column: &str,
@@ -513,46 +522,251 @@ impl<S: HttpSend> RemoteTable<S> {
if row_ids.is_empty() {
return Ok(Vec::new());
}
if !self.server_version.support_blobs() {
return Err(Error::NotSupported {
message: "fetch_blob_files requires LanceDB Cloud server 0.5.0 or newer".into(),
});
}
self.ensure_blob_files_supported()?;
let read_snapshot = self.snapshot_read_state().await;
self.fetch_blob_files_with_snapshot(column, row_ids, read_snapshot)
.await
let mut read_snapshot = self.snapshot_read_state().await;
let sizes = self
.take_blob_descriptor_sizes(column, row_ids, &mut read_snapshot)
.await?;
let version = read_snapshot.version;
let mut files = Vec::with_capacity(row_ids.len());
let mut probe_indices = Vec::new();
let mut probe_requesters = Vec::new();
for (index, row_id) in row_ids.iter().enumerate() {
match sizes[row_id] {
DescriptorSize::Null => files.push(None),
DescriptorSize::Known(size) => {
let requester = self.blob_range_requester(column, *row_id, &read_snapshot);
files.push(Some(RemoteBlobFile::new(requester, size, version).into()));
}
DescriptorSize::Unresolved => {
files.push(None);
probe_indices.push(index);
probe_requesters.push(self.blob_range_requester(
column,
*row_id,
&read_snapshot,
));
}
}
}
let probed = probe_blob_files(probe_requesters).await?;
for (index, file) in probe_indices.into_iter().zip(probed) {
files[index] = file;
}
Ok(files)
}
/// Probe each row once for its size.
///
/// Serves callers that must not issue a query, such as a single blob that
/// the whole-byte route could not return.
async fn fetch_blob_files_with_snapshot(
&self,
column: &str,
row_ids: &[u64],
read_snapshot: ReadSnapshot,
) -> Result<Vec<Option<BlobFile>>> {
let encoded_column = urlencoding::encode(column);
let requesters = row_ids
.iter()
.map(|row_id| {
let path = format!(
"/v1/table/{}/blob/{encoded_column}/{row_id}/bytes",
self.identifier
);
let requester: Arc<dyn BlobRangeRequester> = Arc::new(TableBlobRangeRequester {
client: self.client.clone(),
path,
version: read_snapshot.version,
branch: self.branch.clone(),
freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)),
parent_freshness: self.freshness.clone(),
parent_freshness_request: read_snapshot.freshness,
read_consistency_interval: self.client.read_consistency_interval,
});
requester
})
.map(|row_id| self.blob_range_requester(column, *row_id, &read_snapshot))
.collect();
probe_blob_files(requesters).await
}
/// Read the blob descriptor of every unique row in `row_ids`.
///
/// Pins `read_snapshot` to the version the first take read, so later chunks
/// and the returned handles all see one table version.
async fn take_blob_descriptor_sizes(
&self,
column: &str,
row_ids: &[u64],
read_snapshot: &mut ReadSnapshot,
) -> Result<HashMap<u64, DescriptorSize>> {
let mut unique_row_ids = row_ids.to_vec();
unique_row_ids.sort_unstable();
unique_row_ids.dedup();
let mut sizes = HashMap::with_capacity(unique_row_ids.len());
for chunk in unique_row_ids.chunks(MAX_FETCH_BLOBS_ROW_IDS) {
let (mut chunk_sizes, response_version) = self
.take_blob_descriptor_chunk(column, chunk, *read_snapshot)
.await?;
if read_snapshot.version.is_none() {
match response_version {
Some(version) => read_snapshot.version = Some(version),
None => {
// A server that does not report its read version may have read
// an older version than a describe now returns, so the take is
// repeated at the described version.
read_snapshot.version =
Some(self.describe_read_snapshot(*read_snapshot).await?.version);
(chunk_sizes, _) = self
.take_blob_descriptor_chunk(column, chunk, *read_snapshot)
.await?;
}
}
}
sizes.extend(chunk_sizes);
}
let found = unique_row_ids
.iter()
.filter(|row_id| sizes.contains_key(row_id))
.count();
if found < unique_row_ids.len() {
return Err(Error::InvalidInput {
message: format!(
"blob read for column '{column}' requested {} row ids but only {found} exist \
in the table; pass row ids collected from this table",
unique_row_ids.len()
),
});
}
Ok(sizes)
}
/// Take one chunk of descriptors, also returning the version the server
/// reported reading.
async fn take_blob_descriptor_chunk(
&self,
column: &str,
row_ids: &[u64],
read_snapshot: ReadSnapshot,
) -> Result<(HashMap<u64, DescriptorSize>, Option<u64>)> {
let query = AnyQuery::Query(QueryRequest {
filter: Some(QueryFilter::Datafusion(
col(ROW_ID).in_list(row_ids.iter().map(|id| lit(*id)).collect(), false),
)),
select: Select::columns(&[column]),
with_row_id: true,
// Row ids name base-table rows, and the MemWAL scanner has no stable `_rowid`.
use_lsm: Some(false),
..Default::default()
});
let body = self
.prepare_query_bodies(&query, read_snapshot.version)?
.pop()
.expect("a plain query has one body");
let request = self
.client
.post(&format!("/v1/table/{}/query/", self.identifier))
.json(&body);
let (request_id, response) = self
.send_with_freshness(request, true, read_snapshot.freshness)
.await?;
let response_version = response
.headers()
.get(&VERSION_HEADER)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse().ok());
let mut stream = self.read_arrow_response(&request_id, response).await?;
let mut sizes = HashMap::with_capacity(row_ids.len());
let invalid_response = |message: String| Error::Http {
source: message.into(),
request_id: request_id.clone(),
status_code: None,
};
while let Some(batch) = stream.try_next().await? {
let taken_row_ids = batch
.column_by_name(ROW_ID)
.and_then(|array| array.as_any().downcast_ref::<UInt64Array>())
.ok_or_else(|| {
invalid_response(format!("blob descriptor take is missing UInt64 '{ROW_ID}'"))
})?;
let values = batch.column_by_name(column).ok_or_else(|| {
invalid_response(format!("blob descriptor take is missing '{column}'"))
})?;
// Only blob v2 columns read back as descriptors.
let descriptors = values
.as_any()
.downcast_ref::<StructArray>()
.and_then(BlobDescriptorSizes::try_new)
.ok_or_else(|| Error::InvalidInput {
message: format!("column '{column}' is not a blob column"),
})?;
for (index, row_id) in taken_row_ids.values().iter().enumerate() {
sizes.insert(*row_id, descriptors.size(index)?);
}
}
Ok((sizes, response_version))
}
fn ensure_blob_files_supported(&self) -> Result<()> {
if self.server_version.support_blobs() {
Ok(())
} else {
Err(Error::NotSupported {
message: "fetch_blob_files requires LanceDB Cloud server 0.5.0 or newer".into(),
})
}
}
fn blob_range_requester(
&self,
column: &str,
row_id: u64,
read_snapshot: &ReadSnapshot,
) -> Arc<dyn BlobRangeRequester> {
let path = format!(
"/v1/table/{}/blob/{}/{row_id}/bytes",
self.identifier,
urlencoding::encode(column)
);
Arc::new(TableBlobRangeRequester {
client: self.client.clone(),
path,
version: read_snapshot.version,
branch: self.branch.clone(),
freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)),
parent_freshness: self.freshness.clone(),
parent_freshness_request: read_snapshot.freshness,
read_consistency_interval: self.client.read_consistency_interval,
})
}
}
/// What a blob v2 descriptor says about a row's size.
#[derive(Debug, Clone, Copy)]
enum DescriptorSize {
Null,
Known(u64),
/// An external blob that recorded no size. Its length is the whole
/// object's, which only the server can resolve.
Unresolved,
}
/// The `kind` and `size` children of a blob v2 descriptor array.
struct BlobDescriptorSizes<'a> {
descriptors: &'a StructArray,
kinds: &'a UInt8Array,
sizes: &'a UInt64Array,
}
impl<'a> BlobDescriptorSizes<'a> {
fn try_new(descriptors: &'a StructArray) -> Option<Self> {
let child = |name: &str| descriptors.column_by_name(name);
Some(Self {
descriptors,
kinds: child("kind")?.as_any().downcast_ref::<UInt8Array>()?,
sizes: child("size")?.as_any().downcast_ref::<UInt64Array>()?,
})
}
fn size(&self, index: usize) -> Result<DescriptorSize> {
if self.descriptors.is_null(index) || self.kinds.is_null(index) {
return Ok(DescriptorSize::Null);
}
let kind = BlobKind::try_from(self.kinds.value(index))?;
let size = self.sizes.value(index);
Ok(if kind == BlobKind::External && size == 0 {
DescriptorSize::Unresolved
} else {
DescriptorSize::Known(size)
})
}
}
/// Probe blob sizes while preserving row order.
@@ -676,6 +890,8 @@ mod tests {
use reqwest::Request;
use semver::Version;
use arrow_array::RecordBatch;
use super::*;
const PAYLOAD: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz";
@@ -759,10 +975,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let mut files = table
.fetch_blob_files_impl("image", &[10, 20])
.await
.unwrap();
let mut files = table.probe_files("image", &[10, 20]).await.unwrap();
assert_eq!(files.len(), 2);
assert!(files[1].is_none());
@@ -774,12 +987,368 @@ mod tests {
);
}
/// Descriptors as a remote query returns them, one `(kind, size)` per row.
fn descriptors(rows: &[Option<(BlobKind, u64)>]) -> StructArray {
let kinds = rows
.iter()
.map(|row| row.map_or(0, |(kind, _)| kind as u8))
.collect::<UInt8Array>();
let sizes = rows
.iter()
.map(|row| row.map_or(0, |(_, size)| size))
.collect::<UInt64Array>();
let len = rows.len();
StructArray::new(
lance_core::datatypes::BLOB_V2_DESC_FIELDS.clone(),
vec![
Arc::new(kinds),
Arc::new(UInt64Array::from(vec![0; len])),
Arc::new(sizes),
Arc::new(arrow_array::UInt32Array::from(vec![0; len])),
Arc::new(arrow_array::StringArray::from(vec![""; len])),
],
Some(rows.iter().map(Option::is_some).collect()),
)
}
impl<S: HttpSend> RemoteTable<S> {
/// Open handles through the per-row size probe alone.
async fn probe_files(
&self,
column: &str,
row_ids: &[u64],
) -> Result<Vec<Option<BlobFile>>> {
let read_snapshot = self.snapshot_read_state().await;
self.fetch_blob_files_with_snapshot(column, row_ids, read_snapshot)
.await
}
}
/// A request the take mock received.
#[derive(Debug, Clone, PartialEq)]
enum TakeMockRequest {
Query {
row_ids: Vec<u64>,
version: Option<u64>,
},
Describe,
Range {
row_id: u64,
range: String,
version: Option<String>,
},
}
/// Row ids named by a `_rowid IN (...)` filter.
fn filter_row_ids(filter: &str) -> Vec<u64> {
let list = filter.split_once('(').unwrap().1.trim_end_matches(')');
list.split(',')
.map(|id| id.trim().parse().unwrap())
.collect()
}
/// A table whose query route answers `_rowid` takes from `rows` and whose
/// byte routes serve [`PAYLOAD`]. Takes return rows in reverse order, and
/// report `query_version` in `x-lancedb-version` when it is set.
fn mock_take_blob_table(
rows: Vec<(u64, Option<(BlobKind, u64)>)>,
query_version: Option<&'static str>,
) -> (
RemoteTable<crate::remote::client::test_utils::MockSender>,
Arc<StdMutex<Vec<TakeMockRequest>>>,
) {
let requests = Arc::new(StdMutex::new(Vec::new()));
let captured = requests.clone();
let table = RemoteTable::new_mock(
"my_table".to_string(),
move |request| {
let path = request.url().path().to_string();
if path == "/v1/table/my_table/query/" {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
.unwrap();
assert_eq!(body["columns"], serde_json::json!(["image"]));
assert_eq!(body["with_row_id"], serde_json::json!(true));
assert_eq!(body["use_lsm"], serde_json::json!(false));
let requested = filter_row_ids(body["filter"].as_str().unwrap());
captured.lock().unwrap().push(TakeMockRequest::Query {
row_ids: requested.clone(),
version: body["version"].as_u64(),
});
let taken = rows
.iter()
.rev()
.filter(|(row_id, _)| requested.contains(row_id))
.collect::<Vec<_>>();
let batch = RecordBatch::try_from_iter([
(
"image",
Arc::new(descriptors(
&taken.iter().map(|(_, row)| *row).collect::<Vec<_>>(),
)) as Arc<dyn Array>,
),
(
ROW_ID,
Arc::new(UInt64Array::from_iter_values(
taken.iter().map(|(row_id, _)| *row_id),
)) as Arc<dyn Array>,
),
])
.unwrap();
let mut body = Vec::new();
let mut writer =
arrow_ipc::writer::FileWriter::try_new(&mut body, &batch.schema()).unwrap();
writer.write(&batch).unwrap();
writer.finish().unwrap();
drop(writer);
let mut response = http::Response::builder().status(200);
if let Some(version) = query_version {
response = response.header(VERSION_HEADER, version);
}
return response.body(body).unwrap();
}
if path == "/v1/table/my_table/describe/" {
captured.lock().unwrap().push(TakeMockRequest::Describe);
return http::Response::builder()
.status(200)
.body(r#"{"version":9,"schema":{"fields":[]}}"#.as_bytes().to_vec())
.unwrap();
}
let row_id = path
.strip_prefix("/v1/table/my_table/blob/image/")
.and_then(|rest| rest.strip_suffix("/bytes"))
.unwrap_or_else(|| panic!("unexpected path: {path}"))
.parse()
.unwrap();
captured.lock().unwrap().push(TakeMockRequest::Range {
row_id,
range: request.headers()[header::RANGE]
.to_str()
.unwrap()
.to_string(),
version: request
.url()
.query_pairs()
.find(|(name, _)| name == "version")
.map(|(_, value)| value.into_owned()),
});
range_response(&request, PAYLOAD)
},
Some(Version::new(0, 5, 0)),
);
(table, requests)
}
#[tokio::test]
async fn remote_blob_files_are_sized_from_one_descriptor_take() {
let payload_size = PAYLOAD.len() as u64;
let (table, requests) = mock_take_blob_table(
vec![
(10, Some((BlobKind::Inline, payload_size))),
(20, None),
(30, Some((BlobKind::Inline, 0))),
(40, Some((BlobKind::Packed, payload_size))),
(50, Some((BlobKind::Dedicated, payload_size))),
(60, Some((BlobKind::External, 0))),
],
Some("7"),
);
// Reordered and duplicated row ids come back in request order.
let files = table
.fetch_blob_files_impl("image", &[60, 10, 20, 30, 40, 50, 10])
.await
.unwrap();
let sizes = files
.iter()
.map(|file| file.as_ref().map(BlobFile::size))
.collect::<Vec<_>>();
assert_eq!(
sizes,
[
Some(payload_size),
Some(payload_size),
None,
Some(0),
Some(payload_size),
Some(payload_size),
Some(payload_size),
]
);
// One take, then a probe for the external blob that recorded no size.
assert_eq!(
requests.lock().unwrap().as_slice(),
[
TakeMockRequest::Query {
row_ids: vec![10, 20, 30, 40, 50, 60],
version: None,
},
TakeMockRequest::Range {
row_id: 60,
range: "bytes=0-0".into(),
version: Some("7".into()),
},
]
);
// Sized handles read the version the take read.
assert_eq!(
files[1].as_ref().unwrap().read_range(5..12).await.unwrap(),
&PAYLOAD[5..12]
);
assert!(files[3].as_ref().unwrap().read().await.unwrap().is_empty());
assert_eq!(
requests.lock().unwrap().last().unwrap(),
&TakeMockRequest::Range {
row_id: 10,
range: "bytes=5-11".into(),
version: Some("7".into()),
}
);
}
#[tokio::test]
async fn remote_blob_files_take_every_chunk_at_the_first_version() {
let row_ids = (0..MAX_FETCH_BLOBS_ROW_IDS as u64 + 1).collect::<Vec<_>>();
let (table, requests) = mock_take_blob_table(
row_ids.iter().map(|row_id| (*row_id, None)).collect(),
Some("7"),
);
let files = table
.fetch_blob_files_impl("image", &row_ids)
.await
.unwrap();
assert_eq!(files.len(), row_ids.len());
let versions = requests
.lock()
.unwrap()
.iter()
.map(|request| match request {
TakeMockRequest::Query { row_ids, version } => (row_ids.len(), *version),
other => panic!("unexpected request: {other:?}"),
})
.collect::<Vec<_>>();
assert_eq!(versions, [(MAX_FETCH_BLOBS_ROW_IDS, None), (1, Some(7))]);
}
#[tokio::test]
async fn remote_blob_files_retake_at_the_described_version_when_a_take_does_not_report_it() {
let payload_size = PAYLOAD.len() as u64;
let (table, requests) =
mock_take_blob_table(vec![(10, Some((BlobKind::Inline, payload_size)))], None);
let file = table
.fetch_blob_files_impl("image", &[10])
.await
.unwrap()
.pop()
.flatten()
.unwrap();
file.read_range(0..4).await.unwrap();
// The unversioned take is repeated at the described version.
assert_eq!(
requests.lock().unwrap().as_slice(),
[
TakeMockRequest::Query {
row_ids: vec![10],
version: None,
},
TakeMockRequest::Describe,
TakeMockRequest::Query {
row_ids: vec![10],
version: Some(9),
},
TakeMockRequest::Range {
row_id: 10,
range: "bytes=0-3".into(),
version: Some("9".into()),
},
]
);
}
#[tokio::test]
async fn remote_blob_files_take_at_the_checked_out_version() {
let payload_size = PAYLOAD.len() as u64;
let (table, requests) = mock_take_blob_table(
vec![(10, Some((BlobKind::Inline, payload_size)))],
Some("7"),
);
table.checkout(9).await.unwrap();
requests.lock().unwrap().clear();
table.fetch_blob_files_impl("image", &[10]).await.unwrap();
assert_eq!(
requests.lock().unwrap().as_slice(),
[TakeMockRequest::Query {
row_ids: vec![10],
version: Some(9),
}]
);
}
#[tokio::test]
async fn remote_blob_files_reject_row_ids_missing_from_the_take() {
let (table, requests) = mock_take_blob_table(vec![(10, None)], Some("7"));
let error = table
.fetch_blob_files_impl("image", &[10, 99])
.await
.unwrap_err();
assert!(matches!(error, Error::InvalidInput { .. }), "{error}");
assert!(error.to_string().contains("only 1 exist"), "{error}");
assert_eq!(requests.lock().unwrap().len(), 1);
}
#[tokio::test]
async fn remote_blob_files_reject_a_non_blob_column() {
let table = RemoteTable::new_mock(
"my_table".to_string(),
|request| {
assert_eq!(request.url().path(), "/v1/table/my_table/query/");
let batch = RecordBatch::try_from_iter([
(
"id",
Arc::new(arrow_array::Int64Array::from(vec![1])) as Arc<dyn Array>,
),
(
ROW_ID,
Arc::new(UInt64Array::from(vec![10])) as Arc<dyn Array>,
),
])
.unwrap();
let mut body = Vec::new();
let mut writer =
arrow_ipc::writer::FileWriter::try_new(&mut body, &batch.schema()).unwrap();
writer.write(&batch).unwrap();
writer.finish().unwrap();
drop(writer);
http::Response::builder().status(200).body(body).unwrap()
},
Some(Version::new(0, 5, 0)),
);
let error = table.fetch_blob_files_impl("id", &[10]).await.unwrap_err();
assert!(matches!(error, Error::InvalidInput { .. }), "{error}");
assert!(
error.to_string().contains("'id' is not a blob column"),
"{error}"
);
}
#[tokio::test]
async fn remote_blob_file_reads_the_requested_range() {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -817,7 +1386,7 @@ mod tests {
},
);
let files = table
.fetch_blob_files_impl("image", &(0..16).collect::<Vec<_>>())
.probe_files("image", &(0..16).collect::<Vec<_>>())
.await
.unwrap();
let start = Arc::new(tokio::sync::Barrier::new(files.len() + 1));
@@ -852,10 +1421,7 @@ mod tests {
..Default::default()
},
);
let files = table
.fetch_blob_files_impl("image", &[10, 20])
.await
.unwrap();
let files = table.probe_files("image", &[10, 20]).await.unwrap();
let reads = files
.iter()
.map(|file| file.as_ref().unwrap().read_up_to(1));
@@ -891,7 +1457,7 @@ mod tests {
..Default::default()
},
);
let file = table.fetch_blob_files_impl("image", &[10]).await.unwrap();
let file = table.probe_files("image", &[10]).await.unwrap();
let error = file[0]
.as_ref()
.unwrap()
@@ -939,7 +1505,7 @@ mod tests {
);
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -991,7 +1557,7 @@ mod tests {
table.checkout(5).await.unwrap();
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1011,7 +1577,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1034,10 +1600,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let mut files = table
.fetch_blob_files_impl("image", &[10, 20, 30])
.await
.unwrap();
let mut files = table.probe_files("image", &[10, 20, 30]).await.unwrap();
assert_eq!(files.len(), 3);
assert!(files[1].is_none());
@@ -1065,10 +1628,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let error = table
.fetch_blob_files_impl("image", &[10])
.await
.unwrap_err();
let error = table.probe_files("image", &[10]).await.unwrap_err();
assert!(
error
.to_string()
@@ -1092,10 +1652,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let error = table
.fetch_blob_files_impl("image", &[10])
.await
.unwrap_err();
let error = table.probe_files("image", &[10]).await.unwrap_err();
assert!(
error
.to_string()
@@ -1116,10 +1673,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let error = table
.fetch_blob_files_impl("image", &[10])
.await
.unwrap_err();
let error = table.probe_files("image", &[10]).await.unwrap_err();
assert!(
error
.to_string()
@@ -1177,7 +1731,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1217,7 +1771,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1267,7 +1821,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1292,7 +1846,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1371,7 +1925,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap();
let mut files = table.probe_files("image", &[10]).await.unwrap();
let file = files.remove(0).unwrap();
assert_eq!(file.size(), PAYLOAD.len() as u64);
@@ -1386,7 +1940,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap();
let mut files = table.probe_files("image", &[10]).await.unwrap();
let file = files.remove(0).unwrap();
let probe_requests = requests.lock().unwrap().len();
@@ -1411,7 +1965,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap();
let mut files = table.probe_files("image", &[10]).await.unwrap();
let file = files.remove(0).unwrap();
let probe_requests = requests.lock().unwrap().len();
@@ -1463,7 +2017,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap();
let mut files = table.probe_files("image", &[10]).await.unwrap();
let file = files.remove(0).unwrap();
let error = file.read_range(1..3).await.unwrap_err();
@@ -1554,7 +2108,7 @@ mod tests {
let requests = Arc::new(StdMutex::new(Vec::new()));
let table = mock_remote_blob_table(requests.clone());
let file = table
.fetch_blob_files_impl("image", &[10])
.probe_files("image", &[10])
.await
.unwrap()
.pop()
@@ -1600,7 +2154,7 @@ mod tests {
Some(Version::new(0, 5, 0)),
);
let mut files = table.fetch_blob_files_impl("image", &[10]).await.unwrap();
let mut files = table.probe_files("image", &[10]).await.unwrap();
let file = files.remove(0).unwrap();
let error = file.read_range(1..3).await.unwrap_err();
assert!(error.to_string().contains("206"), "got: {error}");
rust/lancedb/src/table.rs
+78 -39
View File
@@ -330,7 +330,8 @@ pub use self::merge::MergeResult;
/// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default
/// `ShardWriter` configuration recorded in the MemWAL index).
///
/// A fresh spec maintains every index on the table, resolved on install.
/// A fresh spec maintains every index on the table, including ones created
/// after it is installed.
///
/// Install a spec with [`Table::set_lsm_write_spec`] and remove it with
/// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch
@@ -348,9 +349,8 @@ pub enum LsmWriteSpec {
num_buckets: u32,
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
/// `None` means every index the table has, including ones created
/// later. `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
@@ -363,9 +363,8 @@ pub enum LsmWriteSpec {
column: String,
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
/// `None` means every index the table has, including ones created
/// later. `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
@@ -374,9 +373,8 @@ pub enum LsmWriteSpec {
Unsharded {
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
/// `None` means every index the table has, including ones created
/// later. `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
@@ -422,14 +420,15 @@ impl LsmWriteSpec {
/// Set which indexes the MemWAL maintains.
///
/// `None` (the default) resolves to every index on the table at install,
/// failing if one cannot be maintained — name the set to install anyway. A
/// list is verbatim: each name must already exist and be maintainable, and
/// an empty list maintains nothing.
/// `None` (the default) is every index on the table, re-read as the table
/// changes, so an index created later is maintained too; one of a kind the
/// MemWAL cannot mirror is skipped rather than failing the table. A list is
/// verbatim: each name must already exist and be maintainable, and an empty
/// list maintains nothing.
///
/// ```
/// # use lancedb::table::LsmWriteSpec;
/// // Every index the table has when the spec is installed:
/// // Every index the table has, now and later:
/// LsmWriteSpec::unsharded().with_maintained_indexes(None);
/// // Exactly these:
/// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]);
@@ -1991,6 +1990,10 @@ impl Table {
/// - [`LsmWriteSpec::identity`] — shard by the raw value of a scalar column.
/// - [`LsmWriteSpec::unsharded`] — route every write to a single shard.
///
/// A table carries one spec: this fails while one is installed, since the
/// generations already written were homed under it.
/// [`Table::unset_lsm_write_spec`] removes one.
///
/// # Example
///
/// ```
@@ -3794,7 +3797,20 @@ impl BaseTable for NativeTable {
let Some(segment) = segments.first() else {
continue;
};
let params = load_segment_params(&dataset, segment).await?;
// The listing itself only needs the manifest. Missing index files must
// not hide every other index, or callers cannot find the one to repair.
let params = match load_segment_params(&dataset, segment).await {
Ok(params) => params,
Err(err) => {
log::warn!(
"Failed to read full text search configuration for index '{}': {}",
index.name,
err
);
index.index_details = None;
continue;
}
};
let details = serde_json::to_string(&params).map_err(|source| Error::Other {
message: format!(
"Failed to serialize full text search configuration for index '{}'",
@@ -5654,29 +5670,22 @@ mod tests {
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Identity sharding round-trips (column recovered from the schema).
// A spec left at its default maintains every index on the table, so it
// reads back naming the one on the table rather than as "infer".
// A spec left at its default round-trips as the default: what is stored
// is the intent to maintain everything, not the set it resolves to now.
let spec = LsmWriteSpec::identity("region");
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(
table.get_lsm_write_spec().await.unwrap(),
Some(spec.with_maintained_indexes(vec![idx_name.clone()]))
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
table.unset_lsm_write_spec().await.unwrap();
// Unsharded round-trips (no routing column).
let spec = LsmWriteSpec::unsharded();
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(
table.get_lsm_write_spec().await.unwrap(),
Some(spec.with_maintained_indexes(vec![idx_name]))
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
}
/// The maintained set defaults to every index on the table, resolved at
/// install. An index the memtable cannot build fails the install rather
/// than being dropped: maintaining it would take the table offline for
/// writes, dropping it would hide that from the caller.
/// The maintained set defaults to every index on the table. A named index
/// the memtable cannot build fails the install, because the caller asked
/// for it; an unnamed one is skipped, because it arrived by existing.
#[tokio::test]
async fn test_set_lsm_write_spec_infers_maintained_indexes() {
let tmp_dir = tempdir().unwrap();
@@ -5729,19 +5738,27 @@ mod tests {
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// The default covers every index, so the bitmap fails it too.
let err = table
// The default names nothing, so the bitmap is skipped rather than
// refusing the table: it is maintained by existing, not by being asked
// for. The intent is what is stored, not the set it resolves to today.
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { ref message }
if message.contains("tag_bitmap") && message.contains("maintained_indexes")),
"expected the inferred set to be rejected, got {err:?}"
.unwrap();
assert_eq!(
table
.get_lsm_write_spec()
.await
.unwrap()
.unwrap()
.maintained_indexes(),
None,
"an unnamed set is stored as the intent to maintain everything"
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Naming the maintainable subset installs.
// Narrowing to a named subset: unset first, since an installed spec
// cannot be set over.
table.unset_lsm_write_spec().await.unwrap();
table
.set_lsm_write_spec(
LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]),
@@ -5773,6 +5790,28 @@ mod tests {
.maintained_indexes(),
Some([].as_slice())
);
// An installed spec is never set over, whatever the new one asks for,
// and the refused call leaves it alone.
let err = table
.set_lsm_write_spec(LsmWriteSpec::bucket("id", 4))
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { ref message }
if message.contains("already set")),
"expected the repeat to be refused, got {err:?}"
);
assert_eq!(
table
.get_lsm_write_spec()
.await
.unwrap()
.unwrap()
.maintained_indexes(),
Some([].as_slice()),
"a refused call changes nothing"
);
}
#[tokio::test]
rust/lancedb/src/table/computed_columns.rs
+15 -7
View File
@@ -704,13 +704,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
object
.get("function")
.ok_or_else(|| invalid_function("Function binding is missing its exact version"))?,
&[
"name",
"object_id",
"location",
"version",
"manifest_digest",
],
crate::function::FUNCTION_VERSION_REF_FIELDS,
"version reference",
)?;
for input in object
@@ -3760,6 +3754,20 @@ mod tests {
ensure_known_binding_shape(&raw_binding).unwrap();
}
/// A binding names its Function by parts, so one declared against a
/// namespaced Function is part of the known contract too.
#[test]
fn test_namespaced_bindings_are_a_known_shape() {
let mut raw_binding: Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json"
))
.unwrap();
raw_binding["function"]["namespace_path"] = serde_json::json!(["analytics", "features"]);
ensure_known_binding_shape(&raw_binding).unwrap();
let binding = FunctionBinding::from_json(&raw_binding.to_string()).unwrap();
assert_eq!(binding.function().namespace_path, ["analytics", "features"]);
}
#[test]
fn test_named_struct_can_be_kept_as_one_nullable_physical_column() {
let application = named_struct_application("{}");
rust/lancedb/src/table/create_index.rs
+50
View File
@@ -160,6 +160,12 @@ impl NativeTable {
builder = builder.name(name);
}
builder.await?;
// A writer already open was built before this index existed, so it has
// to pick it up here or a read needing it is refused until it reopens.
self.dataset
.shard_writer()
.refresh_maintained_indexes(&dataset)
.await;
self.dataset.update(dataset);
Ok(())
}
@@ -2078,6 +2084,50 @@ mod tests {
assert_eq!(index.columns, vec!["tags".to_string()]);
}
#[tokio::test]
async fn test_list_indices_with_missing_fts_index_files() {
let tmp_dir = tempdir().unwrap();
let uri = tmp_dir.path().to_str().unwrap();
let conn = connect(uri).execute().await.unwrap();
let batch = record_batch!(
("id", Int32, [1, 2, 3]),
("text", Utf8, ["alpha", "beta", "gamma"])
)
.unwrap();
let table = conn.create_table("t", batch).execute().await.unwrap();
table
.create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
.execute()
.await
.unwrap();
table
.create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
.execute()
.await
.unwrap();
let fts_uuid = table
.list_indices()
.await
.unwrap()
.into_iter()
.find(|index| index.name == "text_idx")
.and_then(|index| index.index_uuid)
.unwrap();
std::fs::remove_dir_all(tmp_dir.path().join("t.lance/_indices").join(fts_uuid)).unwrap();
// A new connection, so the index cache cannot serve the deleted files.
let conn = connect(uri).execute().await.unwrap();
let table = conn.open_table("t").execute().await.unwrap();
let mut indices = table.list_indices().await.unwrap();
indices.sort_by(|a, b| a.name.cmp(&b.name));
let names = indices.iter().map(|i| i.name.as_str()).collect::<Vec<_>>();
assert_eq!(names, vec!["id_idx", "text_idx"]);
assert_eq!(indices[1].index_type, crate::index::IndexType::FTS);
assert_eq!(indices[1].index_details, None);
assert!(indices[0].index_details.is_some());
}
#[tokio::test]
async fn test_create_inverted_index() {
let conn = connect("memory://").execute().await.unwrap();
rust/lancedb/src/table/merge.rs
+183
View File
@@ -1505,6 +1505,30 @@ mod lsm_tests {
);
}
/// A table with `id`/`text` and a primary key, ready for an LSM write spec.
async fn lsm_text_table(dir: &tempfile::TempDir) -> crate::Table {
let conn = connect(dir.path().to_str().unwrap())
.execute()
.await
.unwrap();
let table = conn
.create_table("t", id_text_reader(vec![(1, "alpha")]))
.execute()
.await
.unwrap();
table.set_unenforced_primary_key(["id"]).await.unwrap();
table
}
/// Upsert `rows` through the LSM write path.
async fn upsert_text(table: &crate::Table, rows: Vec<(i64, &str)>) {
let mut builder = table.merge_insert(&[]);
builder
.when_matched_update_all(None)
.when_not_matched_insert_all();
builder.execute(id_text_reader(rows)).await.unwrap();
}
/// A reader of `[id: Int64, text: Utf8]` rows.
fn id_text_reader(rows: Vec<(i64, &str)>) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
@@ -1524,6 +1548,165 @@ mod lsm_tests {
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
}
/// Under the default set, an index created while a writer is open reaches
/// that writer.
///
/// The default maintains every index the table has, so creating one changes
/// what the table maintains without anyone touching the spec. A writer left
/// on its old configs would keep building MemTables the new index does not
/// cover.
#[tokio::test]
async fn lsm_default_set_picks_up_an_index_created_while_writing() {
use crate::index::Index;
use lance_index::scalar::FullTextSearchQuery;
let dir = tempdir().unwrap();
let table = lsm_text_table(&dir).await;
// The default set, on a table with no FTS index yet.
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
// Opens the writer, whose MemTable carries no FTS index.
upsert_text(&table, vec![(99, "zebra")]).await;
// Created with the writer still open and never reopened.
table
.create_index(&["text"], Index::FTS(Default::default()))
.execute()
.await
.unwrap();
upsert_text(&table, vec![(100, "zebra stripes")]).await;
let query = FullTextSearchQuery::new("stripes".to_string())
.with_column("text".to_string())
.unwrap();
let batches = table
.query()
.full_text_search(query)
.execute()
.await
.expect("the created index reached the open writer")
.try_collect::<Vec<_>>()
.await
.unwrap();
let found: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(found, 1, "the row written after create_index must be found");
}
/// A spec is replaced by unsetting it and setting the new one.
///
/// `unset_lsm_write_spec` drains the open writer, so the rows it held
/// survive the replacement, and the writer the next write opens builds its
/// MemTables from the new set.
#[tokio::test]
async fn lsm_spec_is_replaced_by_unsetting_it_first() {
use crate::index::Index;
use lance_index::scalar::FullTextSearchQuery;
let dir = tempdir().unwrap();
let table = lsm_text_table(&dir).await;
table
.create_index(&["text"], Index::FTS(Default::default()))
.execute()
.await
.unwrap();
let fts_index = table.list_indices().await.unwrap()[0].name.clone();
// Installed maintaining nothing, then written to: the writer opens with
// no FTS index.
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()))
.await
.unwrap();
upsert_text(&table, vec![(99, "zebra")]).await;
// Setting over an installed spec is refused; unset, then set.
let err = table
.set_lsm_write_spec(
LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index.clone()]),
)
.await
.expect_err("an installed spec cannot be set over");
assert!(
err.to_string().contains("already set"),
"unexpected error: {err}"
);
table.unset_lsm_write_spec().await.unwrap();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index]))
.await
.unwrap();
// A row written after the replacement is answered by the index the new
// writer maintains.
upsert_text(&table, vec![(100, "zebra stripes")]).await;
let query = FullTextSearchQuery::new("stripes".to_string())
.with_column("text".to_string())
.unwrap();
let batches = table
.query()
.full_text_search(query)
.execute()
.await
.expect("the replaced spec maintains the index")
.try_collect::<Vec<_>>()
.await
.unwrap();
let found: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(
found, 1,
"the row written after the replacement must be found"
);
}
/// A row written before the index exists is searchable once it does.
///
/// The MemTable holding it was built without the index. Creating the index
/// replaces the writer's configs, which seals that MemTable and waits for
/// its flush, so no resident MemTable is left that cannot answer the read.
#[tokio::test]
async fn lsm_a_row_written_before_the_index_is_searchable_after_it() {
use crate::index::Index;
use lance_index::scalar::FullTextSearchQuery;
let dir = tempdir().unwrap();
let table = lsm_text_table(&dir).await;
// No index yet, and the default set: maintain whatever the table has.
table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap();
// Opens the writer, whose MemTable therefore carries no FTS index.
upsert_text(&table, vec![(99, "zebra")]).await;
// Built afterwards: the spec now resolves to it, the MemTable does not.
table
.create_index(&["text"], Index::FTS(Default::default()))
.execute()
.await
.unwrap();
let query = FullTextSearchQuery::new("zebra".to_string())
.with_column("text".to_string())
.unwrap();
let batches = table
.query()
.full_text_search(query)
.execute()
.await
.expect("no resident MemTable is left that cannot answer")
.try_collect::<Vec<_>>()
.await
.unwrap();
let found: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(found, 1, "the row written before the index must be found");
}
#[tokio::test]
async fn lsm_read_full_text_search() {
use crate::index::Index;
rust/lancedb/src/table/merge/lsm.rs
+81 -55
View File
@@ -77,21 +77,36 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
let dataset = table.dataset.get().await?;
if dataset.mem_wal_index_details().await?.is_some() {
return Err(Error::InvalidInput {
message: "set_lsm_write_spec: an LSM write spec is already set on this table; mutation is not supported".into(),
message: "set_lsm_write_spec: an LSM write spec is already set on this \
table and cannot be changed"
.into(),
});
}
}
// Before the builder borrows the dataset clone. `list_indices` merges an
// index's segments into one entry, so the result needs no dedup.
let maintained_indexes = {
let dataset = table.dataset.get().await?;
resolve_maintained_indexes(
&dataset,
&table.list_indices().await?,
spec.maintained_indexes(),
)
.await?
// A named set is checked against the table here, where the caller can still
// be told which names exist. An unnamed set is not resolved at all: it is an
// intent lance re-reads whenever it builds a MemTable, so an index created
// later is maintained without another call.
let maintained_indexes = match spec.maintained_indexes() {
Some(requested) => {
let indices = table.list_indices().await?;
for name in requested {
if !indices.iter().any(|index| &index.name == name) {
return Err(Error::InvalidInput {
message: format!(
"maintained index '{}' does not exist on this table; it has {}",
name,
index_name_list(&indices),
),
});
}
}
let dataset = table.dataset.get().await?;
validate_maintained_indexes(&dataset, requested).await?;
Some(requested.to_vec())
}
None => None,
};
table.checkout_latest().await?;
@@ -138,7 +153,9 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
writer_config_defaults
}
};
builder = builder.maintained_indexes(maintained_indexes);
if let Some(maintained_indexes) = maintained_indexes {
builder = builder.maintained_indexes(maintained_indexes);
}
for (key, value) in writer_config_defaults {
builder = builder.add_writer_config_default(key, value);
}
@@ -147,48 +164,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
Ok(())
}
/// Resolve a spec's maintained-index selection against `indices`, as reported
/// by [`Table::list_indices`](crate::Table::list_indices).
///
/// `None` means every index on the table, snapshotted now. Lance validates
/// either selection against its shard-writer rules, so a spec that installs is
/// one the MemWAL can open.
///
/// An unmaintainable index fails an inferred set rather than being dropped from
/// it — dropping would leave the caller believing it is maintained.
async fn resolve_maintained_indexes(
dataset: &Dataset,
indices: &[IndexConfig],
requested: Option<&[String]>,
) -> Result<Vec<String>> {
let Some(requested) = requested else {
let all: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
validate_maintained_indexes(dataset, &all)
.await
.map_err(|source| Error::InvalidInput {
message: format!(
"cannot maintain every index on this table: {source}. Set \
maintained_indexes explicitly to choose from {}",
index_name_list(indices),
),
})?;
return Ok(all);
};
for name in requested {
if !indices.iter().any(|index| &index.name == name) {
return Err(Error::InvalidInput {
message: format!(
"maintained index '{}' does not exist on this table; it has {}",
name,
index_name_list(indices),
),
});
}
}
validate_maintained_indexes(dataset, requested).await?;
Ok(requested.to_vec())
}
/// Index names for an error message.
fn index_name_list(indices: &[IndexConfig]) -> String {
if indices.is_empty() {
@@ -298,7 +273,9 @@ fn lsm_write_spec_from_details(
};
Ok(base
.with_maintained_indexes(details.maintained_indexes.clone())
.with_maintained_indexes(
(!details.maintain_all_indexes).then(|| details.maintained_indexes.clone()),
)
.with_writer_config_defaults(details.writer_config_defaults.clone()))
}
@@ -460,6 +437,55 @@ impl ShardWriterCache {
Ok(entry)
}
/// Install `dataset`'s maintained index set on the cached writer, if one is
/// open, so a change reaches a table that is already being written to.
///
/// Reported rather than propagated: the index is committed durably by the
/// time this runs, and the writer this process happens to hold may be fenced
/// or poisoned -- which is not a reason to fail creating an index. Any
/// writer opened afterwards reads the committed set.
///
/// Holds the entry's read lock across the refresh, so a concurrent close
/// waits rather than retiring the writer mid-seal. Writes take the same
/// lock first, so the order matches theirs.
pub(crate) async fn refresh_maintained_indexes(&self, dataset: &Dataset) {
let entry = {
let guard = self.slot.read().await;
guard.as_ref().map(|(_, entry)| entry.clone())
};
let Some(entry) = entry else {
return;
};
let guard = entry.inner.read().await;
let Some(writer) = guard.as_ref() else {
return;
};
// Replacing the configs seals the active memtable so the next one carries
// the new indexes. The outgoing one keeps the set it was built with, so
// its flush has to land before a read: while it is still frozen, an
// indexed read sees a resident memtable that cannot answer and is
// refused.
let sealed = match dataset.refresh_mem_wal_index_configs(writer).await {
Ok(sealed) => sealed,
Err(error) => {
log::warn!(
"the maintained index set changed but the open shard writer could \
not be refreshed, so it keeps the set it opened with: {error}"
);
return;
}
};
if let Some(fence) = sealed
&& let Err(error) = fence.wait().await
{
log::warn!(
"the shard writer took the new maintained index set, but the memtable \
it sealed was not flushed, so an indexed read stays refused until \
it is: {error}"
);
}
}
/// Snapshot the cached writer's shard for the LSM read path: its shard id,
/// authoritative in-memory manifest, and active + frozen memtable refs.
/// Returns `None` when no writer is currently cached (e.g. nothing has been
rust/lancedb/src/table/query/lsm.rs
+80 -32
View File
@@ -33,7 +33,7 @@ use datafusion_physical_plan::expressions::Column;
use datafusion_physical_plan::projection::ProjectionExec;
use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
use lance::Dataset;
use lance::dataset::mem_wal::scanner::InMemoryMemTables;
use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables};
use lance::dataset::mem_wal::{
DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig,
};
@@ -468,20 +468,19 @@ async fn fts_plan(
.to_string(),
})?;
// Without a maintained in-memory FTS index for this column, the active memtable
// arm produces an empty plan (`active_source_can_execute_fts` returns false), so
// the search silently omits un-compacted documents. Reject rather than mislead.
if !index_maintained(
dataset,
column,
&details.maintained_indexes,
"InvertedIndexDetails",
)
.await?
// Two ways the fresh tier cannot answer: the spec maintains no FTS index for
// this column, or a resident MemTable was built before the index joined the
// set. Either leaves the memtable arm an empty plan
// (`active_source_can_execute_fts` returns false), so the search silently
// omits un-compacted documents. Reject rather than mislead.
if !index_maintained(dataset, column, details, "InvertedIndexDetails").await?
|| !resident_memtables_carry(&in_memory, |memtable| {
memtable.index_store.get_fts_by_column(column).is_some()
})
{
return Err(Error::NotSupported {
message: format!(
"the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only"
"the MemWAL LSM scanner full-text search requires the FTS index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise un-compacted documents are omitted. set use_lsm(false) to read the base table only"
),
});
}
@@ -493,6 +492,27 @@ async fn fts_plan(
Ok(scanner.create_plan().await?)
}
/// Whether the spec maintains the index named `name`.
///
/// An unnamed set names nothing and covers everything, so the stored list alone
/// cannot answer this.
fn spec_maintains(details: &MemWalIndexDetails, name: &str) -> bool {
details.maintain_all_indexes || details.maintained_indexes.iter().any(|n| n == name)
}
/// Whether every resident MemTable carries the index an arm needs.
///
/// The spec is intent; a MemTable built before the index joined the set
/// contributes nothing to an indexed read, so its rows would go missing.
fn resident_memtables_carry(
in_memory: &HashMap<Uuid, InMemoryMemTables>,
carries: impl Fn(&InMemoryMemTableRef) -> bool,
) -> bool {
in_memory
.values()
.all(|tables| carries(&tables.active) && tables.frozen.iter().all(&carries))
}
/// Whether an index of `type_url_suffix` covering `column` is in the MemWAL spec's
/// maintained set. Only a maintained index has its catch-up tracked (so exclusion is
/// gated correctly) and its in-memory arm kept current; an unmaintained base index
@@ -502,7 +522,7 @@ async fn fts_plan(
async fn index_maintained(
dataset: &Dataset,
column: &str,
maintained: &[String],
details: &MemWalIndexDetails,
type_url_suffix: &str,
) -> Result<bool> {
use lance::index::DatasetIndexExt;
@@ -512,7 +532,7 @@ async fn index_maintained(
let indices = dataset.load_indices().await?;
Ok(indices.iter().any(|idx| {
idx.fields.contains(&field.id)
&& maintained.iter().any(|m| m == &idx.name)
&& spec_maintains(details, &idx.name)
&& idx
.index_details
.as_ref()
@@ -583,9 +603,7 @@ async fn arm_maintained_index_names(
})
.map(|idx| idx.name.clone())
.collect();
if let Some(name) =
resolve_single_index(segment_names, &details.maintained_indexes, arm, &column)?
{
if let Some(name) = resolve_single_index(segment_names, details, arm, &column)? {
names.push(name);
}
}
@@ -604,7 +622,7 @@ async fn arm_maintained_index_names(
/// compaction watermark).
fn resolve_single_index(
mut names: Vec<String>,
maintained: &[String],
details: &MemWalIndexDetails,
arm: &str,
column: &str,
) -> Result<Option<String>> {
@@ -620,7 +638,7 @@ fn resolve_single_index(
Ok(names
.into_iter()
.next()
.filter(|name| maintained.contains(name)))
.filter(|name| spec_maintains(details, name)))
}
/// Drop the primary-key columns Lance appends internally for dedup when the user's
@@ -688,21 +706,19 @@ async fn vector_plan(
None => default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?,
};
// The base arm relies on the column's vector index (`fast_search`). Unless it is
// maintained, its catch-up is untracked and exclusion falls back to the
// compaction watermark — dropping compacted SSTables the (lagging) base index has
// not re-indexed. Reject rather than silently omit rows, mirroring the FTS arm.
if !index_maintained(
dataset,
&column,
&details.maintained_indexes,
"VectorIndexDetails",
)
.await?
// The base arm relies on the column's vector index (`fast_search`). Unmaintained,
// its catch-up is untracked and exclusion falls back to the compaction watermark,
// dropping compacted SSTables the lagging base index has not re-indexed; and a
// resident MemTable built before the index joined the set carries none either.
// Reject rather than silently omit rows, mirroring the FTS arm.
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await?
|| !resident_memtables_carry(&in_memory, |memtable| {
memtable.index_store.get_hnsw_by_column(&column).is_some()
})
{
return Err(Error::NotSupported {
message: format!(
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes); otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
"the MemWAL LSM scanner requires the vector index on '{column}' to be maintained by the write spec (LsmWriteSpec::with_maintained_indexes) and carried by every resident MemTable; otherwise compacted rows not yet re-indexed are omitted. set use_lsm(false) to read the base table only"
),
});
}
@@ -888,7 +904,10 @@ mod tests {
#[test]
fn resolve_single_index_dedupes_segments() {
let maintained = vec!["fts_idx".to_string()];
let maintained = MemWalIndexDetails {
maintained_indexes: vec!["fts_idx".to_string()],
..Default::default()
};
// Two physical segments of ONE logical index must not count as "multiple".
assert_eq!(
resolve_single_index(
@@ -917,4 +936,33 @@ mod tests {
None
);
}
/// A spec that maintains every index names none of them, so the stored list
/// is empty while every index on the table is in fact maintained. Reading
/// the list alone would refuse the read path on the default spec.
#[test]
fn an_unnamed_set_maintains_every_index() {
let all = MemWalIndexDetails {
maintained_indexes: Vec::new(),
maintain_all_indexes: true,
..Default::default()
};
assert!(spec_maintains(&all, "fts_idx"));
assert_eq!(
resolve_single_index(vec!["fts_idx".to_string()], &all, "full-text", "text").unwrap(),
Some("fts_idx".to_string()),
);
// The same empty list without the flag maintains nothing.
let none = MemWalIndexDetails {
maintained_indexes: Vec::new(),
maintain_all_indexes: false,
..Default::default()
};
assert!(!spec_maintains(&none, "fts_idx"));
assert_eq!(
resolve_single_index(vec!["fts_idx".to_string()], &none, "full-text", "text").unwrap(),
None,
);
}
}
rust/lancedb/tests/first_class_function_slice2.rs
+6 -3
View File
@@ -97,13 +97,16 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
))
.unwrap();
let create_error = connection.create_function_async(request).await.unwrap_err();
let create_error = connection
.create_function_async(request, &[])
.await
.unwrap_err();
let lookup_error = connection
.get_function("normalize_score", "1")
.get_function("normalize_score", "1", &[])
.await
.unwrap_err();
let drop_error = connection
.drop_function("normalize_score", "1")
.drop_function("normalize_score", "1", &[])
.await
.unwrap_err();
for error in [create_error, lookup_error, drop_error] {