rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4273-1

Browse Source 
Preserve blob query URI detection alongside upstream empty-vector validation.
This commit is contained in:
Gatefixer
2026-10-04 04:58:39 +00:00
parent eadc51f213 7f5933594f
commit 47cd488f78
119 changed files with 11143 additions and 2203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.11"
current_version = "0.40.0-beta.12"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/workflows/nodejs.yml
+65
View File
@@ -164,6 +164,71 @@ jobs:
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
.github/workflows/rust.yml
+2 -1
View File
@@ -180,7 +180,8 @@ jobs:
env:
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
GH_TOKEN: ${{ github.token }}
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
Cargo.lock
Generated
+105 -101
View File
@@ -141,7 +141,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -181,9 +181,9 @@ dependencies = [
[[package]]
name = "arc-swap"
version = "1.9.1"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
dependencies = [
"rustversion",
]
@@ -1110,9 +1110,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
version = "1.6.3"
version = "1.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9"
checksum = "0b791f3ac597193fe1d08b82366986eb1f5bc31f2ac6c194c0855276116c76cd"
dependencies = [
"base64-simd",
"bytes",
@@ -1819,7 +1819,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3087,7 +3087,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3315,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4538,7 +4538,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2 0.6.3",
"system-configuration",
"tokio",
"tower-service",
@@ -4880,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arc-swap",
"arrow",
@@ -5161,6 +5161,7 @@ dependencies = [
"async-trait",
"async_cell",
"aws-sdk-dynamodb",
"blake3",
"byteorder",
"bytes",
"chrono",
@@ -5192,7 +5193,7 @@ dependencies = [
"lance-tokenizer",
"log",
"moka",
"object_store 0.14.1",
"object_store 0.14.2",
"permutation",
"pin-project",
"prost",
@@ -5216,8 +5217,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5239,7 +5240,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5253,7 +5254,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5262,8 +5263,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrayref",
"crunchy",
@@ -5273,8 +5274,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5293,7 +5294,7 @@ dependencies = [
"log",
"moka",
"num_cpus",
"object_store 0.14.1",
"object_store 0.14.2",
"pin-project",
"prost",
"quick_cache",
@@ -5311,8 +5312,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5343,8 +5344,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5361,8 +5362,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"proc-macro2",
"quote",
@@ -5371,8 +5372,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5405,8 +5406,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5427,7 +5428,7 @@ dependencies = [
"lance-io",
"log",
"num-traits",
"object_store 0.14.1",
"object_store 0.14.2",
"prost",
"prost-build",
"prost-types",
@@ -5438,8 +5439,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"datafusion",
"geo-traits",
@@ -5453,8 +5454,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arc-swap",
"arrow",
@@ -5481,6 +5482,7 @@ dependencies = [
"geoarrow-array",
"geoarrow-schema",
"half",
"hashbrown 0.16.1",
"hex",
"itertools 0.14.0",
"jieba-rs",
@@ -5503,7 +5505,7 @@ dependencies = [
"log",
"ndarray",
"num-traits",
"object_store 0.14.1",
"object_store 0.14.2",
"prost",
"prost-build",
"prost-types",
@@ -5524,8 +5526,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5543,12 +5545,13 @@ dependencies = [
"roaring",
"serde",
"serde_json",
"tokio",
]
[[package]]
name = "lance-io"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5568,7 +5571,7 @@ dependencies = [
"log",
"metrics",
"moka",
"object_store 0.14.1",
"object_store 0.14.2",
"object_store_opendal",
"opendal",
"opendal-service-goosefs",
@@ -5590,8 +5593,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5605,8 +5608,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"async-trait",
@@ -5620,9 +5623,10 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arc-swap",
"arrow",
"arrow-ipc",
"arrow-schema",
@@ -5643,7 +5647,7 @@ dependencies = [
"lance-namespace",
"lance-table",
"log",
"object_store 0.14.1",
"object_store 0.14.2",
"quick-xml 0.40.1",
"rand 0.9.5",
"reqwest 0.12.28",
@@ -5660,9 +5664,9 @@ dependencies = [
[[package]]
name = "lance-namespace-reqwest-client"
version = "0.12.0"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
checksum = "828f291764feda98898452bea4ce3756b73e5d5c7c67b62edda27456b7cc72c1"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -5674,8 +5678,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5689,8 +5693,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5711,7 +5715,7 @@ dependencies = [
"lance-io",
"lance-select",
"log",
"object_store 0.14.1",
"object_store 0.14.2",
"prost",
"prost-build",
"prost-types",
@@ -5730,8 +5734,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5744,8 +5748,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
version = "13.1.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v13.1.0-beta.1#e5a3553b1699a7062ff7afe4ee268d596bbf2e0d"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5758,7 +5762,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
dependencies = [
"ahash",
"anyhow",
@@ -5822,7 +5826,7 @@ dependencies = [
"moka",
"num-traits",
"oauth2",
"object_store 0.14.1",
"object_store 0.14.2",
"pin-project",
"polars",
"polars-arrow",
@@ -5855,7 +5859,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5880,7 +5884,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
dependencies = [
"arc-swap",
"arrow",
@@ -6471,9 +6475,9 @@ dependencies = [
[[package]]
name = "napi"
version = "3.12.2"
version = "3.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
checksum = "f0c007d4a8ead952a81887661d41fd56b7e7a70d3d4d921f445fb37a8f6efa1e"
dependencies = [
"bitflags 2.11.1",
"chrono",
@@ -6491,15 +6495,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.2"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "860e7c40864f95cfb83cde99f9ebadd88ef3d9bdccd7dd2cee0cc96a2dd4ffa7"
checksum = "0941983559cd62bfa7f04646a16428967a847365adeef27711e4e3927046bbed"
[[package]]
name = "napi-derive"
version = "3.6.5"
version = "3.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350057056a30368aa76c11a0d406b0aa61321710be2c36656ab4f6efe0b785a2"
checksum = "6b1d232c24dc6754979b27efdf2047a5849efc7c92c1245de87e6bedbb9f5c3c"
dependencies = [
"convert_case",
"ctor",
@@ -6511,9 +6515,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.3"
version = "6.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c1c87a71568f3fe5c736b10878ff55064b250bb4ea4b48c8055713e07b9e463"
checksum = "fd6856fe1686867da90ed0dc292d7c0c87a695eb7c578b913360f33619ee01e3"
dependencies = [
"convert_case",
"proc-macro2",
@@ -6524,9 +6528,9 @@ dependencies = [
[[package]]
name = "napi-sys"
version = "3.3.0"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85fbf1fa9f1babfe396d74bbbf52b3643770243e8f5b0b46715d4caf7f0dfc9a"
checksum = "e22a4f25c16a5c5411d987cd6fbd48313522ae1789ade2d6dd3efdc6d40a0fc8"
dependencies = [
"libloading",
]
@@ -6633,7 +6637,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6762,7 +6766,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.21.7",
"base64 0.22.1",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -6879,13 +6883,13 @@ dependencies = [
[[package]]
name = "object_store"
version = "0.14.1"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"
checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94"
dependencies = [
"async-trait",
"aws-lc-rs",
"base64 0.22.1",
"base64 0.23.1",
"bytes",
"chrono",
"crc-fast",
@@ -6931,7 +6935,7 @@ dependencies = [
"bytes",
"chrono",
"futures",
"object_store 0.14.1",
"object_store 0.14.2",
"opendal",
"pin-project",
"tokio",
@@ -8136,8 +8140,8 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.4.1",
"itertools 0.11.0",
"heck 0.5.0",
"itertools 0.14.0",
"log",
"multimap",
"petgraph",
@@ -8156,7 +8160,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.11.0",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -8378,7 +8382,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls 0.23.45",
"socket2 0.5.10",
"socket2 0.6.3",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -8416,7 +8420,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.10",
"socket2 0.6.3",
"tracing",
"windows-sys 0.60.2",
]
@@ -9216,7 +9220,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -9287,7 +9291,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -9578,9 +9582,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.23.0"
version = "3.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4"
checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f"
dependencies = [
"base64 0.23.1",
"bs58",
@@ -9599,9 +9603,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.23.0"
version = "3.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc"
checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49"
dependencies = [
"darling 0.24.1",
"proc-macro2",
@@ -9834,7 +9838,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9846,7 +9850,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.4.1",
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9869,7 +9873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -9988,7 +9992,7 @@ dependencies = [
"cfg-if 1.0.4",
"libc",
"psm",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -10280,7 +10284,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11301,7 +11305,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.1.0-beta.1", default-features = false, "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.1.0-beta.1", "tag" = "v13.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.11</version>
<version>0.40.0-beta.12</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+3 -1
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -417,6 +417,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/BlobFile.md
+101 -5
View File
@@ -10,16 +10,52 @@ A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fet
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(): Promise<Buffer>
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor to the end and advances the cursor.
Reads from the cursor and advances the cursor.
A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does
not move the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
@@ -30,7 +66,10 @@ not move the cursor.
### readRange()
```ts
readRange(start, end): Promise<Buffer>
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
@@ -43,12 +82,55 @@ Fails when `end` is past the blob size. Does not move the cursor.
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
@@ -60,3 +142,17 @@ Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Query.md
+3 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -491,6 +491,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/QueryBase.md
+3 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+17 -3
View File
@@ -516,7 +516,10 @@ Drop an index from the table.
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]>
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
@@ -531,6 +534,8 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
@@ -540,7 +545,10 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
### fetchBlobs()
```ts
abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]>
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
@@ -548,6 +556,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
#### Parameters
@@ -555,6 +567,8 @@ duplicates. Null blobs are `null`. Empty blobs are empty buffers.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
@@ -1367,7 +1381,7 @@ repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+3 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -267,6 +267,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
docs/src/js/classes/VectorQuery.md
+5 -1
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -673,7 +675,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -740,6 +742,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/globals.md
+3
View File
@@ -158,8 +158,11 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
docs/src/js/interfaces/AddDataOptions.md
+16
View File
@@ -10,6 +10,22 @@ Options for adding data to a table.
## Properties
### allowExternalBlobOutsideBases?
```ts
optional allowExternalBlobOutsideBases: boolean;
```
Whether blob URIs outside registered external bases may be written.
Defaults to `false`. This option is supported only for local/native
tables; remote tables return an error when it is enabled. An enabled write
stores the absolute URI reference without registering a base or copying
the external object into the database. The object must remain accessible
when the blob is read later.
***
### mode
```ts
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
docs/src/python/python.md
+27
View File
@@ -198,6 +198,33 @@ listing a storage directory.
::: lancedb.materialized_view.MaterializedViewDefinition
::: lancedb.vector_dedup
::: lancedb.materialized_view.VectorDedupSource
For an existing indexed source, declare and materialize a dedup result without
writing SQL. Creation captures the source version once and waits for the complete
result; remote connections use the service's MV jobs.
```python
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
```
To submit without waiting, call `db.create_materialized_view_async(...)` with the
same source expression and wait on the returned job. To declare only, pass
`with_no_data=True`, then use `view.refresh_async().wait()` when ready. Pass
`dataset_version=...` to `vector_dedup` for an explicit snapshot. The default
policy retains direct representatives (A-B-C without A-C keeps A and C), preserves
the source, and materializes its original columns. Only within-segment/partition
pairs are covered, using the index's distance representation.
## Views
::: lancedb.view.ViewDescription
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.11</version>
<version>0.40.0-beta.12</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.11</version>
<version>0.40.0-beta.12</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>13.0.0-beta.12</lance-core.version>
<lance-core.version>13.1.0-beta.1</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
publish = false
license.workspace = true
description.workspace = true
nodejs/NODEJS_THIRD_PARTY_LICENSES.md
+8
View File
@@ -275,6 +275,7 @@
[@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
[@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
@@ -306,6 +307,9 @@
[apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0
[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0
[argparse@1.0.10](https://github.com/nodeca/argparse) - MIT
[argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0
[array-back@3.1.0](https://github.com/75lb/array-back) - MIT
@@ -348,6 +352,7 @@
[color@4.2.3](https://github.com/Qix-/color) - MIT
[combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT
[command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT
[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT
[command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT
[concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT
[convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT
@@ -398,12 +403,14 @@
[file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT
[fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT
[find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT
[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT
[find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT
[find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT
[flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT
[flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache*
[flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0
[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0
[flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC
[follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT
[foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC
@@ -644,6 +651,7 @@
[uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT
[undici-types@5.26.5](https://github.com/nodejs/undici) - MIT
[undici-types@6.19.8](https://github.com/nodejs/undici) - MIT
[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT
[update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT
[uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause
[uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT
nodejs/__test__/arrow.test.ts
+1415 -1456
View File
File diff suppressed because it is too large Load Diff
nodejs/__test__/arrow_versions.ts
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// Every Apache Arrow release the package accepts as a peer dependency, as
// separately installed copies so the tests can feed objects built by one
// Arrow version into a LanceDB running on another. When a new Arrow major
// is released, add the alias to `package.json` and extend this list; the
// test files pick the change up from here.
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as arrow19 from "apache-arrow-19";
import * as arrow20 from "apache-arrow-20";
import * as arrow21 from "apache-arrow-21";
export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 };
export type ApacheArrow =
| typeof arrow15
| typeof arrow16
| typeof arrow17
| typeof arrow18
| typeof arrow19
| typeof arrow20
| typeof arrow21;
export const arrowVersions: ApacheArrow[] = [
arrow15,
arrow16,
arrow17,
arrow18,
arrow19,
arrow20,
arrow21,
];
/** The newest supported Arrow release, for tests that only need one. */
export const latestArrow = arrow21;
nodejs/__test__/materialized_view.test.ts
+11 -1
View File
@@ -52,9 +52,19 @@ describe("materialized views", () => {
"SELECT * FROM people",
);
for (const [format, fn] of [
[3, "vector_duplicate_pairs"],
[4, "vector_dedup"],
]) {
const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
expect(read(JSON.stringify({ format, query: native })).query).toBe(
native,
);
}
// A newer writer's layout is reported, never guessed at.
for (const newer of [
`{"format":3,"query":${JSON.stringify(query)}}`,
`{"format":5,"query":${JSON.stringify(query)}}`,
'{"kind":"select_v3","source_table":"people"}',
]) {
expect(() => read(newer)).toThrow(/cannot refresh/);
nodejs/__test__/query.test.ts
+123
View File
@@ -110,6 +110,129 @@ describe("Query outputSchema", () => {
});
});
describe("Query empty results", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const schema = new Schema(
[
new Field("id", new Int64(), false),
new Field(
"text",
new Utf8(),
true,
new Map([["description", "searchable text"]]),
),
new Field(
"vector",
new FixedSizeList(2, new Field("item", new Float32())),
true,
),
],
new Map([["source", "empty-query-regression"]]),
);
table = await db.createTable(
"test",
makeArrowTable([{ id: 1n, text: "hello", vector: [1, 2] }], { schema }),
);
});
afterEach(() => {
tmpDir.removeCallback();
});
it.each([
{
name: "filtered scan",
query: (table: Table) => table.query().where("id < 0"),
fields: ["id", "text", "vector"],
},
{
name: "zero limit",
query: (table: Table) => table.query().limit(0),
fields: ["id", "text", "vector"],
},
{
name: "column projection",
query: (table: Table) =>
table.query().where("id < 0").select(["text", "id"]),
fields: ["text", "id"],
},
{
name: "dynamic projection",
query: (table: Table) =>
table.query().where("id < 0").select({ doubled: "id * 2" }),
fields: ["doubled"],
},
{
name: "filtered vector search",
query: (table: Table) =>
table.search([0.5, 0.5]).where("id < 0").limit(3),
fields: ["id", "text", "vector", "_distance"],
},
{
name: "vector projection with row id",
query: (table: Table) =>
table
.vectorSearch([0.5, 0.5])
.where("id < 0")
.select(["id"])
.withRowId(),
fields: ["id", "_distance", "_rowid"],
},
{
name: "fast search without an index",
query: (table: Table) => table.search([0.5, 0.5]).fastSearch(),
fields: ["id", "text", "vector", "_rowid", "_distance"],
},
{
name: "take with no offsets",
query: (table: Table) => table.takeOffsets([]).select(["text"]),
fields: ["text"],
},
])("preserves the output schema for $name", async ({ query, fields }) => {
const result = await query(table).toArrow({ maxBatchLength: 1 });
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((field) => field.name)).toEqual(fields);
for (const name of fields) {
expect(result.getChild(name)).not.toBeNull();
expect(result.getChild(name)?.length).toBe(0);
}
});
it("preserves types, nullability, and metadata", async () => {
const result = await table.query().where("id < 0").toArrow();
const schema = await table.schema();
expect(result.schema).toEqual(schema);
expect(result.getChild("id")?.type.toString()).toBe("Int64");
expect(result.schema.fields[0].nullable).toBe(false);
expect(result.schema.fields[1].metadata.get("description")).toBe(
"searchable text",
);
expect(result.schema.metadata.get("source")).toBe("empty-query-regression");
});
it("preserves the score column for full-text search with no matches", async () => {
await table.createIndex("text", { config: Index.fts() });
const result = await table
.search("missing", "fts")
.select(["id"])
.toArrow();
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((field) => field.name)).toEqual([
"id",
"_score",
]);
expect(result.getChild("_score")?.type.toString()).toBe("Float32");
});
});
describe("Search pagination", () => {
let tmpDir: tmp.DirResult;
let table: Table;
nodejs/__test__/registry.test.ts
+5 -8
View File
@@ -3,13 +3,10 @@
import * as apiArrow from "apache-arrow";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as tmp from "tmp";
import { arrowVersions } from "./arrow_versions";
import { connect } from "../lancedb";
import {
EmbeddingFunction,
@@ -18,7 +15,7 @@ import {
} from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
describe.each(arrowVersions)("LanceSchema", (arrow) => {
test("should preserve input order", async () => {
const schema = LanceSchema({
id: new arrow.Int32(),
@@ -29,7 +26,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("LanceSchema", (arrow) => {
});
});
describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
describe.each(arrowVersions)("Registry", (arrow) => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
@@ -240,7 +237,7 @@ describe("Registry.setVar", () => {
return data.map(() => [1, 2, 3]);
}
embeddingDataType() {
return new arrow18.Float32() as apiArrow.Float;
return new apiArrow.Float32();
}
protected getSensitiveKeys() {
return ["secretKey"];
nodejs/__test__/remote.test.ts
+163
View File
@@ -14,6 +14,14 @@ import {
TlsConfig,
connect,
} from "../lancedb";
import {
Table as ArrowTable,
Field,
Int64,
RecordBatch,
Schema,
tableToIPC,
} from "../lancedb/arrow";
import {
HeaderProvider,
OAuthHeaderProvider,
@@ -85,6 +93,161 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("rejects nprobes(0) before sending a query", async () => {
const requests: string[] = [];
await withMockDatabase(
(req, res) => {
requests.push(req.url ?? "");
if (req.url === "/v1/table/test/describe/") {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(JSON.stringify({ version: 1, schema: { fields: [] } }));
} else {
res.writeHead(404).end();
}
},
async (db) => {
const table = await db.openTable("test");
expect(() => table.vectorSearch([0, 0]).nprobes(0)).toThrow(
"Invalid input, nprobes must be greater than 0",
);
expect(() =>
table.query().nearestTo([0, 0]).fullTextSearch("dog").nprobes(0),
).toThrow("Invalid input, nprobes must be greater than 0");
await expect(
table
.vectorSearch(Promise.resolve([0, 0]))
.nprobes(0)
.toArrow(),
).rejects.toThrow("Invalid input, nprobes must be greater than 0");
},
);
expect(requests).toEqual(["/v1/table/test/describe/"]);
});
it.each([false, true])(
"preserves an empty query's schema with an empty batch: %s",
async (withEmptyBatch) => {
const schema = new Schema(
[new Field("doubled", new Int64(), false)],
new Map([["source", "query-output"]]),
);
const result = new ArrowTable(
schema,
withEmptyBatch ? [new RecordBatch(schema, undefined)] : [],
);
const response = Buffer.from(tableToIPC(result, "stream"));
let queryRequests = 0;
await withMockDatabase(
(req, res) => {
if (req.url?.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "items",
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: false },
],
},
}),
);
} else if (req.url?.endsWith("/query/")) {
queryRequests++;
req.resume();
req.on("end", () => {
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
})
.end(response);
});
} else {
res.writeHead(404).end();
}
},
async (db) => {
const table = await db.openTable("items");
const result = await table
.query()
.where("id < 0")
.select({ doubled: "id * 2" })
.toArrow();
expect(result.numRows).toBe(0);
expect(result.schema).toEqual(schema);
expect(result.getChild("doubled")?.length).toBe(0);
},
);
expect(queryRequests).toBe(1);
},
);
it("rejects the external blob opt-in without blocking regular adds", async () => {
let insertRequests = 0;
const describeRequests: string[] = [];
await withMockDatabase(
(req, res) => {
const requestPath = req.url ?? "";
if (requestPath.endsWith("/describe/")) {
describeRequests.push(requestPath);
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "items",
version: 1,
schema: {
fields: [
{
name: "id",
type: { type: "int64" },
nullable: true,
},
],
},
}),
);
return;
}
if (requestPath.endsWith("/insert/")) {
insertRequests++;
req.resume();
req.on("end", () => {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(JSON.stringify({ version: insertRequests + 1 }));
});
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("items");
await expect(
table.add([{ id: 1n }], {
allowExternalBlobOutsideBases: true,
}),
).rejects.toThrow("only supported on local tables");
expect(insertRequests).toBe(0);
await expect(table.add([{ id: 2n }])).resolves.toMatchObject({
version: 2,
});
await expect(
table.add([{ id: 3n }], {
allowExternalBlobOutsideBases: false,
}),
).resolves.toMatchObject({ version: 3 });
},
);
expect(describeRequests.length).toBeGreaterThan(0);
expect(insertRequests).toBe(2);
});
it("lists materialized views through the namespace route", async () => {
await withMockDatabase(
(req, res) => {
nodejs/__test__/sanitize.test.ts
+113 -1
View File
@@ -2,7 +2,13 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as arrow from "../lancedb/arrow";
import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
import {
sanitizeField,
sanitizeMap,
sanitizeSchema,
sanitizeType,
} from "../lancedb/sanitize";
import { arrow21 } from "./arrow_versions";
describe("sanitize", function () {
describe("sanitizeType function", function () {
@@ -59,6 +65,8 @@ describe("sanitize", function () {
["durationmicrosecond", new arrow.DurationMicrosecond()],
["durationmillisecond", new arrow.DurationMillisecond()],
["durationsecond", new arrow.DurationSecond()],
["largebinary", new arrow.LargeBinary()],
["largeutf8", new arrow.LargeUtf8()],
] as const;
it.each(allTypeNameTestCases)(
@@ -89,6 +97,110 @@ describe("sanitize", function () {
"Unrecognized type name in schema: invalid_type",
);
});
// These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the
// rest), so the objects come from a second copy of the newest release. The
// library itself may be running on an older Arrow (see the CI matrix), in
// which case sanitizing them has to fail with a clear message instead.
const hostArrow = arrow as unknown as Record<string, unknown>;
const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function";
const whenSupported = hostSupportsNewerTypes ? describe : describe.skip;
const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe;
const newerTypeTestCases = [
["Utf8View", () => new arrow21.Utf8View()],
["BinaryView", () => new arrow21.BinaryView()],
["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()],
[
"LargeList",
() =>
new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())),
],
] as const;
whenSupported("types added in Arrow 20 and 21", function () {
it.each(newerTypeTestCases)(
"should sanitize a %s type from another copy of Arrow",
function (name, makeForeign) {
const foreign = makeForeign();
const result = sanitizeType(foreign);
// Interval subclasses share `Type.Interval`, so the sanitized type
// is the generic `Interval` carrying the same unit, as for the
// other units.
const expected = name === "IntervalMonthDayNano" ? "Interval" : name;
expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown);
expect(result.typeId).toBe(foreign.typeId);
expect(result.toString()).toBe(foreign.toString());
},
);
it.each([
["binaryview", "BinaryView"],
["utf8view", "Utf8View"],
["intervalmonthdaynano", "Interval"],
])('should map type name "%s" to %s', function (name, expected) {
expect(sanitizeType(name)).toBeInstanceOf(
hostArrow[expected] as () => unknown,
);
});
it("should sanitize the child field of a LargeList", function () {
const foreign = new arrow21.LargeList(
new arrow21.Field("item", new arrow21.Int32(), false),
);
const result = sanitizeType(foreign) as arrow.List;
expect(result.children[0]).toBeInstanceOf(arrow.Field);
expect(result.children[0].name).toBe("item");
expect(result.children[0].nullable).toBe(false);
const child = result.children[0].type;
expect(arrow.DataType.isInt(child)).toBe(true);
expect((child as arrow.Int).bitWidth).toBe(32);
expect((child as arrow.Int).isSigned).toBe(true);
});
});
whenUnsupported("types newer than the installed Arrow", function () {
it.each(
newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"),
)(
"should explain that %s needs a newer apache-arrow",
function (name, makeForeign) {
expect(() => sanitizeType(makeForeign())).toThrow(
`The Arrow type ${name} requires apache-arrow`,
);
},
);
it("should sanitize IntervalMonthDayNano to a generic Interval", () => {
// The subclass shares `Type.Interval`, which every supported Arrow
// release has; only the unit is new.
const result = sanitizeType(new arrow21.IntervalMonthDayNano());
expect(result).toBeInstanceOf(arrow.Interval);
expect((result as arrow.Interval).unit).toBe(
arrow21.IntervalUnit.MONTH_DAY_NANO,
);
});
});
it("should rebuild a schema from a second copy of the same Arrow version", function () {
// Arrow 21 makes `instanceof Schema` succeed across copies of the library
// but not `instanceof <DataType>`; the schema must still be rebuilt.
const foreign = new arrow21.Schema([
new arrow21.Field("x", new arrow21.Float32(), true),
]);
// Only Arrow 21+ overrides `Symbol.hasInstance` this way.
const crossCopyInstanceof = Object.getOwnPropertySymbols(
arrow.Schema,
).includes(Symbol.hasInstance);
expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof);
const result = sanitizeSchema(foreign);
expect(result).not.toBe(foreign);
const type = result.fields[0].type;
expect(type).toBeInstanceOf(arrow.DataType);
expect(arrow.DataType.isFloat(type)).toBe(true);
expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE);
expect(() => arrow.makeBuilder({ type })).not.toThrow();
});
});
describe("sanitizeField function", function () {
nodejs/__test__/table.test.ts
+466 -12
View File
@@ -2,14 +2,11 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "fs";
import { getEventListeners } from "node:events";
import { pathToFileURL } from "node:url";
import * as path from "path";
import * as tmp from "tmp";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
AutoQuery,
Connection,
@@ -54,19 +51,21 @@ import {
instanceOfFullTextQuery,
} from "../lancedb/query";
import { LocalTable } from "../lancedb/table";
import {
type ApacheArrow,
arrow18,
arrowVersions,
latestArrow,
} from "./arrow_versions";
describe.each([arrow15, arrow16, arrow17, arrow18])(
describe.each(arrowVersions)(
"Given a table",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
let tmpDir: tmp.DirResult;
let table: Table;
const schema:
| import("apache-arrow-15").Schema
| import("apache-arrow-16").Schema
| import("apache-arrow-17").Schema
| import("apache-arrow-18").Schema = new arrow.Schema([
const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
new arrow.Field("id", new arrow.Float64(), true),
]);
@@ -91,6 +90,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await expect(table.countRows()).resolves.toBe(3);
});
it("creates and adds Arrow tables with large string and binary columns", async () => {
const data = new arrow.Table({
text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()),
bytes: arrow.vectorFromArray(
[Buffer.from("one"), Buffer.from("two")],
new arrow.LargeBinary(),
),
});
const conn = await connect(tmpDir.name);
const largeTable = await conn.createTable("large_columns", data);
await largeTable.add(data);
const rows = await largeTable.query().toArray();
expect(rows.map((row) => row.text)).toEqual([
"alpha",
"beta",
"alpha",
"beta",
]);
expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([
"one",
"two",
"one",
"two",
]);
const fields = (await largeTable.schema()).fields;
expect(fields.map((field) => field.type.typeId)).toEqual([
arrow.Type.LargeUtf8,
arrow.Type.LargeBinary,
]);
});
it("should support a foreign Float64 vector schema end to end", async () => {
const conn = await connect(tmpDir.name);
const schema = new arrow.Schema([
@@ -416,6 +447,26 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(await table.countRows("id == 10")).toBe(1);
});
it("should let me update float values to NaN and Infinity", async () => {
const db = await connect(tmpDir.name);
const floats = await db.createTable("floats", [
{ id: 1, price: 1.5 },
{ id: 2, price: 2.5 },
{ id: 3, price: 3.5 },
{ id: 4, price: 4.5 },
]);
await floats.update({ values: { price: NaN }, where: "id = 1" });
await floats.update({ values: { price: Infinity }, where: "id = 2" });
await floats.update({ values: { price: -Infinity }, where: "id = 3" });
const rows = await floats.query().toArray();
const price = (id: number) => rows.find((row) => row.id === id).price;
expect(price(1)).toBeNaN();
expect(price(2)).toBe(Infinity);
expect(price(3)).toBe(-Infinity);
expect(price(4)).toBe(4.5);
});
// https://github.com/lancedb/lancedb/issues/1293
test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
"can create empty table with non default float type: %s",
@@ -688,6 +739,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
// https://github.com/lancedb/lancedb/issues/1963
// These types were added in Arrow 21, so they are built with a second copy of
// that release rather than the per-version matrix above, and reading them back
// needs the library's own Arrow to know them too.
const hostArrowSupportsViewTypes =
typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
(hostArrowSupportsViewTypes ? it : it.skip)(
"creates and adds tables with LargeList, Utf8View and BinaryView columns",
async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const makeData = () =>
new latestArrow.Table({
id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
tags: latestArrow.vectorFromArray(
[["a", "b"], ["c"]],
new latestArrow.LargeList(
new latestArrow.Field("item", new latestArrow.Utf8(), true),
),
),
text: latestArrow.vectorFromArray(
["short", "a string that is longer than the twelve inline bytes"],
new latestArrow.Utf8View(),
),
bytes: latestArrow.vectorFromArray(
[new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
new latestArrow.BinaryView(),
),
});
const table = await db.createTable("views", makeData());
await table.add(makeData());
const schema = await table.schema();
// Take the enum from the Arrow 21 copy: the host Arrow may predate
// `Type.LargeList`, and this file still has to type-check there.
expect(schema.fields.map((f) => f.type.typeId)).toEqual([
latestArrow.Type.Int,
latestArrow.Type.LargeList,
// Lance stores the view types as their offset-based equivalents.
latestArrow.Type.Utf8,
latestArrow.Type.Binary,
]);
const rows = await table.query().toArrow();
expect(rows.numRows).toBe(4);
const row = rows.get(1)!.toJSON();
expect(row.id).toBe(2);
expect(row.tags.toJSON()).toEqual(["c"]);
expect(row.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
// Arrow JS 21 serializes a *sliced* view column with a truncated views
// buffer; the library rebuilds view columns as Utf8/Binary so this works.
const sliced = makeData().slice(1, 2);
expect(sliced.getChild("text")!.data[0].offset).toBe(1);
const slicedTable = await db.createTable("views_sliced", sliced);
await slicedTable.add(sliced);
const slicedRows = await slicedTable.query().toArrow();
expect(slicedRows.numRows).toBe(2);
for (const i of [0, 1]) {
const r = slicedRows.get(i)!.toJSON();
expect(r.id).toBe(2);
expect(r.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(r.bytes)).toEqual(Array(20).fill(7));
}
// Regression: a field with an integer-like name. JavaScript enumerates
// such keys first, so a key-ordered rebuild would pair "1" with the text
// column's values. The batches are built explicitly to pin field order.
const numericSchema = new latestArrow.Schema([
new latestArrow.Field("text", new latestArrow.Utf8View(), true),
new latestArrow.Field("1", new latestArrow.Utf8View(), true),
new latestArrow.Field("n", new latestArrow.Int32(), true),
]);
const column = (
values: unknown[],
type: InstanceType<typeof latestArrow.DataType>,
) => latestArrow.vectorFromArray(values, type).data[0];
const numericBatch = new latestArrow.RecordBatch(
numericSchema,
latestArrow.makeData({
type: new latestArrow.Struct(numericSchema.fields),
length: 3,
nullCount: 0,
children: [
column(["t0", "t1", "t2"], new latestArrow.Utf8View()),
column(["one0", "one1", "one2"], new latestArrow.Utf8View()),
column([0, 1, 2], new latestArrow.Int32()),
],
}),
);
const numericData = new latestArrow.Table(numericSchema, [numericBatch]);
const numericTable = await db.createTable("views_numeric", numericData);
await numericTable.add(numericData.slice(1));
const numericRows = (await numericTable.query().toArrow()).toArray();
expect(numericRows.map((r) => r.toJSON())).toEqual([
{ text: "t0", "1": "one0", n: 0 },
{ text: "t1", "1": "one1", n: 1 },
{ text: "t2", "1": "one2", n: 2 },
{ text: "t1", "1": "one1", n: 1 },
{ text: "t2", "1": "one2", n: 2 },
]);
tmpDir.removeCallback();
},
);
it("should query documents with LangChain PDF metadata", async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
try {
@@ -1052,6 +1211,12 @@ describe("When creating an index", () => {
.toArrow();
expect(rst.numRows).toBe(2);
expect(() => tbl.vectorSearch(queryVec).nprobes(0)).toThrow(
"Invalid input, nprobes must be greater than 0",
);
expect(() =>
tbl.query().nearestTo(queryVec).fullTextSearch("dog").nprobes(0),
).toThrow("Invalid input, nprobes must be greater than 0");
expect(() => tbl.query().nearestTo(queryVec).minimumNprobes(0)).toThrow(
"Invalid input, minimum_nprobes must be greater than 0",
);
@@ -2429,6 +2594,118 @@ describe("when dealing with blob columns", () => {
await expect(table.countRows()).resolves.toBe(0);
});
it("requires an explicit opt-in for blob URIs outside registered bases", async () => {
const { table, uri } = await openExternalBlobTable();
await expect(table.add([{ id: 1n, payload: uri }])).rejects.toThrow(
"allow_external_blob_outside_bases",
);
await expect(table.countRows()).resolves.toBe(0);
await expect(
table.add([{ id: 2n, payload: uri }], {
allowExternalBlobOutsideBases: false,
}),
).rejects.toThrow("allow_external_blob_outside_bases");
await expect(table.countRows()).resolves.toBe(0);
});
it.each([
["URI string", (uri: string) => uri],
["URI struct", (uri: string) => ({ uri })],
])(
"round-trips an external blob from a %s after reopening",
async (_label, blobValue) => {
const { db, table, uri, payload } = await openExternalBlobTable();
const result = await table.add([{ id: 1n, payload: blobValue(uri) }], {
allowExternalBlobOutsideBases: true,
});
expect(result.version).toBeGreaterThan(0);
await expect(table.countRows()).resolves.toBe(1);
table.close();
const reopened = await db.openTable("external_blobs");
const rows = await reopened.query().withRowId().toArray();
expect(rows).toHaveLength(1);
expect(rows[0].payload.blob_uri).toBe(uri);
const bytes = await reopened.fetchBlobs("payload", [
rows[0]._rowid as bigint,
]);
expect(bytes[0]).toEqual(payload);
},
);
it("applies the external blob opt-in to Arrow table input", async () => {
const { table, schema, uri, payload } = await openExternalBlobTable();
const input = makeArrowTable([{ id: 1n, payload: uri }], { schema });
const result = await table.add(input, {
allowExternalBlobOutsideBases: true,
});
expect(result.version).toBeGreaterThan(0);
const rows = await table.query().withRowId().toArray();
expect(rows[0].payload.blob_uri).toBe(uri);
const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
expect(bytes[0]).toEqual(payload);
});
it("applies the external blob opt-in only to the current add", async () => {
const { table, uri } = await openExternalBlobTable();
await table.add([{ id: 1n, payload: uri }], {
allowExternalBlobOutsideBases: true,
});
await expect(table.countRows()).resolves.toBe(1);
await expect(table.add([{ id: 2n, payload: uri }])).rejects.toThrow(
"allow_external_blob_outside_bases",
);
await expect(table.countRows()).resolves.toBe(1);
});
it("supports progress and overwrite with the external blob opt-in", async () => {
const { table, uri, payload } = await openExternalBlobTable();
await table.add([{ id: 1n, payload: Buffer.from("old") }]);
let resolveFinal!: (progress: import("../lancedb").WriteProgress) => void;
let rejectFinal!: (error: Error) => void;
const finalProgress = new Promise<import("../lancedb").WriteProgress>(
(resolve, reject) => {
resolveFinal = resolve;
rejectFinal = reject;
},
);
const timeout = setTimeout(
() => rejectFinal(new Error("timed out waiting for final progress")),
5_000,
);
try {
const result = await table.add([{ id: 2n, payload: uri }], {
mode: "overwrite",
allowExternalBlobOutsideBases: true,
progress: (progress) => {
if (progress.done) {
resolveFinal(progress);
}
},
});
expect(result.version).toBeGreaterThan(0);
const progress = await finalProgress;
expect(progress.outputRows).toBe(1);
expect(progress.totalRows).toBe(1);
} finally {
clearTimeout(timeout);
}
const rows = await table.query().withRowId().toArray();
expect(rows).toHaveLength(1);
expect(rows[0].id).toBe(2n);
const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
expect(bytes[0]).toEqual(payload);
});
it("discovers blob columns", async () => {
const { table } = await openBlobTable();
expect(await table.blobColumns()).toEqual(["image"]);
@@ -2468,6 +2745,27 @@ describe("when dealing with blob columns", () => {
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
});
it("adds a LargeBinary Arrow column to a blob table", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
const table = await db.createTable(
"blobs",
[{ id: 1n, image: Buffer.from("alpha") }],
{ schema },
);
const payload = Buffer.from("beta");
const data = new arrow18.Table({
id: arrow18.vectorFromArray([2n], new arrow18.Int64()),
image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()),
});
await table.add(data);
const rows = await table.query().withRowId().toArray();
const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint;
const [actual] = await table.fetchBlobs("image", [rowId]);
expect(actual).toEqual(payload);
});
it("reads a half-open range", async () => {
const { table, rowIds } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
@@ -2492,6 +2790,148 @@ describe("when dealing with blob columns", () => {
).rejects.toThrow(/exceeds blob size/);
});
it("reads at most maxBytes and advances the cursor", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect((await handle!.read(2n)).toString()).toBe("al");
expect(await handle!.tell()).toBe(2n);
expect((await handle!.read(2n)).toString()).toBe("ph");
expect((await handle!.read(10n)).toString()).toBe("a");
expect(await handle!.tell()).toBe(5n);
expect(await handle!.read(1n)).toEqual(Buffer.alloc(0));
});
it("seeks the cursor", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await handle!.seek(3n);
expect(await handle!.tell()).toBe(3n);
expect((await handle!.read()).toString()).toBe("ha");
await handle!.seek(0n);
expect(await handle!.read()).toEqual(alpha);
});
it("rejects a negative seek position", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await expect(handle!.seek(-1n)).rejects.toThrow(/cannot be negative/);
});
it("readRanges returns one buffer per range in order", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const buffers = await handle!.readRanges([
{ start: 3n, end: 5n },
{ start: 0n, end: 2n },
{ start: 3n, end: 5n },
]);
expect(buffers.map((b) => b.toString())).toEqual(["ha", "al", "ha"]);
expect(await handle!.tell()).toBe(0n);
});
it("fails when a readRanges end is past the blob size", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
await expect(
handle!.readRanges([
{ start: 0n, end: 1n },
{ start: 0n, end: BigInt(alpha.length + 1) },
]),
).rejects.toThrow(/exceeds blob size/);
});
it("fails reads after close and closes once", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
expect(handle!.isClosed()).toBe(false);
await handle!.close();
expect(handle!.isClosed()).toBe(true);
await expect(handle!.close()).resolves.toBeUndefined();
await expect(handle!.read()).rejects.toThrow(/already closed/);
await expect(handle!.read(1n)).rejects.toThrow(/already closed/);
await expect(handle!.readRange(0n, 1n)).rejects.toThrow(/already closed/);
await expect(handle!.readRanges([{ start: 0n, end: 1n }])).rejects.toThrow(
/already closed/,
);
await expect(handle!.seek(0n)).rejects.toThrow(/already closed/);
await expect(handle!.tell()).rejects.toThrow(/already closed/);
expect(handle!.size()).toBe(5n);
});
it("rejects reads with an already aborted signal", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const signal = AbortSignal.abort();
const abortError = { name: "AbortError" };
await expect(handle!.read(undefined, { signal })).rejects.toMatchObject(
abortError,
);
await expect(handle!.read(2n, { signal })).rejects.toMatchObject(
abortError,
);
await expect(handle!.readRange(0n, 1n, { signal })).rejects.toMatchObject(
abortError,
);
await expect(
handle!.readRanges([{ start: 0n, end: 1n }], { signal }),
).rejects.toMatchObject(abortError);
await expect(
table.fetchBlobs("image", rowIds, { signal }),
).rejects.toMatchObject(abortError);
await expect(
table.fetchBlobFiles("image", rowIds, { signal }),
).rejects.toMatchObject(abortError);
expect(await handle!.read()).toEqual(alpha);
});
it("rejects with the abort reason", async () => {
const { table, rowIds } = await openBlobTable();
const [handle] = await table.fetchBlobFiles("image", rowIds);
const reason = new Error("caller gave up");
await expect(
handle!.readRange(0n, 1n, { signal: AbortSignal.abort(reason) }),
).rejects.toBe(reason);
});
it("reads with a signal and removes its abort listener", async () => {
const { table, rowIds, alpha, beta } = await openBlobTable();
const controller = new AbortController();
const { signal } = controller;
const [handle] = await table.fetchBlobFiles("image", rowIds, { signal });
expect((await handle!.readRange(0n, 2n, { signal })).toString()).toBe("al");
expect(await table.fetchBlobs("image", rowIds, { signal })).toEqual([
alpha,
beta,
null,
]);
expect(getEventListeners(signal, "abort")).toHaveLength(0);
controller.abort();
expect(await handle!.read()).toEqual(alpha);
});
it("cancels an in-flight read", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("payload", { inlineSizeThreshold: 0, dedicatedSizeThreshold: 1 }),
]);
const payload = Buffer.alloc(32 * 1024 * 1024, 7);
const table = await db.createTable("large_blobs", [{ id: 1n, payload }], {
schema,
});
const [row] = await table.query().withRowId().toArray();
const [handle] = await table.fetchBlobFiles("payload", [
row._rowid as bigint,
]);
const controller = new AbortController();
const pending = handle!.readRange(0n, handle!.size(), {
signal: controller.signal,
});
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4));
});
it("rejects fetchBlobs on a non-blob column", async () => {
const { table, rowIds } = await openBlobTable();
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
@@ -2688,6 +3128,20 @@ describe("when dealing with blob columns", () => {
const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
return { table, rowIds, alpha, beta };
}
async function openExternalBlobTable() {
const db = await connect(path.join(tmpDir.name, "db"));
const payload = Buffer.from("external-payload");
const payloadPath = path.join(tmpDir.name, "payload.bin");
fs.writeFileSync(payloadPath, payload);
const uri = pathToFileURL(payloadPath).href;
const schema = new Schema([
new Field("id", new Int64(), true),
blob("payload"),
]);
const table = await db.createEmptyTable("external_blobs", schema);
return { db, table, schema, uri, payload };
}
});
describe("when dealing with tags", () => {
@@ -2860,7 +3314,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
});
describe.each([arrow15, arrow16, arrow17, arrow18])(
describe.each(arrowVersions)(
"when optimizing a dataset",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
nodejs/__test__/util.test.ts
+3
View File
@@ -14,6 +14,9 @@ test.each([
[new ArrayBuffer(8), "X'0000000000000000'"],
[Buffer.from("hello"), "X'68656c6c6f'"],
["Hello 'world'", "'Hello ''world'''"],
[NaN, "CAST('NaN' AS DOUBLE)"],
[Infinity, "CAST('Infinity' AS DOUBLE)"],
[-Infinity, "CAST('-Infinity' AS DOUBLE)"],
])("toSQL(%p) === %p", (value, expected) => {
expect(toSQL(value)).toBe(expected);
});
nodejs/lancedb/arrow.ts
+83 -4
View File
@@ -66,11 +66,20 @@ export type SchemaLike =
export type FieldLike =
| Field
| {
type: string;
type: string | DataTypeLike;
name: string;
nullable: boolean;
metadata?: Map<string, string>;
};
/**
* A `DataType` from any copy or version of apache-arrow.
*
* Arrow 21 brands its classes with `unique symbol` properties, so a type
* object from a second copy of the library no longer satisfies the `DataType`
* type of this one even though it is structurally identical. Inputs that only
* need to be sanitized accept this looser shape instead.
*/
export type DataTypeLike = DataType | { readonly typeId: number };
/**
* Create an Arrow field backed by LanceDB's JSON extension type.
@@ -1059,6 +1068,71 @@ export async function fromRecordsToStreamBuffer(
return Buffer.from(await writer.toUint8Array());
}
// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist
// in Arrow 21+, and this module compiles against every supported release.
const UTF8_VIEW_TYPE_ID = 24;
const BINARY_VIEW_TYPE_ID = 23;
/**
* Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`.
*/
function materializeViewData(child: ArrowData, type: DataType): ArrowData {
const builder = makeBuilder({ type, nullValues: [null] });
for (const value of new Vector([child])) {
builder.append(value);
}
return builder.finish().flush();
}
/**
* Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
*
* Lance stores the view types as their offset-based equivalents anyway, so
* nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC
* writer emits a truncated views buffer for a *sliced* view array, which the
* Rust reader rejects with "Need at least N bytes in buffers[0]".
*
* The record batches are rebuilt positionally rather than through a
* `Record<string, Vector>`: JavaScript enumerates integer-like keys first, so
* a field named e.g. `"1"` would otherwise be paired with the wrong column.
*
* Tables without view columns are returned as-is.
*/
function materializeViewColumns(table: ArrowTable): ArrowTable {
const replacements = new Map<number, DataType>();
table.schema.fields.forEach((field, i) => {
if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
replacements.set(i, new Utf8());
} else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
replacements.set(i, new Binary());
}
});
if (replacements.size === 0) {
return table;
}
const fields = table.schema.fields.map((field, i) => {
const type = replacements.get(i);
return type === undefined
? field
: new Field(field.name, type, field.nullable, field.metadata);
});
const schema = new Schema(fields, table.schema.metadata);
const batches = table.batches.map((batch) => {
const children = batch.data.children.map((child, i) => {
const type = replacements.get(i);
return type === undefined ? child : materializeViewData(child, type);
});
const data = makeData({
type: new Struct(fields),
length: batch.numRows,
nullCount: 0,
children,
});
return new RecordBatch(schema, data);
});
return new ArrowTable(schema, batches);
}
/**
* Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
*
@@ -1075,7 +1149,9 @@ export async function fromTableToBuffer(
if (schema !== undefined && schema !== null) {
schema = sanitizeSchema(schema);
}
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
const tableWithEmbeddings = materializeViewColumns(
await applyEmbeddings(table, embeddings, schema),
);
validateBlobSchema(tableWithEmbeddings.schema);
const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
@@ -1146,7 +1222,8 @@ export async function fromRecordBatchToBuffer(
export async function fromRecordBatchToStreamBuffer(
batch: RecordBatch,
): Promise<Buffer> {
const writer = RecordBatchStreamWriter.writeAll([batch]);
const table = materializeViewColumns(new ArrowTable([batch]));
const writer = RecordBatchStreamWriter.writeAll(table);
return Buffer.from(await writer.toUint8Array());
}
@@ -1163,7 +1240,9 @@ export async function fromTableToStreamBuffer(
embeddings?: EmbeddingFunctionConfig,
schema?: SchemaLike,
): Promise<Buffer> {
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
const tableWithEmbeddings = materializeViewColumns(
await applyEmbeddings(table, embeddings, schema),
);
const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
return Buffer.from(await writer.toUint8Array());
}
nodejs/lancedb/blob.ts
+95 -7
View File
@@ -141,13 +141,16 @@ export class BlobFile {
}
/**
* Reads from the cursor to the end and advances the cursor.
* Reads from the cursor and advances the cursor.
*
* A second call returns an empty buffer. {@link BlobFile.readRange} does
* not move the cursor.
* Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
* otherwise. Returns an empty buffer at the end of the blob.
* {@link BlobFile.readRange} does not move the cursor.
*/
read(): Promise<Buffer> {
return this.inner.read();
read(maxBytes?: bigint, options?: BlobReadOptions): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.read(maxBytes, signal),
);
}
/**
@@ -155,11 +158,96 @@ export class BlobFile {
*
* Fails when `end` is past the blob size. Does not move the cursor.
*/
readRange(start: bigint, end: bigint): Promise<Buffer> {
return this.inner.readRange(start, end);
readRange(
start: bigint,
end: bigint,
options?: BlobReadOptions,
): Promise<Buffer> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRange(start, end, signal),
);
}
/**
* Reads several half-open byte ranges. Returns one buffer per range, in
* the order given.
*
* Fails when any `end` is past the blob size. Does not move the cursor.
*/
readRanges(
ranges: BlobRange[],
options?: BlobReadOptions,
): Promise<Buffer[]> {
return runWithSignal(options?.signal, (signal) =>
this.inner.readRanges(ranges, signal),
);
}
/** Moves the cursor to `position`, in bytes from the start of the blob. */
seek(position: bigint): Promise<void> {
return this.inner.seek(position);
}
/** Returns the cursor position in bytes. */
tell(): Promise<bigint> {
return this.inner.tell();
}
/**
* Releases the handle. Reads after `close()` fail. Calling it again does
* nothing.
*/
close(): Promise<void> {
return this.inner.close();
}
/** Returns true after {@link BlobFile.close}. */
isClosed(): boolean {
return this.inner.isClosed();
}
}
/** Options for blob reads. */
export type BlobReadOptions = {
/**
* Cancels the read. The call rejects with `signal.reason`, and the native
* read stops, including in-flight requests to a remote table.
*/
signal?: AbortSignal;
};
/** @ignore */
export async function runWithSignal<T>(
signal: AbortSignal | undefined,
run: (signal: AbortSignal | undefined) => Promise<T>,
): Promise<T> {
if (signal === undefined) {
return run(undefined);
}
signal.throwIfAborted();
// napi-rs replaces `onabort` on the signal it receives, so pass a private
// signal and forward the caller's abort to it.
const controller = new AbortController();
const onAbort = () => controller.abort();
signal.addEventListener("abort", onAbort, { once: true });
try {
return await run(controller.signal);
} catch (err) {
if (signal.aborted) {
throw signal.reason;
}
throw err;
} finally {
signal.removeEventListener("abort", onAbort);
}
}
/** A half-open byte range `[start, end)` for {@link BlobFile.readRanges}. */
export type BlobRange = {
start: bigint;
end: bigint;
};
export function coerceBlobValue(value: unknown): BlobInput | null {
if (value == null) {
return null;
nodejs/lancedb/index.ts
+2 -1
View File
@@ -84,7 +84,7 @@ export {
} from "./arrow";
export { blob, isBlobField, BlobFile } from "./blob";
export type { BlobOptions } from "./blob";
export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob";
export {
Connection,
@@ -195,6 +195,7 @@ export * as rerankers from "./rerankers";
export {
SchemaLike,
TableLike,
DataTypeLike,
FieldLike,
RecordBatchLike,
DataLike,
nodejs/lancedb/materialized_view.ts
+3 -2
View File
@@ -9,9 +9,10 @@ export const DEFINITION_META_KEY = "mv.definition";
/**
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
* format 2 being a query with `GROUP BY`.
* format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
* native dedup results.
*/
export const DEFINITION_FORMAT = 2;
export const DEFINITION_FORMAT = 4;
/**
* The query that defines a materialized view, as stored:
nodejs/lancedb/query.ts
+12 -23
View File
@@ -41,26 +41,6 @@ export async function* RecordBatchIterator(
}
}
class RecordBatchIterable<
NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
> implements AsyncIterable<RecordBatch>
{
private inner: NativeQueryType;
private options?: QueryExecutionOptions;
constructor(inner: NativeQueryType, options?: QueryExecutionOptions) {
this.inner = inner;
this.options = options;
}
// biome-ignore lint/suspicious/noExplicitAny: skip
[Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>, any, undefined> {
return RecordBatchIterator(
this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs),
);
}
}
/**
* Options that control the behavior of a particular query execution
*/
@@ -271,13 +251,20 @@ export class QueryBase<
return RecordBatchIterator(this.nativeExecute());
}
/** Collect the results as an Arrow @see {@link ArrowTable}. */
/**
* Collect the results as an Arrow @see {@link ArrowTable}.
*
* Empty results retain the query's output schema.
*/
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
const batches = [];
const inner = await this.getInner();
for await (const batch of new RecordBatchIterable(inner, options)) {
const iterator = this.nativeExecute(options);
for await (const batch of RecordBatchIterator(iterator)) {
batches.push(batch);
}
if (batches.length === 0) {
return tableFromIPC((await iterator).schema());
}
return new ArrowTable(batches);
}
@@ -532,6 +519,8 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
/**
* Set the number of partitions to search (probe)
*
* The number of probes must be greater than 0.
*
* This argument is only used when the vector column has an IVF PQ index.
* If there is no index then this value is ignored.
*
nodejs/lancedb/sanitize.ts
+137 -15
View File
@@ -9,6 +9,7 @@
// comes from the exact same library instance. This is not always the case
// and so we must sanitize the input to ensure that it is compatible.
import * as arrowModule from "apache-arrow";
import { BufferType, Data, Vector } from "apache-arrow";
import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
import {
@@ -43,6 +44,8 @@ import {
Interval,
IntervalDayTime,
IntervalYearMonth,
LargeBinary,
LargeUtf8,
List,
Map_,
Null,
@@ -74,6 +77,66 @@ import {
Utf8,
} from "./arrow";
/**
* Whether `value` was created by *this* copy of apache-arrow's `ctor`.
*
* Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
* `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
* built by a different copy of the library. Those are exactly the objects
* this module has to rebuild: the `DataType` instances nested inside them are
* deliberately *not* given that treatment upstream, so Arrow's own builders and
* writers reject them. Walk the real prototype chain instead of trusting
* `instanceof`.
*/
function isOwnInstance<T>(
value: unknown,
// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
ctor: abstract new (...args: any[]) => T,
): value is T {
return (
typeof value === "object" &&
value !== null &&
Object.prototype.isPrototypeOf.call(ctor.prototype, value)
);
}
/**
* Types added to Arrow JS after release 18, keyed by the `Type` enum value
* they use. The library compiles and runs against every Arrow release in the
* peer range, so these are looked up by name at runtime instead of imported:
* an older Arrow simply does not have the class.
*/
type NewerTypeName =
| "IntervalMonthDayNano"
| "LargeList"
| "BinaryView"
| "Utf8View";
const NEWER_TYPES: ReadonlyArray<{
name: NewerTypeName;
typeId: number;
since: number;
}> = [
{ name: "IntervalMonthDayNano", typeId: -31, since: 20 },
{ name: "LargeList", typeId: 21, since: 21 },
{ name: "BinaryView", typeId: 23, since: 21 },
{ name: "Utf8View", typeId: 24, since: 21 },
];
// biome-ignore lint/suspicious/noExplicitAny: constructor arity varies by type
type DataTypeConstructor = new (...args: any[]) => DataType;
function newerTypeConstructor(name: NewerTypeName): DataTypeConstructor {
const ctor = (arrowModule as unknown as Record<string, unknown>)[name];
if (typeof ctor !== "function") {
const since = NEWER_TYPES.find((t) => t.name === name)?.since;
throw new Error(
`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
"but the installed apache-arrow does not provide it",
);
}
return ctor as DataTypeConstructor;
}
type SanitizationContext = {
types: WeakMap<object, DataType>;
vectors: WeakMap<object, Vector>;
@@ -210,19 +273,37 @@ export function sanitizeList(typeLike: object) {
return sanitizeListWithContext(typeLike, createSanitizationContext());
}
function sanitizeListChildWithContext(
typeLike: object,
context: SanitizationContext,
typeName: string,
): Field {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
`Expected a ${typeName} type to have an array-like \`children\` property`,
);
}
if (typeLike.children.length !== 1) {
throw Error(`Expected a ${typeName} type to have exactly one child`);
}
return sanitizeFieldWithContext(typeLike.children[0], context);
}
function sanitizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a List type to have an array-like `children` property",
);
}
if (typeLike.children.length !== 1) {
throw Error("Expected a List type to have exactly one child");
}
return new List(sanitizeFieldWithContext(typeLike.children[0], context));
return new List(sanitizeListChildWithContext(typeLike, context, "List"));
}
function sanitizeLargeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
const LargeList = newerTypeConstructor("LargeList");
return new LargeList(
sanitizeListChildWithContext(typeLike, context, "LargeList"),
);
}
export function sanitizeStruct(typeLike: object) {
@@ -449,11 +530,30 @@ function sanitizeTypeWithContext(
return type;
}
function sanitizeNewerTypeById(
typeLike: object,
typeId: number,
context: SanitizationContext,
): DataType | undefined {
const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
if (entry === undefined) {
return undefined;
}
if (entry.name === "LargeList") {
return sanitizeLargeListWithContext(typeLike, context);
}
return new (newerTypeConstructor(entry.name))();
}
function sanitizeTypeById(
typeLike: object,
typeId: Type,
context: SanitizationContext,
): DataType {
const newer = sanitizeNewerTypeById(typeLike, typeId, context);
if (newer !== undefined) {
return newer;
}
switch (typeId) {
case Type.NONE:
throw Error("Received a Type with a typeId of NONE");
@@ -465,8 +565,12 @@ function sanitizeTypeById(
return sanitizeFloat(typeLike);
case Type.Binary:
return new Binary();
case Type.LargeBinary:
return new LargeBinary();
case Type.Utf8:
return new Utf8();
case Type.LargeUtf8:
return new LargeUtf8();
case Type.Bool:
return new Bool();
case Type.Decimal:
@@ -566,7 +670,7 @@ function sanitizeFieldWithContext(
fieldLike: unknown,
context: SanitizationContext,
): Field {
if (fieldLike instanceof Field) {
if (isOwnInstance(fieldLike, Field)) {
return fieldLike;
}
if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -620,7 +724,7 @@ function sanitizeSchemaWithContext(
schemaLike: SchemaLike,
context: SanitizationContext,
): Schema {
if (schemaLike instanceof Schema) {
if (isOwnInstance(schemaLike, Schema)) {
return schemaLike;
}
if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -647,7 +751,7 @@ function sanitizeSchemaWithContext(
}
export function sanitizeTable(tableLike: TableLike): Table {
if (tableLike instanceof Table) {
if (isOwnInstance(tableLike, Table)) {
return tableLike;
}
if (typeof tableLike !== "object" || tableLike === null) {
@@ -675,7 +779,7 @@ function sanitizeRecordBatch(
batchLike: RecordBatchLike,
context: SanitizationContext,
): RecordBatch {
if (batchLike instanceof RecordBatch) {
if (isOwnInstance(batchLike, RecordBatch)) {
return batchLike;
}
if (typeof batchLike !== "object" || batchLike === null) {
@@ -704,11 +808,17 @@ type DictionaryDataLike = DataLike & {
dictionary?: DictionaryVectorLike;
};
// Arrow 21 added `variadicBuffers` to `Data` for the Utf8View / BinaryView
// types. Older Arrow versions do not have the property, so it is optional.
type VariadicDataLike = DataLike & {
variadicBuffers?: ReadonlyArray<Uint8Array>;
};
function sanitizeData(
dataLike: DataLike,
context: SanitizationContext,
): Data<DataType> {
if (dataLike instanceof Data) {
if (isOwnInstance(dataLike, Data)) {
return dataLike;
}
const cachedData = context.data.get(dataLike);
@@ -726,7 +836,12 @@ function sanitizeData(
context.vectors.set(dictionaryLike, dictionary);
}
}
const data = new Data(
// Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
// releases ignore the extra argument but their typings do not declare it.
const DataCtor = Data as unknown as new (
...args: unknown[]
) => Data<DataType>;
const data = new DataCtor(
sanitizeTypeWithContext(dataLike.type, context),
dataLike.offset,
dataLike.length,
@@ -739,6 +854,7 @@ function sanitizeData(
},
dataLike.children.map((child) => sanitizeData(child, context)),
dictionary,
(dataLike as VariadicDataLike).variadicBuffers,
);
context.data.set(dataLike, data);
return data;
@@ -747,7 +863,11 @@ function sanitizeData(
const constructorsByTypeName = {
null: () => new Null(),
binary: () => new Binary(),
largebinary: () => new LargeBinary(),
binaryview: () => new (newerTypeConstructor("BinaryView"))(),
utf8: () => new Utf8(),
largeutf8: () => new LargeUtf8(),
utf8view: () => new (newerTypeConstructor("Utf8View"))(),
bool: () => new Bool(),
int8: () => new Int8(),
int16: () => new Int16(),
@@ -768,6 +888,8 @@ const constructorsByTypeName = {
timesecond: () => new TimeSecond(),
intervaldaytime: () => new IntervalDayTime(),
intervalyearmonth: () => new IntervalYearMonth(),
intervalmonthdaynano: () =>
new (newerTypeConstructor("IntervalMonthDayNano"))(),
durationnanosecond: () => new DurationNanosecond(),
durationmicrosecond: () => new DurationMicrosecond(),
durationmillisecond: () => new DurationMillisecond(),
nodejs/lancedb/table.ts
+31 -6
View File
@@ -17,7 +17,7 @@ import {
tableFromIPC,
} from "./arrow";
import { BlobFile } from "./blob";
import { BlobFile, BlobReadOptions, runWithSignal } from "./blob";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { IndexOptions } from "./indices";
import { Job } from "./job";
@@ -124,6 +124,17 @@ export interface AddDataOptions {
* ```
*/
progress: (progress: WriteProgress) => void;
/**
* Whether blob URIs outside registered external bases may be written.
*
* Defaults to `false`. This option is supported only for local/native
* tables; remote tables return an error when it is enabled. An enabled write
* stores the absolute URI reference without registering a base or copying
* the external object into the database. The object must remain accessible
* when the blob is read later.
*/
allowExternalBlobOutsideBases?: boolean;
}
export interface UpdateOptions {
@@ -526,10 +537,15 @@ export abstract class Table {
* Reads the table's current checkout. IDs from another version can fail after
* compaction unless stable row ids are enabled. Results keep input order and
* duplicates. Null blobs are `null`. Empty blobs are empty buffers.
* Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
* The client splits requests automatically and reads an individual larger
* blob through the Range route. This method still materializes all bytes in
* memory; use {@link Table.fetchBlobFiles} for large values.
*/
abstract fetchBlobs(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(Buffer | null)[]>;
/**
@@ -542,6 +558,7 @@ export abstract class Table {
abstract fetchBlobFiles(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(BlobFile | null)[]>;
/**
@@ -1091,7 +1108,12 @@ export class LocalTable extends Table {
}
}
: undefined;
return await this.inner.add(buffer, mode, progress);
return await this.inner.add(
buffer,
mode,
progress,
options?.allowExternalBlobOutsideBases,
);
}
async update(
@@ -1228,8 +1250,11 @@ export class LocalTable extends Table {
async fetchBlobs(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(Buffer | null)[]> {
const values = await this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds));
const values = await runWithSignal(options?.signal, (signal) =>
this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds), signal),
);
// N-API Option maps missing values to undefined. Collapse those to null.
return values.map((value) => value ?? null);
}
@@ -1237,10 +1262,10 @@ export class LocalTable extends Table {
async fetchBlobFiles(
column: string,
rowIds: readonly (bigint | number)[],
options?: BlobReadOptions,
): Promise<(BlobFile | null)[]> {
const files = await this.inner.fetchBlobFiles(
column,
rowIdsToBigInts(rowIds),
const files = await runWithSignal(options?.signal, (signal) =>
this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal),
);
// N-API Option maps missing values to undefined. Collapse those to null.
return files.map((file) =>
nodejs/lancedb/util.ts
+8
View File
@@ -15,6 +15,14 @@ export function toSQL(value: IntoSql): string {
if (typeof value === "string") {
return `'${value.replace(/'/g, "''")}'`;
} else if (typeof value === "number") {
// toString() gives "NaN" / "Infinity", which SQL reads as column names.
if (Number.isNaN(value)) {
return "CAST('NaN' AS DOUBLE)";
} else if (!Number.isFinite(value)) {
return value > 0
? "CAST('Infinity' AS DOUBLE)"
: "CAST('-Infinity' AS DOUBLE)";
}
return value.toString();
} else if (typeof value === "boolean") {
return value ? "TRUE" : "FALSE";
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+6 -2
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.40.0-beta.11",
"version": "0.40.0-beta.12",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -49,10 +49,14 @@
"@types/jest": "^29.1.2",
"@types/node": "22.7.4",
"@types/tmp": "^0.2.6",
"apache-arrow": "21.2.0",
"apache-arrow-15": "npm:apache-arrow@15.0.0",
"apache-arrow-16": "npm:apache-arrow@16.0.0",
"apache-arrow-17": "npm:apache-arrow@17.0.0",
"apache-arrow-18": "npm:apache-arrow@18.0.0",
"apache-arrow-19": "npm:apache-arrow@19.0.1",
"apache-arrow-20": "npm:apache-arrow@20.0.0",
"apache-arrow-21": "npm:apache-arrow@21.2.0",
"eslint": "^8.57.0",
"jest": "^29.7.0",
"shx": "^0.3.4",
@@ -102,7 +106,7 @@
},
"peerDependencies": {
"@types/node": ">=22",
"apache-arrow": ">=15.0.0 <=18.1.0"
"apache-arrow": ">=15.0.0 <22.0.0"
},
"peerDependenciesMeta": {
"@types/node": {
nodejs/pnpm-lock.yaml
Generated
+91 -7
View File
@@ -14,9 +14,6 @@ importers:
'@opentelemetry/api':
specifier: ^1.9.0
version: 1.9.1
apache-arrow:
specifier: '>=15.0.0 <=18.1.0'
version: 18.1.0
reflect-metadata:
specifier: ^0.2.2
version: 0.2.2
@@ -54,6 +51,9 @@ importers:
'@types/tmp':
specifier: ^0.2.6
version: 0.2.6
apache-arrow:
specifier: 21.2.0
version: 21.2.0
apache-arrow-15:
specifier: npm:apache-arrow@15.0.0
version: apache-arrow@15.0.0
@@ -66,6 +66,15 @@ importers:
apache-arrow-18:
specifier: npm:apache-arrow@18.0.0
version: apache-arrow@18.0.0
apache-arrow-19:
specifier: npm:apache-arrow@19.0.1
version: apache-arrow@19.0.1
apache-arrow-20:
specifier: npm:apache-arrow@20.0.0
version: apache-arrow@20.0.0
apache-arrow-21:
specifier: npm:apache-arrow@21.2.0
version: apache-arrow@21.2.0
eslint:
specifier: ^8.57.0
version: 8.57.1
@@ -1703,6 +1712,9 @@ packages:
'@types/node@22.7.4':
resolution: {integrity: sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==}
'@types/node@25.9.8':
resolution: {integrity: sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==}
'@types/stack-utils@2.0.3':
resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==}
@@ -1840,8 +1852,16 @@ packages:
resolution: {integrity: sha512-gFlPaqN9osetbB83zC29AbbZqGiCuFH1vyyPseJ+B7SIbfBtESV62mMT/CkiIt77W6ykC/nTWFzTXFs0Uldg4g==}
hasBin: true
apache-arrow@18.1.0:
resolution: {integrity: sha512-v/ShMp57iBnBp4lDgV8Jx3d3Q5/Hac25FWmQ98eMahUiHPXcvwIMKJD0hBIgclm/FCG+LwPkAKtkRO1O/W0YGg==}
apache-arrow@19.0.1:
resolution: {integrity: sha512-APmMLzS4qbTivLrPdQXexGM4JRr+0g62QDaobzEvip/FdQIrv2qLy0mD5Qdmw4buydtVJgbFeKR8f59I6PPGDg==}
hasBin: true
apache-arrow@20.0.0:
resolution: {integrity: sha512-JUeK0jFRUd7rbmrhhzR3O2KXjLaZ4YYYFOptyUfxOsMIoZCPi6bZR58gVi/xi3HTBMPseXm9PXyQ2V916930pA==}
hasBin: true
apache-arrow@21.2.0:
resolution: {integrity: sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==}
hasBin: true
argparse@1.0.10:
@@ -2033,6 +2053,15 @@ packages:
resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==}
engines: {node: '>=4.0.0'}
command-line-args@6.0.2:
resolution: {integrity: sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==}
engines: {node: '>=12.20'}
peerDependencies:
'@75lb/nature': latest
peerDependenciesMeta:
'@75lb/nature':
optional: true
command-line-usage@7.0.4:
resolution: {integrity: sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==}
engines: {node: '>=12.20.0'}
@@ -2281,6 +2310,15 @@ packages:
resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==}
engines: {node: '>=4.0.0'}
find-replace@5.0.2:
resolution: {integrity: sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==}
engines: {node: '>=14'}
peerDependencies:
'@75lb/nature': latest
peerDependenciesMeta:
'@75lb/nature':
optional: true
find-up@4.1.0:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
@@ -2302,6 +2340,9 @@ packages:
flatbuffers@24.12.23:
resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==}
flatbuffers@25.9.23:
resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==}
flatted@3.4.2:
resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==}
@@ -3345,6 +3386,9 @@ packages:
undici-types@6.21.0:
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
undici-types@7.24.6:
resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==}
unist-util-is@6.0.1:
resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==}
@@ -5505,6 +5549,10 @@ snapshots:
dependencies:
undici-types: 6.19.8
'@types/node@25.9.8':
dependencies:
undici-types: 7.24.6
'@types/stack-utils@2.0.3': {}
'@types/tmp@0.2.6': {}
@@ -5694,17 +5742,40 @@ snapshots:
json-bignum: 0.0.3
tslib: 2.8.1
apache-arrow@18.1.0:
apache-arrow@19.0.1:
dependencies:
'@swc/helpers': 0.5.21
'@types/command-line-args': 5.2.3
'@types/command-line-usage': 5.0.4
'@types/node': 20.19.41
command-line-args: 5.2.1
command-line-args: 6.0.2
command-line-usage: 7.0.4
flatbuffers: 24.12.23
json-bignum: 0.0.3
tslib: 2.8.1
transitivePeerDependencies:
- '@75lb/nature'
apache-arrow@20.0.0:
dependencies:
'@swc/helpers': 0.5.21
'@types/command-line-args': 5.2.3
'@types/command-line-usage': 5.0.4
'@types/node': 20.19.41
command-line-args: 6.0.2
command-line-usage: 7.0.4
flatbuffers: 25.9.23
json-bignum: 0.0.3
tslib: 2.8.1
transitivePeerDependencies:
- '@75lb/nature'
apache-arrow@21.2.0:
dependencies:
'@types/node': 25.9.8
flatbuffers: 25.9.23
json-with-bigint: 3.5.8
tslib: 2.8.1
argparse@1.0.10:
dependencies:
@@ -5906,6 +5977,13 @@ snapshots:
lodash.camelcase: 4.3.0
typical: 4.0.0
command-line-args@6.0.2:
dependencies:
array-back: 6.2.3
find-replace: 5.0.2
lodash.camelcase: 4.3.0
typical: 7.3.0
command-line-usage@7.0.4:
dependencies:
array-back: 6.2.3
@@ -6178,6 +6256,8 @@ snapshots:
dependencies:
array-back: 3.1.0
find-replace@5.0.2: {}
find-up@4.1.0:
dependencies:
locate-path: 5.0.0
@@ -6201,6 +6281,8 @@ snapshots:
flatbuffers@24.12.23: {}
flatbuffers@25.9.23: {}
flatted@3.4.2: {}
follow-redirects@1.16.0: {}
@@ -7405,6 +7487,8 @@ snapshots:
undici-types@6.21.0: {}
undici-types@7.24.6: {}
unist-util-is@6.0.1:
dependencies:
'@types/unist': 3.0.3
nodejs/src/blob.rs
+133 -10
View File
@@ -1,25 +1,45 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::future::Future;
use std::ops::Range;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use arrow_array::{Array, LargeBinaryArray};
use futures::future::abortable;
use lancedb::blob::BlobFile as LanceBlobFile;
use napi::Env;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::error::convert_error;
#[napi(object)]
pub struct BlobRange {
pub start: BigInt,
pub end: BigInt,
}
#[napi]
pub struct BlobFile {
inner: Arc<LanceBlobFile>,
closed: AtomicBool,
}
impl BlobFile {
pub(crate) fn new(inner: LanceBlobFile) -> Self {
Self {
inner: Arc::new(inner),
closed: AtomicBool::new(false),
}
}
fn ensure_open(&self) -> napi::Result<()> {
if self.closed.load(Ordering::Acquire) {
Err(napi::Error::from_reason("blob file is already closed"))
} else {
Ok(())
}
}
}
@@ -32,21 +52,124 @@ impl BlobFile {
}
#[napi]
pub async fn read(&self) -> napi::Result<Buffer> {
let bytes = self.inner.read().await.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
pub fn read<'env>(
&self,
env: &'env Env,
max_bytes: Option<BigInt>,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Buffer>> {
self.ensure_open()?;
let max_bytes = max_bytes
.map(|max_bytes| {
usize::try_from(parse_u64(max_bytes, "maxBytes")?)
.map_err(|_| napi::Error::from_reason("maxBytes is too large"))
})
.transpose()?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let bytes = match max_bytes {
None => inner.read().await,
Some(max_bytes) => inner.read_up_to(max_bytes).await,
}
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
})
}
#[napi]
pub async fn read_range(&self, start: BigInt, end: BigInt) -> napi::Result<Buffer> {
pub fn read_range<'env>(
&self,
env: &'env Env,
start: BigInt,
end: BigInt,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Buffer>> {
self.ensure_open()?;
let range = bigint_range(start, end)?;
let bytes = self
.inner
.read_range(range)
.await
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let bytes = inner
.read_range(range)
.await
.map_err(|err| convert_error(&err))?;
Ok(Buffer::from(bytes.as_ref()))
})
}
#[napi]
pub fn read_ranges<'env>(
&self,
env: &'env Env,
ranges: Vec<BlobRange>,
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Buffer>>> {
self.ensure_open()?;
let ranges = ranges
.into_iter()
.map(|range| bigint_range(range.start, range.end))
.collect::<napi::Result<Vec<_>>>()?;
let inner = self.inner.clone();
spawn_abortable(env, signal, async move {
let buffers = inner
.read_ranges(&ranges)
.await
.map_err(|err| convert_error(&err))?;
Ok(buffers
.iter()
.map(|bytes| Buffer::from(bytes.as_ref()))
.collect())
})
}
#[napi]
pub async fn seek(&self, position: BigInt) -> napi::Result<()> {
self.ensure_open()?;
let position = parse_u64(position, "position")?;
self.inner
.seek(position)
.await
.map_err(|err| convert_error(&err))
}
#[napi]
pub async fn tell(&self) -> napi::Result<BigInt> {
self.ensure_open()?;
let position = self.inner.tell().await.map_err(|err| convert_error(&err))?;
Ok(BigInt::from(position))
}
#[napi]
pub async fn close(&self) -> napi::Result<()> {
if self.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
self.inner.close().await.map_err(|err| convert_error(&err))
}
#[napi]
pub fn is_closed(&self) -> bool {
self.closed.load(Ordering::Acquire)
}
}
/// Runs `fut` on the napi runtime. Aborting `signal` drops the future, which
/// cancels in-flight reads.
pub fn spawn_abortable<'env, T>(
env: &'env Env,
signal: Option<AbortSignal>,
fut: impl Future<Output = napi::Result<T>> + Send + 'static,
) -> napi::Result<PromiseRaw<'env, T>>
where
T: ToNapiValue + Send + 'static,
{
let (fut, handle) = abortable(fut);
if let Some(signal) = signal {
signal.on_abort(move || handle.abort());
}
env.spawn_future(async move {
fut.await
.unwrap_or_else(|_| Err(napi::Error::from_reason("blob read aborted")))
})
}
fn bigint_range(start: BigInt, end: BigInt) -> napi::Result<Range<u64>> {
nodejs/src/iterator.rs
+8
View File
@@ -7,6 +7,8 @@ use lancedb::ipc::batches_to_ipc_file;
use napi::bindgen_prelude::*;
use napi_derive::napi;
use crate::util::schema_to_buffer;
/** Typescript-style Async Iterator over RecordBatches */
#[napi]
pub struct RecordBatchIterator {
@@ -19,6 +21,12 @@ impl RecordBatchIterator {
Self { inner }
}
/// Return the executed stream's output schema as an Arrow IPC file.
#[napi]
pub fn schema(&self) -> napi::Result<Buffer> {
schema_to_buffer(&self.inner.schema())
}
#[napi(catch_unwind)]
pub async unsafe fn next(&mut self) -> napi::Result<Option<Buffer>> {
if let Some(rst) = self.inner.next().await {
nodejs/src/query.rs
+8 -2
View File
@@ -289,8 +289,13 @@ impl VectorQuery {
}
#[napi]
pub fn nprobes(&mut self, nprobe: u32) {
self.inner = self.inner.clone().nprobes(nprobe as usize);
pub fn nprobes(&mut self, nprobe: u32) -> napi::Result<()> {
self.inner = self
.inner
.clone()
.nprobes(nprobe as usize)
.default_error()?;
Ok(())
}
#[napi]
@@ -658,6 +663,7 @@ impl JsFullTextQuery {
FtsQuery::Phrase(_) => "phrase".to_string(),
FtsQuery::Boost(_) => "boost".to_string(),
FtsQuery::MultiMatch(_) => "multi_match".to_string(),
FtsQuery::CombinedFields(_) => "combined_fields".to_string(),
FtsQuery::Boolean(_) => "boolean".to_string(),
}
}
nodejs/src/table.rs
+34 -21
View File
@@ -11,11 +11,12 @@ use lancedb::table::{
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
};
use napi::Env;
use napi::bindgen_prelude::*;
use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode};
use napi_derive::napi;
use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids};
use crate::blob::{BlobFile, copy_blob_buffers, parse_row_ids, spawn_abortable};
use crate::error::NapiErrorExt;
use crate::index::Index;
use crate::merge::NativeMergeInsertBuilder;
@@ -74,13 +75,14 @@ impl Table {
#[napi(
catch_unwind,
ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void"
ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void, allowExternalBlobOutsideBases?: boolean"
)]
pub async fn add(
&self,
buf: Buffer,
mode: String,
progress_callback: Option<ProgressFn>,
allow_external_blob_outside_bases: Option<bool>,
) -> napi::Result<AddResult> {
let batches = ipc_file_to_batches(buf.to_vec())
.map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
@@ -105,6 +107,9 @@ impl Table {
return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
};
op = op
.allow_external_blob_outside_bases(allow_external_blob_outside_bases.unwrap_or(false));
if let Some(tsfn) = progress_callback {
op = op.progress(move |p| {
// NonBlocking: dispatch onto the JS event loop without
@@ -336,36 +341,44 @@ impl Table {
}
#[napi(catch_unwind)]
pub async fn fetch_blobs(
pub fn fetch_blobs<'env>(
&self,
env: &'env Env,
column: String,
row_ids: Vec<BigInt>,
) -> napi::Result<Vec<Option<Buffer>>> {
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Option<Buffer>>>> {
let row_ids = parse_row_ids(row_ids)?;
let array = self
.inner_ref()?
.fetch_blobs(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(copy_blob_buffers(array))
let table = self.inner_ref()?.clone();
spawn_abortable(env, signal, async move {
let array = table
.fetch_blobs(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(copy_blob_buffers(array))
})
}
#[napi(catch_unwind)]
pub async fn fetch_blob_files(
pub fn fetch_blob_files<'env>(
&self,
env: &'env Env,
column: String,
row_ids: Vec<BigInt>,
) -> napi::Result<Vec<Option<BlobFile>>> {
signal: Option<AbortSignal>,
) -> napi::Result<PromiseRaw<'env, Vec<Option<BlobFile>>>> {
let row_ids = parse_row_ids(row_ids)?;
let files = self
.inner_ref()?
.fetch_blob_files(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(files
.into_iter()
.map(|file| file.map(BlobFile::new))
.collect())
let table = self.inner_ref()?.clone();
spawn_abortable(env, signal, async move {
let files = table
.fetch_blob_files(column.as_str(), &row_ids)
.await
.default_error()?;
Ok(files
.into_iter()
.map(|file| file.map(BlobFile::new))
.collect())
})
}
#[napi(catch_unwind)]
nodejs/tsconfig.json
+4 -1
View File
@@ -10,7 +10,10 @@
"resolveJsonModule": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"moduleResolution": "Node"
"moduleResolution": "Node",
// flatbuffers 25 (via apache-arrow 21) ships declarations that need
// TypeScript 5.7+ generic typed arrays; skip checking third-party .d.ts.
"skipLibCheck": true
},
"exclude": ["./dist/*", "./examples/*"],
"typedocOptions": {
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/__init__.py
+1
View File
@@ -46,6 +46,7 @@ from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewDefinition,
vector_dedup as vector_dedup,
)
from .view import ViewDescription as ViewDescription
from .table import AsyncTable, Table
python/python/lancedb/_lancedb.pyi
+17 -5
View File
@@ -149,12 +149,20 @@ class Connection(object):
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
async def open_job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def create_function_async(
self, request_json: str, namespace_path: Optional[List[str]] = None
) -> Job: ...
async def get_function(
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> str: ...
async def list_functions(
self, namespace_path: Optional[List[str]] = None
) -> List[str]: ...
async def drop_function(
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> bool: ...
async def drop_function_async(
self, name: str, version: str
self, name: str, version: str, namespace_path: Optional[List[str]] = None
) -> Tuple[bool, Job]: ...
async def create_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
@@ -247,6 +255,8 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
*,
vector_source_json: Optional[str] = None,
) -> Table: ...
async def create_materialized_view_async(
self,
@@ -256,6 +266,8 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
*,
vector_source_json: Optional[str] = None,
) -> Job: ...
async def list_materialized_views(self) -> List[str]: ...
async def drop_materialized_view(
python/python/lancedb/db.py
+223 -47
View File
@@ -56,6 +56,8 @@ from .sql import QueryDescription
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewSource,
VectorDedupSource,
SelectArg,
normalize_select,
)
@@ -120,7 +122,48 @@ def _view_description(
class DBConnection(EnforceOverrides):
"""An active LanceDB connection interface."""
"""An active LanceDB connection interface.
Use [close][lancedb.db.DBConnection.close] to release the connection's
underlying resources, or use the connection as a context manager to close it
automatically when leaving the block, including when an exception is raised.
Examples
--------
>>> import lancedb
>>> with lancedb.connect("memory://") as db:
... assert db.is_open()
>>> db.is_open()
False
"""
def __enter__(self) -> DBConnection:
return self
def __exit__(self, *_) -> None:
self.close()
@abstractmethod
def is_open(self) -> bool:
"""Return True if the connection is open."""
pass
@abstractmethod
def close(self) -> None:
"""Close the connection, releasing any underlying resources.
It is safe to call this method multiple times. Database operations on a
closed connection raise ``RuntimeError: Connection is closed``.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> db.close()
>>> db.is_open()
False
"""
pass
def list_namespaces(
self,
@@ -366,9 +409,10 @@ class DBConnection(EnforceOverrides):
- [LanceModel][lancedb.pydantic.LanceModel]
mode: str; default "create"
The mode to use when creating the table.
Can be either "create" or "overwrite".
Can be "create", "overwrite", or "exist_ok".
By default, if the table already exists, an exception is raised.
If you want to overwrite the table, use mode="overwrite".
To open an existing table without adding data, use mode="exist_ok".
exist_ok: bool, default False
If a table by the same name already exists, then raise an exception
if exist_ok=False. If exist_ok=True, then open the existing table;
@@ -555,7 +599,7 @@ class DBConnection(EnforceOverrides):
def create_materialized_view(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -569,17 +613,21 @@ class DBConnection(EnforceOverrides):
table. The view is a normal table: it can be queried, indexed and
searched, and it appears in ``table_names``.
The source table must have stable row ids (create it with the
Ordinary source tables must have stable row ids (create them with the
``new_table_enable_stable_row_ids`` storage option): they keep the
view's provenance valid across source compactions, and cannot be
enabled after a table exists.
enabled after a table exists. Native dedup sources also support physical
row IDs interpreted against their pinned snapshot.
Parameters
----------
name: str
The name of the view.
source: str
The name of the source table, in this database.
source: str or VectorDedupSource
The name of the source table, or an indexed dedup source declared
with [vector_dedup][lancedb.vector_dedup]. Dedup sources capture a
fixed snapshot and preserve the source table. They accept no
additional select, where or limit options.
select: list or dict, optional
The view's columns: column names, ``(alias, SQL expression)``
pairs, or a dict of the same. Omitting it selects every source
@@ -602,7 +650,7 @@ class DBConnection(EnforceOverrides):
def create_materialized_view_async(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -778,6 +826,7 @@ class DBConnection(EnforceOverrides):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Build and register a scalar Python UDF, then return its version.
@@ -794,6 +843,9 @@ class DBConnection(EnforceOverrides):
Function needs, each naming a Secret and the environment variable
its value arrives in. The Function's source is unchanged by this;
it reads the variable the way it already did.
namespace_path : list of str, optional
The namespace holding the Function. None or an empty list is the
root namespace.
Examples
--------
@@ -809,13 +861,16 @@ class DBConnection(EnforceOverrides):
)
```
"""
return self.create_function_async(definition, secrets=secrets).wait()
return self.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
).wait()
def create_function_async(
self,
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
"""Submit a scalar Python UDF for building and registration.
@@ -827,17 +882,26 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def get_function(self, name: str, *, version: str) -> FunctionVersion:
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
"""List every published immutable Function version in a namespace.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Functions in child namespaces are not included. Results are ordered by
Function name then version. Local connections raise
``NotImplementedError``.
Examples
--------
@@ -851,7 +915,13 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
"""Drop a Function name and the object it was bound to.
The requested version must exist in the currently named object. Returns
@@ -864,7 +934,13 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
"""Drop a Function name and return its cleanup job.
The name is unbound before this returns; the object's content may still
@@ -1241,6 +1317,14 @@ class LanceDBConnection(DBConnection):
def __repr__(self) -> str:
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
@override
def is_open(self) -> bool:
return self._conn.is_open()
@override
def close(self) -> None:
self._conn.close()
@override
def serialize(self) -> str:
import json
@@ -1463,8 +1547,6 @@ class LanceDBConnection(DBConnection):
"""
if namespace_path is None:
namespace_path = []
if mode.lower() not in ["create", "overwrite"]:
raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name)
tbl = LanceTable.create(
@@ -1558,7 +1640,7 @@ class LanceDBConnection(DBConnection):
def create_materialized_view(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -1603,7 +1685,7 @@ class LanceDBConnection(DBConnection):
def create_materialized_view_async(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -1802,25 +1884,62 @@ class LanceDBConnection(DBConnection):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
)
)
return Job(job)
@override
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
return LOOP.run(
self._conn.get_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions(namespace_path=namespace_path))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
return LOOP.run(
self._conn.drop_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
dropped, job = LOOP.run(
self._conn.drop_function_async(
name, version=version, namespace_path=namespace_path
)
)
return dropped, Job(job)
@override
@@ -2230,11 +2349,12 @@ class AsyncConnection(object):
- pyarrow.Schema
- [LanceModel][lancedb.pydantic.LanceModel]
mode: Literal["create", "overwrite"]; default "create"
mode: Literal["create", "overwrite", "exist_ok"]; default "create"
The mode to use when creating the table.
Can be either "create" or "overwrite".
Can be "create", "overwrite", or "exist_ok".
By default, if the table already exists, an exception is raised.
If you want to overwrite the table, use mode="overwrite".
To open an existing table without adding data, use mode="exist_ok".
exist_ok: bool, default False
If a table by the same name already exists, then raise an exception
if exist_ok=False. If exist_ok=True, then open the existing table;
@@ -2498,10 +2618,31 @@ class AsyncConnection(object):
await tbl.checkout(version)
return tbl
async def _materialized_view_source(
self,
source: MaterializedViewSource,
select: SelectArg,
where: Optional[str],
limit: Optional[int],
) -> Tuple[str, Dict[str, str]]:
if not isinstance(source, VectorDedupSource):
return source, {}
if select is not None or where is not None or limit is not None:
raise ValueError(
"vector_dedup cannot be combined with select, where or limit"
)
version = source.dataset_version
if version is None:
table = await self.open_table(source.source)
version = await table.version()
return source.source, {
"vector_source_json": source._native_source_json(version)
}
async def create_materialized_view(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -2512,6 +2653,9 @@ class AsyncConnection(object):
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
"""
source, native_options = await self._materialized_view_source(
source, select, where, limit
)
inner = await self._inner.create_materialized_view(
name,
source,
@@ -2519,13 +2663,14 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
**native_options,
)
return AsyncMaterializedView(AsyncTable(inner))
async def create_materialized_view_async(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -2536,6 +2681,9 @@ class AsyncConnection(object):
Wait for the returned job before opening or querying the view.
"""
source, native_options = await self._materialized_view_source(
source, select, where, limit
)
inner = await self._inner.create_materialized_view_async(
name,
source,
@@ -2543,6 +2691,7 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
**native_options,
)
return AsyncJob(inner)
@@ -2752,6 +2901,7 @@ class AsyncConnection(object):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> AsyncJob[FunctionVersion]:
"""Submit a scalar Python UDF for building and registration.
@@ -2759,36 +2909,60 @@ class AsyncConnection(object):
artifact. Waiting on the job returns the immutable Function version.
``secrets`` is a sequence of
[EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and
the environment variable its value arrives in. Local connections raise
``NotImplementedError``.
the environment variable its value arrives in. ``namespace_path`` is
the namespace the Function is registered in; None or an empty list is
the root. Local connections raise ``NotImplementedError``.
"""
if not isinstance(definition, UdfDefinition):
raise TypeError("create_function_async requires a @udf definition")
request = definition.bind_secrets(secrets)
inner = await self._inner.create_function_async(request.to_canonical_json())
inner = await self._inner.create_function_async(
request.to_canonical_json(), namespace_path
)
return _typed_job(inner, FunctionVersion.from_json)
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
async def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
return FunctionVersion.from_json(
await self._inner.get_function(name, version, namespace_path)
)
async def list_functions(self) -> List[FunctionVersion]:
"""List every published immutable Function version.
async def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
"""List every published immutable Function version in a namespace.
Results are ordered by Function name then version. Local connections
raise ``NotImplementedError``.
Functions in child namespaces are not included. Results are ordered by
Function name then version. Local connections raise
``NotImplementedError``.
"""
return [
FunctionVersion.from_json(value)
for value in await self._inner.list_functions()
for value in await self._inner.list_functions(namespace_path)
]
async def drop_function(self, name: str, *, version: str) -> bool:
async def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
"""Drop a Function name and the object it was bound to."""
return await self._inner.drop_function(name, version)
return await self._inner.drop_function(name, version, namespace_path)
async def drop_function_async(
self, name: str, *, version: str
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, AsyncJob]:
"""Drop a Function name and return its cleanup job.
@@ -2796,7 +2970,9 @@ class AsyncConnection(object):
be being deleted. Await :meth:`AsyncJob.wait` to wait for that to
finish.
"""
dropped, job = await self._inner.drop_function_async(name, version)
dropped, job = await self._inner.drop_function_async(
name, version, namespace_path
)
return dropped, AsyncJob(job)
async def create_secret(
python/python/lancedb/embeddings/base.py
+5 -1
View File
@@ -182,7 +182,11 @@ class EmbeddingFunction(BaseModel, ABC):
def VectorField(self, **kwargs):
"""
Creates a pydantic Field that can automatically annotate
the target vector column for this embedding function
the target vector column for this embedding function.
The field can be annotated as ``list[float]`` for compatibility with
static type checkers. LanceDB will infer the fixed vector dimension from
this embedding function.
"""
return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
python/python/lancedb/functions.py
+10 -1
View File
@@ -356,9 +356,16 @@ _ObjectVersion = Annotated[
class FunctionVersion(_RemoteValue):
"""A pinned object revision, independent of its executable image digest."""
"""A pinned object revision, independent of its executable image digest.
``name`` and ``namespace_path`` are the Function's parts, as a
[SecretReference][lancedb.functions.SecretReference] carries a Secret's;
they are what [get_function][lancedb.db.DBConnection.get_function] takes.
An empty ``namespace_path`` is the root namespace.
"""
name: str
namespace_path: tuple[str, ...] = ()
object_id: str
location: str
version: _ObjectVersion
@@ -444,6 +451,7 @@ class FunctionVersion(_RemoteValue):
return FunctionApplication(
function=FunctionVersionRef(
name=self.name,
namespace_path=self.namespace_path,
object_id=self.object_id,
location=self.location,
version=self.version,
@@ -502,6 +510,7 @@ class FunctionRegistrationRequest(_RemoteValue):
class FunctionVersionRef(_OpenRemoteValue):
name: str
namespace_path: tuple[str, ...] = ()
object_id: str
location: str
version: _ObjectVersion
python/python/lancedb/materialized_view.py
+99 -2
View File
@@ -7,6 +7,7 @@ maintained by refresh. See ``DBConnection.create_materialized_view``."""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
@@ -20,6 +21,7 @@ if TYPE_CHECKING:
from .table import AsyncTable, LanceTable
DEFINITION_META_KEY = b"mv.definition"
_MAX_FLOAT32 = float.fromhex("0x1.fffffep+127")
SelectArg = Union[
str,
@@ -29,9 +31,104 @@ SelectArg = Union[
]
DEFINITION_FORMAT = 2
@dataclass(frozen=True)
class VectorDedupSource:
"""A declarative source of retained original rows from an indexed snapshot.
Construct with [vector_dedup][lancedb.vector_dedup] and pass it as the
``source`` of ``create_materialized_view`` or
``create_materialized_view_async``. An omitted ``dataset_version`` is
captured once when creation is submitted, not when this object is made.
Refreshes keep that captured version. The source table is never modified.
"""
source: str
column: str
distance_threshold: float
dataset_version: Optional[int] = None
def __post_init__(self):
for name in ("source", "column"):
value = getattr(self, name)
if not isinstance(value, str) or not value:
raise ValueError(f"{name} must be a non-empty string")
if self.dataset_version is not None and (
type(self.dataset_version) is not int
or not 0 < self.dataset_version < 2**64
):
raise ValueError("dataset_version must be a positive uint64 integer")
value = self.distance_threshold
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise TypeError("distance_threshold must be a number")
try:
value = float(value)
except OverflowError as exc:
raise ValueError("distance_threshold must fit a finite float32") from exc
if not math.isfinite(value) or abs(value) > _MAX_FLOAT32:
raise ValueError("distance_threshold must fit a finite float32")
object.__setattr__(self, "distance_threshold", value)
def _native_source_json(self, version: int) -> str:
return json.dumps(
{
"kind": "Dedup",
"dataset_version": version,
"column": self.column,
"distance_threshold": repr(self.distance_threshold),
},
allow_nan=False,
)
def vector_dedup(
source: str,
*,
column: str,
distance_threshold: float,
dataset_version: Optional[int] = None,
) -> VectorDedupSource:
"""Declare an indexed dedup source for a materialized view.
The default policy keeps the smallest unassigned snapshot row ID and removes
only its direct qualifying neighbors: an A-B-C chain without an A-C edge
retains A and C. Rows with no pair, including null vectors, survive.
Only same-segment/same-IVF-partition pairs are considered; quantized distances
use reconstructed index vectors. The index must cover the selected snapshot.
``source`` is an exact table name in the connection's root namespace, just
as for ``create_materialized_view``. ``dataset_version=None`` captures the
source's version when creation is submitted. Pass an explicit positive
version to reproduce a previous snapshot. The result contains original
source columns and rows. Do not combine this source with ``select``,
``where`` or ``limit``; query the resulting table for further transforms.
Creation follows the usual MV contract: by default it waits for the result.
Use ``with_no_data=True`` to declare only, or
``create_materialized_view_async`` to get a job handle. On Cloud/Enterprise,
execution uses server-side MV registry jobs; local connections run in process.
Examples
--------
Given a connected ``db`` and an indexed ``images.phash`` column::
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
"""
return VectorDedupSource(source, column, distance_threshold, dataset_version)
MaterializedViewSource = Union[str, VectorDedupSource]
DEFINITION_FORMAT = 4
"""The newest stored layout this version reads: ``{"format": N, "query": "<SQL>"}``,
format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for
format 2 being a query with ``GROUP BY``, format 3 native pairs and format 4
native dedup results. A ``kind`` key beside it is for
readers older than the format number."""
python/python/lancedb/namespace.py
+13 -4
View File
@@ -68,6 +68,7 @@ from lance_namespace import (
from lancedb.materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewSource,
SelectArg,
)
from lancedb.table import AsyncTable, LanceTable, Table
@@ -488,6 +489,14 @@ class LanceNamespaceDBConnection(DBConnection):
)
self._uri = self._inner.uri
@override
def is_open(self) -> bool:
return self._inner.is_open()
@override
def close(self) -> None:
self._inner.close()
@override
def serialize(self) -> str:
import json
@@ -632,7 +641,7 @@ class LanceNamespaceDBConnection(DBConnection):
def create_materialized_view(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
@@ -662,7 +671,7 @@ class LanceNamespaceDBConnection(DBConnection):
def create_materialized_view_async(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
@@ -1242,7 +1251,7 @@ class AsyncLanceNamespaceDBConnection:
async def create_materialized_view(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
@@ -1265,7 +1274,7 @@ class AsyncLanceNamespaceDBConnection:
async def create_materialized_view_async(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
python/python/lancedb/pydantic.py
+39
View File
@@ -88,6 +88,17 @@ def Vector(
... pa.field("url", pa.utf8(), False),
... pa.field("embeddings", pa.list_(pa.float32(), 768))
... ])
Notes
-----
``Vector`` creates a type dynamically, so calls such as ``Vector(768)`` are
not valid static type annotations. For an embedding field, use the standard
``list[float]`` annotation when running mypy; ``VectorField`` supplies the
fixed dimension to LanceDB::
class MyModel(LanceModel):
text: str = embeddings.SourceField()
vector: list[float] = embeddings.VectorField()
"""
# TODO: make a public parameterized type.
@@ -313,6 +324,10 @@ def _unwrap_optional_annotation(annotation: Any) -> Any | None:
def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
"""Convert a Pydantic FieldInfo to Arrow DataType"""
embedding_vector_type = _embedding_vector_to_arrow_type(field)
if embedding_vector_type is not None:
return embedding_vector_type
unwrapped = _unwrap_optional_annotation(field.annotation)
if unwrapped is not None:
return _pydantic_type_to_arrow_type(unwrapped, field)
@@ -326,8 +341,32 @@ def _pydantic_to_arrow_type(field: FieldInfo) -> pa.DataType:
return _pydantic_type_to_arrow_type(field.annotation, field)
def _embedding_vector_to_arrow_type(field: FieldInfo) -> pa.DataType | None:
"""Infer a fixed-size vector type from ``VectorField`` metadata."""
if not _is_embedding_vector_annotation(field):
return None
function = get_extras(field, "vector_column_for")
return pa.list_(pa.float32(), function.ndims())
def _is_embedding_vector_annotation(field: FieldInfo) -> bool:
if get_extras(field, "vector_column_for") is None:
return False
annotation = _unwrap_optional_annotation(field.annotation)
if annotation is None:
annotation = field.annotation
origin = getattr(annotation, "__origin__", None)
args = getattr(annotation, "__args__", ())
return origin is list and args == (float,)
def is_nullable(field: FieldInfo) -> bool:
"""Check if a Pydantic FieldInfo is nullable."""
if _is_embedding_vector_annotation(field):
return True
if _unwrap_optional_annotation(field.annotation) is not None:
return True
if isinstance(field.annotation, (_GenericAlias, GenericAlias)):
python/python/lancedb/query.py
+21 -11
View File
@@ -40,7 +40,7 @@ from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .schema import is_blob_like_field, schema_has_blob_field
from .util import flatten_columns, get_uri_scheme
from .util import _validate_query_vector, flatten_columns, get_uri_scheme
from . import _wal_hybrid # WAL-PK-FUSION: delete.
from ._blob import (
BLOB_MODE_TO_HANDLING,
@@ -961,6 +961,9 @@ class LanceQueryBuilder(ABC):
fast_search: bool
Skip flat search of unindexed data.
"""
if query_type != "fts":
_validate_query_vector(query)
if ordering_field_name is not None:
import warnings
@@ -1675,6 +1678,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
str_query: Optional[str] = None,
fast_search: bool = None,
):
_validate_query_vector(query)
super().__init__(table)
self._query = query
self._distance_type = None
@@ -1749,13 +1753,15 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
Parameters
----------
nprobes: int
The number of probes to use.
The number of probes to use. Must be greater than 0.
Returns
-------
LanceVectorQueryBuilder
The LanceQueryBuilder object.
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
return self
@@ -2352,7 +2358,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
norm=self._norm,
fts_query=self._fts_query._query,
reranker=self._reranker,
limit=self._limit,
limit=self._limit or DEFAULT_HYBRID_LIMIT,
with_row_ids=True,
offset=self._offset,
)
@@ -2551,13 +2557,15 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Parameters
----------
nprobes: int
The number of probes to use.
The number of probes to use. Must be greater than 0.
Returns
-------
LanceHybridQueryBuilder
The LanceHybridQueryBuilder object.
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
return self
@@ -2804,13 +2812,13 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
)
# Apply common configurations
if self._limit:
# The final offset/limit window is sliced out of the combined,
# reranked results, so each sub-query must fetch enough rows to
# cover the skipped prefix as well as the window itself.
sub_query_limit = self._limit + (self._offset or 0)
self._vector_query.limit(sub_query_limit)
self._fts_query.limit(sub_query_limit)
# The final offset/limit window is sliced out of the combined,
# reranked results, so each sub-query must fetch enough rows to
# cover the skipped prefix as well as the window itself.
limit = self._limit or DEFAULT_HYBRID_LIMIT
sub_query_limit = limit + (self._offset or 0)
self._vector_query.limit(sub_query_limit)
self._fts_query.limit(sub_query_limit)
# WAL-PK-FUSION: without the fallback, select `self._columns` as is.
self._pk_fusion = None
columns = self._columns
@@ -3721,6 +3729,8 @@ class AsyncVectorQueryBase:
"""
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF-based index.
If there is no index then this value is ignored.
python/python/lancedb/remote/db.py
+55 -14
View File
@@ -39,7 +39,7 @@ from ..functions import FunctionVersion, UdfDefinition
from ..job import AsyncJob, Job
from ..sql import Query as SqlQuery
from ..sql import QueryDescription
from ..materialized_view import MaterializedView, SelectArg
from ..materialized_view import MaterializedView, MaterializedViewSource, SelectArg
from ..secrets import EnvVarSecret, SecretInfo
from ..view import ViewDescription
@@ -713,7 +713,7 @@ class RemoteDBConnection(DBConnection):
def create_materialized_view(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -745,7 +745,7 @@ class RemoteDBConnection(DBConnection):
def create_materialized_view_async(
self,
name: str,
source: str,
source: MaterializedViewSource,
*,
select: SelectArg = None,
where: Optional[str] = None,
@@ -862,25 +862,62 @@ class RemoteDBConnection(DBConnection):
definition: UdfDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
)
)
return Job(job)
@override
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
def get_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
return LOOP.run(
self._conn.get_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def list_functions(self) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions())
def list_functions(
self, *, namespace_path: Optional[List[str]] = None
) -> List[FunctionVersion]:
return LOOP.run(self._conn.list_functions(namespace_path=namespace_path))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
def drop_function(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> bool:
return LOOP.run(
self._conn.drop_function(
name, version=version, namespace_path=namespace_path
)
)
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
def drop_function_async(
self,
name: str,
*,
version: str,
namespace_path: Optional[List[str]] = None,
) -> Tuple[bool, Job]:
dropped, job = LOOP.run(
self._conn.drop_function_async(
name, version=version, namespace_path=namespace_path
)
)
return dropped, Job(job)
@override
@@ -1017,6 +1054,10 @@ class RemoteDBConnection(DBConnection):
"""
return LOOP.run(self._conn.namespace_client())
async def close(self):
"""Close the connection to the database."""
@override
def is_open(self) -> bool:
return self._conn.is_open()
@override
def close(self) -> None:
self._conn.close()
python/python/lancedb/rerankers/cross_encoder.py
+12 -1
View File
@@ -31,6 +31,12 @@ class CrossEncoderReranker(Reranker):
trust_remote_code : bool, default True
If True, will trust the remote code to be safe. If False, will not trust
the remote code and will not run it
batch_size : int, default 32
The number of query-passage pairs scored per forward pass.
**kwargs
Additional keyword arguments passed to
``sentence_transformers.CrossEncoder``, for example ``max_length`` or
``model_kwargs={"torch_dtype": "bfloat16"}``.
"""
def __init__(
@@ -40,6 +46,8 @@ class CrossEncoderReranker(Reranker):
device: Union[str, None] = None,
return_score="relevance",
trust_remote_code: bool = True,
batch_size: int = 32,
**kwargs,
):
super().__init__(return_score)
torch = attempt_import_or_raise("torch")
@@ -47,6 +55,8 @@ class CrossEncoderReranker(Reranker):
self.column = column
self.device = device
self.trust_remote_code = trust_remote_code
self.batch_size = batch_size
self.kwargs = kwargs
if self.device is None:
self.device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -61,6 +71,7 @@ class CrossEncoderReranker(Reranker):
self.model_name,
device=self.device,
trust_remote_code=self.trust_remote_code,
**self.kwargs,
)
return cross_encoder
@@ -71,7 +82,7 @@ class CrossEncoderReranker(Reranker):
return result_set
passages = result_set[self.column].to_pylist()
cross_inp = [[query, passage] for passage in passages]
cross_scores = self.model.predict(cross_inp)
cross_scores = self.model.predict(cross_inp, batch_size=self.batch_size)
result_set = result_set.append_column(
"_relevance_score", pa.array(cross_scores, type=pa.float32())
)
python/python/lancedb/rerankers/rrf.py
+20 -4
View File
@@ -107,8 +107,24 @@ class RRFReranker(Reranker):
`search().with_row_id(True)`"
)
combined = pa.concat_tables(vector_results, **self._concat_tables_args)
empty_table = pa.Table.from_arrays([], names=[])
reranked = self.rerank_hybrid(query, combined, empty_table)
# RRF ranks each result list on its own, so a document's rank is its
# position within the list it came from, not within the concatenation.
rrf_score_map = defaultdict(float)
for result in vector_results:
for i, result_id in enumerate(result["_rowid"].to_pylist(), 1):
rrf_score_map[result_id] += 1 / (i + self.K)
return reranked
combined = pa.concat_tables(vector_results, **self._concat_tables_args)
combined = self._deduplicate(combined)
relevance_scores = [
rrf_score_map[row_id] for row_id in combined["_rowid"].to_pylist()
]
combined = combined.append_column(
"_relevance_score", pa.array(relevance_scores, type=pa.float32())
)
combined = combined.sort_by([("_relevance_score", "descending")])
if self.score == "relevance":
combined = self._keep_relevance_score(combined)
return combined
python/python/lancedb/table.py
+202 -35
View File
@@ -134,6 +134,34 @@ _MODEL_BACKED_TOKENIZER_ERRORS = (
)
def _optimize_cleanup_since_ms(
cleanup_older_than: Optional[timedelta], retrain: bool
) -> Optional[int]:
# Called directly by both the sync and async optimize so stacklevel=3
# names the user's call site rather than the background event loop.
cleanup_since_ms: Optional[int] = None
if cleanup_older_than is not None:
cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
if cleanup_since_ms <= 0:
warnings.warn(
"optimize(cleanup_older_than=0) removes every version except "
"the latest. Any concurrent reader or writer still using an "
"older version will fail. Use a longer cleanup_older_than "
"unless no other process is working on this table.",
UserWarning,
stacklevel=3,
)
if retrain:
warnings.warn(
"The 'retrain' parameter is deprecated and will be removed in a "
"future version.",
DeprecationWarning,
stacklevel=3,
)
return cleanup_since_ms
def _add_unique_note(exception: BaseException, note: str) -> None:
existing_notes = getattr(exception, "__notes__", ()) or ()
message = (
@@ -233,6 +261,101 @@ IndexConfigType = Union[
KNOWN_METRICS = {"l2", "cosine", "dot", "hamming"}
def _blob_value_to_storage(value: Any) -> Optional[dict]:
"""Keep a Python blob's inline data or external URI before Arrow infers it."""
if value is None:
return None
if isinstance(value, (bytes, bytearray, memoryview)):
return {"data": bytes(value)}
if isinstance(value, str):
if not value:
raise ValueError("Blob uri cannot be empty")
return {"uri": value}
if isinstance(value, dict):
unknown = value.keys() - {"data", "uri", "position", "size"}
if unknown:
raise ValueError(f"Unknown blob fields: {sorted(unknown)}")
return value
try:
from lance.blob import Blob
except ModuleNotFoundError as err:
if err.name not in ("lance", "lance.blob"):
raise
else:
if isinstance(value, Blob):
return {
"data": value.data,
"uri": value.uri,
"position": value.position,
"size": value.size,
}
raise TypeError(f"Unsupported blob value: {type(value).__name__}")
def _blob_input_to_arrow(data: Any, schema: Optional[pa.Schema]) -> Optional[pa.Table]:
"""Convert Python rows with blob fields before Arrow loses their value types."""
if schema is None:
return None
if isinstance(data, list) and data and isinstance(data[0], dict):
names = {
field.name
for field in schema
if is_blob_v2_field(field)
and any(isinstance(row, dict) and field.name in row for row in data)
}
if not names:
return None
values = {name: [row.get(name) for row in data] for name in names}
rows = [{**row, **{name: None for name in names}} for row in data]
table = pa.Table.from_pylist(rows)
elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
names = {
field.name
for field in schema
if is_blob_v2_field(field) and field.name in data.columns
}
if not names:
return None
values = {
name: [
None
if value is pd.NA or isinstance(value, float) and np.isnan(value)
else value
for value in data[name]
]
for name in names
}
table = pa.Table.from_pandas(
data.assign(**{name: None for name in names}), preserve_index=False
).replace_schema_metadata(None)
else:
return None
for field in schema:
if field.name not in names:
continue
storage_type = (
field.type.storage_type
if isinstance(field.type, pa.ExtensionType)
else field.type
)
storage = pa.array(
[_blob_value_to_storage(value) for value in values[field.name]],
type=storage_type,
)
column = (
pa.ExtensionArray.from_storage(field.type, storage)
if isinstance(field.type, pa.ExtensionType)
else storage
)
table = table.set_column(
table.schema.get_field_index(field.name), field, column
)
return table
def _into_pyarrow_reader(
data, schema: Optional[pa.Schema] = None
) -> pa.RecordBatchReader:
@@ -275,9 +398,14 @@ def _into_pyarrow_reader(
return pa.Table.from_batches(data).to_reader()
else:
data = _serialize_json_values(data, schema)
return pa.Table.from_pylist(data).to_reader()
table = _blob_input_to_arrow(data, schema)
return (
table if table is not None else pa.Table.from_pylist(data)
).to_reader()
elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
table = pa.Table.from_pandas(data, preserve_index=False)
table = _blob_input_to_arrow(data, schema)
if table is None:
table = pa.Table.from_pandas(data, preserve_index=False)
# Do not serialize Pandas metadata
meta = table.schema.metadata if table.schema.metadata is not None else {}
meta = {k: v for k, v in meta.items() if k != b"pandas"}
@@ -1935,8 +2063,10 @@ class Table(ABC):
``_rowid`` values stay valid after compaction when the table has stable
row ids.
Convenience for small payloads. For large values use
:meth:`fetch_blob_files`.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob
bytes. The client splits requests automatically and reads an individual
larger blob through the Range route. This method still materializes all
bytes in memory; for large values use :meth:`fetch_blob_files`.
"""
@abstractmethod
@@ -2217,6 +2347,13 @@ class Table(ABC):
All files belonging to versions older than this will be removed. Set
to 0 days to remove all versions except the latest. The latest version
is never removed.
.. warning::
Setting this to 0 deletes the data files of every older
version, so any other reader or writer still using an older
version of the table will fail. Only set it to 0 if no other
process is working on this dataset.
delete_unverified: bool, default False
Files leftover from a failed transaction may appear to be part of an
in-progress operation (e.g. appending new data) and these files will not
@@ -2480,6 +2617,10 @@ class Table(ABC):
to nullable. Currently, you cannot change a nullable column to
non-nullable.
Each alteration must specify at least one of "rename", "data_type",
or "nullable". The legacy key "name" is also accepted for renaming.
Unknown keys raise ValueError before any alterations are applied.
Returns
-------
AlterColumnsResult
@@ -4195,9 +4336,9 @@ class LanceTable(Table):
)
if storage_options is None:
storage_options = {}
storage_options["new_table_enable_v2_manifest_paths"] = (
storage_options["new_table_enable_v2_manifest_paths"] = str(
enable_v2_manifest_paths
)
).lower()
self._table = LOOP.run(
self._conn._conn.create_table(
@@ -4423,6 +4564,13 @@ class LanceTable(Table):
All files belonging to versions older than this will be removed. Set
to 0 days to remove all versions except the latest. The latest version
is never removed.
.. warning::
Setting this to 0 deletes the data files of every older
version, so any other reader or writer still using an older
version of the table will fail. Only set it to 0 if no other
process is working on this dataset.
delete_unverified: bool, default False
Files leftover from a failed transaction may appear to be part of an
in-progress operation (e.g. appending new data) and these files will not
@@ -4447,10 +4595,9 @@ class LanceTable(Table):
modification operations.
"""
LOOP.run(
self._table.optimize(
cleanup_older_than=cleanup_older_than,
delete_unverified=delete_unverified,
retrain=retrain,
self._table._do_optimize(
_optimize_cleanup_since_ms(cleanup_older_than, retrain),
delete_unverified,
)
)
@@ -4623,7 +4770,7 @@ class LanceTable(Table):
[LanceTable.uses_v2_manifest_paths][lancedb.table.LanceTable.uses_v2_manifest_paths]
to check if the table is already using the new path style.
"""
LOOP.run(self._table.migrate_v2_manifest_paths())
LOOP.run(self._table.migrate_manifest_paths_v2())
@deprecation.deprecated(
deprecated_in="0.33.1",
@@ -5848,6 +5995,9 @@ class AsyncTable:
"""
schema = await self.schema()
data = _serialize_json_values(data, schema)
blob_table = _blob_input_to_arrow(data, schema)
if blob_table is not None:
data = blob_table
if on_bad_vectors is None:
on_bad_vectors = "error"
if fill_value is None:
@@ -5961,7 +6111,17 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: None = None,
vector_column_name: Optional[str] = None,
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
) -> AsyncQuery: ...
@overload
async def search(
self,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
@@ -5971,7 +6131,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["hybrid"] = ...,
ordering_field_name: Optional[str] = None,
@@ -5981,7 +6141,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[Union[VEC, "PIL.Image.Image", Tuple]] = None,
query: Union[VEC, "PIL.Image.Image", Tuple],
vector_column_name: Optional[str] = None,
query_type: Literal["auto"] = ...,
ordering_field_name: Optional[str] = None,
@@ -5991,7 +6151,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[str] = None,
query: str,
vector_column_name: Optional[str] = None,
query_type: Literal["fts"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6001,9 +6161,7 @@ class AsyncTable:
@overload
async def search(
self,
query: Optional[
Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery]
] = None,
query: Union[VEC, str, "PIL.Image.Image", Tuple, FullTextQuery],
vector_column_name: Optional[str] = None,
query_type: Literal["vector"] = ...,
ordering_field_name: Optional[str] = None,
@@ -6019,7 +6177,7 @@ class AsyncTable:
query_type: QueryType = "auto",
ordering_field_name: Optional[str] = None,
fts_columns: Optional[Union[str, List[str]]] = None,
) -> Union[AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
) -> Union[AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, AsyncVectorQuery]:
"""Create a search query to find the nearest neighbors
of the given query vector. We currently support [vector search](https://lancedb.com/docs/search/vector-search/)
and [full-text search](https://lancedb.com/docs/search/full-text-search/).
@@ -6034,8 +6192,9 @@ class AsyncTable:
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
- If None then the select/where/limit clauses are applied to filter
the table
- If None then a plain [AsyncQuery][lancedb.query.AsyncQuery] is
returned, equivalent to calling [query][lancedb.table.AsyncTable.query].
The select/where/limit clauses are applied to filter the table.
vector_column_name: str, optional
The name of the vector column to search.
@@ -6063,10 +6222,13 @@ class AsyncTable:
Returns
-------
LanceQueryBuilder
AsyncQuery, AsyncHybridQuery, AsyncFTSQuery, or AsyncVectorQuery
A query builder object representing the query.
"""
if query is None:
return self.query()
def is_embedding(query):
return isinstance(query, (list, np.ndarray, pa.Array, pa.ChunkedArray))
@@ -6716,6 +6878,10 @@ class AsyncTable:
to nullable. Currently, you cannot change a nullable column to
non-nullable.
Each alteration must specify at least one of "rename", "data_type",
or "nullable". The legacy key "name" is also accepted for renaming.
Unknown keys raise ValueError before any alterations are applied.
Returns
-------
AlterColumnsResult
@@ -6958,6 +7124,13 @@ class AsyncTable:
All files belonging to versions older than this will be removed. Set
to 0 days to remove all versions except the latest. The latest version
is never removed.
.. warning::
Setting this to 0 deletes the data files of every older
version, so any other reader or writer still using an older
version of the table will fail. Only set it to 0 if no other
process is working on this dataset.
delete_unverified: bool, default False
Files leftover from a failed transaction may appear to be part of an
in-progress operation (e.g. appending new data) and these files will not
@@ -6981,20 +7154,14 @@ class AsyncTable:
you have added or modified 100,000 or more records or run more than 20 data
modification operations.
"""
cleanup_since_ms: Optional[int] = None
if cleanup_older_than is not None:
cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
if retrain:
import warnings
warnings.warn(
"The 'retrain' parameter is deprecated and will be removed in a "
"future version.",
DeprecationWarning,
stacklevel=2,
)
return await self._do_optimize(
_optimize_cleanup_since_ms(cleanup_older_than, retrain),
delete_unverified,
)
async def _do_optimize(
self, cleanup_since_ms: Optional[int], delete_unverified: bool
) -> OptimizeStats:
return await self._inner.optimize(
cleanup_since_ms=cleanup_since_ms,
delete_unverified=delete_unverified,
python/python/lancedb/util.py
+19
View File
@@ -5,6 +5,7 @@
import binascii
import functools
import importlib
import math
import os
import pathlib
import warnings
@@ -279,6 +280,14 @@ def infer_vector_column_dim(data_type: pa.DataType) -> Optional[int]:
return None
def _validate_query_vector(query: Any) -> None:
"""Reject empty vector inputs before vector-column inference or execution."""
if (isinstance(query, list) and not query) or (
isinstance(query, np.ndarray) and query.size == 0
):
raise ValueError("Query vector must not be empty")
def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
if query is None:
return None
@@ -300,6 +309,9 @@ def infer_vector_column_name(
query: Optional[Any], # inferred later in query builder
vector_column_name: Optional[str],
):
if query_type != "fts":
_validate_query_vector(query)
if vector_column_name is not None:
return vector_column_name
@@ -351,6 +363,13 @@ def _(value: int):
@value_to_sql.register(float)
def _(value: float):
# str() gives "nan" / "inf", which SQL would read as column names.
if math.isnan(value):
return "CAST('NaN' AS DOUBLE)"
if math.isinf(value):
return (
"CAST('Infinity' AS DOUBLE)" if value > 0 else "CAST('-Infinity' AS DOUBLE)"
)
return str(value)
python/python/tests/test_blob.py
+128 -1
View File
@@ -7,10 +7,11 @@ import sys
import textwrap
import lance
import pandas as pd
import pyarrow as pa
import pyarrow.compute as pc
import pytest
from lance.blob import BlobType as LanceBlobType
from lance.blob import Blob, BlobType as LanceBlobType
import lancedb
from lancedb._blob import (
@@ -1532,3 +1533,129 @@ def test_add_external_uri_string_round_trips_with_flag(tmp_path):
hits = table.search().to_arrow()
blobs = table.fetch_blobs("image", hits)
assert blobs[0].as_py() == payload
@pytest.mark.parametrize("as_pandas", [False, True])
def test_add_bytes_and_uri_in_one_batch_preserves_external_uri(tmp_path, as_pandas):
payload = b"external-payload"
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(payload)
rows = [
{"id": 1, "image": b"inline"},
{"id": 2, "image": blob_path.as_uri()},
]
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("mixed", schema=schema)
table.add(
pd.DataFrame(rows) if as_pandas else rows,
allow_external_blob_outside_bases=True,
)
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
b"inline",
payload,
]
@pytest.mark.parametrize("as_pandas", [False, True])
def test_add_mixed_python_blob_values_preserves_external_uris(tmp_path, as_pandas):
payload = b"external-payload"
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(payload)
uri = blob_path.as_uri()
rows = [
{"id": 1, "image": b"inline"},
{"id": 2, "image": uri},
{"id": 3, "image": {"data": b"from-dict"}},
{"id": 4, "image": Blob.from_bytes(b"from-blob")},
{"id": 5, "image": Blob.from_uri(uri)},
{"id": 6, "image": {"uri": uri}},
{"id": 7, "image": None},
]
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("mixed", schema=schema)
table.add(
pd.DataFrame(rows) if as_pandas else rows,
allow_external_blob_outside_bases=True,
)
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[i] for i in range(1, 8)]).to_pylist() == [
b"inline",
payload,
b"from-dict",
b"from-blob",
payload,
payload,
None,
]
@pytest.mark.parametrize("as_pandas", [False, True])
def test_merge_insert_mixed_python_blobs_validates_external_uri(tmp_path, as_pandas):
blob_path = tmp_path / "payload.bin"
blob_path.write_bytes(b"external-payload")
table = _blob_table("merge_mixed_uri", [{"id": 1, "image": b"before"}])
rows = [
{"id": 1, "image": b"updated"},
{"id": 2, "image": blob_path.as_uri()},
]
with pytest.raises(ValueError, match="outside registered external bases"):
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(pd.DataFrame(rows) if as_pandas else rows)
)
assert table.count_rows() == 1
@pytest.mark.parametrize("as_pandas", [False, True])
def test_merge_insert_python_blob_dicts_and_objects(as_pandas):
table = _blob_table("merge_python_blobs", [{"id": 1, "image": b"before"}])
rows = [
{"id": 1, "image": b"updated"},
{"id": 2, "image": {"data": b"from-dict"}},
{"id": 3, "image": Blob.from_bytes(b"from-blob")},
]
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(pd.DataFrame(rows) if as_pandas else rows)
)
assert result.num_updated_rows == 1
assert result.num_inserted_rows == 2
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[i] for i in range(1, 4)]).to_pylist() == [
b"updated",
b"from-dict",
b"from-blob",
]
def test_create_table_with_python_blob_dict_and_object():
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table(
"create_python_blobs",
data=[
{"id": 1, "image": {"data": b"from-dict"}},
{"id": 2, "image": Blob.from_bytes(b"from-blob")},
],
schema=schema,
)
by_id = _row_ids_by_id(table)
assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
b"from-dict",
b"from-blob",
]
python/python/tests/test_db.py
+99
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import contextlib
import inspect
import re
import sys
@@ -350,6 +351,40 @@ async def test_create_table_from_iterator_async(mem_db_async: lancedb.AsyncConne
assert await table.count_rows() == 10
def test_create_exist_ok_mode(tmp_db: lancedb.DBConnection):
db = tmp_db
data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}]
table = db.create_table("t1", data, mode="exist_ok")
assert table.count_rows() == 2
original = table.to_arrow()
reopened = db.create_table("t1", [{"vector": [9.0, 8.0]}], mode="exist_ok")
assert reopened.to_arrow().equals(original)
def test_create_empty_exist_ok_mode(tmp_db: lancedb.DBConnection):
schema = pa.schema([pa.field("id", pa.int64())])
table = tmp_db.create_table("empty", schema=schema, mode="exist_ok")
assert table.schema == schema
assert table.count_rows() == 0
table.add([{"id": 7}])
reopened = tmp_db.create_table("empty", schema=schema, mode="exist_ok")
assert reopened.to_arrow().to_pylist() == [{"id": 7}]
bad_schema = pa.schema([pa.field("other", pa.int64())])
with pytest.raises(ValueError):
tmp_db.create_table("empty", schema=bad_schema, mode="exist_ok")
def test_create_table_invalid_mode(tmp_db: lancedb.DBConnection):
db = tmp_db
data = [{"vector": [1.1, 1.2]}, {"vector": [0.2, 1.8]}]
with pytest.raises(ValueError, match="Invalid mode bogus"):
db.create_table("tb", data, mode="bogus")
assert "tb" not in db
def test_create_exist_ok(tmp_db: lancedb.DBConnection):
from conftest import pandas_string_type
@@ -405,6 +440,49 @@ async def test_connect(tmp_path):
assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=5s)"
@pytest.fixture(params=["memory", "local", "namespace", "from_inner"])
def sync_db(request, tmp_path):
if request.param == "memory":
return lancedb.connect("memory://")
if request.param == "namespace":
return lancedb.connect_namespace("dir", {"root": str(tmp_path)})
db = lancedb.connect(tmp_path)
if request.param == "from_inner":
return lancedb.db.LanceDBConnection.from_inner(db._inner, None)
return db
def test_sync_close(sync_db):
assert sync_db.is_open()
assert sync_db.close() is None
assert not sync_db.is_open()
assert sync_db.close() is None
with pytest.warns(DeprecationWarning, match="table_names"):
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.table_names()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.list_tables()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.open_table("test")
@pytest.mark.parametrize("raise_error", [False, True])
def test_sync_context_manager(sync_db, raise_error):
with contextlib.ExitStack() as stack:
if raise_error:
stack.enter_context(pytest.raises(ValueError, match="test error"))
with sync_db as db:
assert db is sync_db
assert db.is_open()
if raise_error:
raise ValueError("test error")
assert not sync_db.is_open()
with pytest.raises(RuntimeError, match="Connection is closed"):
sync_db.list_tables()
@pytest.mark.asyncio
async def test_close(mem_db_async: lancedb.AsyncConnection):
assert mem_db_async.is_open()
@@ -495,6 +573,27 @@ async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection):
# await db.create_table("test", schema=bad_schema, exist_ok=True)
@pytest.mark.parametrize("enable_v2_manifest_paths", [False, True])
@pytest.mark.parametrize("empty", [False, True])
def test_create_table_deprecated_v2_manifest_paths(
tmp_path, enable_v2_manifest_paths, empty
):
db = lancedb.connect(tmp_path)
data = None if empty else [{"id": 1}]
with pytest.warns(DeprecationWarning, match="enable_v2_manifest_paths"):
table = db.create_table(
"test",
data=data,
schema=pa.schema([("id", pa.int64())]),
enable_v2_manifest_paths=enable_v2_manifest_paths,
)
assert table.uses_v2_manifest_paths() == enable_v2_manifest_paths
assert table.to_arrow().to_pylist() == ([] if empty else data)
assert db.open_table("test").uses_v2_manifest_paths() == enable_v2_manifest_paths
@pytest.mark.asyncio
async def test_create_table_v2_manifest_paths_async(tmp_path):
db_with_v2_paths = await lancedb.connect_async(
python/python/tests/test_first_class_function_slice1.py
+23
View File
@@ -162,6 +162,29 @@ def test_function_version_binds_named_columns_as_one_immutable_application():
] == [("text", "column", "documents.body")]
def test_function_version_names_its_namespace_as_a_separate_field():
"""A Function is named by its parts, as a Secret is; a root one states no
namespace."""
value = job_result("remote_function_job.json")
root = FunctionVersion.from_json(json.dumps(value))
assert root.namespace_path == ()
assert "namespace_path" not in json.loads(root.to_canonical_json())
nested = FunctionVersion(**{**value, "namespace_path": ["analytics", "features"]})
assert nested.name == "embed"
assert nested.namespace_path == ("analytics", "features")
assert json.loads(nested.to_canonical_json())["namespace_path"] == [
"analytics",
"features",
]
application = nested(text=col("documents.body"))
assert application.function.name == "embed"
assert application.function.namespace_path == ("analytics", "features")
assert json.loads(application.to_canonical_json())["function"][
"namespace_path"
] == ["analytics", "features"]
def test_function_version_binding_validates_names_and_direct_columns():
version = FunctionVersion.from_json(
json.dumps(job_result("remote_function_job.json"))
python/python/tests/test_first_class_function_slice2.py
+114 -4
View File
@@ -1539,8 +1539,10 @@ def _mock_remote_function_catalog():
else (None, None)
)
if function_action[1] == "create":
*namespace_path, name = function_action[0].split("$")
state["version"] = {
"name": "normalize_score",
"name": name,
**({"namespace_path": namespace_path} if namespace_path else {}),
"version": FUNCTION_VERSION,
"object_id": "fixture",
"location": "memory:///fixture",
@@ -1565,10 +1567,18 @@ def _mock_remote_function_catalog():
"job_state": "DONE",
"result": state["version"],
}
elif self.path == "/v1/function/normalize_score/describe":
# A namespaced Function's id is its namespace path and name joined
# by `$`, so the name is the last segment of it.
elif (
function_action[1] == "describe"
and function_action[0].split("$")[-1] == "normalize_score"
):
assert body == {"version": FUNCTION_VERSION}
response = state["version"]
elif self.path == "/v1/function/normalize_score/drop":
elif (
function_action[1] == "drop"
and function_action[0].split("$")[-1] == "normalize_score"
):
assert body == {"version": FUNCTION_VERSION}
response = {"dropped": True}
elif secret_action[1] in ("create", "alter"):
@@ -1602,7 +1612,8 @@ def _mock_remote_function_catalog():
assert query["page_token"] == "next"
self._write_response(200, {"secrets": [{"name": "hf-prod"}]})
return
if url.path != "/v1/namespace/$/function/list":
parts = url.path.strip("/").split("/")
if parts[:2] != ["v1", "namespace"] or parts[3:] != ["function", "list"]:
self._write_response(404, {"error": "not found"})
return
assert query["include_definition"] == "true"
@@ -1840,3 +1851,102 @@ async def test_async_remote_drop_function_sends_exact_version():
{"version": FUNCTION_VERSION},
)
]
def test_remote_function_verbs_address_a_namespace_in_the_path():
"""A Function's namespace is part of the identifier each route addresses;
no body carries it, so a namespaced request differs only in its path."""
namespace = ["analytics", "features"]
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
created = db.create_function(normalize_score, namespace_path=namespace)
# The returned parts are what the other verbs take, so a created
# Function reopens from its own fields.
assert created.name == "normalize_score"
assert created.namespace_path == tuple(namespace)
reopened = db.get_function(
created.name,
version=created.version,
namespace_path=list(created.namespace_path),
)
assert reopened == created
assert reopened(value=lancedb.col("score")).function.namespace_path == tuple(
namespace
)
db.list_functions(namespace_path=namespace)
assert (
db.drop_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
is True
)
dropped, _ = db.drop_function_async(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
assert dropped is True
assert [path for path, _ in state["requests"]] == [
"/v1/function/analytics$features$normalize_score/create",
"/v1/jobs/describe",
"/v1/function/analytics$features$normalize_score/describe",
"/v1/namespace/analytics$features/function/list",
"/v1/namespace/analytics$features/function/list",
"/v1/function/analytics$features$normalize_score/drop",
"/v1/function/analytics$features$normalize_score/drop",
]
@pytest.mark.asyncio
async def test_async_remote_function_verbs_address_a_namespace_in_the_path():
namespace = ["analytics", "features"]
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
registration = await db.create_function_async(
normalize_score, namespace_path=namespace
)
await registration.wait()
await db.get_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
await db.list_functions(namespace_path=namespace)
assert (
await db.drop_function(
"normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
)
is True
)
assert [path for path, _ in state["requests"]] == [
"/v1/function/analytics$features$normalize_score/create",
"/v1/jobs/describe",
"/v1/function/analytics$features$normalize_score/describe",
"/v1/namespace/analytics$features/function/list",
"/v1/namespace/analytics$features/function/list",
"/v1/function/analytics$features$normalize_score/drop",
]
def test_a_function_namespace_path_must_be_a_list_of_segments():
"""A bare string is refused rather than read as a sequence of one-character
segments, and nothing reaches the service."""
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
with pytest.raises(TypeError):
db.list_functions(namespace_path="analytics")
assert state["requests"] == []
python/python/tests/test_hybrid_query.py
+27
View File
@@ -315,6 +315,33 @@ def test_hybrid_query_offset(sync_table: Table):
assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:]
def test_hybrid_query_default_limit(sync_table: Table):
# The vector and FTS legs match disjoint rows, so fusing their results
# yields more rows than the default limit.
new_rows = []
for i in range(20):
new_rows.append({"text": "close_vec", "vector": [0.1, 0.1]})
new_rows.append({"text": "dog", "vector": [50.0 + i, 50.0 + i]})
sync_table.add(new_rows)
def query():
return (
sync_table.search(query_type="hybrid")
.vector([0.1, 0.1])
.text("dog")
.with_row_id(True)
)
# Like the async hybrid query, the default limit is 10.
result = query().to_arrow()
assert len(result) == 10
# The offset window is taken after the default limit is applied.
full = query().limit(15).to_arrow()
offset_result = query().offset(5).to_arrow()
assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[5:]
def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
# minimum_nprobes(0) must raise the same validation error a plain vector
# query raises, not silently no-op because 0 is falsy.
python/python/tests/test_lsm_write_spec.py
+7 -7
View File
@@ -52,9 +52,9 @@ def test_set_lsm_write_spec_validates(tmp_path):
with pytest.raises(Exception, match="num_buckets"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025))
# Happy path then mutation rejected.
# Happy path, then a set over the installed spec is refused.
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
with pytest.raises(Exception, match="mutation"):
with pytest.raises(Exception, match="already set"):
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
@@ -176,18 +176,18 @@ def test_get_lsm_write_spec(tmp_path):
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema). Leaving the
# maintained set to be inferred picks up the index on the table, so the
# spec reads back naming it rather than as "infer".
# Identity round-trips (column recovered from the schema). An unnamed
# maintained set reads back as None: what is stored is the intent to
# maintain every index, not the set it resolves to now.
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
assert spec.maintained_indexes == [idx_name]
assert spec.maintained_indexes is None
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column). Opting out is distinct from
# the inferred default.
# the unnamed default.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
python/python/tests/test_materialized_views.py
+157 -1
View File
@@ -520,10 +520,166 @@ def test_stored_queries_and_legacy_layouts_are_read():
"SELECT * FROM people"
)
for fmt, function in [(3, "vector_duplicate_pairs"), (4, "vector_dedup")]:
native = f"SELECT * FROM {function}('images', 3, 'phash', 4)"
assert read({"format": fmt, "query": native}).query == native
# A newer writer's layout is reported, never guessed at.
for newer in (
{"format": 3, "query": query},
{"format": 5, "query": query},
{"kind": "select_v3", "source_table": "people"},
):
with pytest.raises(NotImplementedError, match="cannot refresh"):
read(newer)
def indexed_dedup_source(db, name="images"):
import pyarrow as pa
from lancedb.index import IvfFlat
vectors = pa.FixedSizeListArray.from_arrays(
pa.array(
[v for i in range(256) for v in (float(i if i < 3 else i * 10), 0.0)],
type=pa.float32(),
),
2,
)
table = db.create_table(name, pa.table({"id": range(256), "vector": vectors}))
table.create_index("vector", config=IvfFlat(num_partitions=1))
return table
@pytest.mark.parametrize("stable", [False, True])
def test_vector_dedup_source_captures_snapshot_and_preserves_source(tmp_path, stable):
db = lancedb.connect(
tmp_path,
storage_options={"new_table_enable_stable_row_ids": str(stable).lower()},
)
source = indexed_dedup_source(db, "images.with.dot")
version = source.version
expression = lancedb.vector_dedup(
source.name, column="vector", distance_threshold=1
)
view = db.create_materialized_view("clean", expression)
assert "vector_dedup(" in view.definition.query
assert f", {version}, 'vector', 1.0)" in view.definition.query
assert view.table.count_rows() == 255
assert [r["id"] for r in view.table.search().where("id < 3").to_list()] == [0, 2]
assert source.version == version
assert source.count_rows() == 256
source.delete("id = 0")
view.refresh_async().wait()
assert view.table.search().where("id = 0").to_list() == [
{"id": 0, "vector": [0.0, 0.0]}
]
assert expression.dataset_version is None # reusable immutable declaration
def test_vector_dedup_create_job_and_explicit_snapshot(tmp_path):
db = lancedb.connect(tmp_path)
source = indexed_dedup_source(db)
version = source.version
source.delete("id = 0")
job = db.create_materialized_view_async(
"clean",
lancedb.vector_dedup(
"images", column="vector", distance_threshold=1, dataset_version=version
),
)
assert job.wait() is None
view = db.open_materialized_view("clean")
assert view.table.count_rows() == 255
assert len(view.table.search().where("id = 0").to_list()) == 1
@pytest.mark.asyncio
async def test_async_vector_dedup_declaration_and_creation(tmp_path):
sync_db = lancedb.connect(tmp_path)
source = indexed_dedup_source(sync_db)
db = await lancedb.connect_async(tmp_path)
expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
view = await db.create_materialized_view("clean", expression, with_no_data=True)
assert await view.table.count_rows() == 0
# The snapshot is captured at declaration even if refresh comes later.
source.delete("id = 0")
job = await view.refresh_async()
await job.wait()
assert await view.table.count_rows() == 255
another = await db.create_materialized_view_async(
"later", expression, with_no_data=True
)
await another.wait()
later = await db.open_materialized_view("later")
assert (await later.definition()).query != (await view.definition()).query
assert await later.table.count_rows() == 0
@pytest.mark.parametrize(
"options", [{"select": ["id"]}, {"select": []}, {"where": "id > 0"}, {"limit": 1}]
)
def test_vector_dedup_rejects_extra_clauses_before_opening_source(tmp_path, options):
db = lancedb.connect(tmp_path)
expression = lancedb.vector_dedup("missing", column="vector", distance_threshold=1)
with pytest.raises(ValueError, match="cannot be combined"):
db.create_materialized_view("bad", expression, **options)
assert db.list_tables().tables == []
@pytest.mark.parametrize("version", [0, -1, True, 1.5, "2", 2**64])
def test_vector_dedup_rejects_invalid_snapshot(version):
with pytest.raises(ValueError, match="dataset_version"):
lancedb.vector_dedup(
"images", column="vector", distance_threshold=1, dataset_version=version
)
@pytest.mark.parametrize(
"threshold", [float("nan"), float("inf"), -float("inf"), 1e100, 10**1000, True, "4"]
)
def test_vector_dedup_rejects_invalid_threshold(threshold):
with pytest.raises((ValueError, TypeError), match="distance_threshold"):
lancedb.vector_dedup("images", column="vector", distance_threshold=threshold)
def test_vector_dedup_descriptor_is_frozen():
from dataclasses import FrozenInstanceError
expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
with pytest.raises(FrozenInstanceError):
expression.column = "other"
@pytest.mark.parametrize("with_no_data", [False, True])
def test_remote_vector_dedup_uses_mv_endpoint_without_sql_client(with_no_data):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
sql_host_override="grpc://127.0.0.1:1",
client_config={"retry_config": {"retries": 0}},
)
job = db.create_materialized_view_async(
"clean",
lancedb.vector_dedup(
"images.with'dot",
column="vector'field",
distance_threshold=4,
dataset_version=7,
),
with_no_data=with_no_data,
)
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/clean/create",
{
"query": (
"SELECT * FROM vector_dedup('`images.with''dot`', "
"7, 'vector''field', 4.0)"
),
"with_no_data": with_no_data,
},
)
]
python/python/tests/test_pydantic.py
+20
View File
@@ -9,6 +9,7 @@ from typing import List, Optional, Tuple
import pyarrow as pa
import pydantic
import pytest
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.pydantic import (
LanceModel,
MultiVector,
@@ -425,6 +426,25 @@ def test_bare_vector_raises_clear_error():
exec("class TestModel(LanceModel):\n vector: Vector", namespace)
def test_embedding_vector_list_annotation():
embedding = MockTextEmbeddingFunction.create()
class StaticTypingModel(LanceModel):
text: str = embedding.SourceField()
vector: list[float] = embedding.VectorField()
schema = pydantic_to_schema(StaticTypingModel)
assert schema == pa.schema(
[
pa.field("text", pa.utf8(), False),
pa.field("vector", pa.list_(pa.float32(), embedding.ndims()), True),
]
)
model = StaticTypingModel(text="hello", vector=[0.0] * embedding.ndims())
assert model.vector == [0.0] * embedding.ndims()
def test_fixed_size_list_field():
class TestModel(pydantic.BaseModel):
vec: Vector(16)
python/python/tests/test_query.py
+53
View File
@@ -24,6 +24,7 @@ from lancedb.pydantic import LanceModel, Vector
from lancedb.query import (
AsyncFTSQuery,
AsyncHybridQuery,
AsyncQuery,
AsyncQueryBase,
AsyncVectorQuery,
ColumnOrdering,
@@ -1153,6 +1154,19 @@ def test_query_builder_with_filter(table):
assert all(np.array(rs[0]["vector"]) == [3, 4])
@pytest.mark.parametrize("query_type", ["vector", "hybrid"])
@pytest.mark.parametrize("nprobes", [0, -1])
def test_nprobes_nonpositive_sync(table, query_type, nprobes):
if query_type == "hybrid":
query = table.search(query_type="hybrid").vector([0, 0]).text("a")
else:
query = table.search([0, 0])
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(nprobes)
def test_invalid_nprobes_sync(table):
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(0).to_list()
@@ -1184,6 +1198,18 @@ def test_multiple_nprobes_calls_works_sync(table):
).minimum_nprobes(20).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
async def test_nprobes_zero_async(table_async: AsyncTable, hybrid):
query = table_async.query().nearest_to([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
@pytest.mark.asyncio
async def test_invalid_nprobes_async(table_async: AsyncTable):
with pytest.raises(ValueError, match="minimum_nprobes must be greater than 0"):
@@ -1458,6 +1484,27 @@ async def test_query_to_polars_async(table_async: AsyncTable):
assert df.shape == (0, num_columns)
@pytest.mark.asyncio
@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}])
@pytest.mark.parametrize("with_vector", [False, True])
async def test_async_search_without_query(mem_db_async, search_kwargs, with_vector):
data = [{"id": i} for i in range(10)]
if with_vector:
for row in data:
row["vector"] = [float(row["id"]), 1.0]
table = await mem_db_async.create_table("test", data)
query = await table.search(**search_kwargs)
assert isinstance(query, AsyncQuery)
assert await query.limit(3).to_arrow() == await table.query().limit(3).to_arrow()
query = await table.search(**search_kwargs)
assert await query.where("id >= 8").select(["id"]).limit(3).to_list() == [
{"id": 8},
{"id": 9},
]
@pytest.mark.asyncio
async def test_none_query(table_async: AsyncTable):
with pytest.raises(ValueError):
@@ -2270,6 +2317,12 @@ def test_ensure_vector_query_nested_empty_list():
ensure_vector_query([[]])
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
def test_vector_query_builder_empty_vector(table, query):
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
LanceVectorQueryBuilder(table, query, "vector")
def test_fast_search(tmp_path):
db = lancedb.connect(tmp_path)
python/python/tests/test_remote_db.py
+253 -11
View File
@@ -16,8 +16,9 @@ import uuid
from packaging.version import Version
import lancedb
import numpy as np
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.query import ColumnOrdering
from lancedb.query import AsyncQuery, ColumnOrdering
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
import pytest
@@ -119,6 +120,77 @@ async def test_async_remote_db():
assert table_names == []
@pytest.mark.parametrize(
"alteration, match",
[
({"path": "id"}, "One of rename, nullable or data_type"),
(
{"path": "id", "nulable": False}, # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
{
"path": "id",
"rename": "new_id",
"nulable": False, # spellchecker:disable-line
},
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
],
)
def test_remote_alter_columns_rejects_invalid_before_request(alteration, match):
requests = []
def handler(request):
requests.append(request.path)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
requests.clear()
with pytest.raises(ValueError, match=match):
table.alter_columns(alteration)
assert requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize(
"alteration, match",
[
({"path": "id"}, "One of rename, nullable or data_type"),
(
{"path": "id", "nulable": False}, # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
],
)
async def test_async_remote_alter_columns_rejects_invalid_before_request(
alteration, match
):
requests = []
def handler(request):
requests.append(request.path)
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
async with mock_lancedb_connection_async(handler) as db:
table = await db.open_table("test")
requests.clear()
with pytest.raises(ValueError, match=match):
await table.alter_columns(alteration)
assert requests == []
@pytest.mark.asyncio
async def test_async_checkout():
def handler(request):
@@ -1521,6 +1593,60 @@ def test_query_sync_empty_query():
assert data == expected
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
@pytest.mark.parametrize("vector_column_name", [None, "vector"])
@pytest.mark.parametrize("query_type", ["auto", "vector"])
def test_query_sync_empty_vector(query, vector_column_name, query_type):
def handler(body):
pytest.fail("An empty query vector must be rejected before sending a query")
with query_test_table(handler) as table:
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
table.search(
query, vector_column_name=vector_column_name, query_type=query_type
).limit(3).to_arrow()
@pytest.mark.asyncio
@pytest.mark.parametrize("search_kwargs", [{}, {"query": None}])
async def test_async_search_without_query(search_kwargs):
expected = pa.table({"id": [7, 8, 9]})
def handler(request):
if request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
elif request.path == "/v1/table/test/query/":
body = json.loads(
request.rfile.read(int(request.headers["Content-Length"]))
)
assert body == {
"k": 3,
"filter": "id >= 7",
"vector": [],
"columns": ["id"],
"prefilter": True,
"version": None,
}
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
with pa.ipc.new_file(request.wfile, schema=expected.schema) as writer:
writer.write_table(expected)
else:
request.send_response(404)
request.end_headers()
async with mock_lancedb_connection_async(handler) as db:
table = await db.open_table("test")
query = await table.search(**search_kwargs)
assert isinstance(query, AsyncQuery)
result = await query.where("id >= 7").select(["id"]).limit(3).to_arrow()
assert result == expected
def test_query_sync_maximal():
def handler(body):
assert body == {
@@ -1581,6 +1707,53 @@ def test_query_sync_maximal():
)
@pytest.mark.parametrize("hybrid", [False, True])
def test_query_sync_nprobes_zero(hybrid):
query_requests = []
def handler(body):
query_requests.append(body)
return pa.table({"id": []})
with query_test_table(handler) as table:
if hybrid:
query = table.search(query_type="hybrid").vector([1, 2, 3]).text("dog")
else:
query = table.search([1, 2, 3])
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
assert query_requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
async def test_query_async_nprobes_zero(hybrid):
requests = []
def handler(request):
requests.append(request.path)
if request.path == "/v1/table/test/describe/":
send_json(request, {"version": 1, "schema": {"fields": []}})
else:
request.send_response(404)
request.end_headers()
async with mock_lancedb_connection_async(handler) as db:
table = await db.open_table("test")
query = table.query().nearest_to([1, 2, 3])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(
ValueError, match="^Invalid input, nprobes must be greater than 0$"
):
query.nprobes(0)
assert requests == ["/v1/table/test/describe/"]
def test_query_sync_nprobes():
def handler(body):
assert body == {
@@ -2145,15 +2318,46 @@ async def test_header_provider_overrides_static_headers():
def test_close():
"""Test that close() works without AttributeError."""
import asyncio
def handler(req):
req.send_response(200)
req.end_headers()
with mock_lancedb_connection(handler) as db:
asyncio.run(db.close())
assert db.close() is None
assert not db.is_open()
assert db.close() is None
with pytest.warns(DeprecationWarning, match="table_names"):
with pytest.raises(RuntimeError, match="Connection is closed"):
db.table_names()
with pytest.raises(RuntimeError, match="Connection is closed"):
db.list_tables()
with pytest.raises(RuntimeError, match="Connection is closed"):
db.open_table("test")
@pytest.mark.parametrize("raise_error", [False, True])
def test_sync_context_manager(raise_error):
def handler(req):
req.send_response(200)
req.send_header("Content-Type", "application/json")
req.end_headers()
req.wfile.write(b'{"tables": []}')
with mock_lancedb_connection(handler) as db:
with contextlib.ExitStack() as stack:
if raise_error:
stack.enter_context(pytest.raises(ValueError, match="test error"))
with db as conn:
assert conn is db
assert conn.is_open()
assert conn.list_tables().tables == []
if raise_error:
raise ValueError("test error")
assert not db.is_open()
with pytest.raises(RuntimeError, match="Connection is closed"):
db.list_tables()
@pytest.mark.parametrize("exception", [KeyboardInterrupt, SystemExit, GeneratorExit])
@@ -2391,6 +2595,27 @@ def blob_query_response_table():
)
def blob_descriptor_take_table():
"""What `fetch_blob_files` reads for rows 10, 20, 30, and 40."""
queried = blob_query_response_table()
image_field = queried.schema.field("image")
images = pa.StructArray.from_arrays(
[
pa.array([0, 0, 0, 0], type=pa.uint8()),
pa.array([0, 0, 0, 0], type=pa.uint64()),
pa.array([5, 0, 5, 0], type=pa.uint64()),
pa.array([0, 0, 0, 0], type=pa.uint32()),
pa.array(["", "", "", ""], type=pa.string()),
],
fields=image_field.type,
mask=pa.array([False, True, False, False]),
)
return pa.Table.from_arrays(
[images, pa.array([10, 20, 30, 40], type=pa.uint64())],
schema=pa.schema([image_field, pa.field("_rowid", pa.uint64())]),
)
@contextlib.contextmanager
def blob_remote_table(*, server_version=Version("0.5.0")):
def handler(request):
@@ -2403,11 +2628,16 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
elif request.path.startswith("/v1/table/test/blob/image/"):
path = request.path.partition("?")[0]
row_id = int(path.split("/")[-2])
payload = {10: b"alpha", 20: None, 30: b"gamma"}[row_id]
payload = {10: b"alpha", 20: None, 30: b"gamma", 40: b""}[row_id]
if payload is None:
request.send_response(204)
request.end_headers()
return
if not payload:
request.send_response(416)
request.send_header("Content-Range", "bytes */0")
request.end_headers()
return
byte_range = request.headers["Range"].removeprefix("bytes=")
start_text, end_text = byte_range.split("-", maxsplit=1)
start = int(start_text)
@@ -2416,14 +2646,22 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
request.send_response(206)
request.send_header("Content-Range", f"bytes {start}-{end}/{len(payload)}")
request.send_header("Content-Length", str(len(chunk)))
request.send_header(
"x-lancedb-version", str(BLOB_DESCRIBE_RESPONSE["version"])
)
request.end_headers()
request.wfile.write(chunk)
elif request.path == "/v1/table/test/query/":
content_len = int(request.headers.get("Content-Length", 0))
body = json.loads(request.rfile.read(content_len))
assert body["columns"] == ["id", "image"]
assert body["with_row_id"] is True
response_table = blob_query_response_table()
if body["columns"] == ["image"]:
# fetch_blob_files sizes its handles from a descriptor take.
assert body["filter"].startswith("_rowid IN")
response_table = blob_descriptor_take_table()
else:
assert body["columns"] == ["id", "image"]
response_table = blob_query_response_table()
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
@@ -2459,17 +2697,21 @@ def test_remote_blob_columns_and_fetch():
def test_remote_blob_files_are_lazy_seekable_handles():
with blob_remote_table() as table:
files = table.fetch_blob_files("image", [10, 20, 30])
files = table.fetch_blob_files("image", [10, 20, 30, 40])
assert len(files) == 3
alpha, null_row, gamma = files
assert len(files) == 4
alpha, null_row, gamma, empty = files
assert null_row is None
assert alpha is not None
assert gamma is not None
assert empty is not None
assert alpha.size() == 5
assert alpha.read_range(1, 3) == b"lph"
gamma.seek(2)
assert gamma.read() == b"mma"
assert empty.size() == 0
assert empty.read() == b""
assert empty.read_range(0, 0) == b""
alpha.close()
assert alpha.closed
with pytest.raises(RuntimeError, match="already closed"):
python/python/tests/test_rerankers.py
+40
View File
@@ -395,6 +395,38 @@ def test_mrr_multivector_rewards_consensus():
assert result["_rowid"].to_pylist()[0] == 2
def test_rrf_multivector_ranks_each_result_list():
# RRF scores a document by its rank *within each* result list. The top hit
# of the second list must score the same as the top hit of the first list,
# not as if it were ranked after every row of the first list.
reranker = RRFReranker(K=60)
def ranking(row_ids):
return pa.table({"_rowid": pa.array(row_ids, type=pa.int64())})
rs1 = ranking([1, 2, 3])
rs2 = ranking([4, 5, 6])
result = reranker.rerank_multivector([rs1, rs2])
scores = dict(
zip(result["_rowid"].to_pylist(), result["_relevance_score"].to_pylist())
)
assert scores[1] == pytest.approx(1 / 61)
assert scores[4] == pytest.approx(1 / 61)
assert scores[5] == pytest.approx(1 / 62)
# the second-ranked hit of the second list beats the third hit of the first
assert scores[5] > scores[3]
# A document found by both lists sums its reciprocal ranks from each list.
result = reranker.rerank_multivector([ranking([1, 2]), ranking([2, 3])])
scores = dict(
zip(result["_rowid"].to_pylist(), result["_relevance_score"].to_pylist())
)
assert scores[2] == pytest.approx(1 / 62 + 1 / 61)
assert result["_rowid"].to_pylist() == [2, 1, 3]
def test_rrf_reranker_distance():
data = pa.table(
{
@@ -481,6 +513,14 @@ def test_cross_encoder_reranker(tmp_path):
_run_test_reranker(reranker, table, "single player experience", None, schema)
def test_cross_encoder_reranker_kwargs(tmp_path):
pytest.importorskip("sentence_transformers")
reranker = CrossEncoderReranker(batch_size=4, max_length=64)
assert reranker.model.max_length == 64
table, schema = get_test_table(tmp_path)
_run_test_reranker(reranker, table, "single player experience", None, schema)
def test_colbert_reranker(tmp_path):
pytest.importorskip("rerankers")
reranker = ColbertReranker()
python/python/tests/test_table.py
+137 -8
View File
@@ -113,6 +113,32 @@ def test_basic(mem_db: DBConnection):
assert table.to_arrow() == expected_data
@pytest.mark.parametrize("enable_v2", [False, True])
def test_migrate_v2_manifest_paths(tmp_path, enable_v2):
db = lancedb.connect(
tmp_path,
storage_options={"new_table_enable_v2_manifest_paths": str(enable_v2).lower()},
)
table = db.create_table("calls", [{"id": 1, "vector": [1.0, 1.0]}])
table.add([{"id": 2, "vector": [2.0, 2.0]}])
expected_data = table.to_arrow()
expected_versions = table.list_versions()
assert table.uses_v2_manifest_paths() == enable_v2
# Migration is also safe to repeat on a table already using v2 paths.
for _ in range(2):
table.migrate_v2_manifest_paths()
assert table.uses_v2_manifest_paths()
reopened = db.open_table("calls")
assert reopened.uses_v2_manifest_paths()
assert reopened.to_arrow() == expected_data
assert reopened.list_versions() == expected_versions
manifests = list((tmp_path / "calls.lance" / "_versions").glob("*.manifest"))
assert len(manifests) == len(expected_versions)
assert all(len(path.stem) == 20 and path.stem.isdigit() for path in manifests)
def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection):
data = pa.table(
{
@@ -3751,6 +3777,24 @@ def test_empty_query(mem_db: DBConnection):
assert df.num_rows == 42
@pytest.mark.parametrize("query", [[], np.array([], dtype=np.float32)])
@pytest.mark.parametrize("vector_column_name", [None, "vector"])
@pytest.mark.parametrize("query_type", ["auto", "vector"])
@pytest.mark.parametrize("multiple_vector_columns", [False, True])
def test_search_empty_vector(
mem_db, query, vector_column_name, query_type, multiple_vector_columns
):
fields = [pa.field("vector", pa.list_(pa.float32(), 8))]
if multiple_vector_columns:
fields.append(pa.field("vec2", pa.list_(pa.float32(), 4)))
table = mem_db.create_table("empty_vector_query", schema=pa.schema(fields))
with pytest.raises(ValueError, match="^Query vector must not be empty$"):
table.search(
query, vector_column_name=vector_column_name, query_type=query_type
).limit(3).to_arrow()
def test_search_with_schema_inf_single_vector(mem_db: DBConnection):
class MyTable(LanceModel):
text: str
@@ -4144,14 +4188,74 @@ async def test_add_columns_with_schema(mem_db_async: AsyncConnection):
)
def test_alter_columns(mem_db: DBConnection):
@pytest.mark.parametrize("rename_key", ["rename", "name"])
def test_alter_columns(mem_db: DBConnection, rename_key):
data = pa.table({"id": [0, 1]})
table = mem_db.create_table("my_table", data=data)
alter_columns_res = table.alter_columns({"path": "id", "rename": "new_id"})
alter_columns_res = table.alter_columns({"path": "id", rename_key: "new_id"})
assert alter_columns_res.version == 2
assert table.to_arrow().column_names == ["new_id"]
INVALID_COLUMN_ALTERATIONS = [
(({"path": "id"},), "One of rename, nullable or data_type"),
(
({"path": "id", "nulable": False},), # spellchecker:disable-line
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
(
{
"path": "id",
"rename": "new_id",
"nulable": False, # spellchecker:disable-line
},
),
"Unknown column alteration key 'nulable'", # spellchecker:disable-line
),
(
({"path": "id", "rename": "new_id"}, {"path": "id"}),
"One of rename, nullable or data_type",
),
]
def test_alter_columns_nullable_false(mem_db: DBConnection):
table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]}))
result = table.alter_columns({"path": "id", "nullable": False})
assert result.version == 2
assert not table.schema.field("id").nullable
@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS)
def test_alter_columns_rejects_invalid(mem_db: DBConnection, alterations, match):
table = mem_db.create_table("my_table", data=pa.table({"id": [0, 1]}))
initial_version = table.version
initial_schema = table.schema
with pytest.raises(ValueError, match=match):
table.alter_columns(*alterations)
assert table.version == initial_version
assert table.schema == initial_schema
@pytest.mark.asyncio
@pytest.mark.parametrize("alterations, match", INVALID_COLUMN_ALTERATIONS)
async def test_alter_columns_rejects_invalid_async(
mem_db_async: AsyncConnection, alterations, match
):
table = await mem_db_async.create_table("my_table", data=pa.table({"id": [0, 1]}))
initial_version = await table.version()
initial_schema = await table.schema()
with pytest.raises(ValueError, match=match):
await table.alter_columns(*alterations)
assert await table.version() == initial_version
assert await table.schema() == initial_schema
def test_update_field_metadata(mem_db: DBConnection):
data = pa.table({"id": [0, 1], "category": ["a", "b"]})
table = mem_db.create_table("my_table", data=data)
@@ -4177,10 +4281,11 @@ def test_update_field_metadata(mem_db: DBConnection):
@pytest.mark.asyncio
async def test_alter_columns_async(mem_db_async: AsyncConnection):
@pytest.mark.parametrize("rename_key", ["rename", "name"])
async def test_alter_columns_async(mem_db_async: AsyncConnection, rename_key):
data = pa.table({"id": [0, 1]})
table = await mem_db_async.create_table("my_table", data=data)
alter_columns_res = await table.alter_columns({"path": "id", "rename": "new_id"})
alter_columns_res = await table.alter_columns({"path": "id", rename_key: "new_id"})
assert alter_columns_res.version == 2
assert (await table.to_arrow()).column_names == ["new_id"]
alter_columns_res = await table.alter_columns(
@@ -4307,7 +4412,8 @@ async def test_optimize(mem_db_async: AsyncConnection):
assert stats.prune.bytes_removed == 0
assert stats.prune.old_versions_removed == 0
stats = await table.optimize(cleanup_older_than=timedelta(seconds=0))
with pytest.warns(UserWarning, match="concurrent"):
stats = await table.optimize(cleanup_older_than=timedelta(seconds=0))
assert stats.prune.bytes_removed > 0
assert stats.prune.old_versions_removed == 3
@@ -4335,12 +4441,35 @@ async def test_optimize_delete_unverified(tmp_db_async: AsyncConnection, tmp_pat
stats = await table.optimize(delete_unverified=False)
assert stats.prune.old_versions_removed == 0
stats = await table.optimize(
cleanup_older_than=timedelta(seconds=0), delete_unverified=True
)
with pytest.warns(UserWarning, match="concurrent"):
stats = await table.optimize(
cleanup_older_than=timedelta(seconds=0), delete_unverified=True
)
assert stats.prune.old_versions_removed == 2
@pytest.mark.asyncio
async def test_optimize_warns_on_zero_cleanup(mem_db_async: AsyncConnection):
table = await mem_db_async.create_table("test", data=[{"x": [1]}])
with pytest.warns(UserWarning, match="concurrent"):
await table.optimize(cleanup_older_than=timedelta(0))
with warnings.catch_warnings():
warnings.simplefilter("error")
await table.optimize(cleanup_older_than=timedelta(days=1))
await table.optimize()
def test_optimize_warns_on_zero_cleanup_sync(mem_db: DBConnection):
table = mem_db.create_table("test", data=[{"x": [1]}])
with warnings.catch_warnings(record=True) as seen:
warnings.simplefilter("default")
table.optimize(cleanup_older_than=timedelta(0))
table.optimize(cleanup_older_than=timedelta(0))
assert [w.filename for w in seen] == [__file__, __file__]
assert all("concurrent" in str(w.message) for w in seen)
def test_replace_field_metadata(tmp_path):
db = lancedb.connect(tmp_path)
table = db.create_table("my_table", data=[{"x": 0}])
python/python/tests/test_util.py
+29
View File
@@ -213,6 +213,35 @@ def test_value_to_sql_numpy_scalars():
assert value_to_sql(np.bool_(False)) == "FALSE"
def test_value_to_sql_nan_and_infinity(tmp_path):
# str(float("nan")) is "nan", which SQL reads as a column named nan, so
# table.update(values={"x": float("nan")}) failed with "No field named nan".
import math
import numpy as np
db = lancedb.connect(tmp_path)
table = db.create_table(
"test",
pa.table(
{
"id": [1, 2, 3, 4],
"f64": [1.0, 2.0, 3.0, 4.0],
"f32": pa.array([1.0, 2.0, 3.0, 4.0], pa.float32()),
}
),
)
table.update(where="id = 1", values={"f64": float("nan"), "f32": np.nan})
table.update(where="id = 2", values={"f64": math.inf, "f32": np.float32("inf")})
table.update(where="id = 3", values={"f64": -math.inf, "f32": -math.inf})
rows = {row["id"]: row for row in table.to_arrow().to_pylist()}
assert math.isnan(rows[1]["f64"]) and math.isnan(rows[1]["f32"])
assert rows[2]["f64"] == math.inf and rows[2]["f32"] == math.inf
assert rows[3]["f64"] == -math.inf and rows[3]["f32"] == -math.inf
assert rows[4]["f64"] == 4.0 and rows[4]["f32"] == 4.0
def test_append_vector_columns():
registry = EmbeddingFunctionRegistry.get_instance()
registry.register("test")(MockTextEmbeddingFunction)
python/src/connection.rs
+44 -8
View File
@@ -407,7 +407,8 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
@@ -416,10 +417,17 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
vector_source_json: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(source) = vector_source_json {
builder = builder.vector_source(
serde_json::from_str(&source)
.map_err(|e| PyValueError::new_err(e.to_string()))?,
);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -435,7 +443,8 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
pub fn create_materialized_view_async(
self_: PyRef<'_, Self>,
name: String,
@@ -444,10 +453,17 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
vector_source_json: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(source) = vector_source_json {
builder = builder.vector_source(
serde_json::from_str(&source)
.map_err(|e| PyValueError::new_err(e.to_string()))?,
);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -738,31 +754,37 @@ impl Connection {
})
}
#[pyo3(signature = (request_json, namespace_path=None))]
pub fn create_function_async(
self_: PyRef<'_, Self>,
request_json: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
.infer_error()?;
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.create_function_async(request)
.create_function_async(request, &namespace_path)
.await
.infer_error()
.map(crate::job::Job::new_typed)
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn get_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.get_function(name, version)
.get_function(name, version, &namespace_path)
.await
.infer_error()?
.to_canonical_json()
@@ -770,11 +792,16 @@ impl Connection {
})
}
pub fn list_functions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
#[pyo3(signature = (namespace_path=None))]
pub fn list_functions(
self_: PyRef<'_, Self>,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.list_functions()
.list_functions(&namespace_path)
.await
.infer_error()?
.into_iter()
@@ -783,26 +810,35 @@ impl Connection {
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
inner
.drop_function(name, version, &namespace_path)
.await
.infer_error()
})
}
#[pyo3(signature = (name, version, namespace_path=None))]
pub fn drop_function_async(
self_: PyRef<'_, Self>,
name: String,
version: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
.drop_function_async(name, version)
.drop_function_async(name, version, &namespace_path)
.await
.infer_error()
.map(|(dropped, job)| (dropped, crate::job::Job::new(job)))
python/src/query.rs
+8 -4
View File
@@ -180,6 +180,9 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
.expect("Failed to import namespace");
match self.0 {
FtsQuery::CombinedFields(_) => Err(PyValueError::new_err(
"Combined-fields FTS queries are not yet supported in Python",
)),
FtsQuery::Match(query) => {
let kwargs = PyDict::new(py);
kwargs.set_item("boost", query.boost)?;
@@ -992,8 +995,9 @@ impl VectorQuery {
self.inner = self.inner.clone().refine_factor(refine_factor);
}
pub fn nprobes(&mut self, nprobe: u32) {
self.inner = self.inner.clone().nprobes(nprobe as usize);
pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> {
self.inner = self.inner.clone().nprobes(nprobe as usize).infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
@@ -1185,8 +1189,8 @@ impl HybridQuery {
self.inner_vec.refine_factor(refine_factor);
}
pub fn nprobes(&mut self, nprobe: u32) {
self.inner_vec.nprobes(nprobe);
pub fn nprobes(&mut self, nprobe: u32) -> PyResult<()> {
self.inner_vec.nprobes(nprobe)
}
pub fn ef(&mut self, ef: u32) {
python/src/table.rs
+11
View File
@@ -1897,6 +1897,17 @@ impl Table {
let alterations = alterations
.iter()
.map(|alteration| {
for key in alteration.keys().iter() {
let key = key.extract::<String>()?;
if !matches!(
key.as_str(),
"path" | "rename" | "name" | "nullable" | "data_type"
) {
return Err(PyValueError::new_err(format!(
"Unknown column alteration key '{key}'"
)));
}
}
let path = alteration
.get_item("path")?
.ok_or_else(|| PyValueError::new_err("Missing path"))?
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.40.0-beta.11"
version = "0.40.0-beta.12"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/examples/ivf_pq.rs
+1 -1
View File
@@ -142,7 +142,7 @@ async fn search_index(table: &Table) -> Result<()> {
// Override the default of 10 to get more rows
.limit(15)
// Override the default of 20 to search more partitions
.nprobes(30)
.nprobes(30)?
// Override the default of None to apply a refine step
.refine_factor(1)
.execute()
rust/lancedb/src/connection.rs
+78 -9
View File
@@ -568,6 +568,7 @@ impl Connection {
/// Creates a new table by cloning from an existing source table.
/// By default, this performs a shallow clone where the new table shares
/// the underlying data files with the source table.
/// The target table name must be unused. An existing table is left unchanged.
///
/// # Parameters
/// - `target_table_name`: The name of the new table to create
@@ -587,7 +588,8 @@ impl Connection {
)
}
/// Build and register a Python callable as an immutable Function version.
/// Build and register a Python callable as an immutable Function version
/// in `namespace_path`, which is empty for the root namespace.
///
/// The server-side job builds the OCI image, then registers the completed
/// artifact. Waiting on the returned typed job yields the durable
@@ -597,8 +599,11 @@ impl Connection {
pub async fn create_function_async(
&self,
request: crate::function::FunctionRegistrationRequest,
namespace_path: &[String],
) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
self.internal.create_function_async(request).await
self.internal
.create_function_async(request, namespace_path)
.await
}
/// Look up one exact immutable Function version in the remote catalog.
@@ -610,13 +615,15 @@ impl Connection {
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
namespace_path: &[String],
) -> Result<crate::function::FunctionVersion> {
self.internal
.get_function(name.as_ref(), version.as_ref())
.get_function(name.as_ref(), version.as_ref(), namespace_path)
.await
}
/// List every published immutable Function version in the remote catalog.
/// List every published immutable Function version in `namespace_path` of
/// the remote catalog. Functions in child namespaces are not included.
///
/// Results are ordered by Function name then version. The client walks all
/// server pages before returning. Local databases return
@@ -628,14 +635,17 @@ impl Connection {
/// # async fn list_functions(
/// # connection: &lancedb::Connection,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// for function in connection.list_functions().await? {
/// for function in connection.list_functions(&[]).await? {
/// println!("{} {}", function.name(), function.version());
/// }
/// # Ok(())
/// # }
/// ```
pub async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
self.internal.list_functions().await
pub async fn list_functions(
&self,
namespace_path: &[String],
) -> Result<Vec<crate::function::FunctionVersion>> {
self.internal.list_functions(namespace_path).await
}
/// Remove the current Function name binding, retaining the object history.
@@ -647,9 +657,10 @@ impl Connection {
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
namespace_path: &[String],
) -> Result<bool> {
self.internal
.drop_function(name.as_ref(), version.as_ref())
.drop_function(name.as_ref(), version.as_ref(), namespace_path)
.await
}
@@ -663,9 +674,10 @@ impl Connection {
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
namespace_path: &[String],
) -> Result<(bool, crate::job::Job)> {
self.internal
.drop_function_async(name.as_ref(), version.as_ref())
.drop_function_async(name.as_ref(), version.as_ref(), namespace_path)
.await
}
@@ -2182,6 +2194,63 @@ mod tests {
assert_eq!(tables.len(), 0);
}
#[rstest::rstest]
#[tokio::test]
async fn test_clone_table_target_already_exists(
#[values("source", "target")] target: &str,
#[values(false, true)] enable_v2_manifest_paths: bool,
) {
use crate::query::ExecutableQuery;
use futures::TryStreamExt;
let tmp_dir = tempdir().unwrap();
let options = ListingDatabaseOptions::builder()
.enable_v2_manifest_paths(enable_v2_manifest_paths)
.build();
let db = connect(tmp_dir.path().to_str().unwrap())
.database_options(&options)
.execute()
.await
.unwrap();
let source_data = arrow_array::record_batch!(("id", Int32, [0, 1, 2])).unwrap();
let target_data = arrow_array::record_batch!(("id", Int32, [10, 11])).unwrap();
db.create_table("source", source_data.clone())
.execute()
.await
.unwrap();
db.create_table("target", target_data.clone())
.execute()
.await
.unwrap();
let source_uri = tmp_dir.path().join("source.lance");
let err = db
.clone_table(target, source_uri.to_str().unwrap())
.execute()
.await
.unwrap_err();
assert!(
matches!(&err, Error::TableAlreadyExists { name } if name == target),
"unexpected clone error: {err:?}"
);
// Reopen both tables to verify that the failed clone left their data and versions intact.
for (name, expected_data) in [("source", source_data), ("target", target_data)] {
let table = db.open_table(name).execute().await.unwrap();
assert_eq!(table.version().await.unwrap(), 1);
assert_eq!(table.list_versions().await.unwrap().len(), 1);
let batches = table
.query()
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches, vec![expected_data]);
}
}
#[tokio::test]
async fn test_clone_table() {
let tmp_dir = tempdir().unwrap();
rust/lancedb/src/database.rs
+18 -5
View File
@@ -333,10 +333,12 @@ pub trait Database:
///
/// See [`CloneTableRequest`] for detailed documentation and examples.
async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>>;
/// Submit a Function creation job that builds an image and registers it.
/// Submit a Function creation job that builds an image and registers it
/// in `namespace_path`.
async fn create_function_async(
&self,
_request: crate::function::FunctionRegistrationRequest,
_namespace_path: &[String],
) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
function_catalog_not_supported()
}
@@ -412,15 +414,25 @@ pub trait Database:
&self,
_name: &str,
_version: &str,
_namespace_path: &[String],
) -> Result<crate::function::FunctionVersion> {
function_catalog_not_supported()
}
/// List every published immutable Function version in the remote catalog.
async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
/// List every published immutable Function version in one namespace of the
/// remote catalog.
async fn list_functions(
&self,
_namespace_path: &[String],
) -> Result<Vec<crate::function::FunctionVersion>> {
function_catalog_not_supported()
}
/// Remove the current Function name binding, retaining the object history.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
async fn drop_function(
&self,
_name: &str,
_version: &str,
_namespace_path: &[String],
) -> Result<bool> {
function_catalog_not_supported()
}
/// Start dropping a Function and return a handle to the cleanup job.
@@ -431,8 +443,9 @@ pub trait Database:
&self,
name: &str,
version: &str,
namespace_path: &[String],
) -> Result<(bool, crate::job::Job)> {
let dropped = self.drop_function(name, version).await?;
let dropped = self.drop_function(name, version, namespace_path).await?;
Ok((dropped, crate::job::Job::new_done()))
}
/// Create a named Secret in this database. Fails if the name is taken, so
rust/lancedb/src/database/listing.rs
+15 -1
View File
@@ -1107,6 +1107,21 @@ impl Database for ListingDatabase {
..Default::default()
};
let target_uri = self.table_uri(&request.target_table_name)?;
match DatasetBuilder::from_uri(&target_uri)
.with_read_params(read_params.clone())
.load()
.await
{
Ok(_) => {
return Err(Error::TableAlreadyExists {
name: request.target_table_name,
});
}
Err(lance::Error::DatasetNotFound { .. }) => {}
Err(err) => return Err(err.into()),
}
let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri)
.with_read_params(read_params.clone())
.load()
@@ -1122,7 +1137,6 @@ impl Database for ListingDatabase {
}),
}?;
let target_uri = self.table_uri(&request.target_table_name)?;
source_dataset
.shallow_clone(&target_uri, version_ref, Some(storage_params))
.await
rust/lancedb/src/function.rs
+71 -12
View File
@@ -85,22 +85,25 @@ fn has_unknown_keys(value: &Value, allowed: &[&str]) -> bool {
.is_some_and(|object| object.keys().any(|key| !allowed.contains(&key.as_str())))
}
/// Every field a [`FunctionVersionRef`] document may carry. An application or
/// binding with any other key comes from a newer contract.
pub(crate) const FUNCTION_VERSION_REF_FIELDS: &[&str] = &[
"name",
"namespace_path",
"object_id",
"location",
"version",
"manifest_digest",
];
fn application_has_unknown_nested_fields(value: &Value) -> bool {
let Some(application) = value.as_object() else {
return false;
};
if application.get("function").is_some_and(|value| {
has_unknown_keys(
value,
&[
"name",
"object_id",
"location",
"version",
"manifest_digest",
],
)
}) {
if application
.get("function")
.is_some_and(|value| has_unknown_keys(value, FUNCTION_VERSION_REF_FIELDS))
{
return true;
}
if application
@@ -431,9 +434,17 @@ fn deserialize_object_version<'de, D: Deserializer<'de>>(
}
/// One immutable Function object revision and its executable artifact.
///
/// The Function is named by its parts, `name` and `namespace_path`, the way a
/// [`SecretReference`](crate::secrets::SecretReference) names a Secret.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersion {
name: String,
/// The namespace holding the Function. Empty is the root, and is absent
/// from the wire, as a [`SecretReference`](crate::secrets::SecretReference)
/// omits it.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
namespace_path: Vec<String>,
object_id: String,
location: String,
#[serde(deserialize_with = "deserialize_object_version")]
@@ -463,6 +474,7 @@ impl FunctionVersion {
pub fn reference(&self) -> FunctionVersionRef {
FunctionVersionRef {
name: self.name.clone(),
namespace_path: self.namespace_path.clone(),
object_id: self.object_id.clone(),
location: self.location.clone(),
version: self.version.clone(),
@@ -472,6 +484,9 @@ impl FunctionVersion {
pub fn name(&self) -> &str {
&self.name
}
pub fn namespace_path(&self) -> &[String] {
&self.namespace_path
}
pub fn version(&self) -> &str {
&self.version
}
@@ -556,6 +571,10 @@ impl_json!(FunctionRegistrationRequest);
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersionRef {
pub name: String,
/// The namespace holding the Function; empty is the root and is absent
/// from the wire.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub namespace_path: Vec<String>,
pub object_id: String,
pub location: String,
#[serde(deserialize_with = "deserialize_object_version")]
@@ -954,6 +973,46 @@ mod tests {
);
}
/// A Function is named by its parts, and a root one states no namespace.
#[test]
fn function_version_carries_its_namespace_as_a_separate_field() {
const VERSION: &str = include_str!(
"../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json"
);
let root = FunctionVersion::from_json(VERSION).unwrap();
assert!(root.namespace_path().is_empty());
assert_eq!(root.to_canonical_json().unwrap(), VERSION.trim_end());
let mut document: Value = serde_json::from_str(VERSION).unwrap();
document["namespace_path"] = serde_json::json!(["analytics", "features"]);
let nested = FunctionVersion::from_json(&document.to_string()).unwrap();
assert_eq!(nested.name(), "embed");
assert_eq!(nested.namespace_path(), ["analytics", "features"]);
let reference = nested.reference();
assert_eq!(reference.name, "embed");
assert_eq!(reference.namespace_path, ["analytics", "features"]);
let reference: Value = serde_json::to_value(&reference).unwrap();
assert_eq!(
reference["namespace_path"],
serde_json::json!(["analytics", "features"])
);
let root_reference = serde_json::to_value(root.reference()).unwrap();
assert!(root_reference.get("namespace_path").is_none());
// An application carrying the namespace is not from a newer contract.
let application = serde_json::json!({
"function": reference,
"inputs": [],
"output": serde_json::to_value(nested.signature().output.clone()).unwrap(),
});
let application = FunctionApplication::from_json(&application.to_string()).unwrap();
assert!(!application.has_unknown_fields());
assert_eq!(
application.function().namespace_path,
["analytics", "features"]
);
}
/// Arrays are ordered by the caller, so canonicalization must leave them
/// alone -- sorting them would change what a signature means.
#[test]
rust/lancedb/src/materialized_view.rs
+183 -28
View File
@@ -9,8 +9,12 @@
//! query this version cannot maintain reads back as unrefreshable, not as a
//! plain table. Queries, indexes and search work on the view unchanged.
mod duplicate_pairs;
mod grouped;
mod grouped_units;
mod partitioned;
pub(crate) mod vector_dedup;
pub use duplicate_pairs::{VectorSource, VectorSourceKind};
pub use grouped::IVF_PARTITION;
/// Refreshing a view grouped by [`IVF_PARTITION`] in units, one per index
/// partition, so the work can be spread over several processes. The caller
@@ -74,6 +78,10 @@ pub use grouped_units::{
GroupedRefreshPlan, WrittenUnit, commit_grouped_refresh, plan_grouped_refresh,
write_grouped_unit,
};
pub use partitioned::{
PartitionedRefreshPlan, WrittenPartition, commit_partitioned_refresh, plan_partitioned_refresh,
write_refresh_partition, write_refresh_partition_with_inputs,
};
mod query;
pub mod refresh;
@@ -146,18 +154,26 @@ const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions";
/// The newest layout this version reads under [`DEFINITION_META_KEY`]:
/// `{"format": N, "query": "<SQL>"}`, the query as
/// [`MaterializedViewDefinition::to_sql`] renders it. Format 2 is a query
/// with `GROUP BY`; every other query is written as format 1, so a reader
/// that predates grouping reports a grouped view as unrefreshable rather
/// than failing to parse it. A reader refuses a newer format rather than
/// [`MaterializedViewDefinition::to_sql`] renders it. Format 3 identifies a
/// native duplicate-pair source, format 4 a dedup result source, format 2 a query with `GROUP BY`, and
/// format 1 an ordinary query. A reader refuses a newer format rather than
/// guess at it. The layout also carries `"kind": "query"`, which readers
/// older than the format number report as an unrefreshable view instead of
/// failing to read the metadata.
pub const DEFINITION_FORMAT: u64 = 2;
pub const DEFINITION_FORMAT: u64 = 4;
/// The format `definition` is written in; see [`DEFINITION_FORMAT`].
fn format_of(definition: &MaterializedViewDefinition) -> u64 {
if definition.is_grouped() { 2 } else { 1 }
if let Some(source) = &definition.vector_source {
match source.kind {
VectorSourceKind::Pairs => 3,
VectorSourceKind::Dedup => 4,
}
} else if definition.is_grouped() {
2
} else {
1
}
}
/// Legacy `kind` tag of the structured layout written before
@@ -300,6 +316,8 @@ pub fn read_staging(metadata: &HashMap<String, String>) -> Result<Option<Staging
/// for the shape.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MaterializedViewDefinition {
/// A table-bound native vector source. It is never evaluated per source row.
pub vector_source: Option<Box<VectorSource>>,
/// Name of the source table, in the same database as the view.
pub source_table: String,
/// Namespace path holding the source table; empty is the root namespace.
@@ -498,6 +516,7 @@ pub fn read_definition(metadata: &HashMap<String, String>) -> Result<Option<Stor
)));
}
Ok(Some(StoredDefinition::Query(MaterializedViewDefinition {
vector_source: None,
source_table: legacy.source_table,
source_namespace: legacy.source_namespace,
lateral: None,
@@ -554,6 +573,9 @@ pub(crate) fn plan(
definition: &MaterializedViewDefinition,
staging: Option<&StagingBinding>,
) -> Result<Planned> {
if definition.vector_source.is_some() {
return duplicate_pairs::plan(source_schema, definition);
}
let filter = definition
.filter
.as_deref()
@@ -712,6 +734,7 @@ pub(crate) fn plan(
}
let definition = MaterializedViewDefinition {
vector_source: None,
source_table: definition.source_table.clone(),
source_namespace: definition.source_namespace.clone(),
lateral: definition.lateral.clone(),
@@ -1145,6 +1168,11 @@ impl PreparedDeclaration {
/// read: the column the view projects it to, if any, otherwise an
/// internal projection added here, named by [`input_column_name`].
pub fn input_column(&mut self, source_column: &str) -> Result<String> {
if self.definition.vector_source.is_some() {
return Err(Error::InvalidInput {
message: "native vector views have no per-source-row inputs".into(),
});
}
// A grouped view's rows are groups; no source row carries a value into one.
if self.definition.is_grouped() {
return Err(Error::InvalidInput {
@@ -1245,6 +1273,11 @@ impl PreparedDeclaration {
columns: Vec<(usize, ArrowField)>,
bindings: &[FunctionBinding],
) -> Result<Self> {
if self.definition.vector_source.is_some() {
return Err(Error::InvalidInput {
message: "computed columns on native vector views are not supported".into(),
});
}
let invalid = |message: String| Error::InvalidInput { message };
if columns.is_empty() {
return Err(invalid("at least one computed column is needed".into()));
@@ -1451,6 +1484,7 @@ pub async fn prepare_declaration(
limit: Option<u64>,
) -> Result<PreparedDeclaration> {
let definition = MaterializedViewDefinition {
vector_source: None,
source_table: source.name().to_string(),
source_namespace: source.namespace().to_vec(),
lateral: None,
@@ -1512,6 +1546,27 @@ pub async fn prepare_declaration(
/// # Ok(())
/// # }
/// ```
///
/// An indexed, pinned source can materialize retained original rows using the
/// default greedy direct-representative policy. A row is removed only when it
/// directly matches a retained row; an A-B-C chain keeps A and C. The source is
/// unchanged. Hosts distribute this through [`plan_partitioned_refresh`] and
/// [`write_refresh_partition_with_inputs`], with a barrier between selection
/// tasks and source-fragment materialization tasks.
///
/// ```
/// # #![recursion_limit = "256"]
/// use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition};
/// # async fn dedup(images: &lancedb::Table) -> Result<(), Box<dyn std::error::Error>> {
/// let version = images.version().await?;
/// let definition = MaterializedViewDefinition::from_sql(&format!(
/// "SELECT * FROM vector_dedup('images', {version}, 'phash', 4)"
/// ))?;
/// let clean = prepare_definition(images, definition).await?.create("images_clean").await?;
/// clean.refresh().execute().await?;
/// # Ok(())
/// # }
/// ```
pub async fn prepare_definition(
source: &Table,
definition: MaterializedViewDefinition,
@@ -1627,7 +1682,9 @@ async fn prepare_with(
),
});
}
if !native.dataset.get().await?.manifest.uses_stable_row_ids() {
if definition.vector_source.is_none()
&& !native.dataset.get().await?.manifest.uses_stable_row_ids()
{
return Err(Error::InvalidInput {
message: format!(
"materialized views require stable row ids on the source table; \
@@ -1653,7 +1710,22 @@ async fn prepare_with(
message: format!("view column name '{}' is reserved", reserved.output),
});
}
let source_schema = resolved.schema().await?;
let source_schema = if let Some(pairs) = &definition.vector_source {
let ds = native
.dataset
.get()
.await?
.checkout_version(pairs.dataset_version)
.await?;
crate::table::datafusion::udtf::duplicate_pairs::plan_duplicate_pairs(
Arc::new(ds.clone()),
&pairs.config()?,
)
.await?;
Arc::new(ArrowSchema::from(ds.schema()))
} else {
resolved.schema().await?
};
let source_metadata = source_schema.metadata().clone();
let Planned {
definition,
@@ -1670,11 +1742,13 @@ async fn prepare_with(
None => source_schema.clone(),
Some(unnest) => flattened_schema(&source_schema, &unnest)?,
};
fields.push(ArrowField::new(
SOURCE_ROW_ID_COLUMN,
DataType::UInt64,
false,
));
if definition.vector_source.is_none() {
fields.push(ArrowField::new(
SOURCE_ROW_ID_COLUMN,
DataType::UInt64,
false,
));
}
// Only column-describing metadata comes along: structural declarations
// describe how a table is written, and a view is written by refresh alone.
let mut metadata: HashMap<String, String> = HashMap::new();
@@ -1718,6 +1792,7 @@ pub struct CreateMaterializedViewBuilder {
namespace: Vec<String>,
source: String,
source_namespace: Vec<String>,
vector_source: Option<VectorSource>,
projections: Vec<(String, String)>,
filter: Option<String>,
limit: Option<u64>,
@@ -1732,6 +1807,7 @@ impl CreateMaterializedViewBuilder {
namespace: Vec::new(),
source,
source_namespace: Vec::new(),
vector_source: None,
projections: Vec::new(),
filter: None,
limit: None,
@@ -1752,6 +1828,38 @@ impl CreateMaterializedViewBuilder {
self
}
/// Read native pairs or retained original rows from a fixed indexed snapshot.
/// This source selects its full output schema and cannot be combined with
/// projections, filters or limits. Remote creation uses the ordinary MV
/// endpoint and its registry job; it does not execute a SQL query client-side.
pub fn vector_source(mut self, source: VectorSource) -> Self {
self.vector_source = Some(source);
self
}
fn native_definition(&self) -> Result<Option<MaterializedViewDefinition>> {
let Some(source) = &self.vector_source else {
return Ok(None);
};
source.config()?;
if !self.projections.is_empty() || self.filter.is_some() || self.limit.is_some() {
return Err(Error::InvalidInput {
message: "native vector sources cannot be combined with select, where or limit"
.into(),
});
}
Ok(Some(MaterializedViewDefinition {
vector_source: Some(Box::new(source.clone())),
source_table: self.source.clone(),
source_namespace: self.source_namespace.clone(),
projections: vec![ViewProjection::star()],
filter: None,
lateral: None,
group_by: Vec::new(),
limit: None,
}))
}
/// The view's columns, as `(name, SQL expression)` pairs. Not calling
/// this selects every source column, expanded at creation time.
pub fn select(
@@ -1784,7 +1892,10 @@ impl CreateMaterializedViewBuilder {
self
}
fn query(&self) -> String {
fn query(&self) -> Result<String> {
if let Some(definition) = self.native_definition()? {
return Ok(definition.to_sql());
}
fn quote(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
@@ -1813,12 +1924,13 @@ impl CreateMaterializedViewBuilder {
if let Some(limit) = self.limit {
query.push_str(&format!(" LIMIT {limit}"));
}
query
Ok(query)
}
/// Submit creation and initial population, returning a [`Job`] that
/// settles when the view is ready. The source must keep stable row ids --
/// they hold provenance across compaction, and cannot be enabled later.
/// settles when the view is ready. Ordinary sources must keep stable row
/// ids for provenance across compaction. Native vector sources also support
/// physical row IDs because their definitions pin the dataset version.
pub async fn execute_async(self) -> Result<Job> {
if self.connection.uri().starts_with("db://") {
return self
@@ -1827,7 +1939,7 @@ impl CreateMaterializedViewBuilder {
.create_materialized_view_async(CreateMaterializedViewRequest {
name: self.name.clone(),
namespace_path: self.namespace.clone(),
query: self.query(),
query: self.query()?,
with_no_data: self.with_no_data,
})
.await;
@@ -1855,19 +1967,24 @@ impl CreateMaterializedViewBuilder {
}
async fn execute_native(self) -> Result<MaterializedView> {
let native_definition = self.native_definition()?;
let source = self
.connection
.open_table(&self.source)
.namespace(self.source_namespace.clone())
.execute()
.await?;
let prepared = prepare_declaration(
&source,
(!self.projections.is_empty()).then_some(self.projections.as_slice()),
self.filter.as_deref(),
self.limit,
)
.await?;
let prepared = if let Some(definition) = native_definition {
prepare_definition(&source, definition).await?
} else {
prepare_declaration(
&source,
(!self.projections.is_empty()).then_some(self.projections.as_slice()),
self.filter.as_deref(),
self.limit,
)
.await?
};
let view = prepared.create_in(&self.namespace, &self.name).await?;
if !self.with_no_data {
view.refresh().execute().await?;
@@ -2186,6 +2303,7 @@ mod tests {
assert_eq!(
view.definition(),
&MaterializedViewDefinition {
vector_source: None,
source_table: "people".into(),
source_namespace: Vec::new(),
projections: vec![
@@ -2309,13 +2427,49 @@ mod tests {
.select([("double\"age", "age * 2")])
.only_if("age >= 18")
.limit(10)
.query();
.query()
.unwrap();
assert_eq!(
query,
"SELECT age * 2 AS \"double\"\"age\" FROM \"raw data\".\"odd\"\"source\" WHERE age >= 18 LIMIT 10"
);
}
#[tokio::test]
async fn native_vector_builder_preserves_source_and_rejects_extra_clauses() {
let conn = people_db().await;
let source = VectorSource {
kind: VectorSourceKind::Dedup,
dataset_version: 7,
column: "vector'field".into(),
distance_threshold: "4.0".into(),
};
let builder = || {
conn.create_materialized_view("clean", "odd.source'name")
.source_namespace(vec!["raw space".into()])
.vector_source(source.clone())
};
let query = builder().query().unwrap();
let definition = MaterializedViewDefinition::from_sql(&query).unwrap();
assert_eq!(definition.source_table, "odd.source'name");
assert_eq!(definition.source_namespace, ["raw space"]);
assert_eq!(definition.vector_source.as_deref(), Some(&source));
assert!(definition.selects_star());
assert!(builder().select([("id", "id")]).query().is_err());
assert!(builder().only_if("true").query().is_err());
assert!(builder().limit(1).query().is_err());
let invalid_source = VectorSource {
dataset_version: 0,
..source
};
assert!(
conn.create_materialized_view("bad", "people")
.vector_source(invalid_source)
.query()
.is_err()
);
}
/// No projection selects every source column, expanded now: the schema
/// captured at creation is the definition.
#[tokio::test]
@@ -3235,6 +3389,7 @@ mod tests {
fn definition(source_namespace: Vec<String>) -> MaterializedViewDefinition {
MaterializedViewDefinition {
vector_source: None,
source_table: "people".to_string(),
source_namespace,
lateral: None,
@@ -3321,8 +3476,8 @@ mod tests {
#[test]
fn a_newer_format_is_reported_not_guessed() {
assert_eq!(
read(r#"{"format":3,"query":"SELECT name FROM people"}"#).unwrap(),
Some(StoredDefinition::Newer { format: "3".into() })
read(r#"{"format":5,"query":"SELECT name FROM people"}"#).unwrap(),
Some(StoredDefinition::Newer { format: "5".into() })
);
assert_eq!(
read(r#"{"kind":"join"}"#).unwrap(),
rust/lancedb/src/materialized_view/duplicate_pairs.rs
+478
View File
@@ -0,0 +1,478 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Native pair generation as partitioned materialized-view refresh work.
use std::sync::Arc;
use arrow_schema::{Schema, SchemaRef};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_plan::ExecutionPlan;
use datafusion_sql::sqlparser::ast::{
Expr, FunctionArg, FunctionArgExpr, TableFunctionArgs, Value,
};
use futures::TryStreamExt;
use lance::Dataset;
use lance::dataset::transaction::Operation;
use lance::dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams};
use lance_table::format::Fragment;
use serde::{Deserialize, Serialize};
use super::grouped_units::GroupedRefreshPlan;
use super::refresh::{
RefreshMaterializedViewResult, ensure_incarnation, ensure_no_mem_wal, open_source,
};
use super::{MaterializedViewDefinition, Planned, ViewProjection};
use crate::table::Table;
use crate::table::datafusion::udtf::duplicate_pairs::{
DUPLICATE_PAIRS_OPERATOR_VERSION, DuplicatePairTask, DuplicatePairsConfig, DuplicatePairsExec,
duplicate_pair_schema, plan_duplicate_pairs,
};
use crate::{Error, Result};
pub(super) const FUNCTION_NAME: &str = "vector_duplicate_pairs";
pub(super) const DEDUP_FUNCTION_NAME: &str = "vector_dedup";
/// The result of a pinned native vector source.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum VectorSourceKind {
/// The qualifying pair relation.
Pairs,
/// Original source rows retained by greedy direct representatives.
Dedup,
}
/// A native table-function source, pinned for the lifetime of the view.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VectorSource {
pub kind: VectorSourceKind,
pub dataset_version: u64,
pub column: String,
/// Validated numeric SQL spelling; execution and task identity use f32 bits.
pub distance_threshold: String,
}
impl VectorSource {
pub(super) fn function_name(&self) -> &'static str {
match self.kind {
VectorSourceKind::Pairs => FUNCTION_NAME,
VectorSourceKind::Dedup => DEDUP_FUNCTION_NAME,
}
}
pub(crate) fn config(&self) -> Result<DuplicatePairsConfig> {
let threshold: f32 = self
.distance_threshold
.parse()
.map_err(|_| invalid("invalid pair threshold"))?;
if self.dataset_version == 0 || self.column.is_empty() || !threshold.is_finite() {
return Err(invalid(
"duplicate pairs require a positive snapshot, a column and a finite threshold",
));
}
Ok(DuplicatePairsConfig {
dataset_version: self.dataset_version,
column: self.column.clone(),
distance_threshold: threshold,
})
}
}
fn invalid(message: impl Into<String>) -> Error {
Error::InvalidInput {
message: message.into(),
}
}
pub(super) fn source(
args: &TableFunctionArgs,
kind: VectorSourceKind,
) -> Result<(Vec<String>, String, VectorSource)> {
if args.settings.is_some() {
return Err(invalid("duplicate pairs do not accept function settings"));
}
let exprs = args
.args
.iter()
.map(|arg| match arg {
FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Ok(expr),
_ => Err(invalid(
"duplicate pairs require four positional literal arguments",
)),
})
.collect::<Result<Vec<_>>>()?;
let [table, version, column, threshold] = exprs.as_slice() else {
return Err(invalid(
"native vector sources require (table, dataset_version, column, distance_threshold)",
));
};
let string = |expr: &Expr| match expr {
Expr::Value(value) => match &value.value {
Value::SingleQuotedString(value) => Ok(value.clone()),
_ => Err(invalid("table and column must be string literals")),
},
_ => Err(invalid("table and column must be string literals")),
};
let table = string(table)?;
let plain = MaterializedViewDefinition::from_sql(&format!("SELECT * FROM {table}"))?;
if plain.vector_source.is_some()
|| plain.lateral.is_some()
|| plain.filter.is_some()
|| plain.is_grouped()
|| plain.limit.is_some()
{
return Err(invalid("duplicate pairs must name one source table"));
}
let dataset_version = match version {
Expr::Value(value) => match &value.value {
Value::Number(number, _) => number
.parse()
.map_err(|_| invalid("invalid source version"))?,
_ => return Err(invalid("source version must be an integer literal")),
},
_ => return Err(invalid("source version must be an integer literal")),
};
let config = VectorSource {
kind,
dataset_version,
column: string(column)?,
distance_threshold: threshold.to_string(),
};
config.config()?;
Ok((plain.source_namespace, plain.source_table, config))
}
pub(super) fn check_shape(definition: &MaterializedViewDefinition) -> Result<()> {
if definition
.vector_source
.as_ref()
.is_some_and(|s| s.kind == VectorSourceKind::Dedup)
{
if !definition.selects_star()
|| definition.filter.is_some()
|| definition.limit.is_some()
|| definition.lateral.is_some()
|| definition.is_grouped()
{
return Err(invalid(
"a vector_dedup view must select * without other clauses",
));
}
return Ok(());
}
let schema = duplicate_pair_schema();
let projections = schema
.fields()
.iter()
.map(|field| ViewProjection {
output: field.name().clone(),
expression: field.name().clone(),
})
.collect::<Vec<_>>();
if (!definition.selects_star() && definition.projections != projections)
|| definition.filter.is_some()
|| definition.limit.is_some()
|| definition.lateral.is_some()
|| definition.is_grouped()
{
return Err(invalid(
"a native pair view selects *, or row_id_a, row_id_b, distance, without other clauses",
));
}
Ok(())
}
pub(super) fn plan(
source_schema: SchemaRef,
definition: &MaterializedViewDefinition,
) -> Result<Planned> {
check_shape(definition)?;
let config = definition.vector_source.as_ref().expect("native source");
config.config()?;
source_schema.field_with_name(&config.column)?;
if config.kind == VectorSourceKind::Dedup {
return Ok(Planned {
definition: definition.clone(),
fields: source_schema
.fields()
.iter()
.map(|f| super::without_declarations(f))
.collect(),
lineage: source_schema
.fields()
.iter()
.map(|f| (f.name().clone(), vec![f.name().clone()]))
.collect(),
inputs: source_schema
.fields()
.iter()
.map(|f| f.name().clone())
.collect(),
});
}
let schema = duplicate_pair_schema();
let projections = schema
.fields()
.iter()
.map(|field| ViewProjection {
output: field.name().clone(),
expression: field.name().clone(),
})
.collect();
let mut definition = definition.clone();
let inputs = vec![config.column.clone()];
definition.projections = projections;
Ok(Planned {
definition,
fields: schema.fields().iter().map(|f| f.as_ref().clone()).collect(),
lineage: Default::default(),
inputs,
})
}
/// A saved native refresh plan, including every physical partition identity.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DuplicatePairsRefreshPlan {
pub snapshot: GroupedRefreshPlan,
pub source_uri: String,
pub definition_sql: String,
pub operator_version: u32,
pub threshold_bits: u32,
pub tasks: Vec<DuplicatePairTask>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WrittenPairsPartition {
pub(super) unit: u32,
pub(super) plan_id: String,
pub(super) task: DuplicatePairTask,
pub(super) fragments: Vec<Fragment>,
pub(super) rows: u64,
}
impl DuplicatePairsRefreshPlan {
pub(super) fn identity(&self) -> Result<String> {
use sha2::{Digest, Sha256};
let encoded = serde_json::to_vec(&(
&self.snapshot,
&self.source_uri,
&self.definition_sql,
self.operator_version,
self.threshold_bits,
))
.map_err(|e| invalid(e.to_string()))?;
Ok(format!("{:x}", Sha256::digest(encoded)))
}
}
async fn open(view: &Table) -> Result<(Dataset, MaterializedViewDefinition, Arc<Dataset>)> {
let native = view
.as_native()
.ok_or_else(|| invalid("native pair refresh requires a local view"))?;
native.dataset.ensure_mutable()?;
native.dataset.reload().await?;
let view_ds = native.dataset.get().await?.as_ref().clone();
let Some(super::StoredDefinition::Query(definition)) =
super::read_definition(&view_ds.schema().metadata)?
else {
return Err(invalid("view does not carry a supported definition"));
};
let config = definition
.vector_source
.as_ref()
.ok_or_else(|| invalid("not a native pair view"))?
.config()?;
let source = Arc::new(
open_source(view, &definition, None)
.await?
.checkout_version(config.dataset_version)
.await?,
);
ensure_no_mem_wal(&source, "source table", &definition.source_table).await?;
let planned = plan(Arc::new(Schema::from(source.schema())), &definition)?;
let expected = Schema::new(planned.fields);
let physical = Schema::from(view_ds.schema());
if physical.fields().len() != expected.fields().len()
|| physical
.fields()
.iter()
.zip(expected.fields())
.any(|(a, b)| {
a.name() != b.name()
|| a.data_type() != b.data_type()
|| a.is_nullable() != b.is_nullable()
})
{
return Err(invalid("native pair view schema changed since creation"));
}
Ok((view_ds, planned.definition, source))
}
pub(super) async fn plan_refresh(
view: &Table,
pinned: Option<u64>,
) -> Result<DuplicatePairsRefreshPlan> {
let (view_ds, definition, source) = open(view).await?;
let config = definition
.vector_source
.as_ref()
.expect("native")
.config()?;
if pinned.is_some_and(|v| v != config.dataset_version) {
return Err(invalid(
"refresh source version differs from the native pair view's fixed snapshot",
));
}
let tasks = plan_duplicate_pairs(source.clone(), &config).await?;
let units = u32::try_from(tasks.len()).map_err(|_| invalid("too many index partitions"))?;
let incarnation = view_ds
.schema()
.metadata
.get(super::INCARNATION_META_KEY)
.cloned()
.ok_or_else(|| invalid("native pair view has no incarnation"))?;
Ok(DuplicatePairsRefreshPlan {
snapshot: GroupedRefreshPlan {
units,
source_version: config.dataset_version,
view_version: view_ds.version().version,
incarnation,
},
source_uri: source.uri().to_string(),
definition_sql: definition.to_sql(),
operator_version: DUPLICATE_PAIRS_OPERATOR_VERSION,
threshold_bits: config.distance_threshold.to_bits(),
tasks,
})
}
pub(super) async fn open_planned(
view: &Table,
plan: &DuplicatePairsRefreshPlan,
) -> Result<(Dataset, MaterializedViewDefinition, Arc<Dataset>)> {
let (view_ds, definition, source) = open(view).await?;
ensure_incarnation(&view_ds, Some(&plan.snapshot.incarnation), view.name()).await?;
let config = definition
.vector_source
.as_ref()
.expect("native")
.config()?;
if source.uri() != plan.source_uri
|| source.version().version != plan.snapshot.source_version
|| definition.to_sql() != plan.definition_sql
|| config.distance_threshold.to_bits() != plan.threshold_bits
|| plan.operator_version != DUPLICATE_PAIRS_OPERATOR_VERSION
|| plan.tasks.len() != plan.snapshot.units as usize
{
return Err(invalid(
"native pair task belongs to a different snapshot, configuration or operator",
));
}
Ok((view_ds, definition, source))
}
pub(super) async fn write_unit(
view: &Table,
unit: u32,
plan: &DuplicatePairsRefreshPlan,
) -> Result<WrittenPairsPartition> {
let (view_ds, definition, source) = open_planned(view, plan).await?;
let task = plan
.tasks
.get(unit as usize)
.ok_or_else(|| invalid("pair partition is outside the plan"))?
.clone();
let config = definition
.vector_source
.as_ref()
.expect("native")
.config()?;
let exec = DuplicatePairsExec::try_new(source, config, vec![task.clone()])?;
let stream = exec.execute(0, Arc::new(TaskContext::default()))?;
let rows = Arc::new(std::sync::atomic::AtomicU64::new(0));
let count = rows.clone();
let stream = stream.inspect_ok(move |batch| {
count.fetch_add(
batch.num_rows() as u64,
std::sync::atomic::Ordering::Relaxed,
);
});
let stream: SendableRecordBatchStream = Box::pin(
datafusion_physical_plan::stream::RecordBatchStreamAdapter::new(
duplicate_pair_schema(),
stream,
),
);
let txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds)))
.with_params(&WriteParams {
mode: WriteMode::Append,
..Default::default()
})
.execute_uncommitted_stream(stream)
.await?;
let Operation::Append { fragments } = txn.operation else {
return Err(invalid(
"native pair partition did not stage append fragments",
));
};
Ok(WrittenPairsPartition {
unit,
plan_id: plan.identity()?,
task,
fragments,
rows: rows.load(std::sync::atomic::Ordering::Relaxed),
})
}
pub(super) async fn commit(
view: &Table,
plan: &DuplicatePairsRefreshPlan,
units: Vec<WrittenPairsPartition>,
expected: Option<&str>,
) -> Result<RefreshMaterializedViewResult> {
let (_, definition, source) = open_planned(view, plan).await?;
let config = definition
.vector_source
.as_ref()
.expect("native")
.config()?;
if plan_duplicate_pairs(source.clone(), &config).await? != plan.tasks {
return Err(invalid(
"pair plan does not cover the pinned index partitions exactly",
));
}
let identity = plan.identity()?;
let mut ordered = vec![None; plan.snapshot.units as usize];
for written in units {
if written.plan_id != identity
|| plan.tasks.get(written.unit as usize) != Some(&written.task)
{
return Err(invalid("pair partition receipt belongs to another plan"));
}
let slot = ordered
.get_mut(written.unit as usize)
.ok_or_else(|| invalid("unknown pair partition receipt"))?;
if slot.is_some() {
return Err(invalid("duplicate pair partition receipt"));
}
*slot = Some(written);
}
let units = ordered
.into_iter()
.collect::<Option<Vec<_>>>()
.ok_or_else(|| invalid("missing pair partition receipt"))?;
let rows = units.iter().map(|u| u.rows).sum();
let fragments = units.into_iter().flat_map(|u| u.fragments).collect();
super::grouped_units::commit_fragments(
view,
&plan.snapshot,
&definition,
source.manifest.timestamp_nanos,
fragments,
rows,
expected,
)
.await
}
rust/lancedb/src/materialized_view/grouped_units.rs
+42 -8
View File
@@ -53,6 +53,7 @@ const PROBE_ROWS: usize = 256;
/// How a grouped view can be refreshed in units.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GroupedRefreshPlan {
/// Units `0..units`: unit `p` before the last holds index partition
/// `p`'s groups; the last holds the rows the index cannot place (null or
@@ -177,6 +178,7 @@ pub async fn plan_grouped_refresh(
/// accounted for has to be refusable. Travels to whoever commits, so it
/// serializes, but its fields are not a caller's to write.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WrittenUnit {
unit: u32,
plan: GroupedRefreshPlan,
@@ -491,10 +493,38 @@ pub async fn commit_grouped_refresh(
plan: &GroupedRefreshPlan,
units: Vec<WrittenUnit>,
expected_incarnation: Option<&str>,
) -> Result<RefreshMaterializedViewResult> {
let grouped = open_planned(view, plan).await?;
let units = complete_set(view.name(), plan, units)?;
let rows_written = units.iter().map(|unit| unit.rows).sum();
let new_fragments = units.into_iter().flat_map(|unit| unit.fragments).collect();
commit_fragments(
view,
plan,
&grouped.definition,
grouped.source.manifest.timestamp_nanos,
new_fragments,
rows_written,
expected_incarnation,
)
.await
}
/// The common all-or-nothing unit sink, shared by grouped and native views.
#[allow(clippy::too_many_arguments)]
pub(super) async fn commit_fragments(
view: &Table,
plan: &GroupedRefreshPlan,
definition: &MaterializedViewDefinition,
source_ts: u128,
new_fragments: Vec<Fragment>,
rows_written: u64,
expected_incarnation: Option<&str>,
) -> Result<RefreshMaterializedViewResult> {
let view_native = view.as_native().ok_or_else(|| Error::NotSupported {
message: "materialized views are supported only on local tables".into(),
})?;
view_native.dataset.ensure_mutable()?;
let lock = refresh_lock(view_native.dataset.get().await?.uri());
let _guard = lock.lock().await;
if let Some(expected) = expected_incarnation
@@ -508,10 +538,17 @@ pub async fn commit_grouped_refresh(
),
});
}
let grouped = open_planned(view, plan).await?;
let source_ts = grouped.source.manifest.timestamp_nanos;
let ds = Arc::new(grouped.view_ds);
view_native.dataset.reload().await?;
let ds = view_native.dataset.get().await?.as_ref().clone();
ensure_incarnation(&ds, Some(&plan.incarnation), view.name()).await?;
if super::read_definition(&ds.schema().metadata)?
!= Some(super::StoredDefinition::Query(definition.clone()))
{
return Err(Error::InvalidInput {
message: "view definition changed while its units were written".into(),
});
}
let ds = Arc::new(ds);
if ds.version().version != plan.view_version {
return Err(Error::Runtime {
message: format!(
@@ -523,11 +560,8 @@ pub async fn commit_grouped_refresh(
),
});
}
let units = complete_set(view.name(), plan, units)?;
let removed_fragment_ids: Vec<u64> = ds.get_fragments().iter().map(|f| f.id() as u64).collect();
// Fragment ids are assigned at commit; the units' fragments all carry 0.
let rows_written = units.iter().map(|unit| unit.rows).sum();
let new_fragments = units.into_iter().flat_map(|unit| unit.fragments).collect();
let transaction = Transaction::new(
plan.view_version,
Operation::Update {
@@ -566,7 +600,7 @@ pub async fn commit_grouped_refresh(
committed,
plan.source_version,
source_ts,
Some(&grouped.definition),
Some(definition),
expected_incarnation,
)
.await?;
rust/lancedb/src/materialized_view/partitioned.rs
+192
View File
@@ -0,0 +1,192 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Shared plan/write/commit surface for materialized views with independent units.
use serde::{Deserialize, Serialize};
use super::duplicate_pairs::{
self, DuplicatePairsRefreshPlan, VectorSourceKind, WrittenPairsPartition,
};
use super::vector_dedup::{self, VectorDedupPlan, WrittenDedupUnit};
use super::{GroupedRefreshPlan, RefreshMaterializedViewResult, StoredDefinition, WrittenUnit};
use crate::table::Table;
use crate::{Error, Result};
/// Serializable work description consumed by a host's existing job workers.
/// The grouped variant preserves the previous saved-plan representation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum PartitionedRefreshPlan {
VectorDedup(VectorDedupPlan),
DuplicatePairs(DuplicatePairsRefreshPlan),
Grouped(GroupedRefreshPlan),
}
impl PartitionedRefreshPlan {
/// The shared source snapshot and destination generation fence.
pub fn snapshot(&self) -> &GroupedRefreshPlan {
match self {
Self::VectorDedup(plan) => &plan.snapshot,
Self::DuplicatePairs(plan) => &plan.snapshot,
Self::Grouped(plan) => plan,
}
}
/// Selection tasks that must complete before remaining tasks can start.
pub fn dependency_units(&self) -> u32 {
match self {
Self::VectorDedup(plan) => plan.dependencies(),
_ => 0,
}
}
/// Number of independent units in this immutable plan.
pub fn units(&self) -> u32 {
self.snapshot().units
}
}
/// One completed unit's uncommitted fragments. Failed attempts produce no receipt.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum WrittenPartition {
VectorDedup(WrittenDedupUnit),
DuplicatePairs(WrittenPairsPartition),
Grouped(WrittenUnit),
}
impl WrittenPartition {
/// Number of pair or grouped rows staged by this unit.
pub fn rows(&self) -> u64 {
match self {
Self::VectorDedup(unit) => unit.rows,
Self::DuplicatePairs(unit) => unit.rows,
Self::Grouped(unit) => unit.rows(),
}
}
/// Uncommitted output files, retained only after task completion.
pub fn fragments(&self) -> &[lance_table::format::Fragment] {
match self {
Self::VectorDedup(unit) => &unit.fragments,
Self::DuplicatePairs(unit) => &unit.fragments,
Self::Grouped(unit) => unit.fragments(),
}
}
}
/// Plan native pair or indexed grouped work; return `None` for ordinary views.
/// `pinned`, when supplied, must match a native view's declared source version.
pub async fn plan_partitioned_refresh(
view: &Table,
pinned: Option<u64>,
) -> Result<Option<PartitionedRefreshPlan>> {
if let Some(StoredDefinition::Query(definition)) =
super::read_definition(view.schema().await?.metadata())?
&& definition.vector_source.is_some()
{
if definition
.vector_source
.as_ref()
.is_some_and(|s| s.kind == VectorSourceKind::Dedup)
{
return vector_dedup::plan_refresh(view, pinned)
.await
.map(|p| Some(PartitionedRefreshPlan::VectorDedup(p)));
}
return duplicate_pairs::plan_refresh(view, pinned)
.await
.map(|p| Some(PartitionedRefreshPlan::DuplicatePairs(p)));
}
super::plan_grouped_refresh(view, pinned)
.await
.map(|plan| plan.map(PartitionedRefreshPlan::Grouped))
}
/// Stream one unit into uncommitted fragments. Persist the returned receipt only
/// on success. A failed or interrupted call must be retried from the unit's start.
pub async fn write_refresh_partition(
view: &Table,
unit: u32,
plan: &PartitionedRefreshPlan,
) -> Result<WrittenPartition> {
write_refresh_partition_with_inputs(view, unit, plan, &[]).await
}
/// Write a unit after its durable selection dependencies have completed.
/// Supply exactly the first `plan.dependency_units()` receipts for a dependent
/// unit. A missing, duplicate or foreign receipt fails before output is staged.
pub async fn write_refresh_partition_with_inputs(
view: &Table,
unit: u32,
plan: &PartitionedRefreshPlan,
dependencies: &[WrittenPartition],
) -> Result<WrittenPartition> {
match plan {
PartitionedRefreshPlan::VectorDedup(plan) => {
let dependencies = dependencies
.iter()
.map(|unit| match unit {
WrittenPartition::VectorDedup(unit) => Ok(unit.clone()),
_ => Err(Error::InvalidInput {
message: "foreign dedup dependency".into(),
}),
})
.collect::<Result<Vec<_>>>()?;
vector_dedup::write_unit(view, unit, plan, &dependencies)
.await
.map(WrittenPartition::VectorDedup)
}
PartitionedRefreshPlan::DuplicatePairs(plan) => {
duplicate_pairs::write_unit(view, unit, plan)
.await
.map(WrittenPartition::DuplicatePairs)
}
PartitionedRefreshPlan::Grouped(plan) => super::write_grouped_unit(view, unit, plan)
.await
.map(WrittenPartition::Grouped),
}
}
/// Publish a complete set of unit receipts through the common MV commit sink.
/// `expected` fences the destination incarnation. No partial set is publishable.
pub async fn commit_partitioned_refresh(
view: &Table,
plan: &PartitionedRefreshPlan,
units: Vec<WrittenPartition>,
expected: Option<&str>,
) -> Result<RefreshMaterializedViewResult> {
let foreign = || Error::InvalidInput {
message: "refresh receipt does not match the plan kind".into(),
};
match plan {
PartitionedRefreshPlan::VectorDedup(plan) => {
let units = units
.into_iter()
.map(|unit| match unit {
WrittenPartition::VectorDedup(unit) => Ok(unit),
_ => Err(foreign()),
})
.collect::<Result<Vec<_>>>()?;
vector_dedup::commit(view, plan, units, expected).await
}
PartitionedRefreshPlan::DuplicatePairs(plan) => {
let units = units
.into_iter()
.map(|unit| match unit {
WrittenPartition::DuplicatePairs(unit) => Ok(unit),
_ => Err(foreign()),
})
.collect::<Result<Vec<_>>>()?;
duplicate_pairs::commit(view, plan, units, expected).await
}
PartitionedRefreshPlan::Grouped(plan) => {
let units = units
.into_iter()
.map(|unit| match unit {
WrittenPartition::Grouped(unit) => Ok(unit),
_ => Err(foreign()),
})
.collect::<Result<Vec<_>>>()?;
super::commit_grouped_refresh(view, plan, units, expected).await
}
}
}
rust/lancedb/src/materialized_view/query.rs
+84 -1
View File
@@ -264,7 +264,36 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
};
let mut from = select.from.iter();
let mut vector_source = None;
let (source_namespace, source_table) = match from.next().map(|f| &f.relation) {
Some(TableFactor::Table {
name,
args: Some(args),
alias: None,
with_hints,
version: None,
with_ordinality: false,
partitions,
json_path: None,
sample: None,
index_hints,
..
}) if matches!(
name.to_string().as_str(),
super::duplicate_pairs::FUNCTION_NAME | super::duplicate_pairs::DEDUP_FUNCTION_NAME
) && with_hints.is_empty()
&& partitions.is_empty()
&& index_hints.is_empty() =>
{
let kind = if name.to_string() == super::duplicate_pairs::FUNCTION_NAME {
super::VectorSourceKind::Pairs
} else {
super::VectorSourceKind::Dedup
};
let (namespace, table, config) = super::duplicate_pairs::source(args, kind)?;
vector_source = Some(Box::new(config));
(namespace, table)
}
Some(TableFactor::Table { args: Some(_), .. }) => {
return Err(invalid(
"a view reads a table; a Function in FROM position follows it: \
@@ -416,6 +445,7 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
};
let definition = MaterializedViewDefinition {
vector_source,
source_table,
source_namespace,
lateral,
@@ -424,6 +454,9 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
group_by,
limit,
};
if definition.vector_source.is_some() {
super::duplicate_pairs::check_shape(&definition)?;
}
check_grouping(&definition)?;
Ok(definition)
}
@@ -483,7 +516,18 @@ pub fn render(definition: &MaterializedViewDefinition) -> String {
.chain(std::iter::once(&definition.source_table))
.map(|part| ident_sql(part))
.collect();
sql.push_str(&table.join("."));
if let Some(native) = &definition.vector_source {
sql.push_str(&format!(
"{}('{}', {}, '{}', {})",
native.function_name(),
table.join(".").replace('\'', "''"),
native.dataset_version,
native.column.replace('\'', "''"),
native.distance_threshold
));
} else {
sql.push_str(&table.join("."));
}
if let Some(lateral) = &definition.lateral {
match &lateral.source {
LateralSource::Unnest { column } => {
@@ -511,6 +555,45 @@ pub fn render(definition: &MaterializedViewDefinition) -> String {
mod tests {
use super::*;
#[test]
fn native_pairs_are_a_table_source_with_literal_snapshot_arguments() {
for threshold in ["4", "4.0", "-1", "0.05"] {
let sql =
format!("SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', {threshold})");
let definition = parse(&sql).unwrap();
assert_eq!(definition.source_table, "images");
assert!(definition.lateral.is_none());
assert_eq!(
definition.vector_source.as_ref().unwrap().dataset_version,
3
);
assert_eq!(parse(&render(&definition)).unwrap(), definition);
let metadata = std::collections::HashMap::from([(
super::super::DEFINITION_META_KEY.to_string(),
definition.to_json().unwrap(),
)]);
assert_eq!(
super::super::read_definition(&metadata).unwrap(),
Some(super::super::StoredDefinition::Query(definition))
);
}
for sql in [
"SELECT row_id_a FROM vector_duplicate_pairs('images', 3, 'phash', 4)",
"SELECT * FROM vector_duplicate_pairs('images', 0, 'phash', 4)",
"SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 1e100)",
"SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', cutoff)",
"SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) LIMIT 1",
"SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) WHERE distance < 2",
"SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) AS p",
"SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4), f(phash) AS p",
] {
assert!(
parse(sql).is_err(),
"accepted unsupported native view: {sql}"
);
}
}
#[test]
fn a_query_round_trips_through_its_canonical_form() {
for (sql, canonical) in [

Some files were not shown because too many files have changed in this diff Show More