Compare commits

..

5 Commits

Author SHA1 Message Date
Xuanwo 8f843a8469 Merge branch 'main' into gatekeeper/fix-2085-1 2026-08-26 05:09:04 +08:00
Gatefixer 1f4eea1f17 Merge origin/main into gatekeeper/fix-2085-1 2026-08-08 20:10:35 +00:00
Gatefixer 4ba24bf64b test(rust): detect indexed delete compilation regressions 2026-08-08 12:35:32 +00:00
Gatefixer 28b365fc62 Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2085-1 2026-08-08 12:12:43 +00:00
Gatefixer 267577989b test(rust): cover large indexed deletes 2026-08-05 23:26:26 +00:00
115 changed files with 12726 additions and 10558 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0"
current_version = "0.38.0-beta.10"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
-24
View File
@@ -44,27 +44,3 @@ updates:
python-deps:
patterns:
- "*"
# The npm ecosystem covers pnpm lockfiles. There are two separate installs:
# the bindings themselves and the examples, which have their own lockfile.
# As with cargo and pip above, only bump the lockfile — the version ranges
# in package.json are our consumers' constraints, not ours.
- package-ecosystem: npm
directory: /nodejs
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-deps:
patterns:
- "*"
- package-ecosystem: npm
directory: /nodejs/examples
schedule:
interval: weekly
versioning-strategy: lockfile-only
groups:
nodejs-examples-deps:
patterns:
- "*"
+4 -10
View File
@@ -29,14 +29,12 @@ jobs:
steps:
- uses: actions/setup-node@v6
with:
node-version: "24"
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
node-version: "18"
# These rules are disabled because Github will always ensure there
# is a blank line between the title and the body and Github will
# word wrap the description field to ensure a reasonable max line
# length.
- run: npm install @commitlint/config-conventional
- run: >
echo 'module.exports = {
"rules": {
@@ -45,11 +43,7 @@ jobs:
"body-leading-blank": [0, "always"]
}
}' > .commitlintrc.js
- run: >
pnpm dlx
--package @commitlint/cli@21.2.2
--package @commitlint/config-conventional@21.2.2
commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
- run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
env:
COMMIT_MSG: >
${{ github.event.pull_request.title }}
@@ -60,7 +54,7 @@ jobs:
with:
script: |
const message = `**ACTION NEEDED**
Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.
The PR title and description are used as the merge commit message.\
+1 -1
View File
@@ -56,7 +56,7 @@ jobs:
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
with:
# Restricted to http(s) on purpose. Much of docs/src is generated
# API reference (the js/ tree comes from `pnpm run docs` in nodejs)
# API reference (the js/ tree comes from `npm run docs` in nodejs)
# and the hand-written pages use mkdocstrings cross-references and
# nav-relative paths that only resolve in the site mkdocs builds,
# not in this checkout, so relative links would be reported as
+3 -1
View File
@@ -55,7 +55,9 @@ jobs:
- name: Set up node
uses: actions/setup-node@v6
with:
node-version: 24
node-version: 20
cache: 'npm'
cache-dependency-path: docs/package-lock.json
- name: Install node dependencies
working-directory: nodejs
run: |
+13 -11
View File
@@ -47,8 +47,9 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# Build on a supported LTS; the matrix job below covers every
# Node version the library claims to support.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. The library itself still supports Node >= 18
# (see test matrix below).
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -83,7 +84,7 @@ jobs:
timeout-minutes: 30
strategy:
matrix:
node-version: [ "22", "24", "26" ]
node-version: [ "18", "20" ]
runs-on: "ubuntu-22.04"
defaults:
run:
@@ -100,9 +101,9 @@ jobs:
- uses: actions/setup-node@v6
name: Setup Node.js 24 for build
with:
# Build and install once on a fixed version so the generated docs
# are identical across matrix legs; the tests below then run on each
# supported Node version.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October. Build/install runs on Node 24; tests run on the
# matrix version below using direct jest invocation.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -151,9 +152,9 @@ jobs:
S3_TEST: "1"
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the build-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
- name: Test examples
working-directory: ./
env:
@@ -163,7 +164,7 @@ jobs:
run: |
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
npx jest --testEnvironment jest-environment-node-single-context --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -184,7 +185,8 @@ jobs:
version: 11.1.1
- uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
+36 -81
View File
@@ -40,31 +40,40 @@ jobs:
- target: aarch64-apple-darwin
host: macos-latest
features: fp16kernels
# Fat LTO was ~111 of this job's ~113 minutes.
lto: thin
codegen_units: 16
pre_build: |-
brew install protobuf
# Fat LTO (the workspace default in .cargo/config.toml) is
# single-threaded and is the peak-memory step of the build. On
# this runner it accounted for ~111 of the job's ~113 minutes,
# making it the critical path of the entire publish pipeline.
# ThinLTO parallelizes it across the runner's cores, for a few
# percent of runtime performance.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-pc-windows-msvc
host: windows-2025
features: ","
# The lower peak also keeps this on the standard 4-core runner.
lto: thin
codegen_units: 16
pre_build: |-
choco install --no-progress protoc ninja nasm
tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
# There is an issue where choco doesn't add nasm to the path
export PATH="$PATH:/c/Program Files/NASM"
nasm -v
# See the ThinLTO note on aarch64-apple-darwin above. Keeping
# peak memory down is also what lets this run on the standard
# 4-core runner: the 8-core larger runner was only needed to
# stop fat LTO from OOMing rustc-LLVM.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: aarch64-pc-windows-msvc
host: windows-2025
features: ","
lto: thin
codegen_units: 16
pre_build: |-
choco install --no-progress protoc
rustup target add aarch64-pc-windows-msvc
# See the ThinLTO note on aarch64-apple-darwin above.
export CARGO_PROFILE_RELEASE_LTO=thin
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
- target: x86_64-unknown-linux-gnu
host: ubuntu-latest
features: fp16kernels
@@ -94,14 +103,6 @@ jobs:
# https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
features: "fp16kernels"
# Fat LTO OOM-killed rustc every nightly; even with lld it peaked
# at 31391 MiB of the runner's 32 GiB.
lto: thin
codegen_units: 16
# arm64 Linux links through GNU `ld` where x86_64 defaults to
# `rust-lld`, which is why only arm64 OOM'd. lld cut the largest
# linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
linker: /tmp/aarch64-lld-clang
pre_build: |-
set -e &&
apt-get update &&
@@ -111,30 +112,9 @@ jobs:
# AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
rustup target add aarch64-unknown-linux-gnu
# Not `&&`-chained: in dash, errexit does not fire for a
# non-final command in an `&&` list, so failures were ignored.
#
# A wrapper rather than `-C link-arg` because the per-target
# rustflags variable does not reach every unit that links, while
# the linker variable does. `clang` because GCC silently ignores
# `-fuse-ld=lld` unless built with lld support. Two echoes
# because printf's newline escape gets rewritten to `;` between
# here and the container.
echo '#!/bin/sh' > /tmp/aarch64-lld-clang
echo 'exec clang --target=aarch64-unknown-linux-gnu --sysroot=/usr/aarch64-unknown-linux-gnu/aarch64-unknown-linux-gnu/sysroot --gcc-toolchain=/usr/aarch64-unknown-linux-gnu -fuse-ld=lld "$@"' >> /tmp/aarch64-lld-clang
chmod 0755 /tmp/aarch64-lld-clang
# Fail now, not at the cdylib link ~30 minutes later. Linking at
# all also proves lld resolved; clang errors out when it cannot.
echo 'int main(void){return 0;}' > /tmp/probe.c
/tmp/aarch64-lld-clang /tmp/probe.c -o /tmp/probe
readelf -h /tmp/probe | grep AArch64
- target: aarch64-unknown-linux-musl
host: ubuntu-2404-8x-x64
features: ","
# Fat LTO took the whole runner down. lld cannot help: it died
# inside rustc's LLVM, before any linker was spawned.
lto: thin
codegen_units: 16
pre_build: |-
set -e &&
sudo apt-get update &&
@@ -143,19 +123,6 @@ jobs:
export EXTRA_ARGS="-x"
name: build - ${{ matrix.settings.target }}
runs-on: ${{ matrix.settings.host }}
# On the job, not exported from `pre_build`: `Swatinem/rust-cache` hashes
# `CARGO_*` into its cache key before any step runs, so a step-local export
# leaves the key unchanged while cargo still rebuilds cold. The ThinLTO
# legs had been doing that every run.
#
# Not `RUSTFLAGS`: setting it, even to "", discards every config-file
# rustflag, silently dropping .cargo/config.toml's `target-cpu` and
# `target-feature` from the published binaries.
env:
CARGO_PROFILE_RELEASE_LTO: ${{ matrix.settings.lto || 'fat' }}
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: ${{ matrix.settings.codegen_units || '1' }}
# Empty elsewhere: a per-target variable is only read for that triple.
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: ${{ matrix.settings.linker }}
defaults:
run:
working-directory: nodejs
@@ -168,7 +135,8 @@ jobs:
- name: Setup node
uses: actions/setup-node@v6
with:
# pnpm 11 requires Node >= 22.13.
# pnpm 11 requires Node >= 22.13; use 24 since 22 hits EOL
# in October.
node-version: 24
cache: pnpm
cache-dependency-path: nodejs/pnpm-lock.yaml
@@ -201,15 +169,19 @@ jobs:
# creating ref). The nightly cadence also keeps entries inside
# GitHub's 7-day eviction window, which a tag-only trigger would not.
save-if: ${{ github.ref == 'refs/heads/main' }}
# Docker builds can use rust-cache too: the workspace is bind-mounted, so
# `target/` lives on the host and rust-cache's prune keeps the entry
# small.
# Docker builds can use rust-cache too. `target/` already lives on the
# host because the whole workspace is bind-mounted into the container, and
# rust-cache's prune and save run host-side, so they can manage it -- which
# is what keeps the entry to dependency artifacts rather than a multi-GB
# copy of everything.
#
# Two differences from the native builds. The container's CARGO_HOME is
# bind-mounted from `.cargo-cache` rather than ~/.cargo, so that is cached
# explicitly. And the key uses the *host* rustc version, not the compiler
# that built these artifacts -- safe, since cargo fingerprints the real
# one; a base-image bump just costs one cold build.
# bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that
# has to be cached explicitly. And the key is derived from the *host* rustc
# version, which is not the compiler that produced these artifacts; that is
# safe because cargo fingerprints the real compiler and rebuilds on a
# mismatch, it just means a base-image toolchain bump costs one cold build
# instead of invalidating the key.
- name: Cache cargo (docker builds)
uses: Swatinem/rust-cache@v2
if: ${{ matrix.settings.docker }}
@@ -238,19 +210,14 @@ jobs:
# cache step above saves. Previously the registry mounts pointed at
# `.cargo/...`, a path nothing cached, so the container re-downloaded
# the whole crate registry on every run.
#
# `docker run` inherits nothing; `-e NAME` carries the job's `env:` in.
options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
-v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \
-v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \
-e CARGO_PROFILE_RELEASE_LTO \
-e CARGO_PROFILE_RELEASE_CODEGEN_UNITS \
-e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER \
-v ${{ github.workspace }}:/build -w /build/nodejs"
run: |
set -e
${{ matrix.settings.pre_build }}
node_modules/.bin/napi build --platform --release \
npx napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -270,7 +237,7 @@ jobs:
- name: Build
run: |
${{ matrix.settings.pre_build }}
node_modules/.bin/napi build --platform --release \
npx napi build --platform --release \
--features ${{ matrix.settings.features }} \
--target ${{ matrix.settings.target }} \
--dts ../lancedb/native.d.ts \
@@ -289,18 +256,6 @@ jobs:
if: always()
run: df -h
shell: bash
- name: Report peak memory
if: always() && runner.os == 'Linux'
shell: bash
run: |
peak=$(find /sys/fs/cgroup -name memory.peak -readable \
-exec cat {} + 2>/dev/null | sort -n | tail -1)
if [ -n "$peak" ]; then
echo "peak memory: $((peak / 1024 / 1024)) MiB"
else
echo "peak memory: unavailable (no readable cgroup v2 memory.peak)"
fi
free -g || true
- name: Upload artifact
uses: actions/upload-artifact@v7
with:
@@ -338,7 +293,7 @@ jobs:
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-arm64
node:
- '22'
- '20'
runs-on: ${{ matrix.settings.host }}
defaults:
run:
@@ -384,9 +339,9 @@ jobs:
- name: Move built files
run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
- name: Test bindings
# Invoke the installed jest binary directly; the pnpm shim is set up
# against the install-phase Node, not the version selected above.
run: node_modules/.bin/jest --verbose
# Invoke jest directly because pnpm 11 itself requires Node 22+
# while the matrix tests on older Node versions.
run: npx jest --verbose
publish:
name: Publish
runs-on: ubuntu-latest
@@ -0,0 +1,22 @@
name: Update package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
@@ -0,0 +1,22 @@
name: Update NodeJs package-lock.json
on:
workflow_dispatch:
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: main
persist-credentials: false
fetch-depth: 0
lfs: true
- uses: ./.github/workflows/update_package_lock_nodejs
with:
github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
+1 -4
View File
@@ -20,10 +20,7 @@ repos:
hooks:
- id: local-biome-check
name: biome check
# Use the biome from nodejs/package.json rather than a separately
# pinned one: the two drifted apart and disagreed on formatting, so
# this hook rejected code that `pnpm lint` accepted.
entry: nodejs/node_modules/.bin/biome check --config-path nodejs/biome.json nodejs/
entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
language: system
types: [text]
files: "nodejs/.*"
+3 -3
View File
@@ -38,7 +38,7 @@ Before committing changes, run formatting for every language you touched. At min
* Rust changes: run `cargo fmt --all`.
* Python changes: run `ruff format .` and `ruff check .` from the repository root,
and run targeted tests through `cd python && uv run ...`.
* TypeScript changes: run the relevant `pnpm` lint, format, build, and docs commands in `nodejs`.
* TypeScript changes: run the relevant `npm`/`pnpm` lint, format, build, and docs commands in `nodejs`.
Before creating a PR, the exact value passed to `gh pr create --title` must follow
Conventional Commits, such as `fix: support nested field paths in native index creation`
@@ -101,12 +101,12 @@ Python bindings changes:
TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `pnpm build` to generate TypeScript definitions.
2. Run `npm run build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `pnpm run docs` to generate TypeScript documentation.
6. Run `npm run docs` to generate TypeScript documentation.
## Python API reference
Generated
+71 -73
View File
@@ -535,9 +535,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.92"
version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
@@ -1443,9 +1443,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
[[package]]
name = "bytemuck"
version = "1.25.2"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
dependencies = [
"bytemuck_derive",
]
@@ -1597,9 +1597,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chacha20"
version = "0.10.2"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if 1.0.4",
"cpufeatures 0.3.0",
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"proc-macro2",
"quote",
@@ -5041,8 +5041,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -5172,8 +5172,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5236,8 +5236,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5251,8 +5251,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"async-trait",
@@ -5264,8 +5264,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5318,8 +5318,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5333,8 +5333,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5374,8 +5374,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5388,8 +5388,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5402,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0"
version = "0.38.0-beta.10"
dependencies = [
"ahash",
"anyhow",
@@ -5490,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0"
version = "0.38.0-beta.10"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5515,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0"
version = "0.38.0-beta.10"
dependencies = [
"arrow",
"async-trait",
@@ -5748,9 +5748,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.34"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "loom"
@@ -6001,9 +6001,9 @@ dependencies = [
[[package]]
name = "moka"
version = "0.12.16"
version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
dependencies = [
"async-lock",
"crossbeam-channel",
@@ -6097,15 +6097,14 @@ dependencies = [
[[package]]
name = "napi"
version = "3.12.2"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
dependencies = [
"bitflags 2.11.1",
"chrono",
"ctor 1.0.12",
"futures",
"libc",
"napi-build",
"napi-sys",
"nohash-hasher",
@@ -6117,15 +6116,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
[[package]]
name = "napi-derive"
version = "3.6.3"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
dependencies = [
"convert_case",
"ctor 1.0.12",
@@ -6137,9 +6136,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.2"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
dependencies = [
"convert_case",
"proc-macro2",
@@ -8602,9 +8601,9 @@ dependencies = [
[[package]]
name = "roaring"
version = "0.11.5"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
dependencies = [
"bytemuck",
"byteorder",
@@ -9064,9 +9063,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
dependencies = [
"base64 0.22.1",
"bs58",
@@ -9074,7 +9073,6 @@ dependencies = [
"hex",
"indexmap 1.9.3",
"indexmap 2.14.0",
"jiff",
"schemars 0.9.0",
"schemars 1.2.1",
"serde_core",
@@ -9085,9 +9083,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
dependencies = [
"darling 0.23.0",
"proc-macro2",
@@ -10454,9 +10452,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.26.0"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
dependencies = [
"getrandom 0.4.2",
"js-sys",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -5,5 +5,5 @@ licenses:
cd python && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd python && uv sync --all-extras && uv tool run pip-licenses --python .venv/bin/python --format=markdown --with-urls --output-file=PYTHON_THIRD_PARTY_LICENSES.md
cd nodejs && cargo about generate ../about.hbs -o RUST_THIRD_PARTY_LICENSES.html -c ../about.toml
cd nodejs && pnpm dlx license-checker@25 --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd nodejs && npx license-checker --markdown --out NODEJS_THIRD_PARTY_LICENSES.md
cd java && ./mvnw license:aggregate-add-third-party -q
+6 -2
View File
@@ -12,12 +12,16 @@ done
# This updates the lockfile without building
cargo metadata --quiet > /dev/null
pushd nodejs || exit 1
npm install --package-lock-only --silent
popd
if git diff --quiet --exit-code; then
echo "No lockfile changes to commit; skipping amend."
elif $AMEND; then
git add Cargo.lock
git add Cargo.lock nodejs/package-lock.json
git commit --amend --no-edit
else
git add Cargo.lock
git add Cargo.lock nodejs/package-lock.json
git commit -m "Update lockfiles"
fi
+11 -1
View File
@@ -131,13 +131,18 @@ allow = [
"BSD-3-Clause",
"ISC",
"Unicode-3.0",
"Unicode-DFS-2016",
"Zlib",
"CC0-1.0",
"MPL-2.0",
"BSL-1.0",
"OpenSSL",
# 0BSD ("BSD Zero Clause") is effectively public domain — no attribution
# required. Pulled in by `mock_instant`.
"0BSD",
# bzip2-1.0.6 is the permissive upstream bzip2 license (BSD-like). Pulled
# in by `libbz2-rs-sys`, the pure-Rust bzip2 implementation.
"bzip2-1.0.6",
# CDLA-Permissive-2.0 is a permissive data license used by `webpki-roots`
# for the Mozilla CA root bundle. Data-only, distribution-compatible.
"CDLA-Permissive-2.0",
@@ -145,7 +150,12 @@ allow = [
confidence-threshold = 0.8
# Per-crate license exceptions: allow a license for a specific crate only,
# rather than globally via the `allow` list above.
exceptions = []
exceptions = [
# CDDL-1.0 (copyleft) is pulled in only as a dev/profiling dependency via
# `inferno` -> `pprof` -> `lance-testing`; it is a test dependency that we
# do not distribute, so scope the allowance to `inferno` alone.
{ allow = ["CDDL-1.0"], crate = "inferno" },
]
# Crates whose license cannot be determined from Cargo metadata but whose
# license we've manually confirmed from upstream. Keep this list minimal.
[[licenses.clarify]]
+8 -11
View File
@@ -47,24 +47,22 @@ pytest -vv python/tests/docs
### Checking typescript examples
The examples depend on `@lancedb/lancedb` at `file:../dist`, so the package must be
built before running the tests. This uses pnpm; see the
[Typescript contributing guide](../nodejs/CONTRIBUTING.md) for the toolchain setup.
The `@lancedb/lancedb` package must be built before running the tests:
```shell
pushd nodejs
pnpm install
pnpm build
npm ci
npm run build
popd
```
Then you can run the examples by going to the `nodejs/examples` directory, which is a
separate pnpm package with its own lockfile:
Then you can run the examples by going to the `nodejs/examples` directory and
running the tests like a normal npm package:
```shell
pushd nodejs/examples
pnpm install
pnpm test
npm ci
npm test
popd
```
@@ -86,7 +84,6 @@ The new files should be checked into the repository.
```shell
pushd nodejs
# `pnpm docs` would invoke pnpm's built-in `docs` command, not the script.
pnpm run docs
npm run docs
popd
```
+135
View File
@@ -0,0 +1,135 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "lancedb-docs-test",
"version": "1.0.0",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
},
"../node": {
"name": "vectordb",
"version": "0.21.2-beta.0",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux",
"win32"
],
"dependencies": {
"@neon-rs/load": "^0.0.74",
"axios": "^1.4.0"
},
"devDependencies": {
"@neon-rs/cli": "^0.0.160",
"@types/chai": "^4.3.4",
"@types/chai-as-promised": "^7.1.5",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
"@types/sinon": "^10.0.15",
"@types/temp": "^0.9.1",
"@types/uuid": "^9.0.3",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"apache-arrow-old": "npm:apache-arrow@13.0.0",
"cargo-cp-artifact": "^0.1",
"chai": "^4.3.7",
"chai-as-promised": "^7.1.1",
"eslint": "^8.39.0",
"eslint-config-standard-with-typescript": "^34.0.1",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.2.0",
"openai": "^4.24.1",
"sinon": "^15.1.0",
"temp": "^0.9.4",
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typedoc": "^0.24.7",
"typedoc-plugin-markdown": "^3.15.3",
"typescript": "^5.1.0",
"uuid": "^9.0.0"
},
"optionalDependencies": {
"@lancedb/vectordb-darwin-arm64": "0.21.2-beta.0",
"@lancedb/vectordb-darwin-x64": "0.21.2-beta.0",
"@lancedb/vectordb-linux-arm64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-linux-x64-gnu": "0.21.2-beta.0",
"@lancedb/vectordb-win32-x64-msvc": "0.21.2-beta.0"
},
"peerDependencies": {
"@apache-arrow/ts": "^14.0.2",
"apache-arrow": "^14.0.2"
}
},
"../node/node_modules/apache-arrow": {
"version": "14.0.2",
"license": "Apache-2.0",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "20.3.0",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "7.0.1",
"flatbuffers": "23.5.26",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.3"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/@types/node": {
"version": "20.11.8",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.8.tgz",
"integrity": "sha512-i7omyekpPTNdv4Jb/Rgqg0RU8YqLcNsI12quKSDkRXNfx7Wxdm6HhK1awT3xTgEkgxPn3bvnSpiEAc7a7Lpyow==",
"dev": true,
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/apache-arrow": {
"resolved": "../node/node_modules/apache-arrow",
"link": true
},
"node_modules/typescript": {
"version": "5.3.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz",
"integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==",
"dev": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
"node_modules/vectordb": {
"resolved": "../node",
"link": true
}
}
}
+20
View File
@@ -0,0 +1,20 @@
{
"name": "lancedb-docs-test",
"version": "1.0.0",
"description": "auto-generated tests from doc",
"author": "dev@lancedb.com",
"license": "Apache 2",
"dependencies": {
"apache-arrow": "file:../node/node_modules/apache-arrow",
"vectordb": "file:../node"
},
"scripts": {
"build": "tsc -b && cd ../node && npm run build-release",
"example": "npm run build && node",
"test": "npm run build && ls dist/*.js | xargs -n 1 node"
},
"devDependencies": {
"@types/node": "^20.11.8",
"typescript": "^5.3.3"
}
}
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0</version>
<version>0.38.0-beta.10</version>
</dependency>
```
-12
View File
@@ -1292,18 +1292,6 @@ abstract updateFieldMetadata(updates): Promise<UpdateFieldMetadataResult>
Update per-field (column) metadata.
The following keys are treated specially, by convention, and should be
used when appropriate:
- `lancedb:description`: for a human-readable description of a field.
- `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
names the tag category; e.g. `lancedb:tag:model: "clip"`.
- `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
`feature_v2` might be in the same logical column.
- `lancedb:status`: for status options (`production`, `candidate`,
`deprecated`, `archived`) to designate the current life cycle state of
this column.
#### Parameters
* **updates**: [`FieldMetadataUpdate`](../interfaces/FieldMetadataUpdate.md)[]
@@ -17,8 +17,7 @@ metadata: Record<string, null | string>;
```
Metadata key/value pairs. Merged into the field's existing metadata by
default; a value of `null` deletes that key. See
[Table.updateFieldMetadata](../classes/Table.md#updatefieldmetadata) for the conventional `lancedb:*` keys.
default; a value of `null` deletes that key.
***
+2 -8
View File
@@ -159,8 +159,6 @@ and combined with [BooleanQuery][lancedb.query.BooleanQuery].
::: lancedb.query.FullTextOperator
::: lancedb.query.DocumentGranularity
::: lancedb.query.Occur
## Embeddings
@@ -223,14 +221,10 @@ tokens = list(
Blob columns store large binary values out of line so they can be read lazily
instead of being materialized with the rest of the row.
`lancedb.BlobType` is `lance.blob.BlobType` when pylance is installed. Without
pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
still work. Queries return descriptors. Call
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
::: lancedb.blob
::: lancedb.BlobType
::: lancedb._blob.BlobFile
options:
show_root_full_path: false
+17
View File
@@ -0,0 +1,17 @@
{
"include": [
"src/*.ts",
],
"compilerOptions": {
"target": "es2022",
"module": "nodenext",
"declaration": true,
"outDir": "./dist",
"strict": true,
"allowJs": true,
"resolveJsonModule": true,
},
"exclude": [
"./dist/*",
]
}
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-final.0</version>
<version>0.38.0-beta.10</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-final.0</version>
<version>0.38.0-beta.10</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.5</lance-core.version>
<lance-core.version>11.0.0-beta.22</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0"
version = "0.38.0-beta.10"
publish = false
license.workspace = true
description.workspace = true
-58
View File
@@ -1,16 +1,11 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as fs from "node:fs";
import * as vm from "node:vm";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
Field as CurrentField,
LargeBinary as CurrentLargeBinary,
Schema as CurrentSchema,
Vector as CurrentVector,
convertToTable,
tableFromIPC as currentTableFromIPC,
@@ -41,59 +36,6 @@ function sampleRecords(): Array<Record<string, any>> {
},
];
}
it("serializes an Arrow Table created in another JavaScript realm", async () => {
const context = vm.createContext({
TextDecoder,
TextEncoder,
console,
setTimeout,
clearTimeout,
});
vm.runInContext(
fs.readFileSync(
require.resolve("apache-arrow-15/Arrow.es2015.min"),
"utf8",
),
context,
);
const foreignTable: unknown = vm.runInContext(
"Arrow.tableFromArrays({ id: new Int32Array([1, 2, 3]), text: ['foo', 'bar', 'baz'] })",
context,
);
const foreignMetadata = (
foreignTable as { schema: { metadata: Map<string, string> } }
).schema.metadata;
expect(foreignMetadata).not.toBeInstanceOf(Map);
const buf = await fromDataToBuffer(
foreignTable as Parameters<typeof fromDataToBuffer>[0],
);
const actual = currentTableFromIPC(buf);
expect(actual.numRows).toBe(3);
expect(actual.getChild("id")?.toJSON()).toEqual([1, 2, 3]);
expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar", "baz"]);
});
it("preserves field metadata from a provided schema", async function () {
const jsonMetadata = new Map([["ARROW:extension:name", "lance.json"]]);
const schema = new CurrentSchema([
new CurrentField("meta", new CurrentLargeBinary(), true, jsonMetadata),
]);
const table = makeArrowTable(
[{ meta: Buffer.from(JSON.stringify({ source: "test" })) }],
{ schema },
);
expect(table.schema.fields[0].metadata).toEqual(jsonMetadata);
const roundTripped = currentTableFromIPC(await fromTableToBuffer(table));
expect(roundTripped.schema.fields[0].metadata).toEqual(jsonMetadata);
});
describe.each([arrow15, arrow16, arrow17, arrow18])(
"Arrow",
(
-52
View File
@@ -187,58 +187,6 @@ describe("embedding functions", () => {
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
expect(vector0).toEqual([1, 2, 3]);
});
it("should append multiple Python embeddings with the same alias", async () => {
@register("python-mock")
// biome-ignore lint/correctness/noUnusedVariables: the decorator registers this class
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return data.map((value) =>
value === "hello world" ? [1, 2, 3] : [4, 5, 6],
);
}
}
const metadata = new Map([
[
"embedding_functions",
'[{"source_column":"text1","vector_column":"vector1","name":"python-mock","model":{}},{"source_column":"text2","vector_column":"vector2","name":"python-mock","model":{}}]',
],
]);
const schema = new Schema(
[
new Field("text1", new Utf8(), true),
new Field("text2", new Utf8(), true),
new Field(
"vector1",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
new Field(
"vector2",
new FixedSizeList(3, new Field("item", new Float32(), true)),
true,
),
],
metadata,
);
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test", schema);
await table.add([{ text1: "hello world", text2: "goodbye world" }]);
const rows = await table.query().toArray();
expect(JSON.parse(JSON.stringify(rows[0].vector1))).toEqual([1, 2, 3]);
expect(JSON.parse(JSON.stringify(rows[0].vector2))).toEqual([4, 5, 6]);
});
it("should append generated vectors to a non-nullable schema", async () => {
@register("non_nullable_schema_test")
+2 -2
View File
@@ -5,8 +5,8 @@ import packageJson = require("../package.json");
describe("package metadata", () => {
it("requires Node.js type declarations compatible with the runtime", () => {
expect(packageJson.engines.node).toBe(">= 22");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=22");
expect(packageJson.engines.node).toBe(">= 18");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
optional: true,
});
+2 -9
View File
@@ -3,7 +3,6 @@
import * as http from "http";
import { RequestListener } from "http";
import packageJson = require("../package.json");
import {
ClientConfig,
Connection,
@@ -71,13 +70,7 @@ async function withMockDatabase(
try {
await callback(db);
} finally {
// `close()` alone leaves the port bound until keep-alive sockets drain, so
// a single failing test would cascade into EADDRINUSE for every test after
// it. Destroy the connections and wait for the port to actually be free.
await new Promise<void>((resolve) => {
server.closeAllConnections();
server.close(() => resolve());
});
server.close();
}
}
@@ -138,7 +131,7 @@ describe("remote connection", () => {
(req, res) => {
expect(req.headers["x-api-key"]).toEqual("fake");
expect(req.headers["user-agent"]).toEqual(
`LanceDB-Node-Client/${packageJson.version}`,
`LanceDB-Node-Client/${process.env.npm_package_version}`,
);
const body = JSON.stringify({ tables: [] });
+1 -413
View File
@@ -685,56 +685,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
},
);
// https://github.com/lancedb/lancedb/issues/1963
it("should query documents with LangChain PDF metadata", async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
try {
const db = await connect(tmpDir.name);
const documents = [
{
text: "first page",
vector: [1, 0],
source: "first.pdf",
loc: { pageNumber: 1, lines: { from: 1, to: 12 } },
pdf: {
version: "1.10.100",
info: {
format: "PDF 1.7",
producer: "pdf.js",
creator: "Writer",
},
totalPages: 2,
},
},
{
text: "second page",
vector: [0, 1],
source: "second.pdf",
loc: { pageNumber: 2, lines: { from: 13, to: 24 } },
pdf: {
version: "1.10.100",
info: {
format: "PDF 1.7",
producer: "pdf.js",
creator: "Writer",
},
totalPages: 2,
},
},
];
const documentsTable = await db.createTable("documents", documents);
const results = await documentsTable.query().toArray();
expect(results).toHaveLength(2);
expect(results[0].source).toBe("first.pdf");
expect(results[0].pdf.info.producer).toBe("pdf.js");
expect(results[1].loc.pageNumber).toBe(2);
} finally {
tmpDir.removeCallback();
}
});
describe("merge insert", () => {
let tmpDir: tmp.DirResult;
let table: Table;
@@ -2585,24 +2535,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
});
test("full text search if only an unrelated embedding function is registered", async () => {
register("unused")(
class extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType() {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return data.map(() => [1, 2, 3]);
}
},
);
test("full text search if no embedding function provided", async () => {
const db = await connect(tmpDir.name);
const data = [
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
@@ -2624,306 +2557,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(results2[0].text).toBe(data[1].text);
});
test("auto search stays consistent with the active revision", async () => {
let initCalls = 0;
let queryCalls = 0;
let markStarted!: () => void;
const started = new Promise<void>((resolve) => {
markStarted = resolve;
});
let releaseEmbedding!: () => void;
const embeddingReleased = new Promise<void>((resolve) => {
releaseEmbedding = resolve;
});
@register("refresh-test")
class TestEmbedding extends EmbeddingFunction<string> {
async init() {
initCalls += 1;
}
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings(value: string) {
queryCalls += 1;
if (value === "blocked") {
markStarted();
await embeddingReleased;
}
return value === "greetings" ? [0.1] : [0.2];
}
async computeSourceEmbeddings(values: string[]) {
return values.map((value) =>
value === "hello world" ? [0.1] : [0.2],
);
}
}
const writer = await connect(tmpDir.name);
await writer.createTable("test", [{ text: "plain", vector: [0.0] }]);
const reader = await connect(tmpDir.name, {
readConsistencyInterval: 0,
});
const tracked = await reader.openTable("test");
type SnapshotCountingNative = {
querySnapshot: () => Promise<unknown>;
};
const native = (tracked as unknown as { inner: SnapshotCountingNative })
.inner;
const querySnapshot = native.querySnapshot.bind(native);
let snapshotCalls = 0;
native.querySnapshot = async () => {
snapshotCalls += 1;
return await querySnapshot();
};
const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
const func = new TestEmbedding();
const schema = LanceSchema({
text: func.sourceField(new arrow.Utf8()),
vector: func.vectorField(),
});
const data = [{ text: "hello world" }, { text: "goodbye world" }];
await writer.createTable("test", data, { mode: "overwrite", schema });
const baselineInitCalls = initCalls;
expect(
(await tracked.schema()).metadata.get("embedding_functions"),
).toBeDefined();
const results = await autoQuery.toArray();
expect(results[0].text).toBe(data[0].text);
expect(initCalls).toBe(baselineInitCalls + 1);
expect(queryCalls).toBe(1);
expect(snapshotCalls).toBe(1);
const repeatedResults = await autoQuery.toArray();
expect(repeatedResults[0].text).toBe(data[0].text);
expect(initCalls).toBe(baselineInitCalls + 1);
expect(queryCalls).toBe(1);
expect(snapshotCalls).toBe(2);
const pending = tracked
.search("blocked")
.select(["text"])
.limit(1)
.toArray();
await started;
const ftsData = [
{ text: "greetings from full text", vector: [0.0] },
{ text: "blocked from full text", vector: [0.0] },
];
const ftsTable = await writer.createTable("test", ftsData, {
mode: "overwrite",
});
await ftsTable.createIndex("text", { config: Index.fts() });
releaseEmbedding();
const pendingResults = await pending;
expect(pendingResults[0].text).toBe(data[1].text);
expect(
(await tracked.schema()).metadata.get("embedding_functions"),
).toBeUndefined();
const ftsResults = await autoQuery.toArray();
expect(ftsResults[0].text).toBe(ftsData[0].text);
});
test("auto search keeps newer preparation during a revision race", async () => {
let aCalls = 0;
let bCalls = 0;
let markAStarted!: () => void;
const aStarted = new Promise<void>((resolve) => {
markAStarted = resolve;
});
let releaseA!: () => void;
const aReleased = new Promise<void>((resolve) => {
releaseA = resolve;
});
let markBStarted!: () => void;
const bStarted = new Promise<void>((resolve) => {
markBStarted = resolve;
});
let releaseB!: () => void;
const bReleased = new Promise<void>((resolve) => {
releaseB = resolve;
});
@register("race-a")
class EmbeddingA extends EmbeddingFunction<string> {
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings() {
aCalls += 1;
markAStarted();
await aReleased;
return [0.1];
}
async computeSourceEmbeddings(values: string[]) {
return values.map(() => [0.1]);
}
}
@register("race-b")
class EmbeddingB extends EmbeddingFunction<string> {
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings() {
bCalls += 1;
markBStarted();
await bReleased;
return [0.2];
}
async computeSourceEmbeddings(values: string[]) {
return values.map(() => [0.2]);
}
}
const writer = await connect(tmpDir.name);
const embeddingA = new EmbeddingA();
const schemaA = LanceSchema({
text: embeddingA.sourceField(new arrow.Utf8()),
vector: embeddingA.vectorField(),
});
await writer.createTable("race", [{ text: "revision a" }], {
schema: schemaA,
});
const reader = await connect(tmpDir.name, {
readConsistencyInterval: 0,
});
const tracked = await reader.openTable("race");
const query = tracked.search("query");
const first = query.toArray();
await aStarted;
const embeddingB = new EmbeddingB();
const schemaB = LanceSchema({
text: embeddingB.sourceField(new arrow.Utf8()),
vector: embeddingB.vectorField(),
});
await writer.createTable("race", [{ text: "revision b" }], {
mode: "overwrite",
schema: schemaB,
});
const second = query.toArray();
await bStarted;
releaseA();
releaseB();
await Promise.all([first, second]);
expect(aCalls).toBe(1);
expect(bCalls).toBe(1);
});
test("stale FTS routing keeps newer vector preparation", async () => {
let vectorCalls = 0;
let markVectorStarted!: () => void;
const vectorStarted = new Promise<void>((resolve) => {
markVectorStarted = resolve;
});
let releaseVector!: () => void;
const vectorReleased = new Promise<void>((resolve) => {
releaseVector = resolve;
});
@register("stale-fts-race")
class RaceEmbedding extends EmbeddingFunction<string> {
ndims() {
return 1;
}
embeddingDataType() {
return new arrow.Float32();
}
async computeQueryEmbeddings() {
vectorCalls += 1;
markVectorStarted();
await vectorReleased;
return [0.1];
}
async computeSourceEmbeddings(values: string[]) {
return values.map(() => [0.1]);
}
}
const writer = await connect(tmpDir.name);
const ftsTable = await writer.createTable("stale_fts", [
{ text: "hello", vector: [0.0] },
]);
await ftsTable.createIndex("text", { config: Index.fts() });
const reader = await connect(tmpDir.name, {
readConsistencyInterval: 0,
});
const tracked = await reader.openTable("stale_fts");
type Snapshot = {
schema: () => Promise<Buffer>;
};
type NativeWithSnapshot = {
querySnapshot: () => Promise<Snapshot>;
};
const native = (tracked as unknown as { inner: NativeWithSnapshot })
.inner;
const querySnapshot = native.querySnapshot.bind(native);
let snapshotCalls = 0;
let markStaleSchemaStarted!: () => void;
const staleSchemaStarted = new Promise<void>((resolve) => {
markStaleSchemaStarted = resolve;
});
let releaseStaleSchema!: () => void;
const staleSchemaReleased = new Promise<void>((resolve) => {
releaseStaleSchema = resolve;
});
native.querySnapshot = async () => {
const snapshot = await querySnapshot();
snapshotCalls += 1;
if (snapshotCalls === 1) {
const schema = snapshot.schema.bind(snapshot);
snapshot.schema = async () => {
markStaleSchemaStarted();
await staleSchemaReleased;
return await schema();
};
}
return snapshot;
};
const query = tracked.search("hello");
const staleFtsExecution = query.toArray();
await staleSchemaStarted;
const embedding = new RaceEmbedding();
const vectorSchema = LanceSchema({
text: embedding.sourceField(new arrow.Utf8()),
vector: embedding.vectorField(),
});
await writer.createTable("stale_fts", [{ text: "hello" }], {
mode: "overwrite",
schema: vectorSchema,
});
const vectorExecution = query.toArray();
await vectorStarted;
releaseStaleSchema();
await staleFtsExecution;
releaseVector();
await vectorExecution;
await query.toArray();
expect(vectorCalls).toBe(1);
});
test("tokenizes FTS queries by column or index name", async () => {
const db = await connect(tmpDir.name);
const data = [
@@ -3474,30 +3107,6 @@ describe("column name options", () => {
expect(results[1].query_index).toBe(1);
});
test("observes promised additional vectors while the query is pending", async () => {
const initialVector = new Promise<number[]>(() => undefined);
const query = table.query().nearestTo(initialVector);
const unhandled: unknown[] = [];
const onUnhandled = (reason: unknown) => unhandled.push(reason);
process.on("unhandledRejection", onUnhandled);
try {
query.addQueryVector(Promise.reject(new Error("extra vector failed")));
await new Promise<void>((resolve) => setImmediate(resolve));
expect(unhandled).toEqual([]);
const rejectedQuery = table
.query()
.nearestTo([0.1, 0.2])
.addQueryVector(Promise.reject(new Error("consumed vector failed")));
await expect(rejectedQuery.toArray()).rejects.toThrow(
"consumed vector failed",
);
} finally {
process.off("unhandledRejection", onUnhandled);
}
});
test("index and search multivectors", async () => {
const db = await connect(tmpDir.name);
const data = [];
@@ -3561,27 +3170,6 @@ describe("when creating an empty table", () => {
expect((actualSchema.fields[1].type as Float64).precision).toBe(2);
});
it("can add and query JSON data", async () => {
const schema = new Schema([
new Field("id", new Int32(), true),
new Field(
"meta",
new Utf8(),
true,
new Map([["ARROW:extension:name", "arrow.json"]]),
),
]);
const table = await con.createEmptyTable("json", schema);
const meta = JSON.stringify({ x: 1 });
await table.add([{ id: 1, meta }]);
const rows = await table.query().toArray();
expect(rows).toHaveLength(1);
expect(rows[0].id).toBe(1);
expect(rows[0].meta).toBe(meta);
});
it("can create an empty table from schema that specifies field types by name", async () => {
const schemaLike = {
fields: [
+1 -1
View File
@@ -170,7 +170,7 @@ test("basic table examples", async () => {
// --8<-- [end:create_index]
// --8<-- [start:delete_rows]
await tbl.delete("item = 'fizz'");
await tbl.delete('item = "fizz"');
// --8<-- [end:delete_rows]
// --8<-- [start:drop_table]
+1 -2
View File
@@ -8,8 +8,7 @@
"//1": "--experimental-vm-modules is needed to run jest with sentence-transformers",
"//2": "--testEnvironment is needed to run jest with sentence-transformers",
"//3": "See: https://github.com/huggingface/transformers.js/issues/57",
"//4": "jest is invoked by its JS entry, not node_modules/.bin/jest: under pnpm that path is a shell shim, which `node` cannot execute",
"test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testEnvironment jest-environment-node-single-context --verbose",
"test": "node --experimental-vm-modules node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose",
"lint": "biome check *.ts && biome format *.ts",
"lint-ci": "biome ci .",
"lint-fix": "biome check --write *.ts && pnpm format",
+2 -2
View File
@@ -72,7 +72,8 @@ export type FieldLike =
};
export type DataLike =
| import("apache-arrow").Data
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
| import("apache-arrow").Data<Struct<any>>
| {
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
type: any;
@@ -81,7 +82,6 @@ export type DataLike =
stride: number;
nullable: boolean;
children: DataLike[];
dictionary?: { data: readonly DataLike[] };
get nullCount(): number;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
values: Buffers<any>[BufferType.DATA];
+100 -134
View File
@@ -100,29 +100,6 @@ export interface FullTextSearchOptions {
columns?: string | string[];
}
function nearestToNative(
inner: NativeQuery,
vector: Awaited<IntoVector>,
): NativeVectorQuery {
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
if (raw) {
return inner.nearestToRaw(raw.data, raw.dtype);
}
return inner.nearestTo(Float32Array.from(vector as number[]));
}
function addQueryVectorToNative(
inner: NativeVectorQuery,
vector: Awaited<IntoVector>,
) {
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
if (raw) {
inner.addQueryVectorRaw(raw.data, raw.dtype);
} else {
inner.addQueryVector(Float32Array.from(vector as number[]));
}
}
/** Common methods supported by all query types
*
* @see {@link Query}
@@ -522,13 +499,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
super(inner);
}
/**
* @hidden
*/
protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
super.doCall(fn);
}
/**
* Set the number of partitions to search (probe)
*
@@ -556,7 +526,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* the minimum and maximum to the same value.
*/
nprobes(nprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.nprobes(nprobes));
super.doCall((inner) => inner.nprobes(nprobes));
return this;
}
@@ -570,7 +540,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* but will also increase latency.
*/
minimumNprobes(minimumNprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
return this;
}
@@ -584,7 +554,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* potential false negatives.
*/
maximumNprobes(maximumNprobes: number): VectorQuery {
this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
return this;
}
@@ -597,7 +567,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* `undefined` means no lower or upper bound.
*/
distanceRange(lowerBound?: number, upperBound?: number): VectorQuery {
this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
return this;
}
@@ -611,7 +581,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* also increase the latency of your query. The default value is 1.5*limit.
*/
ef(ef: number): VectorQuery {
this.doVectorCall((inner) => inner.ef(ef));
super.doCall((inner) => inner.ef(ef));
return this;
}
@@ -625,7 +595,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* whose data type is a fixed-size-list of floats.
*/
column(column: string): VectorQuery {
this.doVectorCall((inner) => inner.column(column));
super.doCall((inner) => inner.column(column));
return this;
}
@@ -646,7 +616,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
distanceType(
distanceType: Required<IvfPqOptions>["distanceType"],
): VectorQuery {
this.doVectorCall((inner) => inner.distanceType(distanceType));
super.doCall((inner) => inner.distanceType(distanceType));
return this;
}
@@ -680,7 +650,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* distance between the query vector and the actual uncompressed vector.
*/
refineFactor(refineFactor: number): VectorQuery {
this.doVectorCall((inner) => inner.refineFactor(refineFactor));
super.doCall((inner) => inner.refineFactor(refineFactor));
return this;
}
@@ -705,7 +675,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* factor can often help restore some of the results lost by post filtering.
*/
postfilter(): VectorQuery {
this.doVectorCall((inner) => inner.postfilter());
super.doCall((inner) => inner.postfilter());
return this;
}
@@ -719,7 +689,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* calculate your recall to select an appropriate value for nprobes.
*/
bypassVectorIndex(): VectorQuery {
this.doVectorCall((inner) => inner.bypassVectorIndex());
super.doCall((inner) => inner.bypassVectorIndex());
return this;
}
@@ -727,39 +697,43 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
* Add a query vector to the search
*
* This method can be called multiple times to add multiple query vectors
* to the search. A column called `query_index` will be added to indicate the index
* of the query vector that produced the result. Flat searches share one table scan
* across the query vectors, avoiding the scan and memory amplification of running
* multiple queries concurrently. Indexed searches may still perform per-vector
* index work.
* to the search. If multiple query vectors are added, then they will be searched
* in parallel, and the results will be concatenated. A column called `query_index`
* will be added to indicate the index of the query vector that produced the result.
*
* Performance wise, this is equivalent to running multiple queries concurrently.
*/
addQueryVector(vector: IntoVector): VectorQuery {
if (vector instanceof Promise) {
// Observe the promise as soon as it is accepted. The existing native
// query may still be pending, and delaying observation until it resolves
// can otherwise surface a fast rejection as unhandled.
const settledVector = vector.then(
(value) => ({ status: "fulfilled" as const, value }),
(reason) => ({ status: "rejected" as const, reason }),
);
const res = (async () => {
const inner = await this.getInner();
const outcome = await settledVector;
if (outcome.status === "rejected") {
throw outcome.reason;
try {
const v = await vector;
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
const value: any = this.addQueryVector(v);
const inner = value.inner as
| NativeVectorQuery
| Promise<NativeVectorQuery>;
return inner;
} catch (e) {
return Promise.reject(e);
}
addQueryVectorToNative(inner, outcome.value);
return inner;
})();
return new VectorQuery(res);
} else {
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
super.doCall((inner) => {
const raw = Array.isArray(vector) ? null : extractVectorBuffer(vector);
if (raw) {
inner.addQueryVectorRaw(raw.data, raw.dtype);
} else {
inner.addQueryVector(Float32Array.from(vector as number[]));
}
});
return this;
}
}
rerank(reranker: Reranker): VectorQuery {
this.doVectorCall((inner) =>
super.doCall((inner) =>
inner.rerank(async (args) => {
const vecResults = await fromBufferToRecordBatch(args.vecResults);
const ftsResults = await fromBufferToRecordBatch(args.ftsResults);
@@ -778,71 +752,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
}
}
/**
* Create a string query whose vector/FTS routing is resolved against the active
* table schema when the query executes.
*
* @hidden
*/
export function createAutoQuery(
table: NativeTable,
query: string,
columns: string[] | null,
getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
): AutoQuery {
type RouteSnapshot = {
table: NativeTable;
embeddingMetadata: string | undefined;
};
type CachedPreparation = {
metadata: string;
vector: Promise<Awaited<IntoVector>>;
};
let cachedPreparation: CachedPreparation | undefined;
const snapshotRoute = async (): Promise<RouteSnapshot> => {
const snapshot = await table.querySnapshot();
const schema = tableFromIPC(await snapshot.schema()).schema;
return {
table: snapshot,
embeddingMetadata: schema.metadata.get("embedding_functions"),
};
};
const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
const route = await snapshotRoute();
if (route.embeddingMetadata === undefined) {
const inner = route.table.query();
inner.fullTextSearch({ query, columns });
return inner;
}
const metadata = route.embeddingMetadata;
if (cachedPreparation?.metadata !== metadata) {
cachedPreparation = {
metadata,
vector: Promise.resolve().then(() => getVector(metadata)),
};
}
const preparation = cachedPreparation;
let vector: Awaited<IntoVector>;
try {
vector = await preparation.vector;
} catch (error) {
if (cachedPreparation === preparation) {
cachedPreparation = undefined;
}
throw error;
}
return nearestToNative(route.table.query(), vector);
};
return new AutoQuery(createInner);
}
/**
* A query that returns a subset of the rows in the table.
*
@@ -927,6 +836,37 @@ export class Query extends StandardQueryBase<NativeQuery> {
super(tbl.query());
}
/** @hidden */
static autoSearch(
tbl: () => Promise<NativeTable>,
query: string,
vector: (tbl: NativeTable) => Promise<Awaited<IntoVector> | undefined>,
columns?: string[],
): AutoQuery {
const nativeQuery = async () => {
const snapshot = await Promise.resolve(tbl());
const resolved = await vector(snapshot);
const inner = snapshot.query();
if (resolved === undefined) {
inner.fullTextSearch({
query,
columns: columns ?? null,
});
return inner;
}
const raw = Array.isArray(resolved)
? null
: extractVectorBuffer(resolved);
if (raw) {
return inner.nearestToRaw(raw.data, raw.dtype);
}
return inner.nearestTo(Float32Array.from(resolved as number[]));
};
return new AutoQuery(nativeQuery);
}
/**
* Find the nearest vectors to the given query vector.
*
@@ -965,19 +905,45 @@ export class Query extends StandardQueryBase<NativeQuery> {
* a default `limit` of 10 will be used. @see {@link Query#limit}
*/
nearestTo(vector: IntoVector): VectorQuery {
const inner = this.inner;
if (inner instanceof Promise) {
const nativeQuery = inner.then(async (resolvedInner) =>
nearestToNative(resolvedInner, await vector),
);
const callNearestTo = (
inner: NativeQuery,
resolved: Float32Array | Float64Array | Uint8Array | number[],
): NativeVectorQuery => {
const raw = Array.isArray(resolved)
? null
: extractVectorBuffer(resolved);
if (raw) {
return inner.nearestToRaw(raw.data, raw.dtype);
}
return inner.nearestTo(Float32Array.from(resolved as number[]));
};
if (this.inner instanceof Promise) {
const nativeQuery = this.inner.then(async (inner) => {
const resolved = vector instanceof Promise ? await vector : vector;
return callNearestTo(inner, resolved);
});
return new VectorQuery(nativeQuery);
}
if (vector instanceof Promise) {
return new VectorQuery(
vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
);
const res = (async () => {
try {
const v = await vector;
// biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
const value: any = this.nearestTo(v);
const inner = value.inner as
| NativeVectorQuery
| Promise<NativeVectorQuery>;
return inner;
} catch (e) {
return Promise.reject(e);
}
})();
return new VectorQuery(res);
} else {
const vectorQuery = callNearestTo(this.inner, vector);
return new VectorQuery(vectorQuery);
}
return new VectorQuery(nearestToNative(inner, vector));
}
nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
+4 -11
View File
@@ -94,24 +94,17 @@ export function sanitizeMetadata(
if (metadataLike === undefined || metadataLike === null) {
return undefined;
}
let entries: IterableIterator<[unknown, unknown]>;
try {
entries = Map.prototype.entries.call(metadataLike);
} catch {
if (!(metadataLike instanceof Map)) {
throw Error("Expected metadata, if present, to be a Map<string, string>");
}
const metadata = new Map<string, string>();
for (const [key, value] of entries) {
if (typeof key !== "string" || typeof value !== "string") {
for (const item of metadataLike) {
if (typeof item[0] !== "string" || typeof item[1] !== "string") {
throw Error(
"Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values",
);
}
metadata.set(key, value);
}
return metadata;
return metadataLike as Map<string, string>;
}
export function sanitizeInt(typeLike: object) {
+1 -2
View File
@@ -406,11 +406,10 @@ function matchingFields(fields: Field[], tree: FieldTree): Field[] {
field.name,
new Struct(matchingFields(struct.children, value)),
field.nullable,
field.metadata,
),
);
} else {
matches.push(field);
matches.push(new Field(field.name, value as DataType, field.nullable));
}
}
return matches;
+22 -32
View File
@@ -48,7 +48,6 @@ import {
Query,
TakeQuery,
VectorQuery,
createAutoQuery,
instanceOfFullTextQuery,
} from "./query";
import { sanitizeType } from "./sanitize";
@@ -630,18 +629,6 @@ export abstract class Table {
/**
* Update per-field (column) metadata.
*
* The following keys are treated specially, by convention, and should be
* used when appropriate:
*
* - `lancedb:description`: for a human-readable description of a field.
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
* `feature_v2` might be in the same logical column.
* - `lancedb:status`: for status options (`production`, `candidate`,
* `deprecated`, `archived`) to designate the current life cycle state of
* this column.
* @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
* update's metadata is merged into the field's existing metadata by default;
* a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -1190,29 +1177,33 @@ export class LocalTable extends Table {
});
}
if (queryType === "auto") {
if (instanceOfFullTextQuery(query)) {
return this.query().fullTextSearch(query, {
columns: ftsColumns,
});
}
if (queryType === "auto" && typeof query !== "string") {
return this.query().fullTextSearch(query, {
columns: ftsColumns,
});
}
const columns =
typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
return createAutoQuery(this.inner, query, columns, async (metadata) => {
const functions = await getRegistry().parseFunctions(
new Map([["embedding_functions", metadata]]),
);
if (queryType === "auto" && typeof query === "string") {
const vector = async (snapshot: _NativeTable) => {
const functions = await this.getEmbeddingFunctions(snapshot);
// TODO: Support multiple embedding functions
const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
.values()
.next().value;
// The route only calls this callback when embedding metadata exists.
// parseFunctions either yields a provider or reports malformed metadata.
if (!embeddingFunc)
throw new Error("Invalid embedding function metadata");
if (embeddingFunc === undefined) {
return undefined;
}
return await embeddingFunc.function.computeQueryEmbeddings(query);
});
};
const columns =
typeof ftsColumns === "string" ? [ftsColumns] : ftsColumns;
return Query.autoSearch(
() => this.inner.checkoutCurrent(),
query,
vector,
columns,
);
}
const queryPromise = this.getEmbeddingFunctions().then(
@@ -1567,8 +1558,7 @@ export interface FieldMetadataUpdate {
path: string;
/**
* Metadata key/value pairs. Merged into the field's existing metadata by
* default; a value of `null` deletes that key. See
* {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
* default; a value of `null` deletes that key.
*/
metadata: Record<string, string | null>;
/** If true, replace the field's entire metadata map instead of merging. */
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0",
"version": "0.38.0-beta.10",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+11106
View File
File diff suppressed because it is too large Load Diff
+3 -3
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0",
"version": "0.38.0-beta.10",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -67,7 +67,7 @@
"timeout": "3m"
},
"engines": {
"node": ">= 22"
"node": ">= 18"
},
"packageManager": "pnpm@11.1.1",
"cpu": ["x64", "arm64"],
@@ -101,7 +101,7 @@
"openai": "4.29.2"
},
"peerDependencies": {
"@types/node": ">=22",
"@types/node": ">=18",
"apache-arrow": ">=15.0.0 <=18.1.0"
},
"peerDependenciesMeta": {
+1 -5
View File
@@ -664,11 +664,7 @@ impl JsFullTextQuery {
}
fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
// `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
// path, which is only usable from generated `#[napi]` argument conversion. This is a
// manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
// unwraps the class without requiring a borrow scope.
if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
Ok(FullTextSearchQuery::new_query(query.inner.clone()))
} else if let Ok(Some(query_text)) = query.get::<String>("query") {
let mut query_text = query_text;
-7
View File
@@ -278,13 +278,6 @@ impl Table {
Ok(Query::new(self.inner_ref()?.query()))
}
/// Return a read-only table handle pinned to the current query revision.
#[napi(catch_unwind)]
pub async fn query_snapshot(&self) -> napi::Result<Self> {
let snapshot = self.inner_ref()?.query_snapshot().await.default_error()?;
Ok(Self::new(snapshot))
}
#[napi(catch_unwind)]
pub fn take_offsets(&self, offsets: Vec<i64>) -> napi::Result<TakeQuery> {
Ok(TakeQuery::new(
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0"
version = "0.38.0-beta.10"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
+2 -15
View File
@@ -6,7 +6,7 @@ import importlib.metadata
import os
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
from typing import Dict, Optional, Union, Any, List, Iterable, TYPE_CHECKING
from typing import Dict, Optional, Union, Any, List, Iterable
__version__ = importlib.metadata.version("lancedb")
@@ -20,7 +20,7 @@ from .db import AsyncConnection, DBConnection, LanceDBConnection
from .remote import ClientConfig
from .remote.db import RemoteDBConnection
from .expr import Expr, col, lit, func
from .schema import blob, vector
from .schema import blob, vector, BlobType
from .job import AsyncJob, Job
from .functions import (
FunctionArtifactRequest as FunctionArtifactRequest,
@@ -49,19 +49,6 @@ from .namespace import (
)
if TYPE_CHECKING:
from lance.blob import BlobType as BlobType
def __getattr__(name: str):
if name == "BlobType":
from .schema import BlobType
globals()["BlobType"] = BlobType
return BlobType
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def _check_s3_bucket_with_dots(
uri: str, storage_options: Optional[Dict[str, str]]
) -> None:
+5 -9
View File
@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Optional, Union
import pyarrow as pa
from .expr import Expr
from .schema import row_addressable_blob_v2_paths
from .schema import blob_v2_column_paths
from .types import BlobMode, QueryProjection, QueryProjectionSpec
if TYPE_CHECKING:
@@ -119,7 +119,7 @@ def blob_v2_projection_sources(
schema: pa.Schema,
projection: QueryProjection,
) -> dict[str, str]:
blob_columns = row_addressable_blob_v2_paths(schema)
blob_columns = blob_v2_column_paths(schema)
if not blob_columns:
return {}
columns = set(blob_columns)
@@ -140,9 +140,7 @@ def v2_projection_needs_row_id(
) -> bool:
if with_row_id:
return False
return projection_includes_blob_column(
projection, row_addressable_blob_v2_paths(schema)
)
return projection_includes_blob_column(projection, blob_v2_column_paths(schema))
def blob_auto_row_id_for_scan(
@@ -272,8 +270,7 @@ def _iter_projection_pairs(
if isinstance(expr, str):
yield name, expr
elif isinstance(expr, Expr):
source = expr._column_name()
yield name, source if source is not None else expr.to_sql()
yield name, expr.to_sql()
return
for column in projection:
if isinstance(column, str):
@@ -283,8 +280,7 @@ def _iter_projection_pairs(
if isinstance(expr, str):
yield name, expr
elif isinstance(expr, Expr):
source = expr._column_name()
yield name, source if source is not None else expr.to_sql()
yield name, expr.to_sql()
def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
-6
View File
@@ -87,7 +87,6 @@ class PyExpr:
def contains(self, substr: "PyExpr") -> "PyExpr": ...
def isin(self, values: List["PyExpr"]) -> "PyExpr": ...
def cast(self, data_type: pa.DataType) -> "PyExpr": ...
def column_name(self) -> Optional[str]: ...
def to_sql(self) -> str: ...
def expr_col(name: str) -> PyExpr: ...
@@ -150,12 +149,9 @@ class Connection(object):
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def pause_job(self, job_id: str) -> str: ...
async def resume_job(self, job_id: str) -> str: ...
async def job_history(
self, job_id: Optional[str] = None
) -> List[pa.RecordBatch]: ...
@@ -609,11 +605,9 @@ class FullTextQuery:
class PyQueryRequest:
limit: Optional[int]
offset: Optional[int]
take_offsets: Optional[List[int]]
filter: Optional[Union[str, bytes]]
full_text_search: Optional[FullTextQuery]
select: Optional[Union[str, List[str]]]
select_source_columns: Optional[Dict[str, str]]
fast_search: Optional[bool]
with_row_id: Optional[bool]
use_lsm: Optional[bool]
-71
View File
@@ -712,16 +712,6 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog.
Returns True when the version changed to Dropped and False for an
idempotent replay. Local connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
@@ -753,26 +743,6 @@ class DBConnection(EnforceOverrides):
"cancel_job is not supported for this connection type"
)
def pause_job(self, job_id: str) -> str:
"""Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Returns
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
"""
raise NotImplementedError("pause_job is not supported for this connection type")
def resume_job(self, job_id: str) -> str:
"""Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Returns
"resumed", "still_pausing" -- the pause's worker drain is not
confirmed yet; retry shortly -- or "not_paused".
"""
raise NotImplementedError(
"resume_job is not supported for this connection type"
)
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
@@ -1443,10 +1413,6 @@ class LanceDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
@@ -1470,22 +1436,6 @@ class LanceDBConnection(DBConnection):
"""
return LOOP.run(self._conn.cancel_job(job_id))
@override
def pause_job(self, job_id: str) -> str:
"""Pause a server-side job by id.
Returns "pausing", "already_paused", or "committing".
"""
return LOOP.run(self._conn.pause_job(job_id))
@override
def resume_job(self, job_id: str) -> str:
"""Resume a paused server-side job by id.
Returns "resumed", "still_pausing", or "not_paused".
"""
return LOOP.run(self._conn.resume_job(job_id))
@override
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
@@ -2293,10 +2243,6 @@ class AsyncConnection(object):
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
async def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog."""
return await self._inner.drop_function(name, version)
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
@@ -2317,23 +2263,6 @@ class AsyncConnection(object):
"""
return await self._inner.cancel_job(job_id)
async def pause_job(self, job_id: str) -> str:
"""Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Returns
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
"""
return await self._inner.pause_job(job_id)
async def resume_job(self, job_id: str) -> str:
"""Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Returns
"resumed", "still_pausing" -- retry shortly -- or "not_paused".
"""
return await self._inner.resume_job(job_id)
async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
+1 -5
View File
@@ -249,10 +249,6 @@ class Expr:
# ── utilities ────────────────────────────────────────────────────────────
def _column_name(self) -> str | None:
"""Return the source name when this is a bare column expression."""
return self._inner.column_name()
def to_sql(self) -> str:
"""Render the expression as a SQL string (useful for debugging)."""
return self._inner.to_sql()
@@ -316,7 +312,7 @@ def func(name: str, *args: ExprLike) -> Expr:
--------
>>> from lancedb.expr import col, func
>>> func("lower", col("name"))
Expr(lower(`name`))
Expr(lower(name))
"""
inner_args = [_coerce(a)._inner for a in args]
return Expr(expr_func(name, inner_args))
+23 -266
View File
@@ -49,25 +49,11 @@ from pydantic import (
model_validator,
)
from .schema import is_blob_v2_field as _is_blob_v2_field
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
def _validate_gpu_wire_marker(value: Any) -> bool:
if value is not True:
raise ValueError("runtime.gpu must be true")
return True
def _normalize_gpu_marker(value: bool) -> Optional[bool]:
if not isinstance(value, bool):
raise ValueError("gpu must be a boolean")
return True if value else None
class _FrozenDict(dict):
def _immutable(self, *args, **kwargs):
raise TypeError("remote canonical values are immutable")
@@ -236,7 +222,6 @@ class PythonEnvironmentSpec(_RemoteValue):
kind: str
packages: tuple[str, ...] = ()
channels: tuple[str, ...] = ()
path: Optional[str] = None
modules: tuple[str, ...] = ()
image: Optional[str] = None
@@ -253,23 +238,6 @@ class PythonRuntimeSpec(_RemoteValue):
python_version: Optional[str] = None
environment: Optional[PythonEnvironmentSpec] = None
env: Optional[Mapping[str, str]] = None
gpu: Optional[bool] = None
@model_validator(mode="before")
@classmethod
def _discard_unknown_runtime_payload(cls, value):
if isinstance(value, Mapping):
kind = value.get("kind")
if isinstance(kind, str) and kind not in {"python", "python_v2"}:
return {"kind": kind}
return value
@field_validator("gpu", mode="before")
@classmethod
def _validate_gpu_marker(cls, value):
if value is None:
return None
return _validate_gpu_wire_marker(value)
@model_validator(mode="after")
def _validate_runtime_kind(self):
@@ -278,28 +246,18 @@ class PythonRuntimeSpec(_RemoteValue):
raise ValueError("python runtime requires python_version")
if self.environment is None:
raise ValueError("python runtime requires environment")
if self.gpu is not None:
raise ValueError("python runtime with gpu requires kind='python_v2'")
elif self.kind == "python_v2":
if self.python_version is None:
raise ValueError("python_v2 runtime requires python_version")
if self.environment is None:
raise ValueError("python_v2 runtime requires environment")
if self.gpu is None:
raise ValueError("python_v2 runtime requires gpu")
else:
object.__setattr__(self, "python_version", None)
object.__setattr__(self, "environment", None)
object.__setattr__(self, "env", None)
object.__setattr__(self, "gpu", None)
return self
class FunctionVersion(_RemoteValue):
"""An exact immutable Function version returned by Enterprise.
The GPU execution requirement is part of this identity. CPU and memory sizing,
priority, concurrency, and retry policy belong to the execution platform.
Scheduling resources, priority, concurrency, and retry policy belong to
the submitting Job and are not part of this identity.
"""
name: str
@@ -520,7 +478,6 @@ class RefreshColumnResult(_RemoteValue):
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
_GRAMMAR_PRIMITIVES = (
@@ -537,7 +494,6 @@ _GRAMMAR_PRIMITIVES = (
(pa.float32(), "float32"),
(pa.float64(), "float64"),
(pa.string(), "utf8"),
(pa.large_string(), "large_utf8"),
(pa.binary(), "binary"),
(pa.date32(), "date32"),
(pa.date64(), "date64"),
@@ -545,177 +501,31 @@ _GRAMMAR_PRIMITIVES = (
def _canonical_arrow_type(data_type: pa.DataType) -> str:
"""The compact Function grammar, or canonical exact JSON for nested types."""
grammar = _grammar_arrow_type(data_type)
if grammar is not None:
return grammar
exact = _exact_arrow_type(data_type)
return json.dumps(exact, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def _grammar_arrow_type(data_type: pa.DataType) -> Optional[str]:
"""The server's V1 Function type grammar. Anything outside it is rejected
here rather than at registration."""
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return name
if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
item = _grammar_list_item(data_type)
if item is None:
return None
prefix = "list" if pa.types.is_list(data_type) else "large_list"
return f"{prefix}<{item}>"
return f"{prefix}<{_canonical_list_item(data_type)}>"
if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
item = _grammar_list_item(data_type)
if item is not None:
return f"fixed_size_list<{item}, {data_type.list_size}>"
return None
return (
f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>"
)
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
def _grammar_list_item(data_type: pa.DataType) -> Optional[str]:
def _canonical_list_item(data_type: pa.DataType) -> str:
"""The grammar names only the item type; it always means a non-nullable
child called `item`, so other child properties require exact JSON."""
child called `item`, so any other child metadata cannot be represented."""
child = data_type.value_field
if child.name != "item" or child.nullable or child.metadata:
return None
return _grammar_arrow_type(child.type)
def _validate_exact_arrow_field(field: pa.Field) -> None:
if not field.name:
raise TypeError(
"unsupported Arrow type for Function signature: field names "
"must not be empty"
"unsupported Arrow type for Function signature: list items must be a "
f"non-nullable field named 'item', got {child}"
)
if _is_blob_v2_field(field):
if not _has_supported_blob_v2_layout(field):
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
f"requires a supported Blob storage layout, got {field}"
)
elif field.metadata:
raise TypeError(
"unsupported Arrow type for Function signature: field metadata "
f"is not supported, got {field}"
)
def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
data_type = field.type
if isinstance(data_type, pa.ExtensionType):
data_type = data_type.storage_type
if not pa.types.is_struct(data_type):
return False
fields = tuple(data_type)
def matches(spec, compare_nullable) -> bool:
return len(fields) == len(spec) and all(
actual.name == name
and actual.type == expected_type
and (not check_nullable or actual.nullable == nullable)
for actual, (name, expected_type, nullable), check_nullable in zip(
fields, spec, compare_nullable
)
)
logical_minimal = (
("data", pa.large_binary(), True),
("uri", pa.utf8(), True),
)
logical_full = logical_minimal + (
("position", pa.uint64(), True),
("size", pa.uint64(), True),
)
prepared = (
("kind", pa.uint8(), True),
("data", pa.large_binary(), True),
("uri", pa.utf8(), True),
("blob_id", pa.uint32(), True),
("blob_size", pa.uint64(), True),
("position", pa.uint64(), True),
)
descriptor = (
("kind", pa.uint8(), False),
("position", pa.uint64(), False),
("size", pa.uint64(), False),
("blob_id", pa.uint32(), False),
("blob_uri", pa.utf8(), False),
)
return (
matches(logical_minimal, (True, True))
or matches(logical_full, (True, True, False, False))
or matches(prepared, (True,) * len(prepared))
or matches(descriptor, (False,) * len(descriptor))
)
def _canonical_arrow_field(field: pa.Field) -> str:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
return _FUNCTION_BLOB_V2_TYPE
return _canonical_arrow_type(field.type)
def _exact_arrow_field(field: pa.Field) -> dict[str, Any]:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
raise TypeError(
"unsupported Arrow type for Function signature: nested Blob v2 "
"fields are not supported; declare Blob parameters or named result "
"fields directly"
)
value = {
"name": field.name,
"nullable": field.nullable,
"type": _exact_arrow_type(field.type),
}
return value
def _exact_arrow_type(data_type: pa.DataType) -> dict[str, Any]:
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return {"type": name}
if pa.types.is_struct(data_type):
fields = list(data_type)
names = [field.name for field in fields]
if not fields or len(set(names)) != len(names):
raise TypeError(
"unsupported Arrow type for Function signature: structs must have "
"non-empty, uniquely named fields"
)
return {
"type": "struct",
"fields": [_exact_arrow_field(field) for field in fields],
}
if (
pa.types.is_list(data_type)
or pa.types.is_large_list(data_type)
or pa.types.is_fixed_size_list(data_type)
):
if pa.types.is_fixed_size_list(data_type):
if data_type.value_field.name != "item":
raise TypeError(
"unsupported Arrow type for Function signature: fixed-size list "
"items must be named 'item'"
)
if data_type.list_size <= 0:
raise TypeError(
f"unsupported Arrow type for Function signature: {data_type}"
)
value: dict[str, Any] = {
"type": (
"list"
if pa.types.is_list(data_type)
else "large_list"
if pa.types.is_large_list(data_type)
else "fixed_size_list"
),
"fields": [_exact_arrow_field(data_type.value_field)],
}
if pa.types.is_fixed_size_list(data_type):
value["length"] = data_type.list_size
return value
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
return _canonical_arrow_type(child.type)
def _list_of(item: pa.DataType) -> pa.DataType:
@@ -789,15 +599,8 @@ def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Paramete
def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
if isinstance(output, pa.Schema):
if output.metadata:
raise TypeError("Function output schema metadata is not supported")
fields = tuple(output)
elif (
isinstance(output, pa.Field)
and not _is_blob_v2_field(output)
and pa.types.is_struct(output.type)
):
_validate_exact_arrow_field(output)
elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
if output.nullable:
raise ValueError("Function output must be non-nullable")
fields = tuple(output.type)
@@ -813,12 +616,11 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
raise TypeError(
"output_schema must be a PyArrow DataType, Field, or Schema"
)
_validate_exact_arrow_field(field)
if field.nullable:
raise ValueError("Function output must be non-nullable")
return FunctionOutput(
kind="scalar",
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=False,
)
@@ -826,8 +628,6 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
raise ValueError("named-struct Function output must contain at least one field")
if any(field.nullable for field in fields):
raise ValueError("Function output fields must be non-nullable")
for field in fields:
_validate_exact_arrow_field(field)
names = [field.name for field in fields]
if len(set(names)) != len(names):
raise ValueError("Function output field names must be unique")
@@ -836,7 +636,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
fields=tuple(
FunctionResultField(
name=field.name,
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=False,
)
for field in fields
@@ -856,10 +656,6 @@ def _infer_signature(
if input_schema is not None:
if not isinstance(input_schema, pa.Schema):
raise TypeError("input_schema must be a PyArrow Schema")
if input_schema.metadata:
raise TypeError("Function input schema metadata is not supported")
for field in input_schema:
_validate_exact_arrow_field(field)
expected = tuple(parameter.name for parameter in parameters)
actual = tuple(input_schema.names)
if actual != expected:
@@ -870,7 +666,7 @@ def _infer_signature(
inputs = tuple(
FunctionParameter(
name=field.name,
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=field.nullable,
)
for field in input_schema
@@ -893,9 +689,7 @@ def _infer_signature(
inputs.append(
FunctionParameter(
name=parameter.name,
arrow_type=_canonical_arrow_field(
pa.field(parameter.name, data_type, nullable=nullable)
),
arrow_type=_canonical_arrow_type(data_type),
nullable=nullable,
)
)
@@ -1115,26 +909,13 @@ class UdfDefinition:
pip: tuple[str, ...],
env: Mapping[str, str],
python_version: Optional[str],
gpu: bool = False,
conda: tuple[str, ...] = (),
conda_channels: tuple[str, ...] = (),
):
function_name = name or function.__name__
if not _FUNCTION_NAME.fullmatch(function_name):
raise ValueError(f"invalid Function name: {function_name!r}")
if pip and conda:
raise ValueError("a Function environment is pip or conda, not both")
if conda_channels and not conda:
raise ValueError("conda_channels requires conda packages")
packages = tuple(sorted(set(conda if conda else pip)))
packages = tuple(sorted(set(pip)))
if any(not package or package != package.strip() for package in packages):
raise ValueError("package requirements must be non-empty and trimmed")
if conda:
environment_spec = PythonEnvironmentSpec(
kind="conda", packages=packages, channels=tuple(conda_channels)
)
else:
environment_spec = PythonEnvironmentSpec(kind="pip", packages=packages)
raise ValueError("pip requirements must be non-empty and trimmed")
environment = dict(env)
if any(
not isinstance(key, str) or not isinstance(value, str)
@@ -1144,14 +925,12 @@ class UdfDefinition:
signature = _infer_signature(function, input_schema, output_schema)
source = _package_source(function)
digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
gpu_marker = _normalize_gpu_marker(gpu)
runtime = PythonRuntimeSpec(
kind="python_v2" if gpu_marker is not None else "python",
kind="python",
python_version=python_version
or f"{sys.version_info.major}.{sys.version_info.minor}",
environment=environment_spec,
environment=PythonEnvironmentSpec(kind="pip", packages=packages),
env=environment,
gpu=gpu_marker,
)
self._function = function
self._request = FunctionRegistrationRequest(
@@ -1197,9 +976,6 @@ def udf(
pip: tuple[str, ...] | list[str] = (),
env: Optional[Mapping[str, str]] = None,
python_version: Optional[str] = None,
gpu: bool = False,
conda: tuple[str, ...] | list[str] = (),
conda_channels: tuple[str, ...] | list[str] = (),
) -> Callable[[Callable[..., Any]], UdfDefinition]: ...
@@ -1212,9 +988,6 @@ def udf(
pip: tuple[str, ...] | list[str] = (),
env: Optional[Mapping[str, str]] = None,
python_version: Optional[str] = None,
gpu: bool = False,
conda: tuple[str, ...] | list[str] = (),
conda_channels: tuple[str, ...] | list[str] = (),
):
"""Prepare a scalar Python callable for remote Function registration.
@@ -1237,18 +1010,10 @@ def udf(
provided together with ``input_schema``.
pip : sequence of str, optional
Pip requirements for the remote environment.
conda : sequence of str, optional
Conda packages for the remote environment, instead of ``pip``.
conda_channels : sequence of str, optional
Conda channels in priority order; requires ``conda``.
env : mapping of str to str, optional
Environment variables included in the Function definition.
python_version : str, optional
Remote Python major/minor version. Defaults to the client version.
gpu : bool, default False
Whether every remote execution requires a GPU. The execution platform
selects one compatible GPU for each worker. The requirement is part of
the immutable Function version.
The packaged artifact is a snapshot: the function source plus exactly
the module-level names it references (modules as imports, importable
@@ -1273,11 +1038,6 @@ def udf(
... return value * 2
>>> score(1.5)
3.0
>>> @udf(pip=["cupy-cuda12x"], gpu=True)
... def gpu_score(value: int) -> int:
... return value * 2
>>> gpu_score.registration_request.runtime.gpu
True
"""
def decorate(target: Callable[..., Any]) -> UdfDefinition:
@@ -1289,9 +1049,6 @@ def udf(
pip=tuple(pip),
env={} if env is None else env,
python_version=python_version,
gpu=gpu,
conda=tuple(conda),
conda_channels=tuple(conda_channels),
)
if function is None:
-12
View File
@@ -7,7 +7,6 @@ from typing import List, Literal, Optional
from ._lancedb import (
IndexConfig,
)
from .query import DocumentGranularity
from .types import BaseTokenizerType
lang_mapping = {
@@ -122,11 +121,6 @@ class FTS:
>>> config = FTS(block_size=256)
Create an index that treats each deepest-list element as one document:
>>> from lancedb.query import DocumentGranularity
>>> config = FTS(document_granularity=DocumentGranularity.LIST_ELEMENT)
Attributes
----------
with_position : bool, default False
@@ -178,11 +172,6 @@ class FTS:
roughly half of the available CPU cores. The effective value is
limited by the available compute capacity. This build-only setting is
not persisted with the index and does not apply to remote tables.
document_granularity : DocumentGranularity, default ROW
``ROW`` treats the selected text in one table row as one document.
``LIST_ELEMENT`` treats each element of the deepest list on the indexed
field path as one document and returns its physical coordinates in
``_doc_index`` for matching queries.
Notes
-----
@@ -207,7 +196,6 @@ class FTS:
custom_stop_words: Optional[List[str]] = None
memory_limit: Optional[int] = None
num_workers: Optional[int] = None
document_granularity: DocumentGranularity = DocumentGranularity.ROW
@dataclass
+9 -47
View File
@@ -109,7 +109,6 @@ def _query_is_plain_scan(query: Query) -> bool:
return (
query.vector is None
and query.full_text_query is None
and query.take_offsets is None
and not query.postfilter
and not query.order_by
)
@@ -168,12 +167,6 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
return {"columns": projection}
def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
if req.select_source_columns is not None:
return req.select_source_columns
return req.select
def _scanner_kwargs_for_query(
query: Query,
blob_mode: BlobMode,
@@ -382,13 +375,6 @@ class FullTextOperator(str, Enum):
OR = "OR"
class DocumentGranularity(str, Enum):
"""The unit treated as one full-text-search document."""
ROW = "row"
LIST_ELEMENT = "list_element"
class Occur(str, Enum):
SHOULD = "SHOULD"
MUST = "MUST"
@@ -492,10 +478,6 @@ class MatchQuery(FullTextQuery):
prefix_length : int, optional
The number of beginning characters being unchanged for fuzzy matching.
This is useful to achieve prefix matching.
document_granularity : DocumentGranularity, optional
Explicitly select row or deepest-list-element documents. If omitted,
the indexed granularity is inferred. When both granularities are indexed
for the field, this must be specified. With no index, row granularity is used.
"""
query: str
@@ -505,9 +487,6 @@ class MatchQuery(FullTextQuery):
max_expansions: int = pydantic.Field(50, kw_only=True)
operator: FullTextOperator = pydantic.Field(FullTextOperator.OR, kw_only=True)
prefix_length: int = pydantic.Field(0, kw_only=True)
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
None, kw_only=True
)
def query_type(self) -> FullTextQueryType:
return FullTextQueryType.MATCH
@@ -524,20 +503,11 @@ class PhraseQuery(FullTextQuery):
The query string to match against.
column : str
The name of the column to match against.
slop : int, default 0
The maximum number of intervening positions permitted in the phrase.
document_granularity : DocumentGranularity, optional
Explicitly select row or deepest-list-element documents. If omitted,
the indexed granularity is inferred. When both granularities are indexed
for the field, this must be specified. With no index, row granularity is used.
"""
query: str
column: str
slop: int = pydantic.Field(0, kw_only=True)
document_granularity: Optional[DocumentGranularity] = pydantic.Field(
None, kw_only=True
)
def query_type(self) -> FullTextQueryType:
return FullTextQueryType.MATCH_PHRASE
@@ -805,10 +775,6 @@ class Query(pydantic.BaseModel):
# offset to start fetching results from
offset: Optional[int] = None
# Dataset offsets whose duplicate occurrences must be restored after lookup.
# This is populated when a take query is converted to this serializable form.
take_offsets: Optional[List[int]] = None
# if true, will only search the indexed data
fast_search: Optional[bool] = None
@@ -830,7 +796,6 @@ class Query(pydantic.BaseModel):
query = cls()
query.limit = req.limit
query.offset = req.offset
query.take_offsets = req.take_offsets
query.filter = req.filter
query.full_text_query = req.full_text_search
query.columns = req.select
@@ -2811,16 +2776,15 @@ class AsyncQueryBase(object):
req = self._inner.to_query_request()
schema = await self._table.schema()
projection = _query_request_projection(req)
self._blob_auto_row_id = blob_auto_row_id_for_scan(
schema,
projection,
req.select,
with_row_id=self._with_row_id,
)
if not self._blob_auto_row_id:
self._blob_paths = ()
return
self._blob_paths = tuple(blob_v2_projection_sources(schema, projection).keys())
self._blob_paths = tuple(blob_v2_projection_sources(schema, req.select).keys())
self._inner.with_row_id()
def select(self, columns: Union[List[str], dict[str, str]]) -> Self:
@@ -3414,10 +3378,9 @@ class AsyncQuery(AsyncStandardQuery):
pass in multiple vectors. When multiple vectors are passed in, if the vector
column is with multivector type, then the vectors will be treated as a single
query. Or the vectors will be treated as multiple queries, this can be useful
if you want to find the nearest vectors to multiple query vectors. Flat
searches share one table scan across the query vectors, avoiding the scan
and memory amplification of making multiple queries concurrently. If
multiple vectors are passed in then
if you want to find the nearest vectors to multiple query vectors.
This is not expected to be faster than making multiple queries concurrently;
it is just a convenience method. If multiple vectors are passed in then
an additional column `query_index` will be added to the results. This column
will contain the index of the query vector that the result is nearest to.
"""
@@ -3546,8 +3509,8 @@ class AsyncFTSQuery(AsyncStandardQuery):
Typically, a single vector is passed in as the query. However, you can also
pass in multiple vectors. This can be useful if you want to find the nearest
vectors to multiple query vectors. Flat searches share one table scan across
the query vectors instead of issuing concurrent full scans.
vectors to multiple query vectors. This is not expected to be faster than
making multiple queries concurrently; it is just a convenience method.
If multiple vectors are passed in then an additional column `query_index`
will be added to the results. This column will contain the index of the
query vector that the result is nearest to.
@@ -3907,15 +3870,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
blob_paths: tuple[str, ...] = ()
if self._table is not None:
schema = await self._table.schema()
projection = _query_request_projection(req)
blob_auto_row_id = blob_auto_row_id_for_scan(
schema,
projection,
req.select,
with_row_id=self._with_row_id,
)
if blob_auto_row_id:
blob_paths = tuple(
blob_v2_projection_sources(schema, projection).keys()
blob_v2_projection_sources(schema, req.select).keys()
)
self._blob_auto_row_id = blob_auto_row_id
self._blob_paths = blob_paths
-20
View File
@@ -749,10 +749,6 @@ class RemoteDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
@@ -776,22 +772,6 @@ class RemoteDBConnection(DBConnection):
"""
return LOOP.run(self._conn.cancel_job(job_id))
@override
def pause_job(self, job_id: str) -> str:
"""Pause a server-side job by id.
Returns "pausing", "already_paused", or "committing".
"""
return LOOP.run(self._conn.pause_job(job_id))
@override
def resume_job(self, job_id: str) -> str:
"""Resume a paused server-side job by id.
Returns "resumed", "still_pausing", or "not_paused".
"""
return LOOP.run(self._conn.resume_job(job_id))
@override
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
+5 -19
View File
@@ -36,7 +36,6 @@ from lancedb._lancedb import (
UpdateResult,
)
from lancedb.embeddings.base import EmbeddingFunctionConfig
from lancedb.expr import Expr
from lancedb.index import (
FTS,
BTree,
@@ -62,20 +61,11 @@ from lancedb.table import _normalize_progress
from ..query import (
AnalyzePlanDistributedMetrics,
DocumentGranularity,
LanceQueryBuilder,
LanceTakeQueryBuilder,
LanceVectorQueryBuilder,
)
from ..table import (
AsyncTable,
BlobMode,
Branches,
IndexStatistics,
Query,
Table,
Tags,
)
from ..table import AsyncTable, BlobMode, Branches, IndexStatistics, Query, Table, Tags
from ..types import BaseTokenizerType
@@ -359,7 +349,6 @@ class RemoteTable(Table):
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -382,7 +371,6 @@ class RemoteTable(Table):
ngram_max_length=ngram_max_length,
prefix_only=prefix_only,
block_size=block_size,
document_granularity=document_granularity,
)
LOOP.run(
self._table.create_index(
@@ -872,7 +860,7 @@ class RemoteTable(Table):
def update(
self,
where: Optional[Union[str, Expr]] = None,
where: Optional[str] = None,
values: Optional[dict] = None,
*,
values_sql: Optional[Dict[str, str]] = None,
@@ -883,11 +871,9 @@ class RemoteTable(Table):
Parameters
----------
where: str or [Expr][lancedb.expr.Expr], optional
The filter condition. Can be a SQL string or a type-safe
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
[lit][lancedb.expr.lit]. The filter must not be empty, or it will
error.
where: str, optional
The SQL where clause to use when updating rows. For example, 'x = 2'
or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
values: dict, optional
The values to update. The keys are the column names and the values
are the values to set.
+34 -101
View File
@@ -4,34 +4,30 @@
"""Schema helpers for Lance blob columns."""
import importlib
from typing import TYPE_CHECKING
import pyarrow as pa
import pyarrow.ipc
if TYPE_CHECKING:
from lance.blob import BlobType as BlobType
_BLOB_EXTENSION_NAME = "lance.blob.v2"
_BLOB_V1_KEY = "lance-encoding:blob"
_ARROW_EXT_NAME_KEY = "ARROW:extension:name"
_BLOB_V2_STORAGE_TYPE = pa.struct(
[
pa.field("data", pa.large_binary(), nullable=True),
pa.field("uri", pa.utf8(), nullable=True),
pa.field("position", pa.uint64(), nullable=True),
pa.field("size", pa.uint64(), nullable=True),
]
)
_resolved_blob_type = None
class _FallbackBlobType(pa.ExtensionType):
"""lance.blob.v2 extension type used when pylance is not installed."""
class BlobType(pa.ExtensionType):
"""PyArrow extension type for a Lance blob v2 column.
Queries return descriptors; call :meth:`~lancedb.table.Table.fetch_blob_files`
for lazy reads or :meth:`~lancedb.table.Table.fetch_blobs` for eager bytes.
"""
def __init__(self) -> None:
pa.ExtensionType.__init__(self, _BLOB_V2_STORAGE_TYPE, _BLOB_EXTENSION_NAME)
storage_type = pa.struct(
[
pa.field("data", pa.large_binary(), nullable=True),
pa.field("uri", pa.utf8(), nullable=True),
pa.field("position", pa.uint64(), nullable=True),
pa.field("size", pa.uint64(), nullable=True),
]
)
super().__init__(storage_type, _BLOB_EXTENSION_NAME)
def __arrow_ext_serialize__(self) -> bytes:
return b""
@@ -39,16 +35,23 @@ class _FallbackBlobType(pa.ExtensionType):
@classmethod
def __arrow_ext_deserialize__(
cls, storage_type: pa.DataType, serialized: bytes
) -> "_FallbackBlobType":
) -> "BlobType":
return cls()
def __reduce__(self):
# Ensure pickle round-trips on older pyarrow (apache/arrow#35599).
return type(self).__arrow_ext_deserialize__, (
self.storage_type,
self.__arrow_ext_serialize__(),
)
try:
pa.register_extension_type(BlobType()) # type: ignore[arg-type]
except pa.ArrowKeyError:
pass
def _metadata_value(metadata: dict, key: str):
return metadata.get(key.encode()) or metadata.get(key)
@@ -89,105 +92,43 @@ def is_blob_like_field(field: pa.Field) -> bool:
return is_blob_v2_field(field) or _metadata_marks_legacy_blob(field.metadata or {})
def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[tuple[str, bool]]:
"""Walk the schema and return (path, has_list_ancestor) for each blob field."""
paths: list[tuple[str, bool]] = []
def _collect_blob_paths(schema: pa.Schema, is_blob) -> list[str]:
paths: list[str] = []
def walk(fields, prefix: str, has_list_ancestor: bool) -> None:
def walk(fields, prefix: str) -> None:
for field in fields:
path = f"{prefix}.{field.name}" if prefix else field.name
if is_blob(field):
paths.append((path, has_list_ancestor))
paths.append(path)
elif pa.types.is_struct(field.type):
walk(field.type, path, has_list_ancestor)
walk(field.type, path)
elif (
pa.types.is_list(field.type)
or pa.types.is_large_list(field.type)
or pa.types.is_fixed_size_list(field.type)
):
walk([field.type.value_field], path, True)
walk([field.type.value_field], path)
walk(schema, "", False)
walk(schema, "")
return paths
def blob_column_paths(schema: pa.Schema) -> list[str]:
"""Dotted paths of blob-like columns (v2 extension or legacy metadata)."""
return [path for path, _ in _collect_blob_paths(schema, is_blob_like_field)]
return _collect_blob_paths(schema, is_blob_like_field)
def blob_v2_column_paths(schema: pa.Schema) -> list[str]:
return [path for path, _ in _collect_blob_paths(schema, is_blob_v2_field)]
def row_addressable_blob_v2_paths(schema: pa.Schema) -> list[str]:
"""Blob v2 paths with one blob addressable by table row id.
``fetch_blobs`` and the descriptor row-id ride-along address one blob per
row, so a blob inside a list container has no row-id slot and no fetch
path. Those columns still store and query as raw descriptors.
"""
return [
path
for path, has_list_ancestor in _collect_blob_paths(schema, is_blob_v2_field)
if not has_list_ancestor
]
return _collect_blob_paths(schema, is_blob_v2_field)
def schema_has_blob_field(schema: pa.Schema) -> bool:
return bool(blob_column_paths(schema))
def _deserialize_registered_type(extension_type: pa.ExtensionType) -> pa.DataType:
"""Return the type Arrow reconstructs for this extension name."""
schema = pa.schema([pa.field("value", extension_type)])
restored = pa.ipc.read_schema(schema.serialize())
return restored.field("value").type
def _resolve_blob_type():
"""Return the BlobType class this process should use.
pylance's class when it owns the lance.blob.v2 registry entry,
otherwise LanceDB's fallback. A different registered class is an error.
"""
global _resolved_blob_type
if _resolved_blob_type is not None:
return _resolved_blob_type
try:
blob_module = importlib.import_module("lance.blob")
except ModuleNotFoundError as err:
if err.name not in ("lance", "lance.blob"):
raise
else:
blob_type = getattr(blob_module, "BlobType", None)
if blob_type is not None:
registered_type = _deserialize_registered_type(blob_type())
if type(registered_type) is not blob_type:
registered_cls = type(registered_type)
raise ValueError(
"lance.blob.v2 is already registered by "
f"{registered_cls.__module__}.{registered_cls.__qualname__}"
)
_resolved_blob_type = blob_type
return blob_type
try:
pa.register_extension_type(_FallbackBlobType()) # type: ignore[arg-type]
except pa.ArrowKeyError as err:
raise ValueError(
"lance.blob.v2 is already registered by another extension class"
) from err
_resolved_blob_type = _FallbackBlobType
return _resolved_blob_type
def blob(name: str, nullable: bool = True) -> pa.Field:
"""Create a Lance blob v2 column field.
When pylance is installed this is ``lance.blob.BlobType``.
"""
blob_type = _resolve_blob_type()
return pa.field(name, blob_type(), nullable=nullable)
"""Create a Lance blob v2 column field."""
return pa.field(name, BlobType(), nullable=nullable)
def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
@@ -214,11 +155,3 @@ def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataTyp
... ])
"""
return pa.list_(value_type, dimension)
def __getattr__(name: str):
if name == "BlobType":
blob_type = _resolve_blob_type()
globals()["BlobType"] = blob_type
return blob_type
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+86 -323
View File
@@ -85,7 +85,6 @@ from .query import (
AsyncQuery,
AsyncTakeQuery,
AsyncVectorQuery,
DocumentGranularity,
FullTextQuery,
LanceEmptyQueryBuilder,
LanceFtsQueryBuilder,
@@ -104,12 +103,7 @@ from .util import (
value_to_sql,
)
from .index import lang_mapping
from .schema import (
blob_v2_column_paths,
is_blob_v2_field,
row_addressable_blob_v2_paths,
schema_has_blob_field,
)
from .schema import blob_v2_column_paths, schema_has_blob_field
def _should_push_down_query_table(
@@ -431,7 +425,6 @@ def _cast_to_target_schema(
def gen():
for batch in reader:
batch = _coerce_blob_write_columns(batch, reordered_schema)
# Table but not RecordBatch has cast.
cast_batches = (
pa.Table.from_batches([batch]).cast(reordered_schema).to_batches()
@@ -444,166 +437,6 @@ def _cast_to_target_schema(
return pa.RecordBatchReader.from_batches(reordered_schema, gen())
def _coerce_blob_write_columns(
batch: pa.RecordBatch, target_schema: pa.Schema
) -> pa.RecordBatch:
"""Materialize blob storage structs before the stream leaves Python.
merge_insert requires its source reader to already match the table's
physical schema. Unlike add and insert, it does not pass through
LanceDB's Rust blob coercion, so preserving binary input here would
reach Lance as binary and fail the schema check.
"""
columns = []
fields = []
changed = False
for field, column in zip(batch.schema, batch.columns):
target_field = target_schema.field(field.name)
coerced = _coerce_blob_value(column, target_field)
if coerced is not column:
column = coerced
field = pa.field(
field.name,
coerced.type,
field.nullable,
target_field.metadata,
)
changed = True
columns.append(column)
fields.append(field)
if not changed:
return batch
return pa.RecordBatch.from_arrays(
columns, schema=pa.schema(fields, metadata=batch.schema.metadata)
)
def _coerce_blob_value(column: pa.Array, target_field: pa.Field) -> pa.Array:
if is_blob_v2_field(target_field) and _can_coerce_to_blob(column.type):
return _coerce_value_to_blob(column, target_field)
target_type = target_field.type
if pa.types.is_struct(target_type) and pa.types.is_struct(column.type):
children = []
fields = []
changed = False
for source_field in column.type:
source_column = column.field(source_field.name)
nested_target = next(
(field for field in target_type if field.name == source_field.name),
None,
)
if nested_target is None:
children.append(source_column)
fields.append(source_field)
continue
coerced = _coerce_blob_value(source_column, nested_target)
if coerced is not source_column:
changed = True
child_array, child_type = _physical_array_and_type(coerced)
children.append(child_array)
fields.append(
pa.field(
source_field.name,
child_type,
source_field.nullable,
nested_target.metadata,
)
)
if not changed:
return column
return pa.StructArray.from_arrays(
children,
fields=fields,
mask=column.is_null() if column.null_count else None,
)
if _is_list_like(target_type) and _is_list_like(column.type):
return _coerce_blob_list_values(column, target_type.value_field)
return column
def _coerce_blob_list_values(
column: pa.Array, target_value_field: pa.Field
) -> pa.Array:
"""Coerce blob values inside a list column, preserving offsets and nulls.
Works on the raw child values window instead of ``pc.list_flatten`` because
flatten drops values spanned by null slots, which would misalign offsets.
"""
mask = column.is_null() if column.null_count else None
if pa.types.is_fixed_size_list(column.type):
list_size = column.type.list_size
values = column.values.slice(column.offset * list_size, len(column) * list_size)
coerced = _coerce_blob_value(values, target_value_field)
if coerced is values:
return column
physical_values, _ = _physical_array_and_type(coerced)
return pa.FixedSizeListArray.from_arrays(physical_values, list_size, mask=mask)
offsets = column.offsets
first_offset = offsets[0].as_py()
values = column.values.slice(
first_offset,
offsets[-1].as_py() - first_offset,
)
coerced = _coerce_blob_value(values, target_value_field)
if coerced is values:
return column
physical_values, _ = _physical_array_and_type(coerced)
if first_offset:
offsets = pc.subtract(offsets, pa.scalar(first_offset, offsets.type))
if pa.types.is_large_list(column.type):
return pa.LargeListArray.from_arrays(offsets, physical_values, mask=mask)
return pa.ListArray.from_arrays(offsets, physical_values, mask=mask)
def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
if pa.types.is_null(values.type):
data = pa.nulls(len(values), type=pa.large_binary())
elif pa.types.is_large_binary(values.type):
data = values
else:
data = values.cast(pa.large_binary())
length = len(values)
storage_type = target_field.type
if isinstance(storage_type, pa.ExtensionType):
storage_type = storage_type.storage_type
storage_fields = list(storage_type)
children = []
for storage_field in storage_fields:
if storage_field.name == "data":
children.append(data)
else:
children.append(pa.nulls(length, type=storage_field.type))
storage = pa.StructArray.from_arrays(
children,
fields=storage_fields,
mask=values.is_null() if values.null_count else None,
)
if isinstance(target_field.type, pa.ExtensionType):
return pa.ExtensionArray.from_storage(target_field.type, storage)
return storage
def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
if isinstance(array.type, pa.ExtensionType):
return array.storage, array.type.storage_type
return array, array.type
def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
return _is_binary_like(data_type) or pa.types.is_null(data_type)
def _is_binary_like(data_type: pa.DataType) -> bool:
return (
pa.types.is_binary(data_type)
or pa.types.is_large_binary(data_type)
or pa.types.is_binary_view(data_type)
)
def _field_extension_name(field: pa.Field) -> Optional[str]:
extension_name = getattr(field.type, "extension_name", None)
if extension_name is not None:
@@ -630,73 +463,65 @@ def _align_field_types(
target_field = next((f for f in target_fields if f.name == field.name), None)
if target_field is None:
raise ValueError(f"Field '{field.name}' not found in target schema")
new_fields.append(_align_field(field, target_field))
# Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
# JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
# input to that storage type here merely relabels the raw JSON bytes as
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
if (
_field_extension_name(field) == "arrow.json"
and _field_extension_name(target_field) == "lance.json"
):
new_fields.append(field)
continue
if pa.types.is_struct(target_field.type):
if pa.types.is_struct(field.type):
new_type = pa.struct(
_align_field_types(
field.type.fields,
target_field.type.fields,
)
)
else:
new_type = target_field.type
elif pa.types.is_list(target_field.type):
if _is_list_like(field.type):
new_type = pa.list_(
_align_field_types(
[field.type.value_field],
[target_field.type.value_field],
)[0]
)
else:
new_type = target_field.type
elif pa.types.is_large_list(target_field.type):
if _is_list_like(field.type):
new_type = pa.large_list(
_align_field_types(
[field.type.value_field],
[target_field.type.value_field],
)[0]
)
else:
new_type = target_field.type
elif pa.types.is_fixed_size_list(target_field.type):
if _is_list_like(field.type):
new_type = pa.list_(
_align_field_types(
[field.type.value_field],
[target_field.type.value_field],
)[0],
target_field.type.list_size,
)
else:
new_type = target_field.type
else:
new_type = target_field.type
new_fields.append(
pa.field(field.name, new_type, field.nullable, target_field.metadata)
)
return new_fields
def _align_list_value_field(
value_field: pa.Field, target_value_field: pa.Field
) -> pa.Field:
# A list has exactly one child, so the inferred child name ("item") aligns
# positionally and adopts the table's child name; pa.Table.cast renames it.
return _align_field(value_field, target_value_field).with_name(
target_value_field.name
)
def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
# Preserve arrow.json input until it reaches Lance. LanceDB exposes stored
# JSON columns as lance.json (JSONB-backed LargeBinary), but casting the
# input to that storage type here merely relabels the raw JSON bytes as
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
if (
_field_extension_name(field) == "arrow.json"
and _field_extension_name(target_field) == "lance.json"
):
return field
if pa.types.is_struct(target_field.type):
if pa.types.is_struct(field.type):
new_type = pa.struct(
_align_field_types(
field.type.fields,
target_field.type.fields,
)
)
else:
new_type = target_field.type
elif pa.types.is_list(target_field.type):
if _is_list_like(field.type):
new_type = pa.list_(
_align_list_value_field(
field.type.value_field, target_field.type.value_field
)
)
else:
new_type = target_field.type
elif pa.types.is_large_list(target_field.type):
if _is_list_like(field.type):
new_type = pa.large_list(
_align_list_value_field(
field.type.value_field, target_field.type.value_field
)
)
else:
new_type = target_field.type
elif pa.types.is_fixed_size_list(target_field.type):
if _is_list_like(field.type):
new_type = pa.list_(
_align_list_value_field(
field.type.value_field, target_field.type.value_field
),
target_field.type.list_size,
)
else:
new_type = target_field.type
else:
new_type = target_field.type
return pa.field(field.name, new_type, field.nullable, target_field.metadata)
def _infer_subschema(
schema: List[pa.Field],
reference_fields: List[pa.Field],
@@ -763,7 +588,7 @@ def sanitize_create_table(
schema = data.schema
else:
if schema is not None:
data = pa.Table.from_batches([], schema=schema)
data = pa.Table.from_pylist([], schema)
if schema is None:
if data is None:
raise ValueError("Either data or schema must be provided")
@@ -1343,7 +1168,6 @@ class Table(ABC):
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
):
@@ -1422,11 +1246,6 @@ class Table(ABC):
The number of documents per compressed posting block. Must be 128
or 256. A value of 256 uses the experimental FTS V3 format and
may introduce breaking changes.
document_granularity: DocumentGranularity, default ROW
``ROW`` treats the selected text in one table row as one document.
``LIST_ELEMENT`` treats each element of the deepest list on the field
path as one document and returns its physical coordinates in
``_doc_index`` for matching queries.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
@@ -1678,9 +1497,9 @@ class Table(ABC):
Offsets are mostly useful for sampling as the set of all valid offsets is easily
known in advance to be [0, len(table)).
No guarantees are made regarding the order in which results are returned.
Repeated offsets produce repeated rows, which makes this method suitable for
sampling with replacement.
No guarantees are made regarding the order in which results are returned. If
you desire an output order that matches the order of the given offsets, you will
need to add the row offset column to the output and align it yourself.
Parameters
----------
@@ -1918,7 +1737,7 @@ class Table(ABC):
@abstractmethod
def update(
self,
where: Optional[Union[str, Expr]] = None,
where: Optional[str] = None,
values: Optional[dict] = None,
*,
values_sql: Optional[Dict[str, str]] = None,
@@ -1933,11 +1752,9 @@ class Table(ABC):
Parameters
----------
where: str or [Expr][lancedb.expr.Expr], optional
The filter condition. Can be a SQL string or a type-safe
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
[lit][lancedb.expr.lit]. The filter must not be empty, or it will
error.
where: str, optional
The SQL where clause to use when updating rows. For example, 'x = 2'
or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
values: dict, optional
The values to update. The keys are the column names and the values
are the values to set.
@@ -1955,7 +1772,6 @@ class Table(ABC):
Examples
--------
>>> import lancedb
>>> from lancedb.expr import col
>>> import pandas as pd
>>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
>>> db = lancedb.connect("./.lancedb")
@@ -1965,7 +1781,7 @@ class Table(ABC):
0 1 [1.0, 2.0]
1 2 [3.0, 4.0]
2 3 [5.0, 6.0]
>>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
>>> table.update(where="x = 2", values={"vector": [10.0, 10]})
UpdateResult(rows_updated=1, version=2)
>>> table.to_pandas()
x vector
@@ -2165,11 +1981,9 @@ class Table(ABC):
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str], optional
A mapping from output column names to SQL expressions derives each
output field from its expression. A direct projection of a Blob v2
field inherits Blob v2 semantics; other expressions derive their
ordinary Arrow type. Mapping order is declaration and dependency
order.
A map of column name to a SQL expression defining the column. The
column's type and inputs are derived from the expression, so no
data type is supplied.
Unlike ``transforms``, the expression is stored rather than
evaluated now: the column is committed with no values, and rows get
@@ -2306,25 +2120,12 @@ class Table(ABC):
----------
updates : dict
One or more dicts, each with:
- "path": str dot-path to the field (e.g. "embedding" or "a.b.c").
- "metadata": dict[str, str | None] keys to set; a value of ``None``
deletes that key.
- "replace": bool, optional replace the field's whole metadata map
instead of merging (default False).
The following keys are treated specially, by convention, and should
be used when appropriate:
- "lancedb:description": for a human-readable description of a field.
- ``"lancedb:tag:<name>"`` for a user-defined key-value tag, where the
suffix names the tag category; e.g. "lancedb:tag:model": "clip".
- "lancedb:logical-column" for a column grouping; e.g. "feature_v1"
and "feature_v2" might be in the same logical column.
- "lancedb:status" for status options ("production", "candidate",
"deprecated", "archived") to designate the current life cycle
state of this column.
Returns
-------
UpdateFieldMetadataResult
@@ -2874,7 +2675,7 @@ class LanceTable(Table):
arrow_tbl = self.to_arrow()
if blob_mode == "descriptions":
arrow_tbl = strip_auto_row_ids(
arrow_tbl, row_addressable_blob_v2_paths(self.schema)
arrow_tbl, blob_v2_column_paths(self.schema)
)
return arrow_tbl.to_pandas(**kwargs)
@@ -3472,7 +3273,6 @@ class LanceTable(Table):
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
document_granularity: DocumentGranularity = DocumentGranularity.ROW,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -3524,11 +3324,7 @@ class LanceTable(Table):
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
tokenizer_configs["custom_stop_words"] = custom_stop_words
config = FTS(
block_size=block_size,
document_granularity=document_granularity,
**tokenizer_configs,
)
config = FTS(block_size=block_size, **tokenizer_configs)
try:
LOOP.run(
@@ -4020,7 +3816,7 @@ class LanceTable(Table):
def update(
self,
where: Optional[Union[str, Expr]] = None,
where: Optional[str] = None,
values: Optional[dict] = None,
*,
values_sql: Optional[Dict[str, str]] = None,
@@ -4031,11 +3827,9 @@ class LanceTable(Table):
Parameters
----------
where: str or [Expr][lancedb.expr.Expr], optional
The filter condition. Can be a SQL string or a type-safe
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
[lit][lancedb.expr.lit]. The filter must not be empty, or it will
error.
where: str, optional
The SQL where clause to use when updating rows. For example, 'x = 2'
or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
values: dict, optional
The values to update. The keys are the column names and the values
are the values to set.
@@ -4053,7 +3847,6 @@ class LanceTable(Table):
Examples
--------
>>> import lancedb
>>> from lancedb.expr import col
>>> import pandas as pd
>>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})
>>> db = lancedb.connect("./.lancedb")
@@ -4063,7 +3856,7 @@ class LanceTable(Table):
0 1 [1.0, 2.0]
1 2 [3.0, 4.0]
2 3 [5.0, 6.0]
>>> table.update(where=col("x") == 2, values={"vector": [10.0, 10]})
>>> table.update(where="x = 2", values={"vector": [10.0, 10]})
UpdateResult(rows_updated=1, version=2)
>>> table.to_pandas()
x vector
@@ -4090,7 +3883,6 @@ class LanceTable(Table):
)
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
):
from lancedb.namespace import _execute_server_side_query
@@ -5279,9 +5071,7 @@ class AsyncTable:
if blob_mode == "descriptions" or not schema_has_blob_field(schema):
arrow_tbl = await self.to_arrow()
if blob_mode == "descriptions":
arrow_tbl = strip_auto_row_ids(
arrow_tbl, row_addressable_blob_v2_paths(schema)
)
arrow_tbl = strip_auto_row_ids(arrow_tbl, blob_v2_column_paths(schema))
return arrow_tbl.to_pandas(**kwargs)
if blob_mode == "lazy" and get_uri_scheme(await self.uri()) == "memory":
@@ -5984,23 +5774,7 @@ class AsyncTable:
def _sync_query_to_async(
self, query: Query
) -> (
AsyncHybridQuery
| AsyncFTSQuery
| AsyncVectorQuery
| AsyncQuery
| AsyncTakeQuery
):
if query.take_offsets is not None:
take_query = self.take_offsets(query.take_offsets)
if query.columns:
take_query = take_query.select(query.columns)
if query.use_lsm is not None:
take_query = take_query.use_lsm(query.use_lsm)
if query.with_row_id:
take_query = take_query.with_row_id()
return take_query
) -> AsyncHybridQuery | AsyncFTSQuery | AsyncVectorQuery | AsyncQuery:
async_query = self.query()
if query.limit is not None:
async_query = async_query.limit(query.limit)
@@ -6065,7 +5839,6 @@ class AsyncTable:
self._namespace_client, self._pushdown_operations
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
):
from lancedb.namespace import _execute_server_side_query
@@ -6197,7 +5970,7 @@ class AsyncTable:
self,
updates: Optional[Dict[str, Any]] = None,
*,
where: Optional[Union[str, Expr]] = None,
where: Optional[str] = None,
updates_sql: Optional[Dict[str, str]] = None,
) -> UpdateResult:
"""
@@ -6212,11 +5985,9 @@ class AsyncTable:
The updates to apply. The keys should be the name of the column to
update. The values should be the new values to assign. This is
required unless updates_sql is supplied.
where: str or [Expr][lancedb.expr.Expr], optional
The filter condition. Can be a SQL string or a type-safe
[Expr][lancedb.expr.Expr] built with [col][lancedb.expr.col] and
[lit][lancedb.expr.lit]. Only rows that satisfy this filter will
be updated.
where: str, optional
An SQL filter that controls which rows are updated. For example, 'x = 2'
or 'x IN (1, 2, 3)'. Only rows that satisfy this filter will be udpated.
updates_sql: dict, optional
The updates to apply, expressed as SQL expression strings. The keys should
be column names. The values should be SQL expressions. These can be SQL
@@ -6234,14 +6005,13 @@ class AsyncTable:
--------
>>> import asyncio
>>> import lancedb
>>> from lancedb.expr import col
>>> import pandas as pd
>>> async def demo_update():
... data = pd.DataFrame({"x": [1, 2], "vector": [[1, 2], [3, 4]]})
... db = await lancedb.connect_async("./.lancedb")
... table = await db.create_table("my_table", data)
... # x is [1, 2], vector is [[1, 2], [3, 4]]
... await table.update({"vector": [10, 10]}, where=col("x") == 2)
... await table.update({"vector": [10, 10]}, where="x = 2")
... # x is [1, 2], vector is [[1, 2], [10, 10]]
... await table.update(updates_sql={"x": "x + 1"})
... # x is [2, 3], vector is [[1, 2], [10, 10]]
@@ -6255,8 +6025,7 @@ class AsyncTable:
if updates is not None:
updates_sql = {k: value_to_sql(v) for k, v in updates.items()}
predicate = where.to_sql() if isinstance(where, Expr) else where
return await self._inner.update(updates_sql, predicate)
return await self._inner.update(updates_sql, where)
async def add_columns(
self,
@@ -6288,11 +6057,8 @@ class AsyncTable:
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str], optional
A mapping from output column names to SQL expressions derives each
output field from its expression. A direct projection of a Blob v2
field inherits Blob v2 semantics; other expressions derive their
ordinary Arrow type. Mapping order is declaration and dependency
order.
A map of column name to a SQL expression defining the column. The
column's type and inputs are derived from the expression.
Unlike ``transforms``, the expression is stored rather than
evaluated now: the column is committed with no values, and rows get
@@ -6563,9 +6329,6 @@ class AsyncTable:
Offsets are mostly useful for sampling as the set of all valid offsets is easily
known in advance to be [0, len(table)).
No guarantees are made regarding the order in which results are returned.
Repeated offsets produce repeated rows.
Parameters
----------
offsets: list[int]
+2 -2
View File
@@ -105,7 +105,7 @@ def test_quickstart(tmp_path):
tbl.create_index(num_sub_vectors=1)
# --8<-- [end:create_index]
# --8<-- [start:delete_rows]
tbl.delete("item = 'fizz'")
tbl.delete('item = "fizz"')
# --8<-- [end:delete_rows]
# --8<-- [start:drop_table]
db.drop_table("my_table")
@@ -201,7 +201,7 @@ async def test_quickstart_async(tmp_path):
await tbl.create_index("vector")
# --8<-- [end:create_index_async]
# --8<-- [start:delete_rows_async]
await tbl.delete("item = 'fizz'")
await tbl.delete('item = "fizz"')
# --8<-- [end:delete_rows_async]
# --8<-- [start:drop_table_async]
await db.drop_table("my_table_async")
@@ -266,7 +266,7 @@ def test_table():
tbl.add(pydantic_model_items)
# --8<-- [end:add_table_from_pydantic]
# --8<-- [start:delete_row]
tbl.delete("item = 'fizz'")
tbl.delete('item = "fizz"')
# --8<-- [end:delete_row]
# --8<-- [start:delete_specific_row]
data = [
@@ -538,7 +538,7 @@ async def test_table_async():
await async_tbl.add(pydantic_model_items)
# --8<-- [end:add_table_async_from_pydantic]
# --8<-- [start:delete_row_async]
await async_tbl.delete("item = 'fizz'")
await async_tbl.delete('item = "fizz"')
# --8<-- [end:delete_row_async]
# --8<-- [start:delete_specific_row_async]
data = [
+1 -552
View File
@@ -2,41 +2,17 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import io
import subprocess
import sys
import textwrap
import lance
import pyarrow as pa
import pyarrow.compute as pc
import pytest
from lance.blob import BlobType as LanceBlobType
import lancedb
from lancedb._blob import (
blob_v2_projection_sources,
read_row_ids_from_hits,
stash_auto_row_ids,
)
from lancedb.expr import col
from lancedb._blob import read_row_ids_from_hits, stash_auto_row_ids
from lancedb.index import FTS
from lancedb.schema import blob_column_paths, blob_v2_column_paths
_HIDE_LANCE_BLOB = """\
import importlib.abc
import sys
class _MissingLanceBlob(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path, target=None):
if fullname == "lance.blob" or fullname.startswith("lance.blob."):
raise ModuleNotFoundError(fullname, name="lance.blob")
sys.modules.pop("lance.blob", None)
sys.meta_path.insert(0, _MissingLanceBlob())
"""
def _blob_table(name, rows):
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
@@ -70,181 +46,6 @@ def test_blob_factory_declares_v2_field():
field = lancedb.blob("image")
assert isinstance(field.type, pa.ExtensionType)
assert field.type.extension_name == "lance.blob.v2"
assert lancedb.BlobType is LanceBlobType
assert type(field.type) is LanceBlobType
def test_blob_type_works_without_pylance():
script = _HIDE_LANCE_BLOB + textwrap.dedent(
"""\
import lancedb
import pyarrow as pa
field = lancedb.blob("image")
if not isinstance(field.type, pa.ExtensionType):
raise SystemExit("expected an extension type")
if field.type.extension_name != "lance.blob.v2":
raise SystemExit(field.type.extension_name)
if lancedb.BlobType is not type(field.type):
raise SystemExit("BlobType is not the field type class")
if lancedb.BlobType.__module__ != "lancedb.schema":
raise SystemExit(lancedb.BlobType.__module__)
db = lancedb.connect("memory:///")
table = db.create_table(
"images",
schema=pa.schema([pa.field("id", pa.int64()), field]),
)
table.add([{"id": 1, "image": b"hello"}])
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
)
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
raise SystemExit(
f"merge_insert rows updated={result.num_updated_rows} "
f"inserted={result.num_inserted_rows}"
)
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_blob_resolves_pylance_type_without_eager_import():
script = textwrap.dedent(
"""\
import sys
import lancedb
if "lance.blob" in sys.modules:
raise SystemExit("import lancedb imported lance.blob")
field = lancedb.blob("image")
from lance.blob import BlobType
if type(field.type) is not BlobType:
raise SystemExit(f"{type(field.type)} is not {BlobType}")
import lance
image = lance.blob_array([b"x"])
if type(image.type) is not BlobType:
raise SystemExit("blob_array used a different class")
if type(image.type) is not type(field.type):
raise SystemExit("field and array classes differ")
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_blob_fallback_fails_if_name_already_registered():
script = _HIDE_LANCE_BLOB + textwrap.dedent(
"""\
import pyarrow as pa
class OtherBlobType(pa.ExtensionType):
def __init__(self):
super().__init__(
pa.struct([pa.field("data", pa.large_binary())]),
"lance.blob.v2",
)
def __arrow_ext_serialize__(self):
return b""
@classmethod
def __arrow_ext_deserialize__(cls, storage_type, serialized):
return cls()
pa.register_extension_type(OtherBlobType())
import lancedb
try:
lancedb.blob("image")
except ValueError as err:
if "already registered" not in str(err):
raise SystemExit(err)
else:
raise SystemExit("expected ValueError")
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_blob_type_rejects_competing_registration_with_pylance():
script = textwrap.dedent(
"""\
import pyarrow as pa
import pyarrow.ipc
class OtherBlobType(pa.ExtensionType):
def __init__(self):
super().__init__(
pa.struct(
[
pa.field("data", pa.large_binary()),
pa.field("uri", pa.utf8()),
pa.field("position", pa.uint64()),
pa.field("size", pa.uint64()),
]
),
"lance.blob.v2",
)
def __arrow_ext_serialize__(self):
return b""
@classmethod
def __arrow_ext_deserialize__(cls, storage_type, serialized):
return cls()
pa.register_extension_type(OtherBlobType())
from lance.blob import BlobType
if BlobType is OtherBlobType:
raise SystemExit("pylance BlobType was replaced")
schema = pa.schema([pa.field("value", BlobType())])
restored = pa.ipc.read_schema(schema.serialize())
if type(restored.field("value").type) is not OtherBlobType:
raise SystemExit(type(restored.field("value").type))
import lancedb
try:
lancedb.blob("image")
except ValueError as err:
if "__main__.OtherBlobType" not in str(err):
raise SystemExit(err)
else:
raise SystemExit("expected ValueError")
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_blob_v2_column_paths_include_list_children():
@@ -269,14 +70,6 @@ def test_blob_v2_column_paths_include_list_children():
]
def test_blob_v2_projection_sources_use_typed_column_name():
schema = pa.schema([lancedb.blob("blob")])
assert blob_v2_projection_sources(schema, {"blob_alias": col("blob")}) == {
"blob_alias": "blob"
}
def _legacy_v1_table(name):
db = lancedb.connect("memory:///")
schema = pa.schema(
@@ -373,20 +166,6 @@ async def test_async_table_to_pandas_descriptions_mode_omits_row_id():
assert set(descriptor.keys()) == {"kind", "position", "size", "blob_id", "blob_uri"}
@pytest.mark.asyncio
async def test_async_typed_blob_projection_preserves_source_column():
db = await lancedb.connect_async("memory:///typed_blob_projection")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
table = await db.create_table("typed_blob_projection", schema=schema)
await table.add([{"id": 1, "blob": b"alpha"}])
hits = await table.query().select({"blob_alias": col("blob")}).to_arrow()
assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
blobs = await table.fetch_blobs("blob", hits)
assert blobs.to_pylist() == [b"alpha"]
def test_fetch_blobs_round_trip():
table = _blob_table(
"round_trip",
@@ -397,292 +176,6 @@ def test_fetch_blobs_round_trip():
assert [blobs[0].as_py(), blobs[1].as_py()] == [b"alpha", b"beta"]
def test_merge_insert_writes_python_bytes():
table = _blob_table("merge_bytes", [{"id": 1, "image": b"before"}])
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute([{"id": 1, "image": b"updated"}, {"id": 2, "image": b"inserted"}])
)
assert result.num_updated_rows == 1
assert result.num_inserted_rows == 1
by_id = _row_ids_by_id(table)
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
assert blobs.to_pylist() == [b"updated", b"inserted"]
def test_merge_insert_bytes_after_reopen_without_touching_blob_type(tmp_path):
db = lancedb.connect(tmp_path)
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("images", schema=schema)
table.add([{"id": 1, "image": b"hello"}])
script = textwrap.dedent(
f"""\
import lancedb
db = lancedb.connect({str(tmp_path)!r})
table = db.open_table("images")
image_type = table.schema.field("image").type
if type(image_type).__name__ != "StructType":
raise SystemExit(f"expected StructType, got {{type(image_type)}}")
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(
[{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
)
)
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
raise SystemExit(
f"rows updated={{result.num_updated_rows}} "
f"inserted={{result.num_inserted_rows}}"
)
hits = table.search().with_row_id(True).limit(10).to_arrow()
by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
if blobs.to_pylist() != [b"updated", b"inserted"]:
raise SystemExit(blobs.to_pylist())
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_merge_insert_bytes_after_reopen_without_pylance(tmp_path):
db = lancedb.connect(tmp_path)
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("images", schema=schema)
table.add([{"id": 1, "image": b"hello"}])
script = _HIDE_LANCE_BLOB + textwrap.dedent(
f"""\
import lancedb
db = lancedb.connect({str(tmp_path)!r})
table = db.open_table("images")
image_type = table.schema.field("image").type
if type(image_type).__name__ != "StructType":
raise SystemExit(f"expected StructType, got {{type(image_type)}}")
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(
[{{"id": 1, "image": b"updated"}}, {{"id": 2, "image": b"inserted"}}]
)
)
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
raise SystemExit(
f"rows updated={{result.num_updated_rows}} "
f"inserted={{result.num_inserted_rows}}"
)
hits = table.search().with_row_id(True).limit(10).to_arrow()
by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
if blobs.to_pylist() != [b"updated", b"inserted"]:
raise SystemExit(blobs.to_pylist())
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_merge_insert_blob_array_into_reopened_unregistered_table(tmp_path):
db = lancedb.connect(tmp_path)
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("images", schema=schema)
table.add([{"id": 1, "image": b"before"}])
script = textwrap.dedent(
f"""\
import pyarrow as pa
import lancedb
db = lancedb.connect({str(tmp_path)!r})
table = db.open_table("images")
image_type = table.schema.field("image").type
if type(image_type).__name__ != "StructType":
raise SystemExit(
f"expected StructType before lance import, got {{type(image_type)}}"
)
import lance
updates = pa.Table.from_arrays(
[
pa.array([1, 2], type=pa.int64()),
lance.blob_array([b"updated", b"inserted"]),
],
names=["id", "image"],
)
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(updates)
)
if result.num_updated_rows != 1 or result.num_inserted_rows != 1:
raise SystemExit(
f"rows updated={{result.num_updated_rows}} "
f"inserted={{result.num_inserted_rows}}"
)
hits = table.search().with_row_id(True).limit(10).to_arrow()
by_id = dict(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
if blobs.to_pylist() != [b"updated", b"inserted"]:
raise SystemExit(blobs.to_pylist())
"""
)
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_add_all_null_blob_column():
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("all_null", schema=schema)
table.add([{"id": 1, "image": None}, {"id": 2, "image": None}])
by_id = _row_ids_by_id(table)
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
assert blobs.to_pylist() == [None, None]
def test_create_table_nested_blob_schema_without_rows():
db = lancedb.connect("memory:///")
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("info", pa.struct([lancedb.blob("blob")])),
pa.field("images", pa.list_(lancedb.blob("image"))),
]
)
table = db.create_table("nested_empty", schema=schema)
assert table.count_rows() == 0
def test_merge_insert_nested_blob_dicts():
db = lancedb.connect("memory:///")
info = pa.StructArray.from_arrays(
[
pa.array(["first"], type=pa.string()),
_blob_array("blob", [b"before"]),
],
names=["name", "blob"],
)
data = pa.Table.from_arrays(
[pa.array([1], type=pa.int64()), info],
names=["id", "info"],
)
table = db.create_table("nested_merge", data=data)
result = (
table.merge_insert("id")
.when_matched_update_all()
.execute([{"id": 1, "info": {"name": "first", "blob": b"after"}}])
)
assert result.num_updated_rows == 1
by_id = _row_ids_by_id(table)
blobs = table.fetch_blobs("info.blob", [by_id[1]])
assert blobs.to_pylist() == [b"after"]
def _list_blob_table(name):
db = lancedb.connect("memory:///")
blob_field = lancedb.blob("image")
images = pa.ListArray.from_arrays(
pa.array([0, 1], type=pa.int32()), _blob_array("image", [b"before"])
)
data = pa.Table.from_arrays(
[pa.array([1], type=pa.int64()), images],
schema=pa.schema(
[pa.field("id", pa.int64()), pa.field("images", pa.list_(blob_field))]
),
)
return db.create_table(name, data=data)
def test_merge_insert_list_blob_dicts():
table = _list_blob_table("list_merge")
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute([{"id": 1, "images": [b"one", b"two"]}, {"id": 2, "images": None}])
)
assert result.num_updated_rows == 1
assert result.num_inserted_rows == 1
hits = table.search().limit(10).to_arrow()
sizes = {
row["id"]: None if row["images"] is None else [d["size"] for d in row["images"]]
for row in hits.to_pylist()
}
assert sizes == {1: [3, 3], 2: None}
def test_list_blob_column_queries_as_raw_descriptors():
table = _list_blob_table("list_query")
hits = table.search().limit(10).to_arrow()
element = hits.schema.field("images").type.value_type
assert pa.types.is_struct(element)
assert "_lance_row_id" not in element.names
with pytest.raises(ValueError, match="expected struct before segment"):
table.fetch_blobs("images.image", [0])
def test_row_addressable_paths_exclude_list_children():
from lancedb.schema import row_addressable_blob_v2_paths
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("info", pa.struct([lancedb.blob("blob")])),
pa.field("images", pa.list_(lancedb.blob("image"))),
]
)
assert blob_v2_column_paths(schema) == ["info.blob", "images.image"]
assert row_addressable_blob_v2_paths(schema) == ["info.blob"]
def test_merge_insert_writes_pylance_blob_array():
table = _blob_table("merge_pylance", [{"id": 1, "image": b"before"}])
image = lance.blob_array([b"updated", b"inserted"])
assert type(image.type) is LanceBlobType
assert type(image.type) is type(lancedb.BlobType())
updates = pa.Table.from_arrays(
[pa.array([1, 2], type=pa.int64()), image], names=["id", "image"]
)
result = (
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(updates)
)
assert result.num_updated_rows == 1
assert result.num_inserted_rows == 1
by_id = _row_ids_by_id(table)
blobs = table.fetch_blobs("image", [by_id[1], by_id[2]])
assert blobs.to_pylist() == [b"updated", b"inserted"]
def test_fetch_blobs_accepts_query_result():
table = _blob_table("from_result", [{"id": 1, "image": b"gamma"}])
hits = table.search().limit(10).to_arrow()
@@ -910,50 +403,6 @@ async def test_blob_v2_hybrid_fetch_blobs_async():
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
@pytest.mark.asyncio
async def test_async_hybrid_typed_blob_projection_preserves_source_column():
db = await lancedb.connect_async("memory:///hybrid_typed_blob")
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("text", pa.utf8()),
pa.field("vector", pa.list_(pa.float32(), list_size=2)),
lancedb.blob("blob"),
]
)
table = await db.create_table("hybrid_typed_blob", schema=schema)
await table.add(
[
{
"id": 1,
"text": "hello alpha",
"vector": [1.0, 0.0],
"blob": b"alpha",
},
{
"id": 2,
"text": "hello beta",
"vector": [0.9, 0.1],
"blob": b"beta",
},
]
)
await table.create_index("text", config=FTS(with_position=False))
hits = await (
table.query()
.nearest_to([1.0, 0.0])
.nearest_to_text("hello")
.select({"blob_alias": col("blob")})
.limit(2)
.to_arrow()
)
assert "_lance_row_id" in hits.schema.field("blob_alias").type.names
blobs = await table.fetch_blobs("blob", hits)
assert {blobs[i].as_py() for i in range(len(blobs))} == {b"alpha", b"beta"}
def test_blob_file_seek_read_and_read_range():
payload = _identifiable_payload(1024)
table = _blob_table("seek_read", [{"id": 1, "image": payload}])
+21 -21
View File
@@ -52,7 +52,7 @@ class TestExprConstruction:
def test_func(self):
e = func("lower", col("name"))
assert isinstance(e, Expr)
assert e.to_sql() == "lower(`name`)"
assert e.to_sql() == "lower(name)"
def test_func_unknown_raises(self):
with pytest.raises(Exception):
@@ -115,7 +115,7 @@ class TestExprOperators:
def test_and_operator(self):
e = (col("age") > lit(18)) & (col("status") == lit("active"))
assert isinstance(e, Expr)
assert e.to_sql() == "((age > 18) AND (`status` = 'active'))"
assert e.to_sql() == "((age > 18) AND (status = 'active'))"
def test_or_operator(self):
e = (col("a") == lit(1)) | (col("b") == lit(2))
@@ -166,7 +166,7 @@ class TestExprOperators:
def test_coerce_plain_str(self):
e = col("name") == "alice"
assert isinstance(e, Expr)
assert e.to_sql() == "(`name` = 'alice')"
assert e.to_sql() == "(name = 'alice')"
def test_reflexive_comparisons(self):
# 10 < col("age") swaps to col("age") > 10
@@ -198,85 +198,85 @@ class TestExprBytesLiteral:
def test_bytes_equality_expr_sql(self):
e = col("data") == lit(b"\xca\xfe")
assert e.to_sql() == "(`data` = X'CAFE')"
assert e.to_sql() == "(data = X'CAFE')"
def test_bytes_ne_expr_sql(self):
e = col("data") != lit(b"\xff")
assert e.to_sql() == "(`data` <> X'FF')"
assert e.to_sql() == "(data <> X'FF')"
def test_bytes_compound_expr_sql(self):
e = (col("data") == lit(b"\x01")) & (col("id") > lit(5))
assert e.to_sql() == "((`data` = X'01') AND (id > 5))"
assert e.to_sql() == "((data = X'01') AND (id > 5))"
def test_bytes_in_function_call(self):
# Regression test: binary literals inside scalar function calls
# used to fail because DataFusion's unparser does not support Binary
# scalars. Now handled via a placeholder-substitution rewrite.
e = func("contains", col("data"), lit(b"\xff"))
assert e.to_sql() == "contains(`data`, X'FF')"
assert e.to_sql() == "contains(data, X'FF')"
def test_bytes_in_not(self):
e = ~(col("data") == lit(b"\xff"))
assert e.to_sql() == "NOT (`data` = X'FF')"
assert e.to_sql() == "NOT (data = X'FF')"
class TestExprStringMethods:
def test_lower(self):
e = col("name").lower()
assert isinstance(e, Expr)
assert e.to_sql() == "lower(`name`)"
assert e.to_sql() == "lower(name)"
def test_upper(self):
e = col("name").upper()
assert isinstance(e, Expr)
assert e.to_sql() == "upper(`name`)"
assert e.to_sql() == "upper(name)"
def test_contains(self):
e = col("text").contains(lit("hello"))
assert isinstance(e, Expr)
assert e.to_sql() == "contains(`text`, 'hello')"
assert e.to_sql() == "contains(text, 'hello')"
def test_contains_with_str_coerce(self):
e = col("text").contains("hello")
assert isinstance(e, Expr)
assert e.to_sql() == "contains(`text`, 'hello')"
assert e.to_sql() == "contains(text, 'hello')"
def test_chained_lower_eq(self):
e = col("name").lower() == lit("alice")
assert isinstance(e, Expr)
assert e.to_sql() == "(lower(`name`) = 'alice')"
assert e.to_sql() == "(lower(name) = 'alice')"
class TestExprCast:
def test_cast_string(self):
e = col("id").cast("string")
assert isinstance(e, Expr)
assert e.to_sql() == "arrow_cast(id, 'Utf8')"
assert e.to_sql() == "CAST(id AS VARCHAR)"
def test_cast_int32(self):
e = col("score").cast("int32")
assert isinstance(e, Expr)
assert e.to_sql() == "arrow_cast(score, 'Int32')"
assert e.to_sql() == "CAST(score AS INTEGER)"
def test_cast_float64(self):
e = col("val").cast("float64")
assert isinstance(e, Expr)
assert e.to_sql() == "arrow_cast(val, 'Float64')"
assert e.to_sql() == "CAST(val AS DOUBLE)"
def test_cast_pyarrow_type(self):
e = col("score").cast(pa.int32())
assert isinstance(e, Expr)
assert e.to_sql() == "arrow_cast(score, 'Int32')"
assert e.to_sql() == "CAST(score AS INTEGER)"
def test_cast_pyarrow_float64(self):
e = col("val").cast(pa.float64())
assert isinstance(e, Expr)
assert e.to_sql() == "arrow_cast(val, 'Float64')"
assert e.to_sql() == "CAST(val AS DOUBLE)"
def test_cast_pyarrow_string(self):
e = col("id").cast(pa.string())
assert isinstance(e, Expr)
assert e.to_sql() == "arrow_cast(id, 'Utf8')"
assert e.to_sql() == "CAST(id AS VARCHAR)"
def test_cast_pyarrow_and_string_equivalent(self):
# pa.int32() and "int32" should produce equivalent SQL
@@ -597,14 +597,14 @@ class TestExprIsin:
def test_isin_strs(self):
assert (
col("status").isin(["active", "pending"]).to_sql()
== "`status` IN ('active', 'pending')"
== "status IN ('active', 'pending')"
)
def test_isin_coerces_and_mixes(self):
assert col("id").isin([lit(1), 2]).to_sql() == "id IN (1, 2)"
def test_isin_empty(self):
assert col("id").isin([]).to_sql() == "false"
assert col("id").isin([]).to_sql() == "id IN ()"
def test_isin_filter(self, simple_table):
result = simple_table.search().where(col("id").isin([1, 3, 5])).to_arrow()
@@ -12,8 +12,6 @@ from datetime import date
import http.server
import json
from pathlib import Path
import subprocess
import sys
import threading
from typing import Optional
@@ -21,13 +19,7 @@ import pyarrow as pa
import pytest
import lancedb
from lancedb.functions import (
PythonRuntimeSpec,
UdfDefinition,
_canonical_arrow_type,
_GRAMMAR_PRIMITIVES,
udf,
)
from lancedb.functions import UdfDefinition, udf
THRESHOLD = 20
_CACHE = None
@@ -69,80 +61,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
}
def _main_udf_source(
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
) -> str:
return (
"from __future__ import annotations\n"
"from lancedb.functions import udf\n"
f"THRESHOLD = {threshold}\n"
"\n"
"@udf\n"
f"def label(value: {input_annotation}) -> str:\n"
f" return 'big' if value {comparison} THRESHOLD else 'small'\n"
"\n"
"assert label.__module__ == '__main__'\n"
"print(label.registration_request.to_canonical_json())\n"
)
def _run_main_udf(path: Path, source: str) -> dict:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source)
result = subprocess.run(
[sys.executable, str(path)],
check=True,
capture_output=True,
text=True,
)
return json.loads(result.stdout)
def test_main_udf_registration_identity_is_stable_across_processes_and_paths(
tmp_path,
):
source = _main_udf_source()
original_path = tmp_path / "original" / "job.py"
moved_path = tmp_path / "moved" / "renamed_job.py"
original_runs = [_run_main_udf(original_path, source) for _ in range(2)]
moved_run = _run_main_udf(moved_path, source)
assert len({run["artifact"]["digest"] for run in [*original_runs, moved_run]}) == 1
assert all(
run["signature"] == original_runs[0]["signature"]
for run in [original_runs[1], moved_run]
)
assert original_runs[0] == original_runs[1] == moved_run
body_change = _run_main_udf(
tmp_path / "changes" / "body.py", _main_udf_source(comparison=">")
)
global_change = _run_main_udf(
tmp_path / "changes" / "global.py", _main_udf_source(threshold=21)
)
annotation_change = _run_main_udf(
tmp_path / "changes" / "annotation.py",
_main_udf_source(input_annotation="float"),
)
baseline = original_runs[0]
assert baseline["signature"] == body_change["signature"]
assert baseline["signature"] == global_change["signature"]
assert baseline["signature"] != annotation_change["signature"]
assert (
len(
{
baseline["artifact"]["digest"],
body_change["artifact"]["digest"],
global_change["artifact"]["digest"],
annotation_change["artifact"]["digest"],
}
)
== 4
)
def _run_packaged(definition, *args):
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
source = base64.b64decode(definition.registration_request.artifact.content.data)
@@ -151,78 +69,6 @@ def _run_packaged(definition, *args):
return namespace[definition.registration_request.artifact.entrypoint](*args)
def test_udf_conda_environment():
@udf(conda=["scipy", "numpy"], conda_channels=["conda-forge", "defaults"])
def halve(value: float) -> float:
return value / 2
request = json.loads(halve.registration_request.to_canonical_json())
assert request["runtime"]["environment"] == {
"kind": "conda",
"packages": ["numpy", "scipy"],
"channels": ["conda-forge", "defaults"],
}
pip_request = json.loads(normalize_score.registration_request.to_canonical_json())
assert "channels" not in pip_request["runtime"]["environment"]
with pytest.raises(ValueError, match="not both"):
udf(name="both", pip=["numpy"], conda=["numpy"])(lambda value: value)
with pytest.raises(ValueError, match="requires conda"):
udf(name="channels", conda_channels=["conda-forge"])(lambda value: value)
def test_udf_gpu_marker_uses_gpu_runtime():
@udf(pip=["cupy-cuda12x"], gpu=True)
def double_on_gpu(value: int) -> int:
return value * 2
request = json.loads(double_on_gpu.registration_request.to_canonical_json())
assert request["runtime"]["kind"] == "python_v2"
assert request["runtime"]["gpu"] is True
@udf(pip=["pyarrow"])
def cpu_function(value: int) -> int:
return value
cpu_runtime = json.loads(cpu_function.registration_request.to_canonical_json())[
"runtime"
]
assert cpu_runtime["kind"] == "python"
assert "gpu" not in cpu_runtime
def identity(value: int) -> int:
return value
for invalid in [None, 0, 1, -1, 1.5, "", "true", "1", "H100"]:
with pytest.raises(ValueError, match="gpu must be a boolean"):
udf(name="invalid_gpu", gpu=invalid)(identity)
base_runtime = {
"kind": "python_v2",
"python_version": "3.12",
"environment": {"kind": "pip"},
}
runtime = PythonRuntimeSpec.model_validate({**base_runtime, "gpu": True})
assert runtime.gpu is True
for invalid in [False, 1, 0, "", "true", "1", "H100"]:
with pytest.raises(ValueError, match="runtime.gpu must be true"):
PythonRuntimeSpec.model_validate({**base_runtime, "gpu": invalid})
def test_unknown_runtime_discards_payload_before_known_field_validation():
for payload in [
{"kind": "python_v3", "gpu": {"model": "H100"}},
{"kind": "python_v3", "resources": []},
{
"kind": "python_v3",
"environment": {"kind": []},
"python_version": 3.15,
},
]:
runtime = PythonRuntimeSpec.model_validate(payload)
assert runtime.to_canonical_json() == '{"kind":"python_v3"}'
def test_udf_packages_attribute_access_and_body_imports():
@udf
def word_norm(body: str) -> float:
@@ -302,7 +148,9 @@ def test_udf_resolves_module_globals_before_builtins(tmp_path):
udf(module.uses_callable_shadow)
def test_canonical_arrow_type_prefers_the_compact_grammar():
def test_canonical_arrow_type_is_exactly_the_grammar():
from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
golden = json.loads(
(
Path(__file__).parents[3]
@@ -313,19 +161,14 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"]
]
assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives
assert _canonical_arrow_type(pa.list_(pa.field("item", pa.float32(), False))) == (
"list<float32>"
)
assert (
_canonical_arrow_type(pa.large_list(pa.field("item", pa.float32(), False)))
== "large_list<float32>"
)
for outside in [
pa.timestamp("us"),
pa.decimal128(10, 2),
pa.large_string(),
pa.large_binary(),
pa.binary(4),
pa.duration("s"),
pa.struct([pa.field("a", pa.int32())]),
pa.list_(pa.float32(), 0),
pa.list_(pa.timestamp("us")),
]:
@@ -515,27 +358,14 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
udf(raw_fact)
def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
nullable = pa.list_(pa.float32())
assert json.loads(_canonical_arrow_type(nullable)) == {
"type": "list",
"fields": [
{
"name": "item",
"nullable": True,
"type": {"type": "float32"},
}
],
}
named = pa.list_(pa.field("custom", pa.float32(), nullable=False))
assert json.loads(_canonical_arrow_type(named))["fields"][0]["name"] == "custom"
def test_canonical_arrow_type_rejects_unrepresentable_list_children():
from lancedb.functions import _canonical_arrow_type
for outside in [
pa.list_(pa.float32()), # pyarrow default: nullable child
pa.list_(pa.field("custom", pa.float32(), nullable=False)),
pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})),
pa.list_(pa.field("item", pa.float32(), nullable=False), 0),
pa.list_(
pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"}), 3
),
pa.list_(pa.field("custom", pa.float32(), nullable=False), 3),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(outside)
@@ -545,29 +375,6 @@ def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
)
== "fixed_size_list<float32, 3>"
)
fixed = json.loads(_canonical_arrow_type(pa.list_(pa.float32(), 3)))
assert fixed == {
"type": "fixed_size_list",
"fields": [
{
"name": "item",
"nullable": True,
"type": {"type": "float32"},
}
],
"length": 3,
}
large = json.loads(_canonical_arrow_type(pa.large_list(pa.float32())))
assert large["type"] == "large_list"
assert large["fields"][0]["nullable"] is True
for invalid_struct in [
pa.struct([]),
pa.struct([pa.field("a", pa.int32()), pa.field("a", pa.int64())]),
pa.struct([pa.field("", pa.int32())]),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
_canonical_arrow_type(invalid_struct)
def _calls_missing(value: int) -> int:
@@ -605,7 +412,6 @@ def _arrow_type_from_golden(spec: dict) -> pa.DataType:
"null": pa.null(),
"bool": pa.bool_(),
"utf8": pa.string(),
"large_utf8": pa.large_string(),
"binary": pa.binary(),
"float16": pa.float16(),
"float32": pa.float32(),
@@ -622,6 +428,8 @@ def test_arrow_type_grammar_matches_the_shared_golden():
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
).read_text()
)
from lancedb.functions import _canonical_arrow_type
emitted = {
case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
for case in golden["valid"]
@@ -654,250 +462,6 @@ def test_explicit_arrow_schema_is_deterministic():
assert signature.output.nullable is False
def test_blob_fields_use_the_scalar_function_semantic_type():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=lancedb.blob("result", nullable=False),
)
def copy_blob(image):
return image
signature = copy_blob.registration_request.signature
assert signature.inputs[0].arrow_type == "blob_v2"
assert signature.output.kind == "scalar"
assert signature.output.arrow_type == "blob_v2"
def test_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=pa.schema(
[
lancedb.blob("thumbnail", nullable=False),
pa.field("width", pa.int32(), nullable=False),
]
),
)
def inspect_blob(image):
return {"thumbnail": image, "width": 1}
output = inspect_blob.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.arrow_type) for field in output.fields] == [
("thumbnail", "blob_v2"),
("width", "int32"),
]
def test_metadata_marked_blob_field_uses_the_semantic_type():
extension = lancedb.blob("image", nullable=False).type
storage = (
extension.storage_type if isinstance(extension, pa.ExtensionType) else extension
)
metadata_blob = pa.field(
"image",
storage,
nullable=False,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
@udf(
input_schema=pa.schema([metadata_blob]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(image):
return len(image)
assert blob_size.registration_request.signature.inputs[0].arrow_type == "blob_v2"
def test_blob_marker_rejects_invalid_storage_layout():
malformed = pa.field(
"image",
pa.int64(),
nullable=False,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
with pytest.raises(TypeError, match="requires a supported Blob storage layout"):
@udf(
input_schema=pa.schema([malformed]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(image):
return len(image)
def test_nested_blob_signature_field_has_a_clear_error():
nested = pa.field(
"value",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="nested Blob v2 fields are not supported"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["image"])
def test_nested_non_blob_extension_is_not_silently_unwrapped():
class TestExtension(pa.ExtensionType):
def __init__(self):
super().__init__(pa.int64(), "test.function.extension")
def __arrow_ext_serialize__(self):
return b""
@classmethod
def __arrow_ext_deserialize__(cls, storage_type, serialized):
return cls()
nested = pa.field(
"value",
pa.struct([pa.field("extended", TestExtension(), nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("result", pa.int64(), nullable=False),
)
def extension_value(value):
return value["extended"]
def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
input_schema = pa.schema([pa.field("text", pa.large_string(), nullable=True)])
output_schema = pa.field("result", pa.large_string(), nullable=False)
@udf(input_schema=input_schema, output_schema=output_schema)
def preserve(text):
return text
signature = preserve.registration_request.signature
assert signature.inputs[0].arrow_type == "large_utf8"
assert signature.inputs[0].nullable is True
assert signature.output.arrow_type == "large_utf8"
assert signature.output.nullable is False
nested = pa.struct([pa.field("text", pa.large_string(), nullable=True)])
assert json.loads(_canonical_arrow_type(nested)) == {
"type": "struct",
"fields": [
{
"name": "text",
"nullable": True,
"type": {"type": "large_utf8"},
}
],
}
def test_nested_struct_output_uses_canonical_exact_json():
token = pa.struct(
[
pa.field("position", pa.int32(), nullable=False),
pa.field("value", pa.string(), nullable=False),
pa.field("length", pa.int32(), nullable=False),
]
)
analysis = pa.struct(
[
pa.field("normalized_text", pa.string(), nullable=False),
pa.field("has_content", pa.bool_(), nullable=False),
pa.field(
"metrics",
pa.struct(
[
pa.field("character_count", pa.int64(), nullable=False),
pa.field("word_count", pa.int32(), nullable=False),
pa.field("average_word_length", pa.float64(), nullable=False),
]
),
nullable=False,
),
pa.field(
"diagnostics",
pa.struct(
[
pa.field("status", pa.string(), nullable=False),
pa.field(
"normalization",
pa.struct(
[
pa.field("changed", pa.bool_(), nullable=False),
pa.field(
"original_length", pa.int64(), nullable=False
),
]
),
nullable=False,
),
]
),
nullable=False,
),
pa.field(
"token_preview",
pa.list_(pa.field("item", token, nullable=False)),
nullable=False,
),
]
)
@udf(
input_schema=pa.schema([pa.field("text", pa.string(), nullable=False)]),
output_schema=pa.field("analysis", analysis, nullable=False),
)
def analyze(text):
return {"normalized_text": text}
output = analyze.registration_request.signature.output
assert output.kind == "named_struct"
assert [field.name for field in output.fields] == [
"normalized_text",
"has_content",
"metrics",
"diagnostics",
"token_preview",
]
metrics = json.loads(output.fields[2].arrow_type)
assert metrics == {
"type": "struct",
"fields": [
{
"name": "character_count",
"nullable": False,
"type": {"type": "int64"},
},
{
"name": "word_count",
"nullable": False,
"type": {"type": "int32"},
},
{
"name": "average_word_length",
"nullable": False,
"type": {"type": "float64"},
},
],
}
preview = json.loads(output.fields[4].arrow_type)
assert preview["type"] == "list"
assert preview["fields"][0]["type"]["type"] == "struct"
assert [field["name"] for field in preview["fields"][0]["type"]["fields"]] == [
"position",
"value",
"length",
]
def test_annotation_and_explicit_schema_validation_fail_closed():
with pytest.raises(TypeError, match="missing Function annotations"):
@@ -941,72 +505,6 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
def nullable_explicit(value):
return value
for invalid_field in [
pa.field("", pa.int32(), nullable=False),
pa.field("result", pa.int32(), nullable=False, metadata={"k": "v"}),
]:
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([pa.field("value", pa.int64())]),
output_schema=pa.schema([invalid_field]),
)
def invalid_explicit_field(value):
return value
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema(
[pa.field("value", pa.int64(), metadata={"k": "v"})]
),
output_schema=pa.int64(),
)
def input_field_metadata(value):
return value
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([pa.field("value", pa.int64())]),
output_schema=pa.field(
"result", pa.int64(), nullable=False, metadata={"k": "v"}
),
)
def scalar_output_field_metadata(value):
return value
struct_type = pa.struct([pa.field("value", pa.int64(), nullable=False)])
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([pa.field("value", pa.int64())]),
output_schema=pa.field(
"result", struct_type, nullable=False, metadata={"k": "v"}
),
)
def struct_output_field_metadata(value):
return {"value": value}
for input_schema, output_schema in [
(
pa.schema([pa.field("value", pa.int64())], metadata={"k": "v"}),
pa.int64(),
),
(
pa.schema([pa.field("value", pa.int64())]),
pa.schema(
[pa.field("result", pa.int64(), nullable=False)],
metadata={"k": "v"},
),
),
]:
with pytest.raises(TypeError, match="schema metadata"):
@udf(input_schema=input_schema, output_schema=output_schema)
def schema_metadata(value):
return value
def test_local_function_catalog_operations_are_not_supported(tmp_path):
db = lancedb.connect(tmp_path)
@@ -1017,8 +515,6 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
db.create_function_async(normalize_score)
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version="fv_exact")
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version="fv_exact")
@contextlib.contextmanager
@@ -1064,12 +560,6 @@ def _mock_remote_function_catalog():
"version": "fv_exact",
}
response = state["version"]
elif self.path == "/v1/functions/drop":
assert body == {
"name": "normalize_score",
"version": "fv_exact",
}
response = {"dropped": True}
else:
status = 404
response = {"error": "not found"}
@@ -1128,40 +618,3 @@ def test_blocking_remote_registration_returns_function_version():
"/v1/functions/create",
"/v1/jobs/describe",
]
def test_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
@pytest.mark.asyncio
async def test_async_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert await db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
-77
View File
@@ -25,7 +25,6 @@ from lancedb.db import DBConnection
from lancedb.index import FTS
from lancedb.query import (
BoostQuery,
DocumentGranularity,
MatchQuery,
MultiMatchQuery,
PhraseQuery,
@@ -246,55 +245,6 @@ def test_create_inverted_index_rejects_invalid_block_size(table):
table.create_index("text", config=FTS(block_size=129))
def test_list_element_document_granularity(tmp_path):
docs_type = pa.list_(pa.struct([pa.field("content", pa.string())]))
docs = pa.array(
[
[
{"content": "alpha beta"},
None,
{"content": ""},
{"content": "the and"},
{"content": "alpha beta"},
]
],
type=docs_type,
)
table = ldb.connect(tmp_path).create_table(
"list_element_docs", pa.table({"id": [0], "docs": docs})
)
row_table = ldb.connect(tmp_path).create_table(
"row_docs", pa.table({"id": [0], "docs": docs})
)
row_table.create_index("docs.content", config=FTS())
row_result = row_table.search(MatchQuery("alpha", "docs.content")).to_arrow()
assert row_result.num_rows == 1
assert "_doc_index" not in row_result.column_names
granularity = DocumentGranularity.LIST_ELEMENT
table.create_index(
"docs.content",
config=FTS(with_position=True, document_granularity=granularity),
)
assert table.list_indices()[0].columns == ["docs.content"]
def coordinates(query):
result = table.search(query).limit(10).to_arrow()
doc_index_type = result.schema.field("_doc_index").type
assert pa.types.is_list(doc_index_type)
assert doc_index_type.value_type == pa.uint32()
return sorted(result["_doc_index"].to_pylist())
assert coordinates(
MatchQuery("alpha", "docs.content", document_granularity=granularity)
) == [[0], [4]]
assert coordinates(
PhraseQuery("alpha beta", "docs.content", document_granularity=granularity)
) == [[0], [4]]
assert coordinates(MatchQuery("alpha", "docs.content")) == [[0], [4]]
assert FTS().document_granularity is DocumentGranularity.ROW
def test_create_inverted_index_respects_build_memory_limit(table):
with pytest.raises(ValueError, match="exceeds worker memory limit"):
table.create_index(
@@ -1139,20 +1089,6 @@ def test_fts_query_to_json():
)
assert json_str == expected
# Test MatchQuery with list-element document granularity
match_query = MatchQuery(
"hello world",
"text",
document_granularity=DocumentGranularity.LIST_ELEMENT,
)
json_str = match_query.to_json()
expected = (
'{"match":{"column":"text","terms":"hello world","boost":1.0,'
'"fuzziness":0,"max_expansions":50,"operator":"Or","prefix_length":0,'
'"document_granularity":"list_element"}}'
)
assert json_str == expected
# Test MatchQuery with options
match_query = MatchQuery("puppy", "text", fuzziness=2, boost=1.5, prefix_length=3)
json_str = match_query.to_json()
@@ -1162,19 +1098,6 @@ def test_fts_query_to_json():
)
assert json_str == expected
# Test PhraseQuery with list-element document granularity
phrase_query = PhraseQuery(
"quick brown fox",
"title",
document_granularity=DocumentGranularity.LIST_ELEMENT,
)
json_str = phrase_query.to_json()
expected = (
'{"phrase":{"column":"title","terms":"quick brown fox","slop":0,'
'"document_granularity":"list_element"}}'
)
assert json_str == expected
# Test PhraseQuery
phrase_query = PhraseQuery("quick brown fox", "title")
json_str = phrase_query.to_json()
-47
View File
@@ -675,21 +675,6 @@ def test_distance_range(table: lancedb.table.Table):
assert res["_distance"].to_pylist() == [min_dist, max_dist]
@pytest.mark.parametrize("expression", ["1 - _distance", "1.0 - _distance"])
def test_select_arithmetic_with_distance(table, expression):
result = (
table.search([10, 10])
.select({"similarity": expression, "_distance": "_distance"})
.distance_type("cosine")
.to_arrow()
)
assert result.schema.field("similarity").type == pa.float32()
assert result["similarity"].to_pylist() == pytest.approx(
[1 - distance for distance in result["_distance"].to_pylist()]
)
@pytest.mark.asyncio
async def test_distance_range_async(table_async: AsyncTable):
q = [0, 0]
@@ -912,23 +897,6 @@ def test_query_builder_batches(table):
assert rs_list["id"][1] == 2
def test_batch_vector_query_shares_filtered_flat_scan(table):
query = (
table.search([[1.0, 2.0], [3.0, 4.0]])
.where("id > 0", prefilter=True)
.limit(1)
.select(["id"])
)
plan = query.explain_plan(verbose=True)
assert "KNNVectorDistance: queries=2" in plan
assert "UnionExec" not in plan
results = query.to_arrow()
assert len(results) == 2
assert results["query_index"].to_pylist() == [0, 1]
def test_dynamic_projection(table):
rs = (
LanceVectorQueryBuilder(table, [0, 0], "vector")
@@ -1923,21 +1891,6 @@ def test_take_queries(tmp_path):
17,
]
# Duplicate offsets are occurrences, not set members. Ordering is unspecified.
assert sorted(table.take_offsets([5, 2, 5, 17]).to_pandas()["idx"].to_list()) == [
2,
5,
5,
17,
]
# Converting a take builder to its serializable query representation must
# retain occurrence metadata and execute with the same multiplicity.
query = table.take_offsets([5, 2, 5, 17]).select(["idx"]).to_query_object()
assert query.take_offsets == [5, 2, 5, 17]
converted = table._execute_query(query).read_all()
assert sorted(converted["idx"].to_pylist()) == [2, 5, 5, 17]
# Take by row id
assert list(
sorted(table.take_row_ids([5, 2, 17]).to_pandas()["idx"].to_list())
+5 -96
View File
@@ -479,49 +479,24 @@ def test_remote_permutation_is_picklable():
match = re.search(
r"_rowoffset\s+in\s+\((.*?)\)", body["filter"], re.IGNORECASE
)
offsets = list(
dict.fromkeys(int(o.strip()) for o in match.group(1).split(","))
)
offsets = [int(o.strip()) for o in match.group(1).split(",")]
else:
offsets = list(range(len(rows)))
columns = body.get("columns") or ["a"]
table = pa.table(
{
column: (
[rows[offset] for offset in offsets]
if column == "a"
else offsets
)
for column in columns
}
)
table = pa.table({"a": [rows[offset] for offset in offsets]})
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.file")
request.end_headers()
with pa.ipc.new_file(request.wfile, schema=table.schema) as writer:
writer.write_table(table, max_chunksize=2)
writer.write_table(table)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
assert table.take_offsets([0, 2, 0, 4]).to_list() == [
{"a": 0},
{"a": 0},
{"a": 2},
{"a": 4},
]
permutation = Permutation.identity(table)
permutation = Permutation.identity(db.open_table("test"))
restored = pickle.loads(pickle.dumps(permutation))
assert restored.__getitems__([0, 2, 0, 4]) == [
{"a": 0},
{"a": 2},
{"a": 0},
{"a": 4},
]
assert restored.__getitems__([0, 2, 4]) == [{"a": 0}, {"a": 2}, {"a": 4}]
def test_create_table_exist_ok():
@@ -1643,49 +1618,6 @@ def test_query_sync_fts():
)
def test_query_sync_fts_document_granularity():
from lancedb.query import DocumentGranularity, MatchQuery
def handler(body):
assert body == {
"full_text_query": {
"query": {
"match": {
"column": "docs.content",
"terms": "alpha",
"boost": 1.0,
"fuzziness": 0,
"max_expansions": 50,
"operator": "Or",
"prefix_length": 0,
"document_granularity": "list_element",
}
}
},
"k": 10,
"prefilter": True,
"vector": [],
"version": None,
}
return pa.table(
{
"id": [1, 1],
"_doc_index": pa.array([[0], [4]], type=pa.list_(pa.uint32())),
}
)
with query_test_table(handler, server_version=Version("0.6.0")) as table:
result = table.search(
MatchQuery(
"alpha",
"docs.content",
document_granularity=DocumentGranularity.LIST_ELEMENT,
)
).to_arrow()
assert result["_doc_index"].to_pylist() == [[0], [4]]
def test_query_sync_hybrid():
def handler(body):
if "full_text_query" in body:
@@ -2534,26 +2466,6 @@ def test_remote_connection_jobs_surface():
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"job_id": "job-1"}')
elif request.path == "/v1/jobs/pause":
if payload["job_id"] != "job-1":
request.send_response(404)
request.end_headers()
return
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"job_id": "job-1", "paused": true}')
elif request.path == "/v1/jobs/resume":
if payload["job_id"] != "job-1":
request.send_response(404)
request.end_headers()
return
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
b'{"job_id": "job-1", "resumed": false, "still_pausing": true}'
)
elif request.path == "/v1/jobs/query_events":
assert payload["job_id"] == "job-1"
request.send_response(200)
@@ -2582,9 +2494,6 @@ def test_remote_connection_jobs_surface():
assert db.cancel_job("job-1") is True
assert db.cancel_job("missing") is False
assert db.pause_job("job-1") == "pausing"
assert db.resume_job("job-1") == "still_pausing"
batches = db.job_history("job-1")
assert len(batches) == 1
assert batches[0].num_rows == 2
-20
View File
@@ -4,7 +4,6 @@
import asyncio
import copy
from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
import threading
@@ -87,25 +86,6 @@ def test_s3_lifecycle(s3_bucket: str):
asyncio.run(test())
@pytest.mark.s3_test
def test_concurrent_open_table(s3_bucket: str):
uri = f"s3://{s3_bucket}/test_concurrent_open_table"
db = lancedb.connect(uri, storage_options=copy.copy(CONFIG))
db.create_table("test", pa.table({"x": [1, 2, 3]}))
num_workers = 32
barrier = threading.Barrier(num_workers)
def open_and_count(_):
barrier.wait()
return db.open_table("test").count_rows()
with ThreadPoolExecutor(max_workers=num_workers) as pool:
row_counts = list(pool.map(open_and_count, range(num_workers)))
assert row_counts == [3] * num_workers
@pytest.fixture()
def kms_key():
kms = get_boto3_client("kms", endpoint_url=CONFIG["aws_endpoint"])
-181
View File
@@ -11,7 +11,6 @@ import warnings
import weakref
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from decimal import Decimal
from time import sleep
from typing import List
from unittest.mock import patch
@@ -337,21 +336,6 @@ async def test_update_async(mem_db_async: AsyncConnection):
assert await table.count_rows("id == 10") == 1
@pytest.mark.asyncio
async def test_update_expr_filter_literals_async(mem_db_async: AsyncConnection):
values = ["5", "4.66e-84", "it's"]
table = await mem_db_async.create_table(
"update_expr_literals",
data=[{"field": value, "result": "original"} for value in values],
)
for value in values:
update_res = await table.update({"result": value}, where=col("field") == value)
assert update_res.rows_updated == 1
assert (await table.to_arrow())["result"].to_pylist() == values
def test_create_table(mem_db: DBConnection):
schema = pa.schema(
{
@@ -2359,148 +2343,6 @@ def test_update(mem_db: DBConnection):
assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
def test_update_expr_filter_literals(mem_db: DBConnection):
values = ["5", "4.66e-84", "it's"]
table = mem_db.create_table(
"update_expr_literals",
data=[{"field": value, "result": "original"} for value in values],
)
for value in values:
update_res = table.update(where=col("field") == value, values={"result": value})
assert update_res.rows_updated == 1
assert table.to_arrow()["result"].to_pylist() == values
def test_update_expr_filter_preserves_typed_semantics(mem_db: DBConnection):
low = Decimal("1.234567890123456789")
high = Decimal("1.234567890123456790")
decimal_schema = pa.schema(
[("val", pa.decimal128(19, 18)), ("result", pa.string())]
)
decimal_table = mem_db.create_table(
"update_expr_decimal",
pa.table(
{"val": [low, high], "result": ["old", "old"]},
schema=decimal_schema,
),
)
predicate = col("val") < lit(high)
assert decimal_table.search().where(predicate).to_arrow().num_rows == 1
result = decimal_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 1
keyword_table = mem_db.create_table(
"update_expr_keyword", [{"null": 1, "result": "old"}]
)
predicate = col("null") == 1
assert keyword_table.search().where(predicate).to_arrow().num_rows == 1
result = keyword_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 1
empty_in_table = mem_db.create_table(
"update_expr_empty_in", [{"id": 1, "result": "old"}]
)
predicate = col("id").isin([])
assert empty_in_table.search().where(predicate).to_arrow().num_rows == 0
result = empty_in_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 0
marker = "__lancedb_binary_placeholder_0__"
binary_schema = pa.schema(
[("payload", pa.binary()), ("text", pa.string()), ("result", pa.string())]
)
binary_table = mem_db.create_table(
"update_expr_binary",
pa.table(
{
"payload": [b"\x01", b"\x02"],
"text": ["other", marker],
"result": ["old", "old"],
},
schema=binary_schema,
),
)
predicate = (col("payload") == lit(b"\x01")) | (col("text") == marker)
assert binary_table.search().where(predicate).to_arrow().num_rows == 2
result = binary_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 2
nonfinite_table = mem_db.create_table(
"update_expr_nonfinite",
[{"x": 1.0, "result": "old"}, {"x": 2.0, "result": "old"}],
)
predicate = col("x") < float("inf")
assert nonfinite_table.search().where(predicate).to_arrow().num_rows == 2
result = nonfinite_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 2
float16_table = mem_db.create_table(
"update_expr_float16",
[{"x": 1.0, "result": "old"}, {"x": 3.0, "result": "old"}],
)
predicate = col("x").cast(pa.float16()) < 2.0
assert float16_table.search().where(predicate).to_arrow().num_rows == 1
result = float16_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 1
string_cast_table = mem_db.create_table(
"update_expr_string_cast",
[{"x": 1, "result": "old"}, {"x": 2, "result": "old"}],
)
predicate = col("x").cast("string") == "1"
assert string_cast_table.search().where(predicate).to_arrow().num_rows == 1
result = string_cast_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 1
quoted_identifier_schema = pa.schema(
[("payload", pa.binary()), ("odd'name", pa.int64()), ("result", pa.string())]
)
quoted_identifier_table = mem_db.create_table(
"update_expr_quoted_identifier",
pa.table(
{"payload": [b"\x01"], "odd'name": [1], "result": ["old"]},
schema=quoted_identifier_schema,
),
)
predicate = (col("payload") == lit(b"\x01")) & (col("odd'name") == 1)
assert quoted_identifier_table.search().where(predicate).to_arrow().num_rows == 1
result = quoted_identifier_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 1
decimal256_schema = pa.schema(
[("val", pa.decimal256(40, 2)), ("result", pa.string())]
)
decimal256_table = mem_db.create_table(
"update_expr_decimal256",
pa.table(
{
"val": [Decimal("1.00"), Decimal("3.00")],
"result": ["old", "old"],
},
schema=decimal256_schema,
),
)
predicate = col("val") < lit(Decimal("2.00")).cast(pa.decimal256(40, 2))
assert decimal256_table.search().where(predicate).to_arrow().num_rows == 1
result = decimal256_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 1
binary_empty_table = mem_db.create_table(
"update_expr_binary_empty",
pa.table(
{"payload": [b"\x01", b"\x02"], "result": ["old", "old"]},
schema=pa.schema([("payload", pa.binary()), ("result", pa.string())]),
),
)
predicate = (col("payload") == lit(b"\x01")).isin([])
assert binary_empty_table.search().where(predicate).to_arrow().num_rows == 0
assert predicate.to_sql() == "false"
result = binary_empty_table.update(where=predicate, values={"result": "new"})
assert result.rows_updated == 0
def test_update_with_arrow_scalar(mem_db: DBConnection):
schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
table = mem_db.create_table("my_table", schema=schema)
@@ -4087,29 +3929,6 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
def test_computed_column_blob_projection_inherits_semantics(tmp_path):
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
db = lancedb.connect(tmp_path)
table = db.create_table("computed_column_blob", schema=schema)
table.add(
[
{"id": 1, "image": b"hello"},
{"id": 2, "image": b""},
{"id": 3, "image": None},
]
)
table.add_columns(computed={"image_copy": "image", "second_copy": "image_copy"})
assert table.refresh_column("image_copy").rows_filled == 2
assert table.refresh_column("second_copy").rows_filled == 2
assert table.blob_columns() == ["image", "image_copy", "second_copy"]
hits = table.search().with_row_id(True).limit(10).to_arrow()
rows = sorted(zip(hits["id"].to_pylist(), hits["_rowid"].to_pylist()))
copied = table.fetch_blobs("second_copy", [row_id for _, row_id in rows])
assert copied.to_pylist() == [b"hello", b"", None]
@pytest.mark.asyncio
async def test_computed_column_async(tmp_path):
db = await lancedb.connect_async(tmp_path)
-160
View File
@@ -7,7 +7,6 @@ import pathlib
from typing import Optional
import lance
from lance.blob import BlobType as LanceBlobType
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.embeddings.base import EmbeddingFunctionConfig
from lancedb.embeddings.registry import EmbeddingFunctionRegistry
@@ -908,165 +907,6 @@ def test_cast_to_target_schema():
assert output == expected
def test_cast_to_target_schema_coerces_binary_to_blob_v2():
data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
target = pa.schema([lancedb.blob("image")])
output = _cast_to_target_schema(data.to_reader(), target).read_all()
image = output["image"].chunk(0)
assert type(image.type) is lancedb.BlobType
assert image.storage.to_pylist() == [
{"data": b"hello", "uri": None, "position": None, "size": None},
None,
]
def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
storage = lancedb.blob("image").type.storage_type
target = pa.schema(
[
pa.field(
"image",
storage,
metadata={
b"ARROW:extension:name": b"lance.blob.v2",
b"ARROW:extension:metadata": b"",
},
)
]
)
data = pa.table({"image": pa.array([b"hello", None], type=pa.binary())})
output = _cast_to_target_schema(data.to_reader(), target).read_all()
image = output["image"].chunk(0)
assert not isinstance(image.type, pa.ExtensionType)
assert image.to_pylist() == [
{"data": b"hello", "uri": None, "position": None, "size": None},
None,
]
def test_cast_to_target_schema_coerces_nested_binary_blob():
data = pa.table(
{
"info": pa.array(
[{"blob": b"hello"}, {"blob": None}],
type=pa.struct([pa.field("blob", pa.binary())]),
)
}
)
target = pa.schema([pa.field("info", pa.struct([lancedb.blob("blob")]))])
output = _cast_to_target_schema(data.to_reader(), target).read_all()
blob = output["info"].chunk(0).field("blob")
assert type(blob.type) is lancedb.BlobType
assert blob.storage.to_pylist() == [
{"data": b"hello", "uri": None, "position": None, "size": None},
None,
]
def test_cast_to_target_schema_coerces_list_binary_blob_with_inferred_child_name():
data = pa.table(
{"images": pa.array([[b"a", b"b"], None], type=pa.list_(pa.binary()))}
)
target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
output = _cast_to_target_schema(data.to_reader(), target).read_all()
images = output["images"].chunk(0)
assert images.type.value_field.name == "image"
assert type(images.type.value_type) is lancedb.BlobType
assert images.to_pylist()[1] is None
assert images.values.storage.to_pylist() == [
{"data": b"a", "uri": None, "position": None, "size": None},
{"data": b"b", "uri": None, "position": None, "size": None},
]
def test_list_blob_coercion_preserves_null_slots_with_nonzero_extent():
child = pa.field("image", pa.binary())
source = pa.ListArray.from_arrays(
pa.array([0, 2, 4], type=pa.int32()),
pa.array([b"a", b"b", b"dead", b"beef"], type=pa.binary()),
mask=pa.array([False, True]),
).cast(pa.list_(child))
target = pa.schema([pa.field("images", pa.list_(lancedb.blob("image")))])
output = _cast_to_target_schema(
pa.table({"images": source}).to_reader(), target
).read_all()
images = output["images"].chunk(0)
assert images.to_pylist()[1] is None
assert [b["data"] for b in images.to_pylist()[0]] == [b"a", b"b"]
def test_fixed_size_list_blob_coercion_keeps_null_rows():
child = pa.field("frame", pa.binary())
source = (
pa.FixedSizeListArray.from_arrays(
pa.array([b"a", b"b", b"c", b"d"], type=pa.binary()), 2
)
.take(pa.array([0, None], type=pa.int32()))
.cast(pa.list_(child, 2))
)
target = pa.schema([pa.field("frames", pa.list_(lancedb.blob("frame"), 2))])
output = _cast_to_target_schema(
pa.table({"frames": source}).to_reader(), target
).read_all()
frames = output["frames"].chunk(0)
assert frames.to_pylist()[1] is None
assert [b["data"] for b in frames.to_pylist()[0]] == [b"a", b"b"]
def test_cast_to_target_schema_accepts_pylance_blob_v2():
target_type = lancedb.BlobType()
source = lance.blob_array([b"hello", None])
assert type(source.type) is LanceBlobType
assert type(source.type) is type(target_type)
data = pa.table({"image": source})
target = pa.schema([pa.field("image", target_type)])
output = _cast_to_target_schema(data.to_reader(), target).read_all()
image = output["image"].chunk(0)
assert type(image.type) is LanceBlobType
assert image.type == target_type
assert image.storage.to_pylist() == [
{"data": b"hello", "uri": None, "position": None, "size": None},
None,
]
def test_cast_to_target_schema_rejects_different_blob_v2_class():
class OtherBlobType(pa.ExtensionType):
def __init__(self):
super().__init__(lancedb.BlobType().storage_type, "lance.blob.v2")
def __arrow_ext_serialize__(self) -> bytes:
return b""
@classmethod
def __arrow_ext_deserialize__(
cls, storage_type: pa.DataType, serialized: bytes
) -> "OtherBlobType":
return cls()
storage = lance.blob_array([b"hello"]).storage
source = pa.ExtensionArray.from_storage(OtherBlobType(), storage)
data = pa.table({"image": source})
target = pa.schema([lancedb.blob("image")])
with pytest.raises(pa.ArrowTypeError, match="different extension type"):
_cast_to_target_schema(data.to_reader(), target).read_all()
def test_sanitize_data_stream():
# Make sure we don't collect the whole stream when running sanitize_data
schema = pa.schema({"a": pa.int32()})
-35
View File
@@ -629,17 +629,6 @@ impl Connection {
})
}
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
})
}
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
@@ -666,30 +655,6 @@ impl Connection {
})
}
pub fn pause_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let status = inner.pause_job(&job_id).await.infer_error()?;
Ok(match status {
lancedb::database::PauseJobStatus::Pausing => "pausing",
lancedb::database::PauseJobStatus::AlreadyPaused => "already_paused",
lancedb::database::PauseJobStatus::Committing => "committing",
})
})
}
pub fn resume_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let status = inner.resume_job(&job_id).await.infer_error()?;
Ok(match status {
lancedb::database::ResumeJobStatus::Resumed => "resumed",
lancedb::database::ResumeJobStatus::StillPausing => "still_pausing",
lancedb::database::ResumeJobStatus::NotPaused => "not_paused",
})
})
}
#[pyo3(signature = (job_id=None))]
pub fn job_history(
self_: PyRef<'_, Self>,
-8
View File
@@ -130,14 +130,6 @@ impl PyExpr {
// ── utilities ────────────────────────────────────────────────────────────
/// Return the referenced column name for a bare column expression.
fn column_name(&self) -> Option<String> {
match &self.0 {
DfExpr::Column(column) if column.relation.is_none() => Some(column.name.clone()),
_ => None,
}
}
/// Render the expression as a SQL string (useful for debugging).
fn to_sql(&self) -> PyResult<String> {
lancedb::expr::expr_to_sql_string(&self.0).map_err(|e| PyValueError::new_err(e.to_string()))
+2 -8
View File
@@ -8,7 +8,7 @@ use lancedb::index::vector::{
};
use lancedb::index::{
Index as LanceDbIndex,
scalar::{BTreeIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder},
scalar::{BTreeIndexBuilder, FmIndexBuilder, FtsIndexBuilder},
};
use pyo3::IntoPyObject;
use pyo3::types::PyStringMethods;
@@ -60,11 +60,7 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
.ngram_min_length(params.ngram_min_length)
.ngram_max_length(params.ngram_max_length)
.ngram_prefix_only(params.prefix_only)
.custom_stop_words(params.custom_stop_words)
.document_granularity(
DocumentGranularity::try_from(params.document_granularity.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))?,
);
.custom_stop_words(params.custom_stop_words);
if let Some(memory_limit) = params.memory_limit {
inner_opts = inner_opts.memory_limit_mb(memory_limit);
}
@@ -225,7 +221,6 @@ struct FtsParams {
block_size: usize,
memory_limit: Option<u64>,
num_workers: Option<usize>,
document_granularity: String,
}
#[derive(FromPyObject)]
@@ -486,7 +481,6 @@ mod tests {
block_size = 128
memory_limit = 2048
num_workers = 7
document_granularity = 'row'
config = FTS()",
None,
+12 -71
View File
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
@@ -17,8 +16,8 @@ use arrow::pyarrow::FromPyArrow;
use arrow::pyarrow::IntoPyArrow;
use arrow::pyarrow::ToPyArrow;
use lancedb::index::scalar::{
BooleanQuery, BoostQuery, DocumentGranularity, FtsQuery, FullTextSearchQuery, MatchQuery,
MultiMatchQuery, Occur, Operator, PhraseQuery,
BooleanQuery, BoostQuery, FtsQuery, FullTextSearchQuery, MatchQuery, MultiMatchQuery, Occur,
Operator, PhraseQuery,
};
use lancedb::query::AnalyzePlanDistributedMetrics;
use lancedb::query::QueryBase;
@@ -77,16 +76,8 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let max_expansions = ob.getattr("max_expansions")?.extract()?;
let operator = ob.getattr("operator")?.extract::<String>()?;
let prefix_length = ob.getattr("prefix_length")?.extract()?;
let document_granularity = ob
.getattr("document_granularity")?
.extract::<Option<String>>()?
.map(|value| {
DocumentGranularity::try_from(value.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))
})
.transpose()?;
let mut query =
Ok(Self(
MatchQuery::new(query)
.with_column(Some(column))
.with_boost(boost)
@@ -95,32 +86,21 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
.with_operator(Operator::try_from(operator.as_str()).map_err(|e| {
PyValueError::new_err(format!("Invalid operator: {}", e))
})?)
.with_prefix_length(prefix_length);
if let Some(document_granularity) = document_granularity {
query = query.with_document_granularity(document_granularity);
}
Ok(Self(query.into()))
.with_prefix_length(prefix_length)
.into(),
))
}
"PhraseQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let slop = ob.getattr("slop")?.extract()?;
let document_granularity = ob
.getattr("document_granularity")?
.extract::<Option<String>>()?
.map(|value| {
DocumentGranularity::try_from(value.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))
})
.transpose()?;
let mut query = PhraseQuery::new(query)
.with_column(Some(column))
.with_slop(slop);
if let Some(document_granularity) = document_granularity {
query = query.with_document_granularity(document_granularity);
}
Ok(Self(query.into()))
Ok(Self(
PhraseQuery::new(query)
.with_column(Some(column))
.with_slop(slop)
.into(),
))
}
"BoostQuery" => {
let positive: Self = ob.getattr("positive")?.extract()?;
@@ -187,13 +167,6 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
kwargs.set_item("max_expansions", query.max_expansions)?;
kwargs.set_item::<_, &str>("operator", query.operator.into())?;
kwargs.set_item("prefix_length", query.prefix_length)?;
if let Some(document_granularity) = query.document_granularity {
let value = match document_granularity {
DocumentGranularity::Row => "row",
DocumentGranularity::ListElement => "list_element",
};
kwargs.set_item("document_granularity", value)?;
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
@@ -201,13 +174,6 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
kwargs.set_item("slop", query.slop)?;
if let Some(document_granularity) = query.document_granularity {
let value = match document_granularity {
DocumentGranularity::Row => "row",
DocumentGranularity::ListElement => "list_element",
};
kwargs.set_item("document_granularity", value)?;
}
namespace
.getattr(intern!(py, "PhraseQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
@@ -323,11 +289,9 @@ impl<'py> IntoPyObject<'py> for PyQueryVectors {
pub struct PyQueryRequest {
pub limit: Option<usize>,
pub offset: Option<usize>,
pub take_offsets: Option<Vec<u64>>,
pub filter: Option<PyQueryFilter>,
pub full_text_search: Option<PyLanceDB<FtsQuery>>,
pub select: PySelect,
pub select_source_columns: Option<HashMap<String, String>>,
pub fast_search: Option<bool>,
pub with_row_id: Option<bool>,
pub use_lsm: Option<bool>,
@@ -354,12 +318,10 @@ impl From<AnyQuery> for PyQueryRequest {
AnyQuery::Query(query_request) => Self {
limit: query_request.limit,
offset: query_request.offset,
take_offsets: query_request.take_offsets,
filter: query_request.filter.map(PyQueryFilter),
full_text_search: query_request
.full_text_search
.map(|fts| PyLanceDB(fts.query)),
select_source_columns: PySelect::source_columns(&query_request.select),
select: PySelect(query_request.select),
fast_search: Some(query_request.fast_search),
with_row_id: Some(query_request.with_row_id),
@@ -383,10 +345,8 @@ impl From<AnyQuery> for PyQueryRequest {
AnyQuery::VectorQuery(vector_query) => Self {
limit: vector_query.base.limit,
offset: vector_query.base.offset,
take_offsets: vector_query.base.take_offsets,
filter: vector_query.base.filter.map(PyQueryFilter),
full_text_search: None,
select_source_columns: PySelect::source_columns(&vector_query.base.select),
select: PySelect(vector_query.base.select),
fast_search: Some(vector_query.base.fast_search),
with_row_id: Some(vector_query.base.with_row_id),
@@ -419,25 +379,6 @@ impl From<AnyQuery> for PyQueryRequest {
#[derive(Clone)]
pub struct PySelect(Select);
impl PySelect {
fn source_columns(select: &Select) -> Option<HashMap<String, String>> {
match select {
Select::Expr(pairs) => Some(
pairs
.iter()
.filter_map(|(output, expr)| match expr {
lancedb::expr::DfExpr::Column(column) if column.relation.is_none() => {
Some((output.clone(), column.name.clone()))
}
_ => None,
})
.collect(),
),
_ => None,
}
}
}
impl<'py> IntoPyObject<'py> for PySelect {
type Target = PyAny;
type Output = Bound<'py, Self::Target>;
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.38.0"
version = "0.38.0-beta.10"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
+1 -28
View File
@@ -24,7 +24,7 @@ use crate::data::scannable::Scannable;
use crate::database::listing::ListingDatabase;
use crate::database::{
CloneTableRequest, Database, DatabaseOptions, JobDescription, JobInfo, OpenTableRequest,
PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
ReadConsistency, TableNamesRequest,
};
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
use crate::error::{Error, Result};
@@ -523,21 +523,6 @@ impl Connection {
.await
}
/// Drop one exact immutable Function version from the remote catalog.
///
/// Returns `true` when the server appended a Dropped transition and
/// `false` for an idempotent replay. Local databases return
/// [`Error::NotSupported`].
pub async fn drop_function(
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
) -> Result<bool> {
self.internal
.drop_function(name.as_ref(), version.as_ref())
.await
}
/// Rename a table in the database.
///
/// This is only supported in LanceDB Cloud.
@@ -590,18 +575,6 @@ impl Connection {
self.internal.cancel_job(job_id.as_ref()).await
}
/// Pause a server-side job by id. Its workers drain and it stays parked
/// until resumed; see [`PauseJobStatus`] for the outcomes.
pub async fn pause_job(&self, job_id: impl AsRef<str>) -> Result<PauseJobStatus> {
self.internal.pause_job(job_id.as_ref()).await
}
/// Resume a paused server-side job by id. Its workers pick their work
/// back up from checkpoints; see [`ResumeJobStatus`] for the outcomes.
pub async fn resume_job(&self, job_id: impl AsRef<str>) -> Result<ResumeJobStatus> {
self.internal.resume_job(job_id.as_ref()).await
}
/// The lifecycle event history of a server-side job (all jobs when
/// `job_id` is `None`), as recorded Arrow batches.
pub async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
-37
View File
@@ -235,29 +235,6 @@ pub struct JobDescription {
pub failure: Option<crate::error::JobFailure>,
}
/// The server's answer to a pause request.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PauseJobStatus {
/// The pause was accepted; workers drain and the job stays parked.
Pausing,
/// The job was already paused, so a repeated pause changed nothing.
AlreadyPaused,
/// The job is finalizing its results and cannot be parked right now.
/// The commit is the short tail of a long job; retry shortly.
Committing,
}
/// The server's answer to a resume request.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResumeJobStatus {
/// The job re-entered the queue and will run again.
Resumed,
/// The pause's worker drain is not confirmed yet; retry shortly.
StillPausing,
/// The job was not paused, so there was nothing to resume.
NotPaused,
}
fn job_op_not_supported<T>(what: &str) -> Result<T> {
Err(crate::error::Error::NotSupported {
message: format!("{} is not supported by this database", what),
@@ -330,10 +307,6 @@ pub trait Database:
) -> Result<crate::function::FunctionVersion> {
function_catalog_not_supported()
}
/// Drop one exact immutable Function version from the remote catalog.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
/// waiting on or cancelling the job. The handle is constructed without a
/// server round trip; an unknown id surfaces when the handle is used.
@@ -354,16 +327,6 @@ pub trait Database:
async fn cancel_job(&self, _job_id: &str) -> Result<bool> {
job_op_not_supported("cancel_job")
}
/// Pause a job by id. The job's workers drain and it stays parked until
/// resumed; see [`PauseJobStatus`] for the outcomes.
async fn pause_job(&self, _job_id: &str) -> Result<PauseJobStatus> {
job_op_not_supported("pause_job")
}
/// Resume a paused job by id. It re-enters the queue and its workers pick
/// their work back up from checkpoints; see [`ResumeJobStatus`].
async fn resume_job(&self, _job_id: &str) -> Result<ResumeJobStatus> {
job_op_not_supported("resume_job")
}
/// The lifecycle event history of a job (all jobs when `job_id` is
/// `None`), as recorded Arrow batches.
async fn job_history(&self, _job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
+36 -302
View File
@@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
use lance_datafusion::utils::StreamingWriteSource;
use lance_file::version::LanceFileVersion;
use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider};
use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
use lance_table::io::commit::commit_handler_from_url;
use object_store::local::LocalFileSystem;
use snafu::ResultExt;
@@ -281,22 +281,6 @@ impl std::fmt::Display for ListingDatabase {
}
const LANCE_EXTENSION: &str = "lance";
/// The table a listed child of the database names, or `None` if the child is not a table.
///
/// A table is the directory `<name>.lance`; a loose file or any other directory under the
/// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the
/// caller rather than per child.
/// The table a listed child directory holds, or `None` if it is not a table at all.
///
/// Only directories are considered, so a loose object named like a table is not one.
fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<String> {
location
.filename()?
.strip_suffix(dir_suffix)
.map(String::from)
.filter(|name| !name.is_empty())
}
const ENGINE: &str = "engine";
const MIRRORED_STORE: &str = "mirroredStore";
@@ -960,72 +944,51 @@ impl Database for ListingDatabase {
Ok(f)
}
/// List the tables in the database, a page at a time.
///
/// The page_token is opaque, unlike the `start_after` parameter of [`Self::table_names()`].
///
/// When there are no more results, the returned page_token will be None.
///
/// `limit` is the maximum number of tables to return in the response. But it is possible
/// for the response to contain fewer than `limit` tables, even when there are more tables
/// to return. Clients should check the returned page_token to determine if there are
/// more results, rather than relying on the number of tables returned.
///
/// The order that results are returned in not guaranteed to be stable across calls,
/// so clients should not rely on it.
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
return self.namespace_database().list_tables(request).await;
}
let limit = request.limit.map(|limit| limit.max(0) as usize);
let dir_suffix = format!(".{LANCE_EXTENSION}");
let mut tables = Vec::new();
let mut page_token = request.page_token.filter(|token| !token.is_empty());
let mut f = self
.object_store
.read_dir(self.base_path.clone())
.await?
.iter()
.map(Path::new)
.filter(|path| {
let is_lance = path
.extension()
.and_then(|e| e.to_str())
.map(|e| e == LANCE_EXTENSION);
is_lance.unwrap_or(false)
})
.filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
.collect::<Vec<String>>();
f.sort();
// A page of nothing: the store rejects a limit of zero, and no table was handed over
// for a token to resume after.
if limit == Some(0) {
return Ok(ListTablesResponse {
context: None,
tables,
page_token: None,
});
// Handle pagination with page_token
if let Some(ref page_token) = request.page_token {
let index = f
.iter()
.position(|name| name.as_str() > page_token.as_str())
.unwrap_or(f.len());
f.drain(0..index);
}
loop {
// Ask only for what the page still has room for, so a database holding more
// than one page costs one request per page rather than one per table.
let listing = self
.object_store
.read_dir_page(
self.base_path.clone(),
ReadDirOptions {
page_token: page_token.take(),
limit: limit.map(|limit| limit - tables.len()),
},
)
.await?;
page_token = listing.page_token;
// Only child directories can be tables, and the store already separates them
// out, so the objects in the page are not looked at.
tables.extend(
listing
.result
.common_prefixes
.iter()
.filter_map(|location| table_name(location, &dir_suffix)),
);
// Children that are not tables leave the page short of the limit, so keep
// going until the page is full or the database runs out.
if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) {
break;
// Determine if there's a next page. The token is the last name of this page,
// not the first of the next one: the next page resumes strictly after the
// token, so naming the next page's first entry would skip it.
let next_page_token = match request.limit {
Some(limit) if f.len() > limit as usize => {
f.truncate(limit as usize);
f.last().cloned()
}
}
_ => None,
};
Ok(ListTablesResponse {
context: None,
tables,
page_token,
tables: f,
page_token: next_page_token,
})
}
@@ -1513,7 +1476,7 @@ mod tests {
use crate::table::{AnyQuery, WriteOptions};
use arrow_array::{Int32Array, RecordBatch, StringArray};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use futures::{TryStreamExt, future::try_join_all, stream::once};
use futures::{TryStreamExt, stream::once};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
@@ -1521,182 +1484,6 @@ mod tests {
use tokio::sync::Barrier;
use tokio::time::timeout;
async fn create_tables(db: &ListingDatabase, names: &[&str]) {
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
for name in names {
db.create_table(CreateTableRequest {
name: name.to_string(),
namespace_path: vec![],
data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
})
.await
.unwrap();
}
}
/// Every table in the database, taken `limit` at a time, which is how a caller walks a
/// listing: the token ends the walk, never a short page.
async fn walk(db: &ListingDatabase, limit: Option<i32>) -> Vec<String> {
let mut seen = Vec::new();
let mut page_token = None;
loop {
let page = db
.list_tables(ListTablesRequest {
limit,
page_token,
..Default::default()
})
.await
.unwrap();
seen.extend(page.tables);
page_token = page.page_token;
if page_token.is_none() {
return seen;
}
assert!(
seen.len() < 100,
"the walk is serving tables more than once"
);
}
}
/// Paging with the returned token has to visit every table exactly once, whatever the
/// page size, with nothing lost or repeated at a boundary.
#[rstest::rstest]
#[tokio::test]
async fn test_list_tables_pages_over_every_table_once(#[values(1, 2, 3, 5, 10)] limit: i32) {
let (_tempdir, db) = setup_database().await;
create_tables(&db, &["a", "b", "c", "d", "e"]).await;
assert_eq!(walk(&db, Some(limit)).await, vec!["a", "b", "c", "d", "e"]);
}
/// The token is opaque: it is whatever resumes the store the database sits on, not a
/// table name. Callers hand it back and nothing else.
///
/// Nothing validates a token, so one invented by a caller is read as a position rather
/// than refused — which is why the token has to come back from a previous page.
#[tokio::test]
async fn test_the_page_token_is_not_a_table_name() {
let (_tempdir, db) = setup_database().await;
create_tables(&db, &["a", "b", "c"]).await;
let page = db
.list_tables(ListTablesRequest {
limit: Some(1),
..Default::default()
})
.await
.unwrap();
assert_eq!(page.tables, vec!["a"]);
let token = page.page_token.expect("two tables are still to come");
assert_ne!(token, "a");
// Handing it back is the only thing a caller does with it, and it resumes.
let rest = db
.list_tables(ListTablesRequest {
page_token: Some(token),
..Default::default()
})
.await
.unwrap();
assert_eq!(rest.tables, vec!["b", "c"]);
}
/// A limit the listing does not fill leaves no token behind, so a caller paging by token
/// stops without asking for an empty page.
#[tokio::test]
async fn test_a_listing_that_runs_out_has_no_token() {
let (_tempdir, db) = setup_database().await;
create_tables(&db, &["a", "b"]).await;
let page = db
.list_tables(ListTablesRequest {
limit: Some(10),
..Default::default()
})
.await
.unwrap();
assert_eq!(page.tables, vec!["a", "b"]);
assert_eq!(page.page_token, None);
}
/// An empty page token means "from the start", which is how a client looping on a token
/// spells its first request.
#[tokio::test]
async fn test_an_empty_page_token_lists_from_the_start() {
let (_tempdir, db) = setup_database().await;
create_tables(&db, &["a", "b"]).await;
let page = db
.list_tables(ListTablesRequest {
page_token: Some(String::new()),
..Default::default()
})
.await
.unwrap();
assert_eq!(page.tables, vec!["a", "b"]);
}
/// Listing follows the order the object store lists directories in, so a name that
/// extends another comes first: the `-` of `users-archive.lance` sorts below the `.` of
/// `users.lance`. Pagination pushes its cursor into the list request, so it cannot report
/// an order other than the one it resumes in.
#[tokio::test]
async fn test_listing_order_follows_the_store_not_the_table_name() {
let (_tempdir, db) = setup_database().await;
create_tables(&db, &["users", "users-archive", "users.old"]).await;
assert_eq!(
walk(&db, None).await,
vec!["users-archive", "users", "users.old"]
);
// And paging reports the same order, so a walk sees each table once.
assert_eq!(
walk(&db, Some(1)).await,
vec!["users-archive", "users", "users.old"]
);
}
/// Only directories named `<name>.lance` are tables; loose files and other directories
/// under the database prefix are not. A page spent on them is filled from the next one,
/// so a page holding only non-tables does not read as an empty database.
#[tokio::test]
async fn test_listing_ignores_non_table_children() {
let (tempdir, db) = setup_database().await;
create_tables(&db, &["real"]).await;
std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap();
create_dir_all(tempdir.path().join("aaa-scratch")).unwrap();
let page = db
.list_tables(ListTablesRequest {
limit: Some(1),
..Default::default()
})
.await
.unwrap();
assert_eq!(page.tables, vec!["real"]);
}
#[tokio::test]
async fn listing_ignores_empty_table_name() {
let (tempdir, db) = setup_database().await;
create_dir_all(tempdir.path().join(".lance")).unwrap();
let page = db.list_tables(ListTablesRequest::default()).await.unwrap();
assert!(
page.tables.is_empty(),
"invalid empty table name was listed"
);
}
async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
let tempdir = tempdir().unwrap();
let uri = tempdir.path().to_str().unwrap();
@@ -1827,59 +1614,6 @@ mod tests {
);
}
#[tokio::test]
async fn test_concurrent_open_table_reuses_connection_object_store() {
let tempdir = tempdir().unwrap();
let uri = tempdir.path().to_str().unwrap();
let session = Arc::new(lance::session::Session::default());
let request = ConnectRequest {
uri: uri.to_string(),
#[cfg(feature = "remote")]
client_config: Default::default(),
options: Default::default(),
namespace_client_properties: Default::default(),
manifest_enabled: false,
read_consistency_interval: None,
session: Some(session.clone()),
};
let db = ListingDatabase::connect_with_options(&request)
.await
.unwrap();
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
db.create_table(CreateTableRequest {
name: "test".to_string(),
namespace_path: vec![],
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
})
.await
.unwrap();
let before = session.store_registry().stats();
let opened_tables = try_join_all((0..32).map(|_| {
db.open_table(OpenTableRequest {
name: "test".to_string(),
namespace_path: vec![],
index_cache_size: None,
lance_read_params: None,
location: None,
namespace_client: None,
managed_versioning: None,
})
}))
.await
.unwrap();
let after = session.store_registry().stats();
assert_eq!(opened_tables.len(), 32);
assert_eq!(after.misses, before.misses);
assert_eq!(after.active_stores, before.active_stores);
assert!(after.hits >= before.hits + 32);
}
#[tokio::test]
async fn test_listing_database_root_ops_do_not_create_manifest() {
let tempdir = tempdir().unwrap();
+3 -10
View File
@@ -539,7 +539,9 @@ impl Database for LanceNamespaceDatabase {
self.namespace
.drop_table(drop_request)
.await
.map_err(|e| map_namespace_lance_error(e, name))?;
.map_err(|e| Error::Runtime {
message: format!("Failed to drop table: {}", e),
})?;
Ok(())
}
@@ -1493,15 +1495,6 @@ mod tests {
.expect("Failed to list tables");
assert!(!table_names_after.contains(&"drop_test".to_string()));
let error = conn
.drop_table("drop_test", &["test_ns".into()])
.await
.expect_err("dropping a missing table should fail");
assert!(
matches!(error, Error::TableNotFound { ref name, .. } if name == "drop_test"),
"expected TableNotFound, got: {error:?}"
);
// Verify: Cannot open dropped table
let open_result = conn.open_table("drop_test").execute().await;
assert!(open_result.is_err());
@@ -31,7 +31,7 @@ use lance::io::RecordBatchStream;
use lance_arrow::RecordBatchExt;
use lance_core::ROW_ID;
use lance_core::error::LanceOptionExt;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
/// Reads a permutation of a source table based on row IDs stored in a separate table
@@ -234,14 +234,7 @@ impl PermutationReader {
.expect_ok()?
.values();
let mut unique_row_ids = HashSet::with_capacity(num_rows);
let in_list: Vec<Expr> = row_ids
.iter()
.copied()
.filter(|row_id| unique_row_ids.insert(*row_id))
.map(lit)
.collect();
let num_unique_row_ids = unique_row_ids.len();
let in_list: Vec<Expr> = row_ids.iter().map(|id| lit(*id)).collect();
let base_query = QueryRequest {
filter: Some(QueryFilter::Datafusion(col(ROW_ID).in_list(in_list, false))),
@@ -254,7 +247,7 @@ impl PermutationReader {
.query(
&AnyQuery::Query(base_query),
QueryExecutionOptions {
max_batch_length: num_unique_row_ids as u32,
max_batch_length: num_rows as u32,
..Default::default()
},
)
@@ -269,9 +262,9 @@ impl PermutationReader {
});
}
if batches.iter().map(|b| b.num_rows()).sum::<usize>() != num_unique_row_ids {
if batches.iter().map(|b| b.num_rows()).sum::<usize>() != num_rows {
return Err(Error::InvalidInput {
message: "Base table returned a different number of rows than the number of unique row IDs"
message: "Base table returned different number of rows than the number of row IDs"
.to_string(),
});
}
@@ -511,7 +504,6 @@ impl PermutationReader {
let table = Table::from(self.base_table.clone());
let batches = table
.take_offsets(offsets.to_vec())
.preserve_order()
.select(selection.clone())
.execute()
.await?
@@ -811,10 +803,10 @@ mod tests {
.unwrap();
// Take offsets in reverse order and verify returned rows match that order
let offsets = vec![5, 3, 5, 1, 0];
let offsets = vec![5, 3, 1, 0];
let batch = reader.take_offsets(&offsets, Select::All).await.unwrap();
assert_eq!(batch.num_rows(), 5);
assert_eq!(batch.num_rows(), 4);
let idx_values = batch
.column(0)
@@ -828,52 +820,6 @@ mod tests {
assert_eq!(idx_values, expected);
}
#[tokio::test]
async fn test_take_offsets_preserves_repeated_rows_in_permutation() {
let base_table = lance_datagen::gen_batch()
.col("idx", lance_datagen::array::step::<Int32Type>())
.into_mem_table("tbl", RowCount::from(5), BatchCount::from(1))
.await;
let base_row_ids = collect_column::<UInt64Type>(&base_table, "_rowid").await;
let permutation_row_ids = vec![
base_row_ids[3],
base_row_ids[1],
base_row_ids[3],
base_row_ids[2],
];
let permutation_batch = RecordBatch::try_new(
Arc::new(Schema::new(vec![
Field::new("row_id", DataType::UInt64, false),
Field::new(SPLIT_ID_COLUMN, DataType::UInt64, false),
])),
vec![
Arc::new(UInt64Array::from(permutation_row_ids)),
Arc::new(UInt64Array::from(vec![0; 4])),
],
)
.unwrap();
let permutation_table = virtual_table("row_ids", &permutation_batch).await;
let reader = PermutationReader::try_from_tables(
base_table.base_table().clone(),
permutation_table.base_table().clone(),
0,
)
.await
.unwrap();
let batch = reader
.take_offsets(&[0, 1, 2, 3], Select::All)
.await
.unwrap();
let idx_values = batch
.column(0)
.as_primitive::<Int32Type>()
.values()
.to_vec();
assert_eq!(idx_values, vec![3, 1, 3, 2]);
}
#[tokio::test]
async fn test_take_offsets_with_column_selection() {
let (base_table, row_ids_table, row_ids) = setup_permutation_tables(10).await;
@@ -937,17 +883,17 @@ mod tests {
.unwrap();
// With no permutation table, take_offsets uses the base table directly
let offsets = vec![0, 2, 0, 4, 6];
let offsets = vec![0, 2, 4, 6];
let batch = reader.take_offsets(&offsets, Select::All).await.unwrap();
assert_eq!(batch.num_rows(), 5);
assert_eq!(batch.num_rows(), 4);
let idx_values = batch
.column(0)
.as_primitive::<Int32Type>()
.values()
.to_vec();
assert_eq!(idx_values, vec![0, 2, 0, 4, 6]);
assert_eq!(idx_values, vec![0, 2, 4, 6]);
}
#[tokio::test]
+4 -121
View File
@@ -19,7 +19,6 @@
mod sql;
pub(crate) use sql::canonicalize_sql_predicate;
pub use sql::expr_to_sql_string;
use std::sync::Arc;
@@ -157,7 +156,7 @@ mod tests {
use datafusion_common::ScalarValue;
let expr = col("data").eq(lit(ScalarValue::Binary(Some(vec![0xca, 0xfe]))));
let sql = expr_to_sql_string(&expr).unwrap();
assert_eq!(sql, "(`data` = X'CAFE')");
assert_eq!(sql, "(data = X'CAFE')");
}
#[test]
@@ -167,7 +166,7 @@ mod tests {
let int_expr = col("id").gt(lit(5i64));
let combined = bin_expr.and(int_expr);
let sql = expr_to_sql_string(&combined).unwrap();
assert_eq!(sql, "((`data` = X'01') AND (id > 5))");
assert_eq!(sql, "((data = X'01') AND (id > 5))");
}
#[test]
@@ -185,7 +184,7 @@ mod tests {
// serialized correctly (regression test for placeholder rewrite path).
let expr = contains(col("data"), lit(ScalarValue::Binary(Some(vec![0xff]))));
let sql = expr_to_sql_string(&expr).unwrap();
assert_eq!(sql, "contains(`data`, X'FF')");
assert_eq!(sql, "contains(data, X'FF')");
}
#[test]
@@ -196,7 +195,7 @@ mod tests {
.eq(lit(ScalarValue::Binary(Some(vec![0xab, 0xcd]))))
.not();
let sql = expr_to_sql_string(&expr).unwrap();
assert_eq!(sql, "NOT (`data` = X'ABCD')");
assert_eq!(sql, "NOT (data = X'ABCD')");
}
#[test]
@@ -206,122 +205,6 @@ mod tests {
assert!(sql.contains("IN"), "expected IN in: {}", sql);
}
#[test]
fn test_empty_is_in() {
let expr = is_in(col("id"), vec![]);
assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
}
#[test]
fn test_empty_is_in_discards_binary_children() {
use datafusion_common::ScalarValue;
let expr = is_in(
col("payload").eq(lit(ScalarValue::Binary(Some(vec![0x01])))),
vec![],
);
assert_eq!(expr_to_sql_string(&expr).unwrap(), "false");
}
#[test]
fn test_keyword_identifier() {
let expr = col("null").eq(lit(1i64));
assert_eq!(expr_to_sql_string(&expr).unwrap(), "(`null` = 1)");
}
#[test]
fn test_decimal_literal_preserves_type() {
use datafusion_common::ScalarValue;
let expr = col("val").lt(lit(ScalarValue::Decimal128(
Some(1_234_567_890_123_456_790),
19,
18,
)));
let sql = expr_to_sql_string(&expr).unwrap();
assert_eq!(
sql,
"(val < arrow_cast('1.234567890123456790', 'Decimal128(19, 18)'))"
);
}
#[test]
fn test_non_finite_float_literal_preserves_type() {
let expr = col("x").lt(lit(f64::INFINITY));
assert_eq!(
expr_to_sql_string(&expr).unwrap(),
"(x < arrow_cast('inf', 'Float64'))"
);
}
#[test]
fn test_cast_uses_arrow_type_name() {
let string = expr_cast(col("x"), DataType::Utf8);
assert_eq!(
expr_to_sql_string(&string).unwrap(),
"arrow_cast(x, 'Utf8')"
);
let int32 = expr_cast(col("x"), DataType::Int32);
assert_eq!(
expr_to_sql_string(&int32).unwrap(),
"arrow_cast(x, 'Int32')"
);
let expr = expr_cast(col("x"), DataType::Float16).lt(lit(2.0));
assert_eq!(
expr_to_sql_string(&expr).unwrap(),
"(arrow_cast(x, 'Float16') < 2.0)"
);
let decimal = expr_cast(lit("2.00"), DataType::Decimal256(40, 2));
assert_eq!(
expr_to_sql_string(&decimal).unwrap(),
"arrow_cast('2.00', 'Decimal256(40, 2)')"
);
}
#[test]
fn test_binary_placeholder_does_not_rewrite_user_string() {
use datafusion_common::ScalarValue;
let marker = "__lancedb_binary_placeholder_0__";
let expr = col("payload")
.eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
.or(col("text").eq(lit(marker)));
assert_eq!(
expr_to_sql_string(&expr).unwrap(),
"((payload = X'01') OR (`text` = '__lancedb_binary_placeholder_0__'))"
);
}
#[test]
fn test_binary_binding_skips_quoted_identifiers() {
use datafusion_common::ScalarValue;
let expr = col("payload")
.eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
.and(col("odd'name").eq(lit(1i64)))
.and(col("odd`'name").eq(lit(2i64)));
assert_eq!(
expr_to_sql_string(&expr).unwrap(),
"(((payload = X'01') AND (`odd'name` = 1)) AND (`odd``'name` = 2))"
);
}
#[test]
fn test_binary_placeholder_collision_search_is_linear() {
use datafusion_common::ScalarValue;
let collision_shaped = format!("__lancedb_binary_placeholder_0__{}", "_".repeat(64_000));
let expr = col("payload")
.eq(lit(ScalarValue::Binary(Some(vec![0x01]))))
.and(col("text").eq(lit(collision_shaped.clone())));
let sql = expr_to_sql_string(&expr).unwrap();
assert!(sql.contains("X'01'"));
assert!(sql.contains(&format!("'{collision_shaped}'")));
}
#[test]
fn test_multiple_binary_literals() {
use datafusion_common::ScalarValue;
+43 -330
View File
@@ -1,27 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{
any::TypeId,
collections::{HashMap, HashSet},
};
use arrow_array::types::{
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType,
};
use arrow_schema::DataType;
use datafusion_common::ScalarValue;
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_expr::Expr;
use datafusion_functions::core::expr_fn::{
arrow_cast as datafusion_arrow_cast, arrow_try_cast as datafusion_arrow_try_cast,
};
use datafusion_sql::sqlparser::{
dialect::{Dialect as SqlParserDialect, GenericDialect},
keywords::ALL_KEYWORDS,
tokenizer::{Token, Tokenizer},
};
use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
use datafusion_sql::unparser::{self, dialect::Dialect};
/// Unparser dialect that matches the quoting style expected by the Lance SQL
/// parser. Lance uses backtick (`` ` ``) as the only delimited-identifier
@@ -36,74 +19,17 @@ use datafusion_sql::unparser::{self, dialect::Dialect as UnparserDialect};
/// lower-case by the SQL parser, which would break case-sensitive schemas).
struct LanceSqlDialect;
impl UnparserDialect for LanceSqlDialect {
impl Dialect for LanceSqlDialect {
fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
let identifier_upper = identifier.to_ascii_uppercase();
let needs_quote =
(identifier_upper != "ID" && ALL_KEYWORDS.contains(&identifier_upper.as_str()))
|| identifier.chars().any(|c| c.is_ascii_uppercase())
|| !identifier.chars().enumerate().all(|(i, c)| {
c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())
});
let needs_quote = identifier.chars().any(|c| c.is_ascii_uppercase())
|| !identifier
.chars()
.enumerate()
.all(|(i, c)| c == '_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit()));
if needs_quote { Some('`') } else { None }
}
}
/// Lance's tokenizer dialect with SQL-standard double-quoted identifiers added.
///
/// Keep this deliberately small: Lance's parser wraps `GenericDialect` and
/// delegates only identifier recognition, leaving every other dialect option at
/// its default. In particular, `/*! ... */` remains an ordinary block comment.
#[derive(Debug, Default)]
struct PredicateDialect(GenericDialect);
impl SqlParserDialect for PredicateDialect {
fn dialect(&self) -> TypeId {
self.0.dialect()
}
fn is_identifier_start(&self, ch: char) -> bool {
self.0.is_identifier_start(ch)
}
fn is_identifier_part(&self, ch: char) -> bool {
self.0.is_identifier_part(ch)
}
fn is_delimited_identifier_start(&self, ch: char) -> bool {
ch == '"' || ch == '`'
}
}
/// Canonicalize a raw SQL predicate for Lance's parser.
///
/// Lance wraps [`GenericDialect`] for identifier recognition while retaining the
/// default dialect behavior for every other lexical option. [`PredicateDialect`]
/// mirrors that contract and additionally recognizes `"` as an identifier
/// delimiter, allowing this function to rewrite only those identifier tokens.
pub fn canonicalize_sql_predicate(predicate: &str) -> crate::Result<String> {
let dialect = PredicateDialect::default();
let tokens = Tokenizer::new(&dialect, predicate)
.with_unescape(false)
.tokenize()
.map_err(|err| crate::Error::InvalidInput {
message: format!("invalid SQL predicate: {err}"),
})?;
Ok(tokens
.into_iter()
.map(|token| match token {
Token::Word(word) if word.quote_style == Some('"') => {
// with_unescape(false) retains doubled double quotes. Decode
// those before escaping any backticks for Lance's delimiter.
let identifier = word.value.replace("\"\"", "\"").replace('`', "``");
format!("`{identifier}`")
}
other => other.to_string(),
})
.collect())
}
/// Prefix for placeholder strings inserted in place of binary literals. Chosen
/// to be extremely unlikely to occur in user data.
const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_";
@@ -113,128 +39,24 @@ fn bytes_to_hex_sql(bytes: &[u8]) -> String {
format!("X'{hex}'")
}
fn string_literals(expr: &Expr) -> HashSet<String> {
let mut literals = HashSet::new();
/// Returns true if *expr* contains a `Binary` or `LargeBinary` scalar literal
/// anywhere in its subtree. DataFusion's SQL unparser cannot serialize those
/// variants, so we route such expressions through a placeholder-substitution
/// path that emits SQL `X'...'` byte-string literals.
fn has_binary_literal(expr: &Expr) -> bool {
let mut found = false;
let _ = expr.apply(&mut |e: &Expr| {
if let Expr::Literal(
ScalarValue::Utf8(Some(value))
| ScalarValue::LargeUtf8(Some(value))
| ScalarValue::Utf8View(Some(value)),
_,
) = e
{
literals.insert(value.clone());
}
Ok(TreeNodeRecursion::Continue)
});
literals
}
fn typed_string_literal(value: String, data_type: DataType) -> Expr {
datafusion_arrow_cast(
Expr::Literal(ScalarValue::Utf8(Some(value)), None),
Expr::Literal(ScalarValue::Utf8(Some(data_type.to_string())), None),
)
}
fn next_binary_placeholder(user_strings: &HashSet<String>, next_id: &mut usize) -> String {
loop {
let placeholder = format!("{BINARY_PLACEHOLDER_PREFIX}{}__", *next_id);
*next_id += 1;
if !user_strings.contains(&placeholder) {
return placeholder;
}
}
}
fn bind_binary_literals(
sql: &str,
mut bindings: HashMap<String, Vec<u8>>,
) -> crate::Result<String> {
let bytes = sql.as_bytes();
let mut output = Vec::with_capacity(bytes.len());
let mut index = 0;
// Walk SQL string tokens once. Placeholders are plain, unescaped string
// literals, so this remains linear even when user strings are large or
// deliberately resemble the placeholder prefix.
while index < bytes.len() {
if bytes[index] == b'`' {
let identifier_start = index;
index += 1;
let mut identifier_end = None;
while index < bytes.len() {
if bytes[index] == b'`' {
if index + 1 < bytes.len() && bytes[index + 1] == b'`' {
index += 2;
} else {
index += 1;
identifier_end = Some(index);
break;
}
} else {
index += 1;
}
}
let Some(identifier_end) = identifier_end else {
return Err(crate::Error::InvalidInput {
message: "unterminated identifier while binding binary literal".to_string(),
});
};
output.extend_from_slice(&bytes[identifier_start..identifier_end]);
continue;
}
if bytes[index] != b'\'' {
output.push(bytes[index]);
index += 1;
continue;
}
let literal_start = index;
index += 1;
let content_start = index;
let mut escaped = false;
let mut content_end = None;
while index < bytes.len() {
if bytes[index] == b'\'' {
if index + 1 < bytes.len() && bytes[index + 1] == b'\'' {
escaped = true;
index += 2;
} else {
content_end = Some(index);
index += 1;
break;
}
} else {
index += 1;
}
}
let Some(content_end) = content_end else {
return Err(crate::Error::InvalidInput {
message: "unterminated string while binding binary literal".to_string(),
});
};
let placeholder = &sql[content_start..content_end];
if !escaped && let Some(value) = bindings.remove(placeholder) {
output.extend_from_slice(bytes_to_hex_sql(&value).as_bytes());
if matches!(
e,
Expr::Literal(ScalarValue::Binary(_) | ScalarValue::LargeBinary(_), _)
) {
found = true;
Ok(TreeNodeRecursion::Stop)
} else {
output.extend_from_slice(&bytes[literal_start..index]);
Ok(TreeNodeRecursion::Continue)
}
}
if !bindings.is_empty() {
return Err(crate::Error::InvalidInput {
message: "failed to bind binary literal while serializing expression".to_string(),
});
}
String::from_utf8(output).map_err(|e| crate::Error::InvalidInput {
message: format!("failed to bind binary literal: {e}"),
})
});
found
}
fn run_unparser(expr: &Expr) -> crate::Result<String> {
@@ -247,37 +69,25 @@ fn run_unparser(expr: &Expr) -> crate::Result<String> {
}
pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
// DataFusion's unparser needs a few adaptations before its SQL can be
// reparsed by Lance without changing the typed expression's semantics:
//
// * decimal literals need an explicit cast to preserve precision and scale;
// * casts need exact Arrow type names rather than SQL type aliases;
// * an empty IN list is valid in DataFusion but invalid SQL;
// * binary literals are unsupported by the unparser and need placeholders.
// Eliminate empty membership expressions before visiting their children.
// Otherwise a discarded binary child could leave behind a stale binding.
// Fast path: no binary literals — DataFusion's unparser handles everything.
if !has_binary_literal(expr) {
return run_unparser(expr);
}
// Slow path: DataFusion's unparser cannot serialize `Binary`/`LargeBinary`
// scalars, so we rewrite each one to a unique string-literal placeholder,
// let the unparser do the rest of the work, then substitute the SQL
// `X'...'` byte-string literal back in. This keeps the operator/function
// serialization logic centralized in DataFusion and works for every
// expression node type the unparser supports.
let mut bindings: Vec<Vec<u8>> = Vec::new();
let rewritten = expr
.clone()
.transform(|e: Expr| match e {
Expr::InList(in_list) if in_list.list.is_empty() => Ok(Transformed::yes(
Expr::Literal(ScalarValue::Boolean(Some(in_list.negated)), None),
)),
other => Ok(Transformed::no(other)),
})
.map_err(|e| crate::Error::InvalidInput {
message: format!("failed to rewrite expression: {e}"),
})?
.data;
let user_strings = string_literals(&rewritten);
let mut next_placeholder_id = 0;
let mut binary_bindings = HashMap::new();
let rewritten = rewritten
.transform(|e: Expr| match e {
Expr::Literal(ScalarValue::Binary(Some(bytes)), m)
| Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => {
let placeholder = next_binary_placeholder(&user_strings, &mut next_placeholder_id);
binary_bindings.insert(placeholder.clone(), bytes);
let placeholder = format!("{}{}__", BINARY_PLACEHOLDER_PREFIX, bindings.len());
bindings.push(bytes);
Ok(Transformed::yes(Expr::Literal(
ScalarValue::Utf8(Some(placeholder)),
m,
@@ -287,57 +97,6 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
| Expr::Literal(ScalarValue::LargeBinary(None), m) => {
Ok(Transformed::yes(Expr::Literal(ScalarValue::Null, m)))
}
Expr::Literal(ScalarValue::Decimal32(Some(value), precision, scale), _m) => {
let value = Decimal32Type::format_decimal(value, precision, scale);
Ok(Transformed::yes(typed_string_literal(
value,
DataType::Decimal32(precision, scale),
)))
}
Expr::Literal(ScalarValue::Decimal64(Some(value), precision, scale), _m) => {
let value = Decimal64Type::format_decimal(value, precision, scale);
Ok(Transformed::yes(typed_string_literal(
value,
DataType::Decimal64(precision, scale),
)))
}
Expr::Literal(ScalarValue::Decimal128(Some(value), precision, scale), _m) => {
let value = Decimal128Type::format_decimal(value, precision, scale);
Ok(Transformed::yes(typed_string_literal(
value,
DataType::Decimal128(precision, scale),
)))
}
Expr::Literal(ScalarValue::Decimal256(Some(value), precision, scale), _m) => {
let value = Decimal256Type::format_decimal(value, precision, scale);
Ok(Transformed::yes(typed_string_literal(
value,
DataType::Decimal256(precision, scale),
)))
}
Expr::Literal(ScalarValue::Float16(Some(value)), _m) if !value.is_finite() => Ok(
Transformed::yes(typed_string_literal(value.to_string(), DataType::Float16)),
),
Expr::Literal(ScalarValue::Float32(Some(value)), _m) if !value.is_finite() => Ok(
Transformed::yes(typed_string_literal(value.to_string(), DataType::Float32)),
),
Expr::Literal(ScalarValue::Float64(Some(value)), _m) if !value.is_finite() => Ok(
Transformed::yes(typed_string_literal(value.to_string(), DataType::Float64)),
),
Expr::Cast(cast) => Ok(Transformed::yes(datafusion_arrow_cast(
*cast.expr,
Expr::Literal(
ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
None,
),
))),
Expr::TryCast(cast) => Ok(Transformed::yes(datafusion_arrow_try_cast(
*cast.expr,
Expr::Literal(
ScalarValue::Utf8(Some(cast.field.data_type().to_string())),
None,
),
))),
other => Ok(Transformed::no(other)),
})
.map_err(|e| crate::Error::InvalidInput {
@@ -345,58 +104,12 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result<String> {
})?
.data;
let sql = run_unparser(&rewritten)?;
if binary_bindings.is_empty() {
Ok(sql)
} else {
bind_binary_literals(&sql, binary_bindings)
}
}
#[cfg(test)]
mod tests {
use super::canonicalize_sql_predicate;
#[test]
fn normalizes_double_quoted_identifiers() {
assert_eq!(
canonicalize_sql_predicate(r#""PartyAbbrev" = 'D'"#).unwrap(),
"`PartyAbbrev` = 'D'"
);
assert_eq!(
canonicalize_sql_predicate(r#""MetaData"."userId" = 5"#).unwrap(),
"`MetaData`.`userId` = 5"
);
assert_eq!(
canonicalize_sql_predicate(r#""a""b" = 1"#).unwrap(),
"`a\"b` = 1"
);
}
#[test]
fn preserves_quotes_inside_literals_and_backticks() {
let filter = r#"name = 'Alice "Ace"' AND `quoted"field` = 1"#;
assert_eq!(canonicalize_sql_predicate(filter).unwrap(), filter);
}
#[test]
fn preserves_literals_and_comments_using_lance_dialect_rules() {
let predicate = r#"path = '\' AND "PartyAbbrev" = 'D' -- unmatched " in comment"#;
assert_eq!(
canonicalize_sql_predicate(predicate).unwrap(),
r#"path = '\' AND `PartyAbbrev` = 'D' -- unmatched " in comment"#
);
let predicate = r#"id = 1 /* unmatched " in block comment */"#;
assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
let predicate = r#"id = 1 /*! OR "PartyAbbrev" = 'D' */"#;
assert_eq!(canonicalize_sql_predicate(predicate).unwrap(), predicate);
}
#[test]
fn rejects_unterminated_double_quoted_identifier() {
let error = canonicalize_sql_predicate(r#""PartyAbbrev = 'D'"#).unwrap_err();
assert!(matches!(error, crate::Error::InvalidInput { .. }));
let mut sql = run_unparser(&rewritten)?;
for (i, bytes) in bindings.iter().enumerate() {
// The unparser quotes string literals with single quotes, so the
// placeholder appears as `'__lancedb_binary_placeholder_<i>__'`.
let quoted = format!("'{}{}__'", BINARY_PLACEHOLDER_PREFIX, i);
sql = sql.replace(&quoted, &bytes_to_hex_sql(bytes));
}
Ok(sql)
}
+26 -178
View File
@@ -15,9 +15,6 @@ use serde_json::Value;
use crate::{Error, Result};
/// Semantic Function type for a Blob v2 value.
pub const FUNCTION_BLOB_V2_TYPE: &str = "blob_v2";
fn invalid_json(error: impl std::fmt::Display) -> Error {
Error::InvalidInput {
message: format!("invalid remote Function JSON: {error}"),
@@ -189,9 +186,6 @@ pub struct PythonEnvironmentSpec {
pub kind: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub packages: Vec<String>,
/// Conda channels in priority order; conda environments only.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub channels: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
@@ -210,33 +204,6 @@ pub enum PythonRuntimeSpec {
environment: PythonEnvironmentSpec,
env: BTreeMap<String, String>,
},
/// The GPU-enabled Sophon-managed Python runtime.
///
/// # Examples
///
/// ```
/// use std::collections::BTreeMap;
/// use lancedb::function::{PythonEnvironmentSpec, PythonRuntimeSpec};
///
/// let runtime = PythonRuntimeSpec::PythonV2 {
/// python_version: "3.12".to_string(),
/// environment: PythonEnvironmentSpec {
/// kind: "pip".to_string(),
/// packages: vec!["cupy-cuda12x".to_string()],
/// channels: Vec::new(),
/// path: None,
/// modules: Vec::new(),
/// image: None,
/// },
/// env: BTreeMap::new(),
/// };
/// assert!(runtime.requires_gpu());
/// ```
PythonV2 {
python_version: String,
environment: PythonEnvironmentSpec,
env: BTreeMap<String, String>,
},
/// A runtime kind introduced by a newer server.
///
/// Unknown payload fields are intentionally not retained because the
@@ -249,27 +216,22 @@ impl PythonRuntimeSpec {
pub fn kind(&self) -> &str {
match self {
Self::Python { .. } => "python",
Self::PythonV2 { .. } => "python_v2",
Self::Unrecognized { kind } => kind,
}
}
/// The Python version for a known Python runtime, or `None` for an unknown kind.
/// The Python version for the V1 runtime, or `None` for an unknown kind.
pub fn python_version(&self) -> Option<&str> {
match self {
Self::Python { python_version, .. } | Self::PythonV2 { python_version, .. } => {
Some(python_version)
}
Self::Python { python_version, .. } => Some(python_version),
Self::Unrecognized { .. } => None,
}
}
/// The Python environment for a known Python runtime, or `None` for an unknown kind.
/// The Python environment for the V1 runtime, or `None` for an unknown kind.
pub fn environment(&self) -> Option<&PythonEnvironmentSpec> {
match self {
Self::Python { environment, .. } | Self::PythonV2 { environment, .. } => {
Some(environment)
}
Self::Python { environment, .. } => Some(environment),
Self::Unrecognized { .. } => None,
}
}
@@ -277,73 +239,38 @@ impl PythonRuntimeSpec {
/// Environment variables, or `None` for an unknown kind.
pub fn env(&self) -> Option<&BTreeMap<String, String>> {
match self {
Self::Python { env, .. } | Self::PythonV2 { env, .. } => Some(env),
Self::Python { env, .. } => Some(env),
Self::Unrecognized { .. } => None,
}
}
/// Whether the runtime requires a GPU selected by the execution platform.
pub fn requires_gpu(&self) -> bool {
matches!(self, Self::PythonV2 { .. })
}
}
#[derive(Deserialize)]
struct PythonRuntimeV1Wire {
python_version: String,
environment: PythonEnvironmentSpec,
struct PythonRuntimeWire {
kind: String,
#[serde(default)]
python_version: Option<String>,
#[serde(default)]
environment: Option<PythonEnvironmentSpec>,
#[serde(default)]
env: BTreeMap<String, String>,
#[serde(default)]
gpu: Option<Value>,
}
#[derive(Deserialize)]
struct PythonRuntimeV2Wire {
python_version: String,
environment: PythonEnvironmentSpec,
#[serde(default)]
env: BTreeMap<String, String>,
gpu: bool,
}
impl<'de> Deserialize<'de> for PythonRuntimeSpec {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> std::result::Result<Self, D::Error> {
let value = Value::deserialize(deserializer)?;
let kind = value
.get("kind")
.ok_or_else(|| de::Error::missing_field("kind"))?
.as_str()
.ok_or_else(|| de::Error::custom("runtime.kind must be a string"))?
.to_string();
match kind.as_str() {
"python" => {
let wire: PythonRuntimeV1Wire =
serde_json::from_value(value).map_err(de::Error::custom)?;
if wire.gpu.is_some() {
return Err(de::Error::custom(
"python runtime with gpu requires kind='python_v2'",
));
}
Ok(Self::Python {
python_version: wire.python_version,
environment: wire.environment,
env: wire.env,
})
}
"python_v2" => {
let wire: PythonRuntimeV2Wire =
serde_json::from_value(value).map_err(de::Error::custom)?;
if !wire.gpu {
return Err(de::Error::custom("runtime.gpu must be true"));
}
Ok(Self::PythonV2 {
python_version: wire.python_version,
environment: wire.environment,
env: wire.env,
})
}
_ => Ok(Self::Unrecognized { kind }),
let wire = PythonRuntimeWire::deserialize(deserializer)?;
if wire.kind == "python" {
Ok(Self::Python {
python_version: wire
.python_version
.ok_or_else(|| de::Error::missing_field("python_version"))?,
environment: wire
.environment
.ok_or_else(|| de::Error::missing_field("environment"))?,
env: wire.env,
})
} else {
Ok(Self::Unrecognized { kind: wire.kind })
}
}
}
@@ -357,8 +284,6 @@ impl Serialize for PythonRuntimeSpec {
environment: &'a PythonEnvironmentSpec,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
env: &'a BTreeMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
gpu: Option<bool>,
}
#[derive(Serialize)]
@@ -376,19 +301,6 @@ impl Serialize for PythonRuntimeSpec {
python_version,
environment,
env,
gpu: None,
}
.serialize(serializer),
Self::PythonV2 {
python_version,
environment,
env,
} => PythonRuntimeRef {
kind: "python_v2",
python_version,
environment,
env,
gpu: Some(true),
}
.serialize(serializer),
Self::Unrecognized { kind } => UnrecognizedRuntimeRef { kind }.serialize(serializer),
@@ -398,8 +310,8 @@ impl Serialize for PythonRuntimeSpec {
/// Immutable Function version returned by the Enterprise catalog.
///
/// The GPU execution requirement is part of this identity. CPU and memory sizing,
/// priority, concurrency, and retry policy belong to the execution platform.
/// Scheduling resources, priority, concurrency, and retry policy belong to
/// the submitting Job and are not part of this identity.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersion {
name: String,
@@ -671,67 +583,3 @@ impl RefreshColumnResult {
}
impl_json!(RefreshColumnResult);
#[cfg(test)]
mod conda_environment_tests {
use super::{PythonEnvironmentSpec, PythonRuntimeSpec};
#[test]
fn conda_channels_round_trip_and_pip_stays_bare() {
let conda: PythonEnvironmentSpec = serde_json::from_str(
r#"{"kind":"conda","packages":["numpy"],"channels":["conda-forge"]}"#,
)
.unwrap();
assert_eq!(conda.channels, ["conda-forge"]);
assert!(
serde_json::to_string(&conda)
.unwrap()
.contains(r#""channels":["conda-forge"]"#)
);
let pip: PythonEnvironmentSpec =
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
assert!(!serde_json::to_string(&pip).unwrap().contains("channels"));
}
#[test]
fn gpu_python_runtime_marker_round_trips_and_validates() {
let runtime: PythonRuntimeSpec = serde_json::from_str(
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
)
.unwrap();
assert_eq!(runtime.kind(), "python_v2");
assert!(runtime.requires_gpu());
assert_eq!(
super::canonical_json(&runtime).unwrap(),
r#"{"environment":{"kind":"pip"},"gpu":true,"kind":"python_v2","python_version":"3.12"}"#
);
for invalid in [
r#"{"kind":"python","python_version":"3.12","environment":{"kind":"pip"},"gpu":true}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"}}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":1}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":false}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"true"}"#,
r#"{"kind":"python_v2","python_version":"3.12","environment":{"kind":"pip"},"gpu":"H100"}"#,
] {
assert!(serde_json::from_str::<PythonRuntimeSpec>(invalid).is_err());
}
}
#[test]
fn unknown_runtime_discards_payload_before_known_field_validation() {
for encoded in [
r#"{"kind":"python_v3","gpu":{"model":"H100"}}"#,
r#"{"kind":"python_v3","resources":[]}"#,
r#"{"kind":"python_v3","python_version":3.15,"environment":{"kind":[]}}"#,
] {
let runtime: PythonRuntimeSpec = serde_json::from_str(encoded).unwrap();
assert_eq!(runtime.kind(), "python_v3");
assert_eq!(
super::canonical_json(&runtime).unwrap(),
r#"{"kind":"python_v3"}"#
);
}
}
}
-1
View File
@@ -63,5 +63,4 @@ pub struct FmIndexBuilder {}
pub use lance_index::scalar::FullTextSearchQuery;
pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder;
pub use lance_index::scalar::InvertedIndexParams;
pub use lance_index::scalar::inverted::DocumentGranularity;
pub use lance_index::scalar::inverted::query::*;
+1 -9
View File
@@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
UploadPart, list::PaginatedListStore, path::Path,
UploadPart, path::Path,
};
use async_trait::async_trait;
@@ -187,14 +187,6 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper {
secondary: self.secondary.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn PaginatedListStore>,
) -> Option<Arc<dyn PaginatedListStore>> {
Some(original)
}
}
// windows pathing can't be simply concatenated
@@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
UploadPart, list::PaginatedListStore, path::Path,
UploadPart, path::Path,
};
#[derive(Debug, Default)]
@@ -57,14 +57,6 @@ impl WrappingObjectStore for IoStatsHolder {
stats: self.0.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn PaginatedListStore>,
) -> Option<Arc<dyn PaginatedListStore>> {
Some(original)
}
}
impl IoTrackingStore {
-4
View File
@@ -47,10 +47,6 @@ impl TerminalResult {
}
}
pub(crate) fn value(&self) -> Option<&Value> {
self.value.as_ref()
}
fn decode<T: DeserializeOwned>(self) -> Result<T> {
let value = self.value.ok_or_else(|| match &self.request_id {
Some(request_id) => Error::Http {
+2 -11
View File
@@ -170,15 +170,6 @@ pub(crate) fn plan(
filter: Option<&str>,
limit: Option<u64>,
) -> Result<(MaterializedViewDefinition, Vec<ArrowField>, Lineage)> {
let filter = filter
.map(crate::expr::canonicalize_sql_predicate)
.transpose()
.map_err(|err| match err {
Error::InvalidInput { message } => Error::InvalidInput {
message: format!("invalid view filter: {message}"),
},
err => err,
})?;
let projections: Vec<(String, String)> = if projections.is_empty() {
source_schema
.fields()
@@ -283,7 +274,7 @@ pub(crate) fn plan(
declared.push(output);
}
if let Some(filter) = filter.as_deref() {
if let Some(filter) = filter {
let expr = planner
.parse_filter(filter)
.map_err(|e| Error::InvalidInput {
@@ -323,7 +314,7 @@ pub(crate) fn plan(
.into_iter()
.map(|(output, expression)| ViewProjection { output, expression })
.collect(),
filter,
filter: filter.map(String::from),
limit,
inputs,
};
+24 -160
View File
@@ -46,9 +46,8 @@ use lance_table::format::Fragment;
use serde::{Deserialize, Serialize};
use super::{
DEFINITION_META_KEY, INCARNATION_META_KEY, MaterializedViewDefinition,
REFRESHED_AT_MS_META_KEY, SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
definition_to_metadata,
INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY,
SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY,
};
use crate::database::OpenTableRequest;
use crate::table::{NativeTable, NativeTableExt, Table};
@@ -198,28 +197,8 @@ pub(crate) async fn execute_refresh(
),
});
}
let definition_changed =
definition.filter != replanned.filter || definition.inputs != replanned.inputs;
let definition = &replanned;
// A watermark written for a legacy raw filter certifies the rows that
// filter produced, not the canonical predicate above. Rebuild instead of
// accepting or advancing it, and persist the migrated definition in the
// same metadata commit that certifies the replacement rows.
if definition_changed {
return rebuild(
view_native,
&view_ds,
&source_ds,
source_version,
source_ts,
definition,
true,
expected_incarnation,
)
.await;
}
let metadata = &view_ds.schema().metadata;
let watermark: Option<u64> = metadata
.get(SOURCE_VERSION_META_KEY)
@@ -278,7 +257,6 @@ pub(crate) async fn execute_refresh(
source_version,
source_ts,
definition,
false,
expected_incarnation,
)
.await
@@ -293,7 +271,6 @@ pub(crate) async fn execute_refresh(
source_version,
source_ts,
definition,
false,
expected_incarnation,
)
.await
@@ -706,7 +683,6 @@ async fn incremental(
view_ds.clone(),
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
@@ -728,7 +704,6 @@ async fn incremental(
published,
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
@@ -800,7 +775,6 @@ async fn incremental(
published,
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
@@ -850,14 +824,12 @@ async fn incremental(
appended,
source_version,
source_ts,
None,
expected_incarnation,
)
.await?;
Ok(Some(result))
}
#[allow(clippy::too_many_arguments)]
async fn rebuild(
view_native: &NativeTable,
view_ds: &Dataset,
@@ -865,7 +837,6 @@ async fn rebuild(
source_version: u64,
source_ts: u128,
definition: &MaterializedViewDefinition,
persist_definition: bool,
expected_incarnation: Option<&str>,
) -> Result<RefreshMaterializedViewResult> {
let rows_written = Arc::new(AtomicU64::new(0));
@@ -896,7 +867,6 @@ async fn rebuild(
replaced,
source_version,
source_ts,
persist_definition.then_some(definition),
expected_incarnation,
)
.await?;
@@ -1011,7 +981,6 @@ async fn stamp_watermark(
mut dataset: Dataset,
source_version: u64,
source_ts: u128,
definition: Option<&MaterializedViewDefinition>,
expected_incarnation: Option<&str>,
) -> Result<u64> {
ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?;
@@ -1024,32 +993,27 @@ async fn stamp_watermark(
.get(INCARNATION_META_KEY)
.cloned()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
let mut metadata = vec![(INCARNATION_META_KEY.to_string(), Some(incarnation))];
if let Some(definition) = definition {
metadata.push((
DEFINITION_META_KEY.to_string(),
Some(definition_to_metadata(definition)?),
));
}
metadata.extend([
(
SOURCE_VERSION_META_KEY.to_string(),
Some(source_version.to_string()),
),
(
SOURCE_VERSION_TS_META_KEY.to_string(),
Some(source_ts.to_string()),
),
(
REFRESHED_AT_MS_META_KEY.to_string(),
Some(now_ms().to_string()),
),
(
VIEW_VERSION_META_KEY.to_string(),
Some(predicted.to_string()),
),
]);
dataset.update_schema_metadata(metadata).await?;
dataset
.update_schema_metadata([
(INCARNATION_META_KEY.to_string(), Some(incarnation)),
(
SOURCE_VERSION_META_KEY.to_string(),
Some(source_version.to_string()),
),
(
SOURCE_VERSION_TS_META_KEY.to_string(),
Some(source_ts.to_string()),
),
(
REFRESHED_AT_MS_META_KEY.to_string(),
Some(now_ms().to_string()),
),
(
VIEW_VERSION_META_KEY.to_string(),
Some(predicted.to_string()),
),
])
.await?;
let actual = dataset.version().version;
if actual != predicted {
return Err(Error::Runtime {
@@ -1621,106 +1585,6 @@ mod tests {
assert_eq!(read(view.table(), "x").await, vec![20, 40]);
}
#[tokio::test]
async fn test_mixed_case_filter_is_canonicalized_for_lineage_and_refresh() {
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(
("id", Int32, [1, 2, 3]),
("PartyAbbrev", Utf8, ["D", "R", "D"])
)
.unwrap();
conn.create_table("src", batch)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
conn.create_materialized_view("democrats", "src")
.select([("id", "id")])
.only_if(r#""PartyAbbrev" = 'D'"#)
.execute()
.await
.unwrap();
// Reopen from schema metadata so these assertions cover the stored
// predicate and lineage, not only the declaration-time handle.
let view = conn.open_materialized_view("democrats").await.unwrap();
assert_eq!(
view.definition().filter.as_deref(),
Some("`PartyAbbrev` = 'D'")
);
assert_eq!(view.definition().inputs, ["PartyAbbrev", "id"]);
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.rows_written, 2);
assert_eq!(read(view.table(), "id").await, vec![1, 3]);
}
#[tokio::test]
async fn test_legacy_raw_filter_rebuilds_and_persists_canonical_definition() {
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(
("id", Int32, [1, 2, 3]),
("PartyAbbrev", Utf8, ["D", "R", "D"])
)
.unwrap();
conn.create_table("legacy_src", batch)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let view = conn
.create_materialized_view("legacy_view", "legacy_src")
.select([("id", "id")])
.only_if(r#""PartyAbbrev" = 'X'"#)
.execute()
.await
.unwrap();
assert_eq!(view.refresh().execute().await.unwrap().rows_written, 0);
// Model a definition and up-to-date watermark written before filter
// canonicalization was applied to materialized views.
let mut legacy = view.definition().clone();
legacy.filter = Some(r#""PartyAbbrev" = 'D'"#.into());
legacy.inputs = vec!["id".into()];
let native = view.table().as_native().unwrap();
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
let predicted = dataset.version().version + 1;
dataset
.update_schema_metadata([
(
DEFINITION_META_KEY.to_string(),
Some(definition_to_metadata(&legacy).unwrap()),
),
(
VIEW_VERSION_META_KEY.to_string(),
Some(predicted.to_string()),
),
])
.await
.unwrap();
native.dataset.update(dataset);
let reopened = conn.open_materialized_view("legacy_view").await.unwrap();
let result = reopened.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Rebuild);
assert_eq!(result.rows_written, 2);
assert_eq!(read(reopened.table(), "id").await, vec![1, 3]);
// A fresh handle proves the migration was stored alongside the new
// watermark and therefore happens only once.
let migrated = conn.open_materialized_view("legacy_view").await.unwrap();
assert_eq!(
migrated.definition().filter.as_deref(),
Some("`PartyAbbrev` = 'D'")
);
assert_eq!(migrated.definition().inputs, ["PartyAbbrev", "id"]);
assert_eq!(
migrated.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
assert_eq!(read(migrated.table(), "id").await, vec![1, 3]);
}
#[tokio::test]
async fn test_append_refreshes_incrementally() {
let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await;
@@ -2903,7 +2767,7 @@ mod tests {
let stale = view_native.dataset.get().await.unwrap().as_ref().clone();
view.table().delete("x = 1").await.unwrap();
let err = stamp_watermark(view_native, stale, 99, 99, None, None).await;
let err = stamp_watermark(view_native, stale, 99, 99, None).await;
assert!(err.is_err());
let result = view.refresh().execute().await.unwrap();
+14 -1108
View File
File diff suppressed because it is too large Load Diff
+1 -118
View File
@@ -26,9 +26,7 @@ use crate::database::{
use crate::error::Result;
use crate::function::{FunctionRegistrationRequest, FunctionVersion};
use crate::job::Job;
use crate::remote::job::{
DescribeJobResponse, PauseJobResponse, RemoteJob, ResumeJobResponse, job_state_to_client,
};
use crate::remote::job::{DescribeJobResponse, RemoteJob, job_state_to_client};
use crate::remote::util::stream_as_body;
use crate::table::BaseTable;
@@ -89,10 +87,6 @@ impl ServerVersion {
pub fn support_blobs(&self) -> bool {
self.0 >= semver::Version::new(0, 5, 0)
}
pub fn support_fts_document_granularity(&self) -> bool {
self.0 >= semver::Version::new(0, 6, 0)
}
}
pub const OPT_REMOTE_PREFIX: &str = "remote_database_";
@@ -535,11 +529,6 @@ struct RemoteListJobsResponse {
page_token: Option<String>,
}
#[derive(serde::Deserialize)]
struct RemoteDropFunctionResponse {
dropped: bool,
}
/// Bound on `list_jobs` page walking; a warning is logged when the listing
/// is truncated at this many pages.
const MAX_LIST_JOBS_PAGES: usize = 100;
@@ -590,20 +579,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
response.json().await.err_to_http(request_id)
}
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
let req = self
.client
.post("/v1/functions/drop")
.json(&serde_json::json!({
"name": name,
"version": version,
}));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?;
Ok(response.dropped)
}
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
self.client.clone(),
@@ -686,40 +661,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
}
async fn pause_job(&self, job_id: &str) -> Result<crate::database::PauseJobStatus> {
let req = self
.client
.post("/v1/jobs/pause")
.json(&serde_json::json!({ "job_id": job_id }));
let (request_id, rsp) = self.client.send(req).await?;
let rsp = self.client.check_response(&request_id, rsp).await?;
let body: PauseJobResponse = rsp.json().await.err_to_http(request_id)?;
Ok(if body.paused {
crate::database::PauseJobStatus::Pausing
} else if body.committing {
crate::database::PauseJobStatus::Committing
} else {
crate::database::PauseJobStatus::AlreadyPaused
})
}
async fn resume_job(&self, job_id: &str) -> Result<crate::database::ResumeJobStatus> {
let req = self
.client
.post("/v1/jobs/resume")
.json(&serde_json::json!({ "job_id": job_id }));
let (request_id, rsp) = self.client.send(req).await?;
let rsp = self.client.check_response(&request_id, rsp).await?;
let body: ResumeJobResponse = rsp.json().await.err_to_http(request_id)?;
Ok(if body.resumed {
crate::database::ResumeJobStatus::Resumed
} else if body.still_pausing {
crate::database::ResumeJobStatus::StillPausing
} else {
crate::database::ResumeJobStatus::NotPaused
})
}
async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<arrow_array::RecordBatch>> {
let mut body = serde_json::json!({});
if let Some(job_id) = job_id {
@@ -2655,45 +2596,6 @@ mod tests {
assert!(!conn.cancel_job("nope").await.unwrap());
}
#[tokio::test]
async fn test_pause_and_resume_job() {
use crate::database::{PauseJobStatus, ResumeJobStatus};
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.url().path(), "/v1/jobs/pause");
http::Response::builder()
.status(200)
.body(r#"{"job_id": "job-1", "paused": true}"#)
.unwrap()
});
assert_eq!(
conn.pause_job("job-1").await.unwrap(),
PauseJobStatus::Pausing
);
let conn = Connection::new_with_handler(|_| {
http::Response::builder()
.status(200)
.body(r#"{"job_id": "job-1", "paused": false, "committing": true}"#)
.unwrap()
});
assert_eq!(
conn.pause_job("job-1").await.unwrap(),
PauseJobStatus::Committing
);
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.url().path(), "/v1/jobs/resume");
http::Response::builder()
.status(200)
.body(r#"{"job_id": "job-1", "resumed": false, "still_pausing": true}"#)
.unwrap()
});
assert_eq!(
conn.resume_job("job-1").await.unwrap(),
ResumeJobStatus::StillPausing
);
}
#[tokio::test]
async fn test_job_history_parses_arrow_stream() {
let schema = Arc::new(Schema::new(vec![Field::new(
@@ -2783,25 +2685,6 @@ mod tests {
assert_eq!(version.version(), "fv_01K3EXACT");
}
#[tokio::test]
async fn test_drop_function_sends_exact_version_and_decodes_replay() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/drop");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
body,
serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"})
);
http::Response::builder()
.status(200)
.body(r#"{"dropped":false}"#)
.unwrap()
});
assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap());
}
#[tokio::test]
async fn test_conn_job_waits_to_done() {
let polls = Arc::new(AtomicUsize::new(0));
-22
View File
@@ -73,28 +73,6 @@ pub(super) struct ReportedFailure {
retryable: Option<bool>,
}
/// Forward-compatible `/v1/jobs/pause` wire envelope.
#[derive(Deserialize)]
pub(super) struct PauseJobResponse {
/// False when the job was already paused, so a repeated pause changed
/// nothing.
#[serde(default)]
pub(super) paused: bool,
/// The job is finalizing its results and cannot be parked right now.
#[serde(default)]
pub(super) committing: bool,
}
/// Forward-compatible `/v1/jobs/resume` wire envelope.
#[derive(Deserialize)]
pub(super) struct ResumeJobResponse {
#[serde(default)]
pub(super) resumed: bool,
/// The pause's worker drain is not confirmed yet.
#[serde(default)]
pub(super) still_pausing: bool,
}
/// Forward-compatible `/v1/jobs/describe` wire envelope.
#[derive(Deserialize)]
pub(super) struct DescribeJobResponse {
File diff suppressed because it is too large Load Diff
+12 -65
View File
@@ -6,7 +6,6 @@
use std::ops::Range;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use arrow_array::{Array, LargeBinaryArray};
use arrow_schema::DataType;
@@ -21,7 +20,7 @@ use crate::error::Result;
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
use crate::table::BaseTable;
use super::{FreshnessHeaders, FreshnessState, RemoteTable, freshness_headers_snapshot};
use super::{FreshnessHeaders, RemoteTable};
#[derive(Debug, Clone, Copy)]
enum RangeRequestMode {
@@ -44,10 +43,7 @@ struct TableBlobRangeRequester<S: HttpSend> {
path: String,
version: Option<u64>,
branch: Option<String>,
freshness: Arc<std::sync::Mutex<FreshnessState>>,
parent_freshness: Arc<std::sync::Mutex<FreshnessState>>,
parent_freshness_request: FreshnessHeaders,
read_consistency_interval: Option<Duration>,
freshness: FreshnessHeaders,
}
#[async_trait::async_trait]
@@ -57,9 +53,8 @@ impl<S: HttpSend> BlobRangeRequester for TableBlobRangeRequester<S> {
range_header: &str,
mode: RangeRequestMode,
) -> Result<(String, Response)> {
let freshness_request =
freshness_headers_snapshot(&self.freshness, self.read_consistency_interval);
let mut request = freshness_request
let mut request = self
.freshness
.apply(self.client.get(&self.path))
.header(header::RANGE, range_header);
if let Some(version) = self.version {
@@ -76,9 +71,6 @@ impl<S: HttpSend> BlobRangeRequester for TableBlobRangeRequester<S> {
return Ok((request_id, response));
}
let response = self.client.check_response(&request_id, response).await?;
freshness_request.observe_headers(&self.freshness, response.headers());
self.parent_freshness_request
.observe_headers(&self.parent_freshness, response.headers());
Ok((request_id, response))
}
}
@@ -369,21 +361,18 @@ impl<S: HttpSend> RemoteTable<S> {
message: "fetch_blobs is not supported on this LanceDB Cloud server".into(),
});
}
let read_snapshot = self.snapshot_read_state().await;
let version = self.current_version().await;
let mut body = serde_json::json!({
"version": read_snapshot.version,
"version": version,
"column": column,
"row_ids": row_ids,
});
self.apply_branch_body(&mut body);
let request = self
.client
.post(&format!("/v1/table/{}/fetch_blobs/", self.identifier))
.post_read(&format!("/v1/table/{}/fetch_blobs/", self.identifier))
.json(&body);
let (request_id, response) = self
.send_with_freshness(request, true, read_snapshot.freshness)
.await?;
let (request_id, response) = self.send(request, true).await?;
let mut stream = self.read_arrow_response(&request_id, response).await?;
let mut blob_chunks: Vec<Arc<dyn Array>> = Vec::new();
@@ -459,7 +448,8 @@ impl<S: HttpSend> RemoteTable<S> {
});
}
let read_snapshot = self.snapshot_read_state().await;
let version = self.current_version().await;
let freshness = self.snapshot_freshness_headers();
let encoded_column = urlencoding::encode(column);
let requesters = row_ids
.iter()
@@ -471,12 +461,9 @@ impl<S: HttpSend> RemoteTable<S> {
let requester: Arc<dyn BlobRangeRequester> = Arc::new(TableBlobRangeRequester {
client: self.client.clone(),
path,
version: read_snapshot.version,
version,
branch: self.branch.clone(),
freshness: Arc::new(std::sync::Mutex::new(read_snapshot.freshness_state)),
parent_freshness: self.freshness.clone(),
parent_freshness_request: read_snapshot.freshness,
read_consistency_interval: self.client.read_consistency_interval,
freshness,
});
requester
})
@@ -698,46 +685,6 @@ mod tests {
assert!(requests.lock().unwrap().contains(&"bytes=5-11".to_string()));
}
#[tokio::test]
async fn remote_blob_file_keeps_the_open_timeline_after_parent_checkout() {
let range_requests = Arc::new(StdMutex::new(Vec::new()));
let captured = range_requests.clone();
let table = RemoteTable::new_mock(
"my_table".to_string(),
move |request| match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(r#"{"version":5,"schema":{"fields":[]}}"#.as_bytes().to_vec())
.unwrap(),
"/v1/table/my_table/blob/image/10/bytes" => {
captured.lock().unwrap().push((
request.url().query().unwrap_or_default().to_string(),
request.headers().clone(),
));
range_response(&request, PAYLOAD)
}
path => panic!("unexpected path: {path}"),
},
Some(Version::new(0, 5, 0)),
);
table.checkout(5).await.unwrap();
let file = table
.fetch_blob_files_impl("image", &[10])
.await
.unwrap()
.pop()
.flatten()
.unwrap();
table.checkout_latest().await.unwrap();
file.read_range(5..12).await.unwrap();
let requests = range_requests.lock().unwrap();
let (query, headers) = requests.last().unwrap();
assert!(query.contains("version=5"));
assert!(!headers.contains_key("x-lancedb-min-timestamp"));
}
#[tokio::test]
async fn remote_blob_file_reuses_sequential_response_until_seek() {
let requests = Arc::new(StdMutex::new(Vec::new()));
+9 -74
View File
@@ -24,10 +24,7 @@ use lance::io::exec::utils::InstrumentedRecordBatchStreamAdapter;
use crate::Error;
use crate::remote::ARROW_STREAM_CONTENT_TYPE;
use crate::remote::client::{HttpSend, RestfulLanceDbClient, Sender};
use crate::remote::table::{
FreshnessHeaders, FreshnessState, MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable,
freshness_headers_snapshot,
};
use crate::remote::table::{MergeInsertRequest, REQUEST_TIMEOUT_HEADER, RemoteTable};
use crate::table::datafusion::insert::COUNT_SCHEMA;
use crate::table::write_progress::WriteProgressTracker;
use crate::table::{AddResult, MergeResult};
@@ -57,38 +54,6 @@ pub enum WriteResult {
Merge(MergeResult),
}
#[derive(Debug, Clone, Default)]
struct WriteFreshness {
state: Option<Arc<Mutex<FreshnessState>>>,
read_consistency_interval: Option<Duration>,
}
impl WriteFreshness {
fn prepare(
&self,
request: reqwest::RequestBuilder,
) -> (reqwest::RequestBuilder, Option<FreshnessHeaders>) {
match &self.state {
Some(state) => {
let freshness_request =
freshness_headers_snapshot(state, self.read_consistency_interval);
(freshness_request.apply(request), Some(freshness_request))
}
None => (request, None),
}
}
fn observe(
&self,
freshness_request: Option<FreshnessHeaders>,
headers: &reqwest::header::HeaderMap,
) {
if let (Some(state), Some(freshness_request)) = (&self.state, freshness_request) {
freshness_request.observe_headers(state, headers);
}
}
}
/// ExecutionPlan for streaming a write (add or merge_insert) to a remote
/// LanceDB table.
///
@@ -106,7 +71,6 @@ pub struct RemoteWriteExec<S: HttpSend = Sender> {
table_name: String,
identifier: String,
client: RestfulLanceDbClient<S>,
freshness: WriteFreshness,
input: Arc<dyn ExecutionPlan>,
op: WriteOp,
properties: Arc<PlanProperties>,
@@ -206,7 +170,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
table_name,
identifier,
client,
freshness: WriteFreshness::default(),
input,
op,
properties: Arc::new(properties),
@@ -220,18 +183,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
}
}
pub(super) fn with_freshness(
mut self,
state: Arc<Mutex<FreshnessState>>,
read_consistency_interval: Option<Duration>,
) -> Self {
self.freshness = WriteFreshness {
state: Some(state),
read_consistency_interval,
};
self
}
/// Get the add result after execution, if this exec ran an insert.
pub fn add_result(&self) -> Option<AddResult> {
match self
@@ -334,7 +285,6 @@ impl<S: HttpSend + 'static> RemoteWriteExec<S> {
/// each threading the same handful of arguments.
struct PartRequestCtx<'a, S: HttpSend> {
client: &'a RestfulLanceDbClient<S>,
freshness: &'a WriteFreshness,
identifier: &'a str,
table_name: &'a str,
upload_id: &'a str,
@@ -402,11 +352,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
}
/// Build the `/insert` request for a single multipart part.
fn build_part_request(
&self,
part_id: &str,
body: reqwest::Body,
) -> (reqwest::RequestBuilder, Option<FreshnessHeaders>) {
fn build_part_request(&self, part_id: &str, body: reqwest::Body) -> reqwest::RequestBuilder {
let mut request = self
.client
.post(&format!("/v1/table/{}/insert/", self.identifier))
@@ -422,16 +368,12 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
if let Some(b) = self.branch {
request = request.query(&[("branch", b)]);
}
self.freshness.prepare(request.body(body))
request.body(body)
}
/// Send a single part's request and drain the response, mapping HTTP and
/// table-not-found errors into `DataFusionError`.
async fn send_part_request(
&self,
request: reqwest::RequestBuilder,
freshness_request: Option<FreshnessHeaders>,
) -> DataFusionResult<()> {
async fn send_part_request(&self, request: reqwest::RequestBuilder) -> DataFusionResult<()> {
let (request_id, response) = self
.client
.send(request)
@@ -446,8 +388,6 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
.check_response(&request_id, response)
.await
.map_err(|e| DataFusionError::External(Box::new(e)))?;
self.freshness
.observe(freshness_request, response.headers());
response.bytes().await.map_err(|e| {
DataFusionError::External(Box::new(Error::Http {
source: Box::new(e),
@@ -479,7 +419,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
let body = reqwest::Body::wrap_stream(chunk_rx);
let part_id = uuid::Uuid::new_v4().to_string();
let (request, freshness_request) = self.build_part_request(&part_id, body);
let request = self.build_part_request(&part_id, body);
// Measured from just before the request is sent, matching the window the
// client read timeout applies to the upload.
@@ -555,7 +495,7 @@ impl<S: HttpSend + 'static> PartRequestCtx<'_, S> {
Ok::<bool, DataFusionError>(input_ended)
};
let send = self.send_part_request(request, freshness_request);
let send = self.send_part_request(request);
// `join!` rather than `tokio::spawn`: the producer borrows `input` (and
// `schema`), so it cannot satisfy the `'static` bound a spawned task
@@ -629,7 +569,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
// Building a fresh exec (with a new, empty `result`) is what makes the
// outer rescannable retry loop work: `reset_state()` clears the captured
// result so a re-execution starts clean.
let mut exec = Self::new_inner(
Ok(Arc::new(Self::new_inner(
self.table_name.clone(),
self.identifier.clone(),
self.client.clone(),
@@ -640,9 +580,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
self.branch.clone(),
self.max_bytes_per_request,
self.max_request_duration,
);
exec.freshness = self.freshness.clone();
Ok(Arc::new(exec))
)))
}
fn execute(
@@ -675,7 +613,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
&self.metrics,
));
let client = self.client.clone();
let freshness = self.freshness.clone();
let identifier = self.identifier.clone();
let op = self.op.clone();
let result_slot = self.result.clone();
@@ -697,7 +634,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
let overwrite = matches!(op, WriteOp::Insert { overwrite: true });
let ctx = PartRequestCtx {
client: &client,
freshness: &freshness,
identifier: &identifier,
table_name: &table_name,
upload_id,
@@ -752,7 +688,7 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
let (error_tx, mut error_rx) = tokio::sync::oneshot::channel();
let body = Self::stream_as_http_body(input_stream, error_tx, tracker)?;
let (request, freshness_request) = freshness.prepare(request.body(body));
let request = request.body(body);
let result: DataFusionResult<(String, _)> = async {
let (request_id, response) = client
@@ -772,7 +708,6 @@ impl<S: HttpSend + 'static> ExecutionPlan for RemoteWriteExec<S> {
.check_response(&request_id, response)
.await
.map_err(|e| DataFusionError::External(Box::new(e)))?;
freshness.observe(freshness_request, response.headers());
Ok((request_id, response))
}
+11 -107
View File
@@ -59,9 +59,7 @@ use crate::index::{IndexConfig, IndexStatisticsImpl, IndexType};
use crate::job::Job;
use crate::query::{IntoQueryVector, Query, QueryExecutionOptions, TakeQuery, VectorQuery};
use crate::table::datafusion::insert::InsertExec;
use crate::utils::{
PatchReadParam, PatchWriteParam, public_fts_field_path_by_id, resolve_arrow_field_path,
};
use crate::utils::{PatchReadParam, PatchWriteParam, resolve_arrow_field_path};
use self::dataset::DatasetConsistencyWrapper;
use self::merge::MergeInsertBuilder;
@@ -562,13 +560,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
fn id(&self) -> &str;
/// Get the arrow [Schema] of the table.
async fn schema(&self) -> Result<SchemaRef>;
/// Create a read-only handle pinned to the table's current active revision.
///
/// The returned handle is independent from later refreshes or checkouts on
/// this handle. This is used by bindings that must prepare client-side
/// query state from the same revision that the query will execute against.
#[doc(hidden)]
async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>>;
/// Count the number of rows in this table.
async fn count_rows(&self, filter: Option<Filter>) -> Result<usize>;
/// Create a physical plan for the query.
@@ -595,14 +586,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
query: &AnyQuery,
options: QueryExecutionOptions,
) -> Result<String>;
/// Whether [`BaseTable::analyze_plan`] is provided by a remote service.
///
/// Client-side query wrappers use this to preserve backend metrics and
/// distributed-analysis options instead of replacing them with a local plan.
#[doc(hidden)]
fn analyze_plan_is_remote(&self) -> bool {
false
}
/// Add new records to the table.
async fn add(&self, add: AddDataBuilder) -> Result<AddResult>;
@@ -758,8 +741,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
/// Declare computed columns, each defined by a SQL expression.
///
/// Where the declaration is planned depends on the backend: a local table
/// validates and types the expression itself, while a remote one sends the
/// expression for the server to plan.
/// validates and types the expression itself, a remote one sends the text
/// for the server to plan.
async fn add_computed_columns(
&self,
_columns: &[(String, String)],
@@ -1156,26 +1139,13 @@ impl Table {
self.inner.schema().await
}
/// Create a read-only handle pinned to the current active revision.
#[doc(hidden)]
pub async fn query_snapshot(&self) -> Result<Self> {
Ok(Self {
inner: self.inner.query_snapshot().await?,
database: self.database.clone(),
embedding_registry: self.embedding_registry.clone(),
})
}
/// Count the number of rows in this dataset.
///
/// # Arguments
///
/// * `filter` if present, only count rows matching the filter
pub async fn count_rows(&self, filter: Option<String>) -> Result<usize> {
let filter = filter
.map(|predicate| crate::expr::canonicalize_sql_predicate(&predicate).map(Filter::Sql))
.transpose()?;
self.inner.count_rows(filter).await
self.inner.count_rows(filter.map(Filter::Sql)).await
}
/// Names of the blob v2 columns in this table, in declaration order.
@@ -1375,13 +1345,7 @@ impl Table {
/// # });
/// ```
pub async fn delete(&self, predicate: impl Into<Predicate<'_>>) -> Result<DeleteResult> {
match predicate.into() {
Predicate::String(predicate) => {
let predicate = crate::expr::canonicalize_sql_predicate(predicate)?;
self.inner.delete(Predicate::String(&predicate)).await
}
predicate @ Predicate::Expr(_) => self.inner.delete(predicate).await,
}
self.inner.delete(predicate.into()).await
}
/// Create an index on the provided column(s).
@@ -1660,9 +1624,9 @@ impl Table {
/// Offsets are useful for sampling as the set of all valid offsets is easily
/// known in advance to be [0, len(table)).
///
/// No guarantees are made regarding the order in which results are returned.
/// Repeated offsets produce repeated rows, which makes this method suitable for
/// sampling with replacement.
/// No guarantees are made regarding the order in which results are returned. If you
/// desire an output order that matches the order of the given offsets, you will need
/// to add the row offset column to the output and align it yourself.
///
/// Parameters
/// ----------
@@ -1794,23 +1758,7 @@ impl Table {
self.inner.alter_columns(alterations).await
}
/// Update per-field (column) metadata.
///
/// Each [`FieldMetadataUpdate`] is merged into the field's existing metadata
/// by default; use [`FieldMetadataUpdate::remove`] to delete a key, or
/// [`FieldMetadataUpdate::replace`] to swap the field's entire metadata map.
///
/// The following keys are treated specially, by convention, and should be
/// used when appropriate:
///
/// - `lancedb:description`: for a human-readable description of a field.
/// - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
/// names the tag category; e.g. `lancedb:tag:model: "clip"`.
/// - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
/// `feature_v2` might be in the same logical column.
/// - `lancedb:status`: for status options (`production`, `candidate`,
/// `deprecated`, `archived`) to designate the current life cycle state of
/// this column.
/// Update per-field metadata (merges by default).
pub async fn update_field_metadata(
&self,
updates: &[FieldMetadataUpdate],
@@ -3111,17 +3059,6 @@ impl BaseTable for NativeTable {
&self.id
}
async fn query_snapshot(&self) -> Result<Arc<dyn BaseTable>> {
let snapshot = self.dataset.new_query_snapshot().await?;
let mut table = self.with_dataset(snapshot);
// QueryTable requests do not carry a revision. A pinned snapshot must
// execute locally until the namespace API can accept that revision.
table
.pushdown_operations
.remove(&NamespaceClientPushdownOperation::QueryTable);
Ok(Arc::new(table))
}
async fn version(&self) -> Result<u64> {
Ok(self.dataset.get().await?.version().version)
}
@@ -3256,10 +3193,7 @@ impl BaseTable for NativeTable {
let dataset = self.dataset.get().await?;
match filter {
None => Ok(dataset.count_rows(None).await?),
Some(Filter::Sql(sql)) => {
let sql = crate::expr::canonicalize_sql_predicate(&sql)?;
Ok(dataset.count_rows(Some(sql)).await?)
}
Some(Filter::Sql(sql)) => Ok(dataset.count_rows(Some(sql)).await?),
Some(Filter::Datafusion(_)) => Err(Error::NotSupported {
message: "Datafusion filters are not yet supported".to_string(),
}),
@@ -3597,14 +3531,7 @@ impl BaseTable for NativeTable {
let field_ids = idx_desc.field_ids();
let mut columns = Vec::with_capacity(field_ids.len());
for field_id in field_ids {
let field_path = match if index_type == crate::index::IndexType::FTS {
public_fts_field_path_by_id(dataset.schema(), *field_id as i32)
} else {
dataset
.schema()
.field_path(*field_id as i32)
.map_err(Into::into)
} {
let field_path = match dataset.schema().field_path(*field_id as i32) {
Ok(field_path) => field_path,
Err(e) => {
log::warn!(
@@ -4191,14 +4118,6 @@ mod tests {
parent_list_calls: self.parent_list_calls.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
_original: Arc<dyn object_store::list::PaginatedListStore>,
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
None
}
}
#[tokio::test]
@@ -4302,14 +4221,6 @@ mod tests {
self.called.store(true, Ordering::Relaxed);
original
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn object_store::list::PaginatedListStore>,
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
Some(original)
}
}
#[tokio::test]
@@ -5771,13 +5682,6 @@ mod tests {
assert!(index_bytes > 0);
assert_eq!(with_index, data_only + index_bytes);
// Release builds reject unstable overlay datasets unless explicitly opted in.
if !lance_table::feature_flags::can_read_dataset(
lance_table::feature_flags::FLAG_UNSTABLE_DATA_OVERLAY_FILES,
) {
return;
}
// Commit an overlay file supplying new `foo` values for the first three
// rows of fragment 0. There is no high-level API that writes overlays
// yet, so write the overlay's data file and commit the `DataOverlay`

Some files were not shown because too many files have changed in this diff Show More