rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-16 16:52:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into remote-set-unenforced-primary-key

Browse Source 
Keep the remote set_unenforced_primary_key implementation over main's
NotSupported stub, and drop the remote test case that expected it to fail.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-10-15 20:44:52 -05:00
co-authored by Claude Opus 5.5
parent f9dab6c3c8 7b3ba4c437
commit 8b52a75ecc
642 changed files with 117355 additions and 11566 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.39.0-beta.6"
current_version = "0.41.0-beta.7"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/actions/setup-protoc/action.yml
+80
View File
@@ -0,0 +1,80 @@
name: Setup protoc
description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves
inputs:
install:
description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there."
required: false
default: "true"
outputs:
version:
description: "The pinned protoc version, e.g. 36.2"
value: ${{ steps.pin.outputs.version }}
linux-x86_64-sha256:
description: "SHA-256 of the protoc linux-x86_64 zip"
value: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
linux-aarch_64-sha256:
description: "SHA-256 of the protoc linux-aarch_64 zip"
value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
runs:
using: composite
steps:
# protobuf does not publish checksum files for protoc. These are the digests
# GitHub records for the release assets; when bumping, copy them from
# `gh api repos/protocolbuffers/protobuf/releases/tags/v<version>`.
- name: Set protoc pin
id: pin
shell: bash
run: |
{
echo "version=36.2"
echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278"
echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a"
echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b"
echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3"
} >> "$GITHUB_OUTPUT"
- name: Install protoc
if: ${{ inputs.install == 'true' }}
shell: bash
env:
PROTOC_VERSION: ${{ steps.pin.outputs.version }}
RUNNER_OS_NAME: ${{ runner.os }}
RUNNER_ARCH_NAME: ${{ runner.arch }}
LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }}
WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }}
run: |
set -euo pipefail
case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in
Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;;
Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;;
macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;;
Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;;
*) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;;
esac
zip="$RUNNER_TEMP/protoc.zip"
dest="$RUNNER_TEMP/protoc"
curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip"
# Hash stdin rather than a path: Windows paths contain backslashes,
# which make sha256sum escape its output.
if [ "$RUNNER_OS_NAME" = "macOS" ]; then
actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1)
else
actual=$(sha256sum < "$zip" | cut -d' ' -f1)
fi
if [ "$actual" != "$sha256" ]; then
echo "protoc checksum mismatch: expected $sha256, got $actual" >&2
exit 1
fi
if [ "$RUNNER_OS_NAME" = "Windows" ]; then
# Git Bash on Windows runners does not reliably ship unzip.
7z x -y -o"$dest" "$zip" > /dev/null
else
unzip -q -o "$zip" -d "$dest"
fi
rm "$zip"
echo "$dest/bin" >> "$GITHUB_PATH"
"$dest/bin/protoc" --version
.github/release_notes.json
+5 -1
View File
@@ -6,6 +6,10 @@
"title": "## 🏆 Highlights",
"labels": ["highlight"]
},
{
"title": "## 🚨 Critical Fixes",
"labels": ["critical-fix"]
},
{
"title": "## 🛠 Breaking Changes",
"labels": ["breaking-change"]
@@ -38,4 +42,4 @@
"labels": ["ci"]
}
]
}
}
.github/workflows/build_linux_wheel/action.yml
+11 -4
View File
@@ -29,6 +29,11 @@ inputs:
runs:
using: "composite"
steps:
- name: Get protoc version
id: protoc
uses: ./.github/actions/setup-protoc
with:
install: "false"
- name: CONFIRM ARM BUILD
shell: bash
env:
@@ -54,13 +59,14 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
target: x86_64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
@@ -71,14 +77,15 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
target: aarch64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
yum install -y clang
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
.github/workflows/build_mac_wheel/action.yml
+1 -4
View File
@@ -12,10 +12,7 @@ inputs:
runs:
using: "composite"
steps:
- name: Install macos dependency
shell: bash
run: |
brew install protobuf
- uses: ./.github/actions/setup-protoc
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
.github/workflows/build_windows_wheel/action.yml
+1 -9
View File
@@ -12,15 +12,7 @@ inputs:
runs:
using: "composite"
steps:
- name: Install Protoc v21.12
working-directory: C:\
run: |
New-Item -Path 'C:\protoc' -ItemType Directory
Set-Location C:\protoc
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
7z x protoc.zip
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
shell: powershell
- uses: ./.github/actions/setup-protoc
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
.github/workflows/license-header-check.yml
+1
View File
@@ -9,6 +9,7 @@ on:
- python/**
- nodejs/**
- java/**
- docs/web-tests/**
- .github/workflows/license-header-check.yml
permissions:
.github/workflows/nodejs.yml
+66 -1
View File
@@ -164,6 +164,71 @@ jobs:
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
@@ -207,4 +272,4 @@ jobs:
pnpm tsc
- name: Test
run: |
pnpm test
pnpm test --runInBand
.github/workflows/pypi-publish.yml
+10 -10
View File
@@ -11,6 +11,7 @@ on:
- .github/workflows/build_linux_wheel/action.yml
- .github/workflows/build_mac_wheel/action.yml
- .github/workflows/build_windows_wheel/action.yml
- .github/actions/setup-protoc/action.yml
- Cargo.toml # Change in dependency frequently breaks builds
- Cargo.lock
@@ -21,7 +22,7 @@ permissions:
contents: read
# Without this, a force-push to a PR leaves the previous run going -- including
# a ~74 minute Windows job and a billed arm64 wheel build.
# a long fat-LTO Windows job and a billed arm64 wheel build.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
@@ -123,18 +124,17 @@ jobs:
path: target/wheels/lancedb-*.whl
if-no-files-found: error
windows:
timeout-minutes: 90
runs-on: windows-latest
# The fat-LTO build of the `lancedb` crate does not finish within 90 minutes
# on the 4-core standard runner, so use the 8-core larger runner with extra
# headroom.
timeout-minutes: 150
runs-on: windows-2025-8x-x64
env:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
# Fat LTO of the cdylib is single-threaded and the peak-memory step of the
# build. ThinLTO parallelizes it across the runner's cores, at some cost
# to runtime performance on our least performance-sensitive platform.
# Matches what the nodejs Windows builds already do in npm-publish.yml.
CARGO_PROFILE_RELEASE_LTO: thin
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
# Keep the repository's fat LTO / 1 codegen unit release profile to leave
# headroom below PyPI's file-size limit for Windows wheels.
steps:
- uses: actions/checkout@v6
with:
@@ -149,7 +149,7 @@ jobs:
# or the default branch -- so with no run on main there is nothing that
# can populate an entry the release build would be allowed to read. Fixing
# this needs a main/nightly trigger (which would also catch wheel-build
# breakage before a release); the ~74 minutes here is otherwise dominated
# breakage before a release); the build time here is otherwise dominated
# by the fat-LTO link, which no cache avoids.
- uses: ./.github/workflows/build_windows_wheel
with:
.github/workflows/python.yml
+2 -1
View File
@@ -15,6 +15,7 @@ on:
- .github/workflows/build_linux_wheel/**
- .github/workflows/build_mac_wheel/**
- .github/workflows/build_windows_wheel/**
- .github/actions/setup-protoc/**
- .github/workflows/run_tests/**
permissions:
@@ -156,7 +157,7 @@ jobs:
- name: Test without pylance or pandas
run: |
pip uninstall -y pylance pandas
pytest -vv python/tests/test_table.py
pytest -vv python/tests/test_table.py python/tests/test_namespace_no_pylance.py
# Make sure wheels are not included in the Rust cache
- name: Delete wheels
run: rm -rf target/wheels
.github/workflows/rust.yml
+6 -36
View File
@@ -180,7 +180,8 @@ jobs:
env:
CC: clang-18
CXX: clang++-18
GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
GH_TOKEN: ${{ github.token }}
SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
@@ -284,7 +285,7 @@ jobs:
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang-18
@@ -307,40 +308,9 @@ jobs:
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Downgrade dependencies
# These packages have newer requirements for MSRV
run: |
cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
cargo update -p aws-sdk-dynamodb --precise 1.68.0
cargo update -p aws-config --precise 1.6.0
cargo update -p aws-sdk-kms --precise 1.63.0
cargo update -p aws-sdk-s3 --precise 1.79.0
cargo update -p aws-sdk-sso --precise 1.62.0
cargo update -p aws-sdk-ssooidc --precise 1.63.0
cargo update -p aws-sdk-sts --precise 1.63.0
# aws-runtime/sigv4/credential-types/types and the aws-smithy-*
# crates bumped their MSRV to 1.91.1 in late 2026; pin to the last
# 1.91.0-compatible versions. The order matters — each downgrade
# only succeeds once everything that still pins it at a higher
# version has itself been downgraded.
cargo update -p aws-runtime --precise 1.5.12
cargo update -p aws-types --precise 1.3.9
cargo update -p aws-sigv4 --precise 1.3.5
cargo update -p aws-credential-types --precise 1.2.8
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
cargo update -p aws-smithy-checksums --precise 0.63.13
cargo update -p aws-smithy-runtime --precise 1.9.3
cargo update -p aws-smithy-http --precise 0.62.6
cargo update -p aws-smithy-eventstream --precise 0.60.14
cargo update -p aws-smithy-http-client --precise 1.1.3
cargo update -p aws-smithy-observability --precise 0.1.4
cargo update -p aws-smithy-query --precise 0.60.8
cargo update -p aws-smithy-runtime-api --precise 1.9.3
cargo update -p aws-smithy-async --precise 1.2.7
cargo update -p aws-smithy-types --precise 1.3.6
cargo update -p aws-smithy-xml --precise 0.60.11
cargo update -p home --precise 0.5.9
- name: Downgrade dependencies
# aws-smithy-types 1.6.1+ requires rustc 1.94.1
run: cargo update -p aws-smithy-types --precise 1.5.0
- name: cargo +${{ matrix.msrv }} check
env:
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
.typos.toml
+15 -1
View File
@@ -1,5 +1,15 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
extend-ignore-re = [
"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$",
# Conventional import aliases in the docs examples: `import fiftyone as fo`
# and `import torchvision.io as tio`.
"\\bas (fo|tio)\\b",
"\\btio\\.",
# The substring query in the n-gram full-text search example.
'search\("nce"',
# Garbled OCR output quoted verbatim in the VLM fine-tuning tutorial.
"permission nstallation",
]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
@@ -12,6 +22,10 @@ mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
# Superlinked's self-hosted inference engine (SIE), its `sie-lancedb` package,
# and the `sie` embedding function it registers, in
# docs/web/integrations/embedding/superlinked.mdx.
SIE = "SIE"
[files]
extend-exclude = [
CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
CONTRIBUTING.md
+82 -1
View File
@@ -71,7 +71,88 @@ are wrappers around the Rust library.
* `nodejs`: Typescript package source code
* `node`: **Deprecated** Typescript package source code
* `java`: Java package source code
* `docs`: Documentation source code
* `docs/src`: SDK reference, built with mkdocs
* `docs/web`: the open-source pages of [docs.lancedb.com](https://docs.lancedb.com)
* `docs/web-tests`: the tests the documentation's code examples are extracted from
## Documentation
The open-source pages of [docs.lancedb.com](https://docs.lancedb.com) live in
`docs/web`, so a change to behaviour and the change to the page describing it
belong in the same pull request.
Preview them locally — this serves `docs/web` on its own, with no other checkout:
```bash
npm i -g mint # https://mintlify.com/docs/installation
cd docs/web && mint dev
```
This repository holds a page at every path the site publishes, so the whole
navigation resolves locally. Enterprise pages exist here as open-source pages --
what the capability is, and what the embedded form does instead. A separate
private repository supplies the fuller version of each, which replaces the page
at the same path when the published site is assembled.
Four things to know before editing:
**Code examples are not written into the pages.** They live in real tests under
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
imported by the pages. Edit the test, then regenerate from the repository root:
```bash
uv run docs/web-tests/mdx_snippets_gen.py -s docs/web-tests/py -s docs/web-tests/ts -s docs/web-tests/rs -o docs/web/snippets
```
Review the diff before committing: a few modules currently regenerate with
unrelated changes, so commit only the snippets for the examples you changed.
Run the Python, TypeScript and Rust examples in `docs/web-tests/{py,ts,rs}`
locally before submitting the change. These examples are not yet wired into
pull-request CI; that integration remains follow-up work.
**Every heading carries an explicit `{#anchor}`.** Those anchors are how the
Enterprise pages attach their additions to the right section, so they are
assigned once and never regenerated. Leave an existing anchor alone even when
you reword the heading above it; only new headings need a new one.
**A page can declare where it applies.** Its frontmatter says whether LanceDB
OSS and LanceDB Enterprise support the feature: use only `available` or
`unavailable`, and one sentence that explains its scope. Availability describes
product support, not whether a particular deployment has enabled or configured
the feature. A supported feature stays `available` when it requires setup, a
supported SDK or server version, or has differences between offerings. State
those requirements in the summary or the relevant section, including unsupported
subfeatures. Do not use `varies`, infer support from an unknown status, or mark an
absent capability available. Narrow a page to the capability it actually covers
when necessary. For example:
```yaml
availability:
oss: unavailable
enterprise: available
summary: >-
Requests to a deployment carry an API key or an OAuth token. Embedded
LanceDB has no server to sign in to.
```
The assembled site renders this as a label above the page's content and as a
sidebar tag when only one offering has the feature. Do not also open the page
with a badge of its own, and keep a qualification that only one section needs
in that section's text. A `mint dev` preview of `docs/web` alone renders
neither the labels nor the tags.
**A feature can be documented before its release.** List a draft page in
`docs/web/.mintignore`, keep it out of the navigation in `docs.json`, and do not
link to it from published pages. Wrap a draft section in an MDX comment,
`{/* ... */}`, whose first line names the release, such as
`Draft for the Feature X release.` Keep draft table rows and list items in a
comment below the table or list, since a comment between rows or items breaks
it. Drafts are hidden from readers but stay in the source, and availability does
not change what is published. At release, remove the comments and the
`.mintignore` entries, and restore the navigation and links. See
[Writing docs before release](https://github.com/lancedb/docs#writing-docs-before-release)
in the documentation repository's README for previews and release checks.
## Release process
Cargo.lock
Generated
+894 -502
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+25 -18
View File
@@ -10,23 +10,23 @@ repository = "https://github.com/lancedb/lancedb"
description = "Serverless, low-latency vector database for AI applications"
keywords = ["lancedb", "lance", "database", "vector", "search"]
categories = ["database-implementations"]
rust-version = "1.91.0"
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -41,6 +41,8 @@ arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
@@ -60,22 +62,27 @@ log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.13.2"
object_store = "0.14.1"
pin-project = "1.0.7"
rand = "0.9"
snafu = "0.8"
url = "2"
num-traits = "0.2"
oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = "1"
serde_json = { version = "1", features = ["preserve_order"] }
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[patch.crates-io]
# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision.
object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" }
[profile.ci]
debug = "line-tables-only"
inherits = "dev"
README.md
+2 -4
View File
@@ -1,8 +1,6 @@
<a href="https://cloud.lancedb.com" target="_blank">
<img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
</a>
<div align="center">
[![Reverie](docs/src/assets/reverie-banner.png)](https://www.reveriesummit.com/?utm_medium=social&utm_source=github&utm_campaign=reverie-2026)
[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
@@ -13,7 +11,7 @@
<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
# **The Multimodal AI Lakehouse**
# **The Multimodal Lakehouse for AI**
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
deny.toml
+23
View File
@@ -78,6 +78,27 @@ ignore = [
{ id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
{ id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
# rustls 0.23.45: four advisories fixed in 0.23.46 (Debug impls leaking
# TLS secrets, ECH rejection buffering, Brotli certificate decompression
# allocation, ServerHello key-exchange downgrade). 0.23.46 was published
# 2026-10-14 and the dependency age policy requires releases to be 48
# hours old, so the upgrade cannot land yet. Remove these entries and run
# `cargo update -p rustls` once the window has passed. Mirrors lance #9985.
# https://rustsec.org/advisories/RUSTSEC-2026-0346
# https://rustsec.org/advisories/RUSTSEC-2026-0347
# https://rustsec.org/advisories/RUSTSEC-2026-0348
# https://rustsec.org/advisories/RUSTSEC-2026-0349
{ id = "RUSTSEC-2026-0346", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
{ id = "RUSTSEC-2026-0347", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
{ id = "RUSTSEC-2026-0348", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
{ id = "RUSTSEC-2026-0349", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
# unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera
# → serde_yaml_ng. No maintained upgrade exists; clearing it requires
# lindera to move off serde_yaml_ng.
# https://rustsec.org/advisories/RUSTSEC-2024-0449
{ id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" },
# rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
# rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
# we actively use is upgraded to 0.103.13 which contains the fix.
@@ -197,4 +218,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
# releases are cut to crates.io. Allow that specific host.
allow-git = [
"https://github.com/lance-format/lance",
# Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml).
"https://github.com/apache/arrow-rs-object-store",
]
docs/README.md
+7 -4
View File
@@ -15,18 +15,21 @@ unreleased features.
### Preview the docs
Run these commands from the LanceDB repository root. The subshell changes into
`docs` to load `mkdocs.yml`, while leaving your shell in the repository root.
```shell
cd docs
mkdocs serve
(cd docs && PYTHONPATH=. mkdocs serve)
```
If you want to just generate the HTML files:
```shell
PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
(cd docs && PYTHONPATH=. mkdocs build)
```
If successful, you should see a `docs/site` directory that you can verify locally.
If successful, you should see a `docs/site` directory in the repository root that
you can verify locally.
## Adding examples
docs/src/assets/hero-header.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 MiB

After

Width:  |  Height:  |  Size: 1.3 MiB

docs/src/assets/reverie-banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 722 KiB

docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.39.0-beta.6</version>
<version>0.41.0-beta.7</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+3 -1
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -417,6 +417,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/BlobFile.md
+158
View File
@@ -0,0 +1,158 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobFile
# Class: BlobFile
A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fetchblobfiles).
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor and advances the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRange()
```ts
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
Fails when `end` is past the blob size. Does not move the cursor.
#### Parameters
* **start**: `bigint`
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
size(): bigint
```
Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Catalog.md
+118
View File
@@ -0,0 +1,118 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Catalog
# Class: Catalog
A remote catalog manages databases through the server's root namespace.
## Accessors
### uri
```ts
get uri(): string
```
The root namespace endpoint.
#### Returns
`string`
## Methods
### connectDatabase()
```ts
connectDatabase(name): Promise<Connection>
```
Connect to an existing database by its logical name.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### createDatabase()
```ts
createDatabase(name, options): Promise<Connection>
```
Create a database, or open an existing database when existOk is true.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.existOk?**: `boolean`
#### Returns
`Promise`&lt;[`Connection`](Connection.md)&gt;
***
### dropDatabase()
```ts
dropDatabase(name, options): Promise<void>
```
Drop an empty database. The server rejects nonempty databases.
#### Parameters
* **name**: `string`
* **options** = `{}`
* **options.ignoreMissing?**: `boolean`
#### Returns
`Promise`&lt;`void`&gt;
***
### listDatabases()
```ts
listDatabases(options): DatabaseNames
```
Iterate lazily over all database names using `for await...of`.
pageToken resumes from a saved token; omitted starts at the beginning.
pageLimit limits each REST response, not the total; omitted uses the server default.
Request errors are raised during iteration and terminate the iterator.
#### Parameters
* **options** = `{}`
* **options.pageLimit?**: `number`
* **options.pageToken?**: `string`
#### Returns
[`DatabaseNames`](DatabaseNames.md)
#### Example
```ts
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/classes/Connection.md
+239 -13
View File
@@ -180,13 +180,9 @@ abstract createMaterializedView(
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
#### Parameters
@@ -202,6 +198,8 @@ a table exists. Local databases only.
* **options.where?**: `string`
* **options.withNoData?**: `boolean`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
@@ -317,6 +315,38 @@ Creates a new Table and initialize it with new data.
***
### createView()
```ts
abstract createView(
name,
query,
namespacePath?): Promise<ViewDescription>
```
Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned is
rejected now rather than at the first read. A view holds no rows, and its
readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create.
#### Parameters
* **name**: `string`
* **query**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### describeNamespace()
```ts
@@ -340,6 +370,27 @@ The namespace's properties
***
### describeView()
```ts
abstract describeView(name, namespacePath?): Promise<ViewDescription>
```
What this database records about the view named `name`: its defining
query and the schema that query resolved to.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### display()
```ts
@@ -373,6 +424,54 @@ Drop all tables in the database.
***
### dropMaterializedView()
```ts
abstract dropMaterializedView(name, namespacePath?): Promise<void>
```
Drop the materialized view named `name`.
The view may become unavailable before physical cleanup finishes. Use
[dropMaterializedViewAsync](Connection.md#dropmaterializedviewasync) to retain and wait for the cleanup job.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropMaterializedViewAsync()
```ts
abstract dropMaterializedViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the materialized view named `name` and return its cleanup
job without waiting for completion.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### dropNamespace()
```ts
@@ -448,6 +547,53 @@ on the returned job to know when cleanup has finished.
***
### dropView()
```ts
abstract dropView(name, namespacePath?): Promise<void>
```
Drop the view named `name` and wait for its definition to be deleted.
The tables it reads are untouched: a view holds no rows of its own. Use
[dropViewAsync](Connection.md#dropviewasync) to retain the cleanup job instead of waiting on it.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### dropViewAsync()
```ts
abstract dropViewAsync(name, namespacePath?): Promise<Job>
```
Start dropping the view named `name` and return the job deleting its
definition, without waiting for completion.
The name is free before this resolves. When nothing was bound to it, the
returned job is already finished and has no id.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### isOpen()
```ts
@@ -465,30 +611,42 @@ Return true if the connection has not been closed
### listJobs()
```ts
abstract listJobs(): Promise<JobInfo[]>
abstract listJobs(options?): Listing<JobInfo>
```
List server-side jobs across the database's tables.
Iterate lazily over server-side jobs across the database's tables.
Use `for await...of`. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;[`JobInfo`](../interfaces/JobInfo.md)[]&gt;
[`Listing`](Listing.md)&lt;[`JobInfo`](../interfaces/JobInfo.md)&gt;
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
abstract listMaterializedViews(options?): Listing<string>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
Iterate lazily with `for await...of`. Local listings inspect table schemas
one page at a time. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
@@ -586,6 +744,30 @@ A page of table names and an
***
### listViews()
```ts
abstract listViews(namespacePath?, options?): Listing<string>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
Iterate lazily with `for await...of`. See [Listing](Listing.md) for pagination
state and error behavior; options control page size and the starting token.
#### Parameters
* **namespacePath?**: `string`[]
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
[`Listing`](Listing.md)&lt;`string`&gt;
***
### openJob()
```ts
@@ -653,6 +835,28 @@ abstract openTable(
***
### pauseJob()
```ts
abstract pauseJob(jobId): Promise<string>
```
Pause a server-side job by id.
The job's workers drain and it stays parked until resumed. Resolves to
"pausing", "already_paused", or "committing" -- a job finalizing its
results cannot be parked; retry shortly.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### renameTable()
```ts
@@ -686,6 +890,28 @@ a "not supported" error.
***
### resumeJob()
```ts
abstract resumeJob(jobId): Promise<string>
```
Resume a paused server-side job by id.
Its workers pick their work back up from checkpoints. Resolves to
"resumed", "still_pausing" -- the pause's worker drain is not confirmed
yet; retry shortly -- or "not_paused".
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`string`&gt;
***
### ~~tableNames()~~
#### tableNames(options)
docs/src/js/classes/DatabaseNames.md
+79
View File
@@ -0,0 +1,79 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DatabaseNames
# Class: DatabaseNames
A lazy async iterator of database names with pagination state.
Returned by [Catalog.listDatabases](Catalog.md#listdatabases).
## Implements
- `AsyncIterableIterator`&lt;`string`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<string>
```
#### Returns
`AsyncIterableIterator`&lt;`string`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<string, any>>
```
Fetch the next name, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`string`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of names available without another REST request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next REST request. Initially this is the supplied starting token.
Returns undefined after the final page is fetched, even if names remain cached.
Drain the cache before saving a token to avoid skipping names when resuming.
A failed request terminates iteration and retains its token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/Listing.md
+91
View File
@@ -0,0 +1,91 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Listing
# Class: Listing&lt;T&gt;
A lazy async iterator over database resources with pagination state.
Requests occur during iteration; request errors terminate the iterator.
## Example
```ts
for await (const name of connection.listViews([], { pageLimit: 20 })) {
console.log(name);
}
```
## Type Parameters
• **T**
## Implements
- `AsyncIterableIterator`&lt;`T`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<T>
```
#### Returns
`AsyncIterableIterator`&lt;`T`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<T, any>>
```
Fetch the next item, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`T`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of cached items available without another request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next request. Initially this is the supplied starting token.
Returns undefined after fetching the final page, even with cached items.
Drain the cache before saving a token to avoid skipping items on resumption.
Failed requests terminate iteration and retain their token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/MaterializedView.md
+4 -11
View File
@@ -49,7 +49,7 @@ get name(): string
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
The query that defines the view.
#### Returns
@@ -65,21 +65,14 @@ refresh(options?): Promise<RefreshMaterializedViewResult>
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
Every refresh rebuilds the complete view. For local views,
`sourceVersion` refreshes to that source version instead of the latest;
remote SQL refreshes do not support source-version pinning.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
docs/src/js/classes/OAuthSession.md
+105
View File
@@ -0,0 +1,105 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / OAuthSession
# Class: OAuthSession
Explicit OAuth session lifecycle for the persistent token cache: eager
`login`, non-secret `status`, and local `logout`.
A session is built from the same [OAuthConfig](../interfaces/OAuthConfig.md) used to connect
(including its `tokenCache` options). A connection created with the same
configuration shares the cache, so logging in here prepares tokens for
later processes without any database request.
`login` always runs the configured interactive flow and replaces the cached
session (the most recent login wins). `logout` removes only the local
credential; it does not revoke anything with the provider and does not sign
out of a browser SSO session.
## Example
```typescript
const config: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "my-app",
scopes: ["openid", "offline_access"],
flow: OAuthFlowType.DeviceCode,
tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" },
};
const session = new OAuthSession(config);
const status = await session.login();
```
## Constructors
### new OAuthSession()
```ts
new OAuthSession(config): OAuthSession
```
Create a session manager for the given OAuth configuration.
#### Parameters
* **config**: [`OAuthConfig`](../interfaces/OAuthConfig.md)
#### Returns
[`OAuthSession`](OAuthSession.md)
## Methods
### login()
```ts
login(): Promise<SessionStatus>
```
Eagerly run the configured authentication flow and store the session.
A successful login always replaces any prior cached session for this
identity; if the provider does not issue a refresh token (for example
without `offline_access`), the previous record is removed and the status
reports `refreshable == false`.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
***
### logout()
```ts
logout(): Promise<SessionLogout>
```
Remove the matching local cached credential.
This only deletes the local cache entry. It does not revoke the refresh
token with the provider and does not sign out of a browser SSO session.
Repeated calls succeed; `removed` reports whether a credential existed.
#### Returns
`Promise`&lt;[`SessionLogout`](../interfaces/SessionLogout.md)&gt;
***
### status()
```ts
status(): Promise<SessionStatus>
```
Report whether a matching cached session exists, with safe metadata.
This never contacts the identity provider and never exposes token values.
#### Returns
`Promise`&lt;[`SessionStatus`](../interfaces/SessionStatus.md)&gt;
docs/src/js/classes/Query.md
+3 -1
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -491,6 +491,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/classes/QueryBase.md
+3 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+132 -15
View File
@@ -74,10 +74,10 @@ now: the column is committed with no values, and rows get them from
[Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a
large table as on an empty one.
A refresh does not revisit rows it has already filled, so mutating an
input leaves the value computed at fill time; recomputing means dropping
the column and declaring it again. While a declaration reads a column,
that column cannot be renamed, retyped or dropped.
A refresh also recomputes the rows whose inputs changed since they were
computed, so a mutated input is reflected by the next refresh. While a
declaration reads a column, that column cannot be renamed, retyped or
dropped.
On LanceDB Cloud and Enterprise the expression is planned by the
server, and the refresh runs as a server job -- see
@@ -137,6 +137,21 @@ containing the new version number of the table after altering the columns.
***
### blobColumns()
```ts
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns addressable by row ID, including struct-nested dotted paths.
Blobs inside lists are omitted because a row ID does not identify an element.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### branches()
```ts
@@ -499,6 +514,74 @@ Drop an index from the table.
***
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
One missing or deleted row ID rejects the entire batch, including on remote
tables; no partial handles are returned.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
***
### fetchBlobs()
```ts
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
One missing or deleted row ID rejects the entire batch, including on remote
tables; no partial results are returned.
Blobs inside lists cannot be fetched by row ID.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
#### Parameters
* **column**: `string`
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
***
### flushLsm()
```ts
@@ -516,6 +599,41 @@ so this is safe to call repeatedly.
***
### functionErrors()
```ts
abstract functionErrors(options?): Promise<FunctionErrors>
```
The per-row errors Function refreshes recorded on this table.
A refresh running under a skip policy records each row it skipped with
the input that failed and the error. This lists those records, newest
job first, plus a summary for any fragment whose per-row detail was
capped. LanceDB Cloud and Enterprise only; reading errors needs read
access to the table, since a message carries the value that failed.
#### Parameters
* **options?**: [`FunctionErrorsOptions`](../interfaces/FunctionErrorsOptions.md)
Optional filters: `jobId`,
`column`, and `limit` (server default 10000, cap 100000).
#### Returns
`Promise`&lt;[`FunctionErrors`](../interfaces/FunctionErrors.md)&gt;
The records, the capped fragments,
and whether the listing stopped at its limit.
#### Example
```ts
const { records, truncated } = await table.functionErrors({ column: "embedding" });
```
***
### getLsmStats()
```ts
@@ -551,10 +669,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
@@ -854,10 +971,10 @@ abstract refreshColumn(column): Promise<RefreshColumnResult>
Fill the rows of a computed column that hold no value yet.
Rows appended since the last refresh are filled by the next one; rows
already filled are left as they are, so the call is idempotent and does
not observe a mutated input. Local tables only: a remote refresh runs
as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
Rows appended since the last refresh are filled by the next one, and
rows whose inputs changed since they were computed are recomputed;
everything else is left as it is. Local tables only: a remote refresh
runs as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync).
#### Parameters
@@ -996,8 +1113,8 @@ All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
@@ -1270,7 +1387,7 @@ repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+3 -1
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -267,6 +267,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
docs/src/js/classes/VectorQuery.md
+42 -7
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -458,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
For more fine grained control over behavior when you have a very narrow filter
you can use `minimumNprobes` and `maximumNprobes`. This method sets both
the minimum and maximum to the same value.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
#### Parameters
@@ -645,6 +646,38 @@ rerank(reranker): VectorQuery
***
### searchEffort()
```ts
searchEffort(searchEffort): VectorQuery
```
Set the initial IVF search effort to a finite value in [0, 1].
The default, null, uses Auto (internally 0.5). Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The MemWAL LSM
scanner does not yet support this option.
#### Parameters
* **searchEffort**: `null` \| `number`
#### Returns
[`VectorQuery`](VectorQuery.md)
#### Example
```ts
table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
```
***
### select()
```ts
@@ -673,7 +706,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -740,6 +773,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
docs/src/js/enumerations/ClientAuthMethod.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ClientAuthMethod
# Enumeration: ClientAuthMethod
How the client authenticates to the OAuth token endpoint.
The method applies to every OAuth request that carries client
authentication: client-credentials, authorization-code exchange,
refresh-token, and device-authorization requests. The Azure managed
identity flow ignores this option.
## Enumeration Members
### ClientSecretBasic
```ts
ClientSecretBasic: "client_secret_basic";
```
HTTP Basic authentication. This is the RFC 6749 recommended method and
the normal default for confidential clients, including default Okta
applications. Requires `clientSecret`.
***
### ClientSecretPost
```ts
ClientSecretPost: "client_secret_post";
```
Credentials in the request body, for providers configured to require it.
Requires `clientSecret`.
***
### None
```ts
None: "none";
```
No client authentication, for public clients using PKCE or the device
flow. Cannot be combined with `clientSecret`.
docs/src/js/enumerations/OAuthFlowType.md
+20
View File
@@ -10,6 +10,16 @@ OAuth authentication flow types.
## Enumeration Members
### AuthorizationCode
```ts
AuthorizationCode: "authorization_code";
```
Interactive Authorization Code grant, using PKCE by default.
***
### AzureManagedIdentity
```ts
@@ -27,3 +37,13 @@ ClientCredentials: "client_credentials";
```
Client Credentials grant (service-to-service / M2M).
***
### DeviceCode
```ts
DeviceCode: "device_code";
```
Device Authorization grant for CLI and headless environments.
docs/src/js/functions/blob.md
+57
View File
@@ -0,0 +1,57 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / blob
# Function: blob()
```ts
function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
Blobs inside lists can be written and queried as descriptors, but cannot
currently be fetched because a row ID does not identify a list element.
## Parameters
* **name**: `string`
* **options**: [`BlobOptions`](../type-aliases/BlobOptions.md) = `{}`
## Returns
`Field`
## Example
```ts
import { readFile } from "node:fs/promises";
import { Field, Int64, Schema } from "apache-arrow";
import { blob, connect } from "@lancedb/lancedb";
const db = await connect("./data");
const video = await readFile("clip.mp4");
const table = await db.createTable(
"videos",
[{ id: 1n, video }],
{
schema: new Schema([
new Field("id", new Int64()),
blob("video"),
]),
},
);
const rows = await table.query().select(["id"]).withRowId().toArray();
const rowIds = rows.map((row) => row._rowid as bigint);
const bytes = await table.fetchBlobs("video", rowIds);
const [handle] = await table.fetchBlobFiles("video", rowIds);
const size = handle!.size();
const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
```
docs/src/js/functions/connectCatalog.md
+34
View File
@@ -0,0 +1,34 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / connectCatalog
# Function: connectCatalog()
```ts
function connectCatalog(endpoint, options): Promise<Catalog>
```
Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
headers; opened database connections inherit authentication and client options.
## Parameters
* **endpoint**: `string`
* **options**: [`CatalogOptions`](../interfaces/CatalogOptions.md) = `{}`
## Returns
`Promise`&lt;[`Catalog`](../classes/Catalog.md)&gt;
## Example
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/functions/isBlobField.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / isBlobField
# Function: isBlobField()
```ts
function isBlobField(field): boolean
```
Checks for the `lance.blob.v2` extension marker. Does not validate the
field's storage type.
## Parameters
* **field**: `Field`&lt;`any`&gt;
## Returns
`boolean`
docs/src/js/functions/makeJsonField.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / makeJsonField
# Function: makeJsonField()
```ts
function makeJsonField(name, nullable): Field
```
Create an Arrow field backed by LanceDB's JSON extension type.
## Parameters
* **name**: `string`
The field name.
* **nullable**: `boolean` = `true`
Whether the field accepts null values.
## Returns
`Field`
## Example
```ts
import { connect, makeJsonField } from "@lancedb/lancedb";
import { Schema } from "apache-arrow";
const schema = new Schema([makeJsonField("metadata")]);
const db = await connect("/path/to/database");
await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema });
```
docs/src/js/globals.md
+27
View File
@@ -11,6 +11,7 @@
## Enumerations
- [ClientAuthMethod](enumerations/ClientAuthMethod.md)
- [FullTextQueryType](enumerations/FullTextQueryType.md)
- [OAuthFlowType](enumerations/OAuthFlowType.md)
- [Occur](enumerations/Occur.md)
@@ -19,14 +20,18 @@
## Classes
- [AutoQuery](classes/AutoQuery.md)
- [BlobFile](classes/BlobFile.md)
- [BooleanQuery](classes/BooleanQuery.md)
- [BoostQuery](classes/BoostQuery.md)
- [BranchContents](classes/BranchContents.md)
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [DatabaseNames](classes/DatabaseNames.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [Listing](classes/Listing.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
@@ -34,6 +39,7 @@
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
- [OAuthHeaderProvider](classes/OAuthHeaderProvider.md)
- [OAuthSession](classes/OAuthSession.md)
- [PermutationBuilder](classes/PermutationBuilder.md)
- [PhraseQuery](classes/PhraseQuery.md)
- [Query](classes/Query.md)
@@ -61,6 +67,7 @@
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
@@ -87,6 +94,10 @@
- [FtsToken](interfaces/FtsToken.md)
- [FullTextQuery](interfaces/FullTextQuery.md)
- [FullTextSearchOptions](interfaces/FullTextSearchOptions.md)
- [FunctionErrorFragment](interfaces/FunctionErrorFragment.md)
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
- [GenerationStats](interfaces/GenerationStats.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
@@ -103,6 +114,7 @@
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
@@ -121,6 +133,8 @@
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
- [RetryConfig](interfaces/RetryConfig.md)
- [ScannableOptions](interfaces/ScannableOptions.md)
- [SessionLogout](interfaces/SessionLogout.md)
- [SessionStatus](interfaces/SessionStatus.md)
- [ShuffleOptions](interfaces/ShuffleOptions.md)
- [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md)
- [SplitHashOptions](interfaces/SplitHashOptions.md)
@@ -130,12 +144,14 @@
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
- [TokenCacheOptions](interfaces/TokenCacheOptions.md)
- [TokenResponse](interfaces/TokenResponse.md)
- [TokenizeOptions](interfaces/TokenizeOptions.md)
- [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md)
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)
- [ViewDescription](interfaces/ViewDescription.md)
- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
- [WriteProgress](interfaces/WriteProgress.md)
@@ -143,8 +159,15 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [BlobUri](type-aliases/BlobUri.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
@@ -158,10 +181,14 @@
## Functions
- [RecordBatchIterator](functions/RecordBatchIterator.md)
- [blob](functions/blob.md)
- [connect](functions/connect.md)
- [connectCatalog](functions/connectCatalog.md)
- [connectNamespace](functions/connectNamespace.md)
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
- [isBlobField](functions/isBlobField.md)
- [makeArrowTable](functions/makeArrowTable.md)
- [makeJsonField](functions/makeJsonField.md)
- [packBits](functions/packBits.md)
- [permutationBuilder](functions/permutationBuilder.md)
- [tokenize](functions/tokenize.md)
docs/src/js/interfaces/AddDataOptions.md
+16
View File
@@ -10,6 +10,22 @@ Options for adding data to a table.
## Properties
### allowExternalBlobOutsideBases?
```ts
optional allowExternalBlobOutsideBases: boolean;
```
Whether blob URIs outside registered external bases may be written.
Defaults to `false`. This option is supported only for local/native
tables; remote tables return an error when it is enabled. An enabled write
stores the absolute URI reference without registering a base or copying
the external object into the database. The object must remain accessible
when the blob is read later.
***
### mode
```ts
docs/src/js/interfaces/CatalogOptions.md
+81
View File
@@ -0,0 +1,81 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CatalogOptions
# Interface: CatalogOptions
Options shared by a catalog and the database connections it returns.
## Extends
- `Omit`&lt;`NativeCatalogOptions`, `"oauthConfig"`&gt;
## Properties
### apiKey?
```ts
optional apiKey: string;
```
#### Inherited from
`Omit.apiKey`
***
### clientConfig?
```ts
optional clientConfig: ClientConfig;
```
#### Inherited from
`Omit.clientConfig`
***
### headerProvider?
```ts
optional headerProvider: HeaderProvider | () => Record<string, string> | Promise<Record<string, string>>;
```
Called for each request to supply authentication headers.
***
### oauthConfig?
```ts
optional oauthConfig: OAuthConfig;
```
***
### readConsistencyInterval?
```ts
optional readConsistencyInterval: number;
```
#### Inherited from
`Omit.readConsistencyInterval`
***
### sqlHostOverride?
```ts
optional sqlHostOverride: string;
```
SQL service endpoint inherited by database connections.
#### Inherited from
`Omit.sqlHostOverride`
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/ClientConfig.md
+16
View File
@@ -8,6 +8,18 @@
## Properties
### blobRequestConcurrency?
```ts
optional blobRequestConcurrency: number;
```
Maximum number of concurrent blob HTTP requests across all tables and
blob handles on this connection. Defaults to 8. Can also be set with
`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
***
### extraHeaders?
```ts
@@ -22,6 +34,10 @@ optional extraHeaders: Record<string, string>;
optional idDelimiter: string;
```
The delimiter joining a namespace path and a name into one object
identifier. `"$"` is the only supported value, and leaving this unset is
how to get it; anything else is rejected when the connection is created.
***
### retryConfig?
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/interfaces/FtsOptions.md
+2
View File
@@ -37,6 +37,8 @@ The following tokenizers are available:
"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
"icu" - ICU dictionary-based word segmentation.
"icu/split" - ICU segmentation with simple-style delimiter splitting.
docs/src/js/interfaces/FunctionErrorFragment.md
+42
View File
@@ -0,0 +1,42 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorFragment
# Interface: FunctionErrorFragment
A fragment whose per-row error detail was capped: `rowsSkipped` rows
failed, of which only `rowsRecorded` have a record of their own.
## Properties
### fragmentId
```ts
fragmentId: number;
```
***
### jobId
```ts
jobId: string;
```
***
### rowsRecorded
```ts
rowsRecorded: number;
```
***
### rowsSkipped
```ts
rowsSkipped: number;
```
docs/src/js/interfaces/FunctionErrorRecord.md
+92
View File
@@ -0,0 +1,92 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorRecord
# Interface: FunctionErrorRecord
One row a Function refresh skipped, as the server recorded it.
## Properties
### column
```ts
column: string;
```
***
### createdAtMillis
```ts
createdAtMillis: number;
```
***
### errorMessage
```ts
errorMessage: string;
```
***
### errorType
```ts
errorType: string;
```
***
### fragmentId
```ts
fragmentId: number;
```
***
### function
```ts
function: string;
```
***
### functionVersion
```ts
functionVersion: string;
```
***
### jobId
```ts
jobId: string;
```
***
### rowOffset?
```ts
optional rowOffset: number;
```
The row's offset within the fragment; absent when the fragment's
detail was capped.
***
### tableVersion
```ts
tableVersion: number;
```
docs/src/js/interfaces/FunctionErrors.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrors
# Interface: FunctionErrors
A table's per-row Function errors.
## Properties
### fragments
```ts
fragments: FunctionErrorFragment[];
```
Fragments whose detail was capped.
***
### records
```ts
records: FunctionErrorRecord[];
```
The recorded rows, newest job first.
***
### truncated
```ts
truncated: boolean;
```
Whether the listing stopped at its limit.
docs/src/js/interfaces/FunctionErrorsOptions.md
+39
View File
@@ -0,0 +1,39 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / FunctionErrorsOptions
# Interface: FunctionErrorsOptions
Which per-row Function errors to list; every filter is optional.
## Properties
### column?
```ts
optional column: string;
```
Only errors on this column.
***
### jobId?
```ts
optional jobId: string;
```
Only errors recorded by this job.
***
### limit?
```ts
optional limit: number;
```
At most this many records (server default 10000, cap 100000).
docs/src/js/interfaces/ListTablesOptions.md
+1
View File
@@ -15,6 +15,7 @@ optional limit: number;
```
An upper bound on how many tables to return.
Defaults to 100. Zero returns an empty page without a continuation token.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
docs/src/js/interfaces/ListingOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListingOptions
# Interface: ListingOptions
Pagination controls for lazy resource listings.
## Properties
### pageLimit?
```ts
optional pageLimit: number;
```
Maximum results per request, not a limit on the whole listing.
***
### pageToken?
```ts
optional pageToken: string;
```
Starting continuation token; omitted starts at the beginning.
docs/src/js/interfaces/LsmWriteSpec.md
+2 -3
View File
@@ -34,9 +34,8 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
Indexes the MemWAL keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/interfaces/MaterializedViewDefinition.md
+7 -54
View File
@@ -6,64 +6,17 @@
# Interface: MaterializedViewDefinition
The query that defines a materialized view.
The query that defines a materialized view, as stored:
`SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
position yields one row per element it returns.
## Properties
### filter?
### query
```ts
optional filter: string;
query: string;
```
SQL predicate selecting the source rows the view holds.
***
### inputs
```ts
inputs: string[];
```
Source columns the projections and filter read.
***
### limit?
```ts
optional limit: number;
```
Cap on the number of rows the view holds.
***
### projections
```ts
projections: [string, string][];
```
`[output column, SQL expression]` pairs, in view schema order.
***
### sourceNamespace
```ts
sourceNamespace: string[];
```
Namespace holding the source table; empty is the root namespace.
***
### sourceTable
```ts
sourceTable: string;
```
Name of the source table, in the same database as the view.
The defining query, in the canonical spelling the server stores.
docs/src/js/interfaces/NativeOAuthConfig.md
+76 -1
View File
@@ -15,6 +15,39 @@ All token acquisition and refresh is handled in the Rust layer.
## Properties
### audience?
```ts
optional audience: string;
```
Optional provider-specific audience for authorization and token requests.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the authorization_code loopback callback server.
***
### clientAuthMethod?
```ts
optional clientAuthMethod: string;
```
How the client authenticates to the token endpoint: "none",
"client_secret_basic", or "client_secret_post". Defaults to
"client_secret_basic" when a client secret is set, and "none" for
public clients.
***
### clientId
```ts
@@ -41,7 +74,8 @@ Client secret (required for client_credentials).
optional flow: string;
```
Authentication flow: "client_credentials" or "azure_managed_identity"
Authentication flow: "client_credentials", "authorization_code",
"device_code", or "azure_managed_identity"
***
@@ -66,6 +100,16 @@ Client ID for user-assigned managed identity (azure_managed_identity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for authorization_code.
***
### refreshBufferSecs?
```ts
@@ -78,6 +122,16 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Optional resource indicator for authorization and token requests.
***
### scopes
```ts
@@ -86,3 +140,24 @@ scopes: string[];
OAuth scopes to request. For Azure managed identity, exactly one scope
or resource is required. For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse
one session. Only refresh tokens are persisted.
***
### usePkce?
```ts
optional usePkce: boolean;
```
Whether authorization_code uses S256 PKCE (default: true).
docs/src/js/interfaces/OAuthConfig.md
+106
View File
@@ -26,6 +26,18 @@ const config: OAuthConfig = {
};
```
Providers requiring an explicit target can set `resource` and/or `audience`:
```typescript
const targeted: OAuthConfig = {
issuerUrl: "https://issuer.example.com",
clientId: "app-id",
clientSecret: "secret",
scopes: ["read"],
resource: "https://api.example.com",
audience: "lancedb-api",
};
```
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
@@ -35,8 +47,57 @@ const config: OAuthConfig = {
};
```
The authorization URL is written to stderr before LanceDB tries to open a
browser, so it can be copied in headless environments.
```typescript
const config: OAuthConfig = {
issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0",
clientId: "app-id",
scopes: ["openid", "api://lancedb-api/access"],
flow: OAuthFlowType.AuthorizationCode,
};
```
Device Authorization writes the verification URL and user code to stderr
before polling begins.
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience, forwarded to authorization and token endpoints,
including refresh requests. Not supported for Azure managed identity.
***
### callbackPort?
```ts
optional callbackPort: number;
```
Port for the AuthorizationCode loopback callback server (default: 8400).
***
### clientAuthMethod?
```ts
optional clientAuthMethod: ClientAuthMethod;
```
How the client authenticates to the token endpoint (default: auto).
With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`,
which matches the RFC 6749 recommendation and the default configuration
of Okta confidential applications; without a secret the client is public
and no client authentication is sent.
***
### clientId
```ts
@@ -88,6 +149,16 @@ Client ID for user-assigned managed identity (AzureManagedIdentity).
***
### redirectUri?
```ts
optional redirectUri: string;
```
Loopback redirect URI for AuthorizationCode.
***
### refreshBufferSecs?
```ts
@@ -100,6 +171,18 @@ the TTL, each request refreshes the token.
***
### resource?
```ts
optional resource: string;
```
Resource indicator (RFC 8707), forwarded verbatim to authorization and token
endpoints, including refresh requests. Must be an absolute URI without a
fragment. Not supported for Azure managed identity.
***
### scopes
```ts
@@ -109,3 +192,26 @@ scopes: string[];
OAuth scopes to request.
For Azure managed identity, exactly one scope or resource is required.
For example: `["api://{app_id}/.default"]`
***
### tokenCache?
```ts
optional tokenCache: TokenCacheOptions;
```
Opt in to the persistent token cache so short-lived processes reuse one
session. Only refresh tokens are persisted. Only supported by
AuthorizationCode and DeviceCode; Azure managed identity is rejected.
Default: unset (memory only).
***
### usePkce?
```ts
optional usePkce: boolean;
```
Protect AuthorizationCode with S256 PKCE (default: true).
docs/src/js/interfaces/OptimizeOptions.md
+2 -1
View File
@@ -26,7 +26,8 @@ const olderThan = new Date();
olderThan.setDate(olderThan.getDate() - 1));
tbl.optimize({cleanupOlderThan: olderThan});
// Delete all versions except the current version
// Delete versions committed before this point. Versions created by the
// optimize call itself are newer than the cutoff and will be retained.
tbl.optimize({cleanupOlderThan: new Date()});
```
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+1 -1
View File
@@ -14,7 +14,7 @@
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
How the view was brought up to date: "rebuild".
***
docs/src/js/interfaces/SessionLogout.md
+20
View File
@@ -0,0 +1,20 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionLogout
# Interface: SessionLogout
Result of [OAuthSession.logout](../classes/OAuthSession.md#logout).
## Properties
### removed
```ts
removed: boolean;
```
Whether a cached credential was removed. `false` means no matching
session was cached; logout is idempotent.
docs/src/js/interfaces/SessionStatus.md
+94
View File
@@ -0,0 +1,94 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SessionStatus
# Interface: SessionStatus
Safe, non-secret view of a cached OAuth session, returned by
[OAuthSession.status](../classes/OAuthSession.md#status) and [OAuthSession.login](../classes/OAuthSession.md#login).
## Properties
### audience?
```ts
optional audience: string;
```
Provider-specific audience used to obtain the cached session, if configured.
***
### clientId
```ts
clientId: string;
```
Client ID of the cached session.
***
### flow
```ts
flow: string;
```
Flow that produced the cached session.
***
### issuerUrl
```ts
issuerUrl: string;
```
Canonical issuer URL of the cached session.
***
### obtainedAt?
```ts
optional obtainedAt: number;
```
When the cached session was obtained, as Unix seconds.
***
### refreshable
```ts
refreshable: boolean;
```
Whether a cached session exists that can obtain tokens without
interactive authentication. Because access tokens are not persisted,
this is `true` exactly when a refresh token is cached; the next
connection refreshes with it rather than opening a browser or device
prompt.
***
### resource?
```ts
optional resource: string;
```
Resource indicator used to obtain the cached session, if configured.
***
### scopes
```ts
scopes: string[];
```
Canonical (sorted, de-duplicated) scope set of the cached session.
docs/src/js/interfaces/TableNamesOptions.md
+1 -1
View File
@@ -19,7 +19,7 @@ instead.
optional limit: number;
```
An optional limit to the number of results to return.
The maximum number of names to return. Omitted returns all names; zero returns none.
***
docs/src/js/interfaces/TableStatistics.md
+17
View File
@@ -18,6 +18,23 @@ Statistics on table fragments
***
### numDeletedRows?
```ts
optional numDeletedRows: number;
```
The number of rows marked as deleted across all fragments of the table
These rows are not included in `numRows`, but still occupy space on disk
until the table is compacted, so a large value here indicates that the
table should be optimized. Fragments in which every row was deleted are
dropped outright, so their rows are not counted here.
Absent (`undefined`) when the backend does not report deletion counts.
***
### numIndices
```ts
docs/src/js/interfaces/TokenCacheOptions.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / TokenCacheOptions
# Interface: TokenCacheOptions
Options for the persistent OAuth token cache.
The cache is opt-in: it is only used when set as `tokenCache` on
[OAuthConfig](OAuthConfig.md). Only refresh tokens are persisted, in a private
directory with owner-only permissions, so short-lived processes can reuse
an authenticated session instead of re-prompting on every start.
Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication)
get separate cache entries. Within one identity the most recent login wins.
## Properties
### cacheDir?
```ts
optional cacheDir: string;
```
Directory that holds cached credentials. Defaults to
`$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix,
or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created
with owner-only permissions (`0700`) when missing.
***
### lockTimeoutSecs?
```ts
optional lockTimeoutSecs: number;
```
How long to wait for the cross-process refresh lock before failing, in
seconds (default: 30).
docs/src/js/interfaces/ViewDescription.md
+78
View File
@@ -0,0 +1,78 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ViewDescription
# Interface: ViewDescription
What a database records about one view.
A view holds no rows: it stores the statement that defines it and the
schema that statement resolved to, so a reader sees its sources as they
are at read time.
## Properties
### defaultDatabase
```ts
defaultDatabase: string;
```
The database that unqualified table names in [query](ViewDescription.md#query) resolve against.
***
### defaultNamespacePath
```ts
defaultNamespacePath: string[];
```
The namespace path those unqualified names resolve against; empty is the
root namespace.
Recorded with the view because it outlives the session that declared it:
a reader resolving the query against its own default namespace could read
a different table than the view was defined over.
***
### name
```ts
name: string;
```
The view's name within its namespace.
***
### namespacePath
```ts
namespacePath: string[];
```
The namespace holding the view; empty is the root namespace.
***
### query
```ts
query: string;
```
The defining query, as the database stores it.
***
### schema
```ts
schema: Schema<any>;
```
The schema the defining query resolved to when the view was created.
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BaseTokenizer.md
+1
View File
@@ -12,6 +12,7 @@ type BaseTokenizer:
| "whitespace"
| "raw"
| "ngram"
| "code"
| "icu"
| "icu/split"
| `jieba/${string}`
docs/src/js/type-aliases/BlobData.md
+16
View File
@@ -0,0 +1,16 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobData
# Type Alias: BlobData
```ts
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
```
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
docs/src/js/type-aliases/BlobInput.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobInput
# Type Alias: BlobInput
```ts
type BlobInput:
| BlobData
| BlobUri
| object
| object
| null
| undefined;
```
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
## Example
```ts
import { pathToFileURL } from "node:url";
import type { BlobInput } from "@lancedb/lancedb";
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
{ id: 2n, image: pathToFileURL("/data/cat.png") },
{ id: 3n, image: { data: new Blob(["hello"]) } },
];
await table.add(rows);
```
docs/src/js/type-aliases/BlobOptions.md
+48
View File
@@ -0,0 +1,48 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobOptions
# Type Alias: BlobOptions
```ts
type BlobOptions: object;
```
## Type declaration
### dedicatedSizeThreshold?
```ts
optional dedicatedSizeThreshold: number;
```
Max payload bytes stored in a packed sidecar before a dedicated file. Must
be a positive safe integer.
### inlineSizeThreshold?
```ts
optional inlineSizeThreshold: number;
```
Max payload bytes kept inline in the data file. Zero is allowed. Must be a
safe integer.
### nullable?
```ts
optional nullable: boolean;
```
Defaults to true.
### packFileSizeThreshold?
```ts
optional packFileSizeThreshold: number;
```
Max bytes in one packed sidecar before starting another. Must be a positive
safe integer.
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/BlobUri.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobUri
# Type Alias: BlobUri
```ts
type BlobUri: string | URL;
```
A URI accepted for a blob value. A `URL` is stored as its `href`.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
docs/src/python/python.md
+249
View File
@@ -26,8 +26,191 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.db.DBConnection
Resource listings (`list_secrets`, `list_views`, `list_jobs`, `list_functions`, and
`list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to
collect synchronous results, or `[name async for name in db.list_views()]` with
an asynchronous connection. `page_limit` controls each request, and `page_token`
resumes a listing. Drain the iterator's cached results before saving its token.
::: lancedb.listing.Listing
::: lancedb.Session
## Catalogs (Synchronous)
Remote catalogs manage databases through a server's root namespace. Opened databases
are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.connect_catalog
::: lancedb.catalog.Catalog
::: lancedb.catalog.DatabaseNames
## Authorization
Some catalogs support the authorization API. This is an API that can be used to
bind access control lists (ACLs) to individual catalog objects.
Each ACL contains a subject, an object, and privilege.
### Subjects
There are three kinds of subjects: principals, groups, and roles.
A principal uniquely identifies each user accessing the system. Each LanceDB
Enterprise API key corresponds to a unique principal, and each OpenID user has
a unique principal.
All principals have an immutable ID which never changes, and is safe to include
in log files. For example, the API key
`sk_AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPQ====` maps to the
principal ID `pk_aaaqeayeaudaocajbifqydiob4`. APIs such as `list_acls` will
return the principal ID, never the API key.
OpenID principals also typically have a human-readable name associated with
them. For example, a principal might be named `bob@example.com` and might have
a principal ID of `8a004400-9443-4fab-9997-b9372a7ca8fd`. Whenever
possible, APIs will return the principal name for convenience. However,
principal names are mutable and can be changed by the identity provider.
Here is an example of getting information about an API key:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
principal = catalog.authz.get_principal(Subject.principal_api_key(api_key))
print(f"Found principal {principal.principal_id}")
```
OpenID principals can be members of groups. Like principals, groups also have
IDs and human-readable names. It is not possible to modify group membership via
this API; it is managed by the OpenID identity provider.
Here is an example of getting information about a group based on its name:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
group = catalog.authz.get_group(Subject.group_name("Engineering"))
print(f"Found group {group.group_id} with name {group.group_name}")
```
Roles can contain either principals or groups. Unlike groups, they are managed
by the catalog, not the identity provider. Roles can also contain API key
principals.
Here is an example of managing the `reader` role:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
catalog.authz.add_role_binding(subject=Subject.principal_name("Alice"), role="reader")
```
It may take a few milliseconds for changes to role bindings to become visible
to all clients.
More subject types may be added in the future.
### Objects
An object is any resource in the system that we can attach authorization rules to.
The Python builders and Rust client share resource validation and wire encoding.
Methods also accept literal strings such as `table:tenant/db:public$events`;
malformed names for known resource types raise `ValueError` locally before a
request is sent. Unknown resource types are preserved for compatibility with
newer servers.
Object types include:
The System object, a singleton which represents the entire system. You can
refer to it as `Object.system()`.
Database objects which represents specific databases. For example,
`Object.database("analytics")` represents the "analytics" database.
Table objects which represent specific tables. For example,
`Object.table(database="analytics", name="events")` represents the "events"
table inside the "analytics" database. Tables are in the default namespace (which
is named "public") unless a specific namespace is given.
Similarly, View objects represent specific views. For example,
`Object.view(database="analytics", name="recent_events")` represents the
"recent_events" view inside the "analytics" database. Views are in the default
namespace (which is named "public") unless a specific namespace is given.
Secret objects identify named credentials, for example,
`Object.secret(database="analytics", name="api-key")`. Function objects identify
registered functions, for example,
`Object.function(database="analytics", name="caption")`. Both default to the
`public` namespace. Pass `namespace_path=["platform", "ml"]` to select a nested
namespace. These objects identify resources for ACL operations; a secret object
contains the credential's name, not its value.
More object types may be added in the future.
### Access control lists
An access control list attaches a specific set of privileges to a specific
(subject, object) pair. So, for example, we might have an access control list
that gives `bob@example.com` the ability to read the foo table, that looks like
this:
```python
from lancedb.authz import Object, Subject, Privilege
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
catalog.authz.add_acl(
object=Object.table(database="analytics", name="foo"),
subject=Subject.principal_name("bob@example.com"),
privilege=Privilege.SELECT,
)
```
Reading a table generally requires USAGE on its database and namespace, plus
SELECT on the table. Each object has at most one owner.
OWNERSHIP is a special privilege which grants all the other privileges. By
default, the principal who created an object owns it. Only one subject can own
an object; granting OWNERSHIP to a new subject removes it from the previous
owner. OWNERSHIP cannot be deleted.
To delegate secret or function creation, grant `CREATE_SECRET` or
`CREATE_FUNCTION` on the containing namespace. This preserves namespace ownership:
```python
for privilege in [Privilege.CREATE_SECRET, Privilege.CREATE_FUNCTION]:
catalog.authz.add_acl(
object=Object.namespace(database="analytics", namespace_path=["public"]),
subject=Subject.role("developer"),
privilege=privilege,
)
```
Both privileges also accept their string names in synchronous and asynchronous
ACL methods.
By default, the `list_acls` function lists all the access control entries in
the system. By adding a function parameter, the output is filtered to just ACLs
matching that parameter. For example, this code lists all the ACLs for the
"reader" role:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
print(entry)
```
Or, when using the async API:
```python
catalog = await lancedb.connect_catalog_async(endpoint, api_key=api_key)
async for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
print(entry)
```
## Remote SQL
Submit SQL against a remote LanceDB database through the connection.
@@ -66,6 +249,15 @@ print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# Values bind to $1 / $name placeholders and travel as Arrow, so a float32
# stays a float32 and a vector stays a compact fixed-size list:
import numpy as np
reader = db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $vector) LIMIT $k",
parameters={"vector": np.random.rand(768).astype(np.float32), "k": 10},
)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
@@ -125,6 +317,10 @@ listing a storage directory.
::: lancedb.functions.UdfDefinition
::: lancedb.secrets.EnvVarSecret
::: lancedb.secrets.SecretInfo
::: lancedb.functions.FunctionRegistrationRequest
::: lancedb.functions.FunctionArtifactRequest
@@ -133,6 +329,8 @@ listing a storage directory.
::: lancedb.functions.PythonAdapterSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersion
::: lancedb.functions.PythonRuntimeSpec
@@ -153,6 +351,12 @@ listing a storage directory.
::: lancedb.functions.RefreshColumnResult
::: lancedb.FunctionErrors
::: lancedb.FunctionErrorRecord
::: lancedb.FunctionErrorFragment
::: lancedb.job.Job
::: lancedb.job.AsyncJob
@@ -169,12 +373,18 @@ listing a storage directory.
::: lancedb.sql.QueryDescription
::: lancedb.sql.QueryParameters
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
## Views
::: lancedb.view.ViewDescription
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -295,6 +505,12 @@ pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
still work. Queries return descriptors. Call
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
One missing or deleted row ID rejects the entire blob-read batch; the APIs do
not return partial results.
Blobs inside lists can be written and queried as descriptors. They are omitted
from `blob_columns()` because row ids do not identify individual list elements,
so the fetch APIs cannot read their bytes yet.
::: lancedb.blob
@@ -312,6 +528,8 @@ still work. Queries return descriptors. Call
## Exceptions
::: lancedb.remote.errors.HttpError
::: lancedb.exceptions.MissingValueError
::: lancedb.exceptions.MissingColumnError
@@ -352,6 +570,29 @@ still work. Queries return descriptors. Call
## Reranking
`TypeSafeReranker` supports opt-in request batching:
```python
from lancedb.rerankers import TypeSafeReranker
reranker = TypeSafeReranker(batch_size=40, max_concurrency=8)
```
The default `batch_size=1` keeps the query and document in request state and
sends one request per non-null candidate. With `batch_size=40`, 80 non-null
candidates require two requests. `max_concurrency` still limits simultaneous
requests, and the SDK handles retries.
In batched mode, state contains only `{"query": query}`. Each independent
question contains `{"question": instructions, "document": document}` in its
structured instructions, so it sees only its own document and the shared query.
Custom instructions and criteria are kept verbatim: adapt prompts that explicitly
reference request-state fields such as `state.document` before enabling batching.
Null documents retain a zero score without an API call; empty strings are scored.
API failures, mismatched answer IDs, and invalid probabilities raise errors.
Batching changes the payload and can affect model scores; compare quality and
latency on your workload before opting in.
::: lancedb.rerankers
options:
show_root_heading: false
@@ -359,6 +600,12 @@ still work. Queries return descriptors. Call
## Connections (Asynchronous)
::: lancedb.connect_catalog_async
::: lancedb.catalog.AsyncCatalog
::: lancedb.catalog.AsyncDatabaseNames
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
@@ -366,6 +613,8 @@ can be used to create, list, or open tables.
::: lancedb.db.AsyncConnection
::: lancedb.listing.AsyncListing
## Namespaces (Asynchronous)
::: lancedb.connect_namespace_async
docs/web-tests/camelot.json
+66
View File
@@ -0,0 +1,66 @@
[
{
"id": 1,
"name": "King Arthur",
"role": "King of Camelot",
"description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.",
"vector": [0.72, -0.28, 0.60, 0.86],
"stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 2,
"name": "Merlin",
"role": "Wizard and Advisor",
"description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.",
"vector": [0.05, 0.88, 0.62, 0.85],
"stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 }
},
{
"id": 3,
"name": "Queen Guinevere",
"role": "Queen of Camelot",
"description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.",
"vector": [0.22, -0.22, 0.42, 0.82],
"stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 }
},
{
"id": 4,
"name": "Sir Lancelot",
"role": "Knight of the Round Table",
"description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.",
"vector": [0.86, -0.35, 0.38, 0.55],
"stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 }
},
{
"id": 5,
"name": "Sir Gawain",
"role": "Knight of the Round Table",
"description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.",
"vector": [0.82, -0.32, 0.52, 0.60],
"stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 6,
"name": "Sir Galahad",
"role": "Knight of the Round Table",
"description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.",
"vector": [0.80, -0.20, 0.70, 0.78],
"stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 }
},
{
"id": 7,
"name": "Sir Percival",
"role": "Knight of the Round Table",
"description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.",
"vector": [0.78, -0.36, 0.48, 0.52],
"stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 }
},
{
"id": 8,
"name": "Mordred",
"role": "Traitor Knight",
"description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.",
"vector": [0.68, -0.30, -0.65, 0.20],
"stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 }
}
]
docs/web-tests/mdx_snippets_gen.py
+362
View File
@@ -0,0 +1,362 @@
"""
Generate MDX-ready snippet modules that export raw string constants.
The generator groups snippets by the originating test file name rather than by
language. For a source file like `tests/py/test_basic_usage.py`, the snippets are
written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with
a language identifier (Py|Ts|Rs) followed by the snippet name converted to
TitleCase so that docs authors can selectively assemble `<CodeGroup>` blocks.
"""
from __future__ import annotations
import argparse
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple
# Supported languages and relevant metadata.
LANG_ORDER = ("py", "ts", "rs")
LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"}
LANG_EXTENSIONS = {
".py": "py",
".ts": "ts",
".tsx": "ts",
".rs": "rs",
}
LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]}
# Match each export block; we rely on the blank line we emit after each export to
# avoid accidentally truncating on semicolons inside the JSON string literal.
EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL)
DEFAULT_SOURCE_DIRS = (
Path("tests") / "py",
Path("tests") / "ts",
Path("tests") / "rs",
)
@dataclass(frozen=True)
class SnippetRecord:
lang: str
snippet_name: str
export_name: str
text: str
source_rel: str
def build_markers(
comment_markers: Iterable[str],
) -> Tuple[re.Pattern[str], re.Pattern[str]]:
escaped = "|".join(re.escape(c) for c in comment_markers)
start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]")
end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]")
return start_re, end_re
def parse_snippets(
lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str]
) -> Dict[str, List[str]]:
snippets: Dict[str, List[str]] = {}
stack: List[Tuple[str, int]] = []
for idx, line in enumerate(lines):
m_start = start_re.match(line)
if m_start:
name = m_start.group(1).strip()
stack.append((name, idx))
continue
m_end = end_re.match(line)
if m_end:
name = m_end.group(1).strip()
if not stack:
raise ValueError(
f"End marker for '{name}' found at line {idx + 1} without matching start"
)
start_name, start_idx = stack.pop()
if start_name != name:
raise ValueError(
f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, "
f"end '{name}' at line {idx + 1}"
)
snippets[name] = lines[start_idx + 1 : idx]
continue
if stack:
open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack)
raise ValueError(f"Unclosed snippet markers for: {open_names}")
return snippets
def dedent(lines: List[str]) -> List[str]:
min_indent = None
for ln in lines:
if ln.strip() == "":
continue
expanded = ln.expandtabs(4)
indent = len(expanded) - len(expanded.lstrip(" "))
if min_indent is None or indent < min_indent:
min_indent = indent
if not lines:
return []
if min_indent in (None, 0):
return [ln.rstrip("\n") for ln in lines]
dedented: List[str] = []
for ln in lines:
expanded = ln.expandtabs(4)
if expanded.strip() == "":
dedented.append(expanded.rstrip("\n"))
else:
dedented.append(expanded[min_indent:].rstrip("\n"))
return dedented
def ensure_dir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def detect_language(file_path: Path) -> str:
lang = LANG_EXTENSIONS.get(file_path.suffix.lower())
if not lang:
raise ValueError(f"Unsupported file extension for {file_path}")
return lang
def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]:
markers = LANG_MARKERS.get(lang)
if not markers:
raise ValueError(f"Unsupported language for markers: {lang}")
return build_markers(markers)
def normalize_target_name(file_path: Path) -> str:
stem = file_path.stem
if stem.startswith("test_"):
stem = stem[len("test_") :]
if stem.endswith("_test"):
stem = stem[: -len("_test")]
if stem.endswith(".test"):
stem = stem[: -len(".test")]
if stem.endswith(".spec"):
stem = stem[: -len(".spec")]
if stem == "":
stem = file_path.stem
return stem.replace(" ", "_")
def to_title_case(name: str) -> str:
parts = re.split(r"[^0-9a-zA-Z]+", name)
filtered = [p for p in parts if p]
return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet"
def format_export_name(lang: str, snippet_name: str) -> str:
prefix = LANG_PREFIX[lang]
return f"{prefix}{to_title_case(snippet_name)}"
def snippet_text(lines: List[str]) -> str:
if not lines:
return ""
joined = "\n".join(lines).rstrip("\n")
return (joined + "\n") if joined else ""
def to_js_literal(text: str) -> str:
return json.dumps(text, ensure_ascii=False)
def split_export_prefix(export_name: str) -> Tuple[str | None, str]:
for lang, prefix in LANG_PREFIX.items():
if export_name.startswith(prefix):
suffix = export_name[len(prefix) :]
return lang, suffix or export_name
return None, export_name
def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]:
try:
text = path.read_text(encoding="utf-8")
except FileNotFoundError:
return {}
existing: Dict[str, Dict[str, SnippetRecord]] = {}
for match in EXPORT_RE.finditer(text):
export_name, literal = match.group(1), match.group(2).strip()
lang, snippet_suffix = split_export_prefix(export_name)
if not lang:
continue
try:
decoded = json.loads(literal)
except Exception:
decoded = literal.strip().strip(";")
record = SnippetRecord(
lang=lang,
snippet_name=snippet_suffix or export_name,
export_name=export_name,
text=decoded,
source_rel="(existing module)",
)
existing.setdefault(lang, {})[record.snippet_name] = record
return existing
def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]:
for root in source_dirs:
for path in root.rglob("*"):
if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS:
yield path
def write_if_changed(path: Path, content: str) -> bool:
if path.exists():
try:
current = path.read_text(encoding="utf-8")
if current == content:
return False
except Exception:
pass
ensure_dir(path.parent)
path.write_text(content, encoding="utf-8")
return True
def collect_snippets(
source_dirs: Iterable[Path],
) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]:
result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {}
for file_path in iter_source_files(source_dirs):
lang = detect_language(file_path)
start_re, end_re = markers_for_lang(lang)
try:
text = file_path.read_text(encoding="utf-8")
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc
lines = text.splitlines(keepends=True)
snippets = parse_snippets(lines, start_re, end_re)
if not snippets:
continue
target = normalize_target_name(file_path)
lang_map = result.setdefault(target, {}).setdefault(lang, {})
for snippet_name, content_lines in snippets.items():
if snippet_name in lang_map:
raise ValueError(
f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}"
)
dedented = dedent(content_lines)
export_name = format_export_name(lang, snippet_name)
source_rel = os.path.relpath(file_path, Path.cwd())
lang_map[snippet_name] = SnippetRecord(
lang=lang,
snippet_name=snippet_name,
export_name=export_name,
text=snippet_text(dedented),
source_rel=source_rel,
)
return result
def render_module(
target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]]
) -> str:
parts: List[str] = []
parts.append(
"{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n"
)
for lang in LANG_ORDER:
snippets = lang_map.get(lang)
if not snippets:
continue
for snippet_name in sorted(snippets.keys()):
record = snippets[snippet_name]
literal = to_js_literal(record.text)
parts.append(f"export const {record.export_name} = {literal};\n\n")
return "".join(parts)
def generate_modules(
snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]],
output_root: Path,
) -> None:
modules_written = 0
total_exports = 0
for target in sorted(snippets_by_target.keys()):
module_path = output_root / f"{target}.mdx"
lang_map = snippets_by_target[target]
existing_langs = parse_existing_module(module_path)
for lang, snippets in existing_langs.items():
if lang in lang_map:
continue
print(module_path, " -- ", lang)
merged_lang = lang_map.setdefault(lang, {})
existing_export_names = {rec.export_name for rec in merged_lang.values()}
for snippet_name, record in snippets.items():
# If an export with the same name already exists (e.g., regenerated),
# prefer the newly generated snippet and skip the old one to avoid duplicates.
if record.export_name in existing_export_names:
continue
merged_lang.setdefault(snippet_name, record)
total_exports += sum(len(snippets) for snippets in lang_map.values())
module_content = render_module(target, lang_map)
if write_if_changed(module_path, module_content):
modules_written += 1
print(
f"\nGenerated {total_exports} snippets\n---"
)
def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]:
if args_dirs:
dirs = [Path(p) for p in args_dirs]
else:
dirs = list(DEFAULT_SOURCE_DIRS)
missing = [str(p) for p in dirs if not p.exists()]
if missing:
raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}")
return dirs
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate MDX snippet modules grouped by test filename."
)
parser.add_argument(
"-s",
"--source-dir",
action="append",
help="Path(s) to language test directories (default: tests/py, ts, rs)",
)
parser.add_argument(
"-o",
"--output-dir",
default=str(Path("docs") / "snippets"),
help="Directory where snippet modules are written (default: docs/snippets)",
)
args = parser.parse_args()
source_dirs = resolve_source_dirs(args.source_dir)
snippets = collect_snippets(source_dirs)
if not snippets:
print("No snippets found.")
return 0
output_root = Path(args.output_dir)
generate_modules(snippets, output_root)
return 0
if __name__ == "__main__":
raise SystemExit(main())
docs/web-tests/py/conftest.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import shutil
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
from lancedb.db import AsyncConnection, DBConnection
class DatabasePathFactory:
"""Create per-test database directories and ensure they get removed."""
def __init__(self, base_dir: Path) -> None:
self._base_dir = base_dir
self._created: list[Path] = []
def __call__(self, name: str) -> Path:
safe_name = name.replace("/", "_")
path = self._base_dir / safe_name
if path.exists():
shutil.rmtree(path, ignore_errors=True)
path.mkdir(parents=True, exist_ok=True)
self._created.append(path)
return path
def cleanup(self) -> None:
for path in reversed(self._created):
shutil.rmtree(path, ignore_errors=True)
self._created.clear()
@pytest.fixture
def db_path_factory(tmp_path: Path):
factory = DatabasePathFactory(tmp_path)
yield factory
factory.cleanup()
@pytest.fixture
def mem_db() -> DBConnection:
return lancedb.connect("memory://")
@pytest.fixture
def tmp_db(db_path_factory) -> DBConnection:
"""Create a temporary database connection for testing."""
db_path = db_path_factory("tmp_db")
return lancedb.connect(str(db_path))
@pytest_asyncio.fixture
async def mem_db_async() -> AsyncConnection:
return await lancedb.connect_async("memory://")
docs/web-tests/py/provider_examples.py
+355
View File
@@ -0,0 +1,355 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Reader examples with explicit placeholders.
These functions are documentation source. test_provider_examples.py runs the
reranker examples offline, with stand-ins for the model and the provider, to
check that each one writes only to its own new directory. Corresponding opt-in
integration tests remain in test_integrations.py and retain real credential
loading. Replace placeholders before running these examples.
"""
def embedding_ibm_usage():
# --8<-- [start:embedding_ibm_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.pydantic import LanceModel, Vector
registry = EmbeddingFunctionRegistry.get_instance()
registry.set_var("watsonx-api-key", "your-api-key")
watsonx_embed = registry.get("watsonx").create(
name="ibm/slate-125m-english-rtrvr",
api_key="$var:watsonx-api-key",
project_id="your-project-id",
)
class TextModel(LanceModel):
text: str = watsonx_embed.SourceField()
vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo"))
tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
tbl.add(data)
rs = tbl.search("hello").limit(1).to_pandas()
print(rs.head())
# --8<-- [end:embedding_ibm_usage]
def embedding_jina_text():
# --8<-- [start:embedding_jina_text]
import os
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.pydantic import LanceModel, Vector
os.environ["JINA_API_KEY"] = "your-api-key"
jina_embed = (
EmbeddingFunctionRegistry.get_instance()
.get("jina")
.create(name="jina-embeddings-v2-base-en")
)
class TextModel(LanceModel):
text: str = jina_embed.SourceField()
vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
data = [{"text": "hello world"}, {"text": "goodbye world"}]
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text"))
tbl = db.create_table("test", schema=TextModel, mode="overwrite")
tbl.add(data)
# --8<-- [end:embedding_jina_text]
def embedding_jina_multimodal():
# --8<-- [start:embedding_jina_multimodal]
import os
import tempfile
from pathlib import Path
import lancedb
import pandas as pd
import requests
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
os.environ["JINA_API_KEY"] = "your-api-key"
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images"))
func = get_registry().get("jina").create()
class Images(LanceModel):
label: str
image_uri: str = func.SourceField()
image_bytes: bytes = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
vec_from_bytes: Vector(func.ndims()) = func.VectorField()
table = db.create_table("images", schema=Images)
labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
uris = [
"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
]
image_bytes = [requests.get(uri).content for uri in uris]
table.add(
pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
)
# --8<-- [end:embedding_jina_multimodal]
def reranking_cohere_usage():
# --8<-- [start:reranking_cohere_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import CohereReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = CohereReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_cohere_usage]
def reranking_jina_usage():
# --8<-- [start:reranking_jina_usage]
import os
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import JinaReranker
os.environ["JINA_API_KEY"] = "your-api-key"
embedder = get_registry().get("jina").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = JinaReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_jina_usage]
def reranking_watsonx_usage():
# --8<-- [start:reranking_watsonx_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import WatsonxReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
# Supply your API key and project ID (or use space_id instead of project_id).
reranker = WatsonxReranker(
api_key="your-api-key",
project_id="your-project-id",
)
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_watsonx_usage]
def reranking_openai_usage():
# --8<-- [start:reranking_openai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import OpenaiReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = OpenaiReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_openai_usage]
def reranking_voyageai_usage():
# --8<-- [start:reranking_voyageai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import VoyageAIReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_voyageai_usage]
docs/web-tests/py/test_basic_usage.py
+205
View File
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:basic_imports]
import json
import lancedb
import pandas as pd
import polars as pl
import pyarrow as pa
# --8<-- [end:basic_imports]
import pytest
data_path = "tests/camelot.json"
def test_basic_usage(db_path_factory):
uri = "./basic_usage_db"
uri = db_path_factory("basic_usage_db")
db = lancedb.connect(uri)
# --8<-- [start:data_load]
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [end:data_load]
# --8<-- [start:basic_create_table]
table = db.create_table("camelot", data=data, mode="overwrite")
# --8<-- [end:basic_create_table]
assert len(table) == 8
# --8<-- [start:basic_open_table]
table = db.open_table("camelot")
# --8<-- [end:basic_open_table]
# --8<-- [start:basic_create_table_pandas]
pandas_df = pd.DataFrame(data)
table_pd = db.create_table("camelot_pd", data=pandas_df, mode="overwrite")
# --8<-- [end:basic_create_table_pandas]
assert len(table_pd) == 8
db.drop_table("camelot_pd")
# --8<-- [start:basic_create_table_polars]
polars_df = pl.DataFrame(data)
table_pl = db.create_table("camelot_pl", data=polars_df, mode="overwrite")
# --8<-- [end:basic_create_table_polars]
assert len(table_pl) == 8
db.drop_table("camelot_pl")
# --8<-- [start:basic_create_empty_table]
schema = pa.schema(
[
pa.field("id", pa.uint16()),
pa.field("name", pa.string()),
pa.field("role", pa.string()),
pa.field("description", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field(
"stats",
pa.struct(
[
pa.field("strength", pa.int8()),
pa.field("courage", pa.int8()),
pa.field("magic", pa.int8()),
pa.field("wisdom", pa.int8()),
]
),
),
]
)
db.create_table("camelot_pa", schema=schema, mode="overwrite")
# --8<-- [end:basic_create_empty_table]
assert "camelot_pa" in db.list_tables().tables
db.drop_table("camelot_pa")
# --8<-- [start:basic_add_data]
magical_characters = [
{
"id": 9,
"name": "Morgan le Fay",
"role": "Sorceress",
"description": "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
"vector": [0.10, 0.84, 0.25, 0.70],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 4 }
},
{
"id": 10,
"name": "The Lady of the Lake",
"role": "Mystical Guardian",
"description": "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
"vector": [0.00, 0.90, 0.58, 0.88],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 5 }
}
]
table.add(magical_characters)
# --8<-- [end:basic_add_data]
assert len(table) == 10
# --8<-- [start:basic_vector_search]
query_vector = [0.03, 0.85, 0.61, 0.90]
result = table.search(query_vector).limit(5).to_polars()
print(result)
# --8<-- [end:basic_vector_search]
# --8<-- [start:basic_add_columns]
table.add_columns(
{
"power": "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:basic_add_columns]
assert "power" in table.schema.names
# Run examples to illustrate search
# --8<-- [start:basic_vector_search_q1]
# Who are the characters similar to "wizard"?
query_vector_1 = [0.03, 0.85, 0.61, 0.90]
r1 = (
table.search(query_vector_1)
.limit(5)
.select(["name", "role", "description"])
.to_polars()
)
print(r1)
# --8<-- [end:basic_vector_search_q1]
# --8<-- [start:basic_vector_search_q2]
# Who are the characters with high magic stats?
query_vector_2 = [0.03, 0.85, 0.61, 0.90]
r2 = (
table.search(query_vector_2)
.where("stats.magic > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r2)
# --8<-- [end:basic_vector_search_q2]
# --8<-- [start:basic_vector_search_q3]
# Who are the strongest characters?
r3 = (
table.search()
.where("stats.strength > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r3)
# --8<-- [end:basic_vector_search_q3]
# --8<-- [start:basic_vector_search_q4]
# Who are the strongest characters?
r4 = (
table.search()
.select(["name", "role", "description", "power"])
.to_polars()
)
print(r4)
# --8<-- [end:basic_vector_search_q4]
# --8<-- [start:basic_sort_polars]
# Sort Polars DataFrame by power in descending order
print(r4.sort("power", descending=True).limit(5))
# --8<-- [end:basic_sort_polars]
sorted_power = r4.sort("power", descending=True)["power"].to_list()
assert sorted_power == sorted(sorted_power, reverse=True)
assert sorted_power[0] == 4.0
# --8<-- [start:basic_drop_columns]
table.drop_columns(["power"])
# --8<-- [end:basic_drop_columns]
# --8<-- [start:basic_delete_rows]
table.delete('role = "Traitor Knight"')
# --8<-- [end:basic_delete_rows]
assert len(table) == 9
# --8<-- [start:basic_drop_table]
db.drop_table("camelot")
# --8<-- [end:basic_drop_table]
assert "camelot" not in db.list_tables().tables
@pytest.mark.asyncio
async def test_basic_usage_async_api(db_path_factory):
uri = db_path_factory("basic_usage_async_db")
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [start:basic_async_api]
import lancedb
async_db = await lancedb.connect_async(uri)
async_table = await async_db.create_table(
"camelot_async",
data=data,
mode="overwrite",
)
query_vector = [0.03, 0.85, 0.61, 0.90]
async_results = await (
await async_table.search(query_vector)
).limit(5).select(["name", "role", "description"]).to_polars()
print(async_results)
# --8<-- [end:basic_async_api]
assert async_results.height >= 1
docs/web-tests/py/test_build_with_ai_agents.py
+109
View File
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:camelot_schema]
from lancedb.pydantic import LanceModel
from pydantic import ConfigDict
class Stats(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
strength: int
courage: int
magic: int
wisdom: int
class Character(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
id: int
name: str
role: str
description: str
stats: Stats
image_filename: str
image: bytes
# --8<-- [end:camelot_schema]
# --8<-- [start:camelot_batches]
import json
from collections.abc import Iterator
from pathlib import Path
def validated_batches(
input_path: Path, batch_size: int
) -> Iterator[list[dict]]:
raw_records = json.loads(input_path.read_text())
asset_root = input_path.parent.parent
batch: list[dict] = []
for raw in raw_records:
payload = dict(raw)
image_path = asset_root / payload.pop("img")
payload["image_filename"] = image_path.name
payload["image"] = image_path.read_bytes()
character = Character.model_validate(payload)
batch.append(character.model_dump(mode="python"))
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch
# --8<-- [end:camelot_batches]
# --8<-- [start:camelot_oss_ingestion]
import lancedb
def ingest_oss(
input_path: Path,
uri: str = "data/camelot.lancedb",
table_name: str = "camelot_multimodal",
batch_size: int = 64,
):
db = lancedb.connect(uri)
if table_name in db.list_tables():
raise ValueError(
f"Table {table_name!r} already exists. Choose a fresh table name."
)
table = db.create_table(table_name, schema=Character)
for batch in validated_batches(input_path, batch_size):
table.add(batch)
table.optimize()
return table
if __name__ == "__main__":
ingest_oss(Path("data/camelot.json"))
# --8<-- [end:camelot_oss_ingestion]
def test_camelot_oss_ingestion(tmp_path):
source = Path(
"docs/web/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json"
)
table = ingest_oss(
source,
uri=str(tmp_path / "camelot.lancedb"),
table_name="camelot",
)
rows = (
table.search()
.select(["id", "name", "image", "image_filename"])
.limit(8)
.to_list()
)
assert len(rows) == 8
assert all(row["image"] for row in rows)
assert all(row["image_filename"].endswith(".jpg") for row in rows)
docs/web-tests/py/test_connection.py
+210
View File
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from pathlib import Path
import shutil
import pytest
def test_connection():
# --8<-- [start:connect]
import lancedb
uri = "ex_lancedb"
db = lancedb.connect(uri)
# --8<-- [end:connect]
assert db is not None
shutil.rmtree(uri, ignore_errors=True)
assert not Path(uri).exists()
async def connect_async_example():
# --8<-- [start:connect_async]
import lancedb
uri = "ex_lancedb"
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_async]
return async_db
def connect_enterprise_quickstart_config():
# --8<-- [start:connect_enterprise_quickstart]
import os
import lancedb
db = lancedb.connect(
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"], # https://your-endpoint
)
# --8<-- [end:connect_enterprise_quickstart]
return db
def test_connect_enterprise_quickstart(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
monkeypatch.delenv("LANCEDB_REGION", raising=False)
monkeypatch.setenv("LANCEDB_HOST_OVERRIDE", "https://your-endpoint")
assert connect_enterprise_quickstart_config() is not None
assert captured == {
"uri": "db://your-database",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-endpoint",
}
def test_connect_enterprise_quickstart_requires_an_endpoint(monkeypatch):
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
monkeypatch.delenv("LANCEDB_HOST_OVERRIDE", raising=False)
with pytest.raises(KeyError, match="LANCEDB_HOST_OVERRIDE"):
connect_enterprise_quickstart_config()
def connect_object_storage_config():
# --8<-- [start:connect_object_storage]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
db = lancedb.connect(uri)
# --8<-- [end:connect_object_storage]
return db
def namespace_table_ops_example():
# --8<-- [start:namespace_table_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
# Create namespace tree: prod/search
db.create_namespace(["prod"], mode="exist_ok")
db.create_namespace(["prod", "search"], mode="exist_ok")
db.create_namespace(["prod", "recommendations"], mode="exist_ok")
db.create_table(
"user",
data=[{"id": 1, "vector": [0.1, 0.2], "name": "alice"}],
namespace_path=["prod", "search"],
mode="create", # use "overwrite" only if you want to replace existing table
)
db.create_table(
"user",
data=[{"id": 2, "vector": [0.3, 0.4], "name": "bob"}],
namespace_path=["prod", "recommendations"],
mode="create", # use "overwrite" only if you want to replace existing table
)
# Verify: listings return a response with the names and a page token
print(db.list_namespaces().namespaces) # ['prod']
print(db.list_namespaces(namespace_path=["prod"]).namespaces)
# ['recommendations', 'search']
print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user']
print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user']
# --8<-- [end:namespace_table_ops]
def test_namespace_table_ops(tmp_path, monkeypatch, capsys):
monkeypatch.chdir(tmp_path)
namespace_table_ops_example()
assert capsys.readouterr().out.splitlines() == [
"['prod']",
"['recommendations', 'search']",
"['user']",
"['user']",
]
def namespace_admin_ops_example():
# --8<-- [start:namespace_admin_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
namespace = ["prod", "search"]
db.create_namespace(["prod"])
db.create_namespace(["prod", "search"])
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
print(f"Child namespaces under ['prod']: {child_namespaces}")
# Child namespaces under ['prod']: ['search']
metadata = db.describe_namespace(["prod", "search"])
print(f"Metadata for namespace {namespace}: {metadata}")
# Metadata for namespace ['prod', 'search']: properties=None
db.drop_namespace(["prod", "search"], mode="skip")
db.drop_namespace(["prod"], mode="skip")
# --8<-- [end:namespace_admin_ops]
return db, child_namespaces, metadata
def test_namespace_admin_ops(tmp_path, monkeypatch, capsys):
monkeypatch.chdir(tmp_path)
db, child_namespaces, metadata = namespace_admin_ops_example()
assert child_namespaces == ["search"]
assert capsys.readouterr().out.splitlines()[0] == (
"Child namespaces under ['prod']: ['search']"
)
assert db.list_namespaces().namespaces == []
async def connect_object_storage_config_async():
# --8<-- [start:connect_object_storage_async]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_object_storage_async]
return async_db
def test_connect_enterprise_placeholders(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
# --8<-- [start:connect_enterprise_placeholders]
import lancedb
db = lancedb.connect(
uri="db://your-database",
api_key="your-api-key",
region="us-east-1",
host_override="https://your-endpoint",
)
# --8<-- [end:connect_enterprise_placeholders]
assert db is not None
assert captured == {
"uri": "db://your-database",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-endpoint",
}
docs/web-tests/py/test_curation_sampling.py
+27
View File
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_sample_dataset():
# --8<-- [start:sample_dataset]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.permutation import Permutation, permutation_builder
with TemporaryDirectory(prefix="lancedb-sampling-") as uri:
db = lancedb.connect(uri)
table = db.create_table("records", [{"id": i} for i in range(10)])
splits = (
permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()
)
sample = Permutation.from_tables(table, splits, split=0).with_format("arrow")
selected_ids = []
for batch in sample.iter(batch_size=3):
selected_ids.extend(batch.column("id").to_pylist())
print(f"Selected {len(selected_ids)} rows: {selected_ids}")
# --8<-- [end:sample_dataset]
assert len(selected_ids) == 3
assert len(set(selected_ids)) == 3
assert all(0 <= value < 10 for value in selected_ids)
docs/web-tests/py/test_doc_function_examples.py
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from lancedb.index import FTS
def test_python_boolean_query_operators(tmp_db):
table = tmp_db.create_table(
"boolean_docs",
[
{"id": 1, "text": "puppy"},
{"id": 2, "text": "merrily"},
{"id": 3, "text": "puppy merrily"},
{"id": 4, "text": "other"},
],
)
table.create_index("text", config=FTS())
# --8<-- [start:fts_boolean_queries]
from lancedb.query import MatchQuery
intersection = MatchQuery("puppy", "text") & MatchQuery("merrily", "text")
union = MatchQuery("puppy", "text") | MatchQuery("merrily", "text")
# --8<-- [end:fts_boolean_queries]
def ids(query):
return set(table.search(query, query_type="fts").to_arrow()["id"].to_pylist())
assert ids(intersection) == {3}
assert ids(union) == {1, 2, 3}
a = MatchQuery("puppy", "text")
b = MatchQuery("merrily", "text")
assert ids(a and b) == {2, 3}
assert ids(a or b) == {1, 3}
docs/web-tests/py/test_doc_storage_examples.py
+84
View File
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import numpy as np
import pyarrow as pa
import pytest
def test_rebuild_vector_index(tmp_db):
rng = np.random.default_rng(7)
table = tmp_db.create_table(
"rebuild_index",
[{"id": i, "vector": v.tolist()} for i, v in enumerate(rng.random((512, 4)))],
)
table.create_index(
"vector", config=lancedb.index.IvfFlat(distance_type="l2", num_partitions=4)
)
original_index = table.list_indices()[0]
table.add(
[
{"id": 512 + i, "vector": v.tolist()}
for i, v in enumerate(rng.random((64, 4)) + 5)
]
)
assert table.index_stats("vector_idx").num_unindexed_rows == 64
# --8<-- [start:storage_rebuild_vector_index]
from lancedb.index import IvfFlat
table.create_index(
"vector",
config=IvfFlat(distance_type="l2", num_partitions=4),
replace=True,
)
# --8<-- [end:storage_rebuild_vector_index]
indices = table.list_indices()
assert len(indices) == 1
assert indices[0].name == original_index.name
assert indices[0].index_uuid != original_index.index_uuid
stats = table.index_stats("vector_idx")
assert stats.num_indexed_rows == 576
assert stats.num_unindexed_rows == 0
def test_local_shallow_clone_pins_source_version(tmp_path):
db = lancedb.connect(str(tmp_path))
source = db.create_table("source", [{"id": 1}])
clone = db.clone_table("snapshot", source_uri=str(tmp_path / "source.lance"))
source.add([{"id": 2}])
assert source.count_rows() == 2
assert clone.to_arrow().to_pylist() == [{"id": 1}]
def _reader():
return pa.RecordBatchReader.from_batches(
pa.schema([("id", pa.int64())]),
[pa.record_batch([[1, 2]], names=["id"]), pa.record_batch([[3]], names=["id"])],
)
@pytest.mark.parametrize("parallelism", [1, 2])
def test_streaming_write_parallelism(tmp_db, parallelism):
table = tmp_db.create_table("streaming", [{"id": 0}])
progress = []
table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
assert sorted(table.to_arrow()["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
@pytest.mark.asyncio
@pytest.mark.parametrize("parallelism", [1, 2])
async def test_async_streaming_write_parallelism(mem_db_async, parallelism):
table = await mem_db_async.create_table("streaming", [{"id": 0}])
progress = []
await table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
data = await table.to_arrow()
assert sorted(data["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
docs/web-tests/py/test_embedding.py
+141
View File
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
# --8<-- [start:imports]
from lancedb.pydantic import LanceModel, Vector
from lancedb.embeddings import get_registry
# --8<-- [end:imports]
import pytest
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_create_embedding_function():
# --8<-- [start:create_embedding_function]
func = get_registry().get("openai").create(
name="text-embedding-3-small",
max_retries=7,
)
# --8<-- [end:create_embedding_function]
assert func is not None
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_openai():
# --8<-- [start:openai_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = table.search(query).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
@pytest.mark.asyncio
async def test_embeddings_openai_async():
uri = "memory://"
# --8<-- [start:async_openai_embeddings]
db = await lancedb.connect_async(uri)
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = await db.create_table("words", schema=Words, mode="overwrite")
await table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = await (await table.search(query)).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:async_openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_manual_query():
# --8<-- [start:manual_query_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query_vector = func.generate_embeddings(["greetings"])[0]
# --8<-- [start:manual_query_search]
# query_vector is assumed to already be generated by your embedding function
actual = table.search(query_vector).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:manual_query_search]
# --8<-- [end:manual_query_embeddings]
def test_custom_embedding_function():
# --8<-- [start:embedding_function]
from functools import cached_property
from lancedb.embeddings import TextEmbeddingFunction, register
class MyEmbeddingModel:
def __init__(self, model_name: str):
self.model_name = model_name
def encode(self, texts: list[str]) -> list[list[float]]:
return [[1.0, 2.0, 3.0] for _ in texts]
@register("my-embedder")
class MyTextEmbedder(TextEmbeddingFunction):
model_name: str = "my-model"
def generate_embeddings(self, texts: list[str]) -> list[list[float]]:
# Your embedding logic here
return self._model.encode(texts)
def ndims(self) -> int:
# Return the dimensionality of the embeddings
return len(self.generate_embeddings(["test"])[0])
@cached_property
def _model(self) -> MyEmbeddingModel:
# Initialize your model once
return MyEmbeddingModel(self.model_name)
# --8<-- [end:embedding_function]
def test_embeddings_secret():
# --8<-- [start:register_secret]
registry = get_registry()
registry.set_var("api_key", "sk-...")
func = registry.get("openai").create(api_key="$var:api_key")
# --8<-- [end:register_secret]
try:
import torch
except ImportError:
pytest.skip("torch not installed")
# --8<-- [start:register_device]
import torch
registry = get_registry()
if torch.cuda.is_available():
registry.set_var("device", "cuda")
func = registry.get("huggingface").create(device="$var:device:cpu")
# --8<-- [end:register_device]
assert func.device == "cuda" if torch.cuda.is_available() else "cpu"
docs/web-tests/py/test_function.py
+335
View File
@@ -0,0 +1,335 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/management/function.mdx.
Registering a Function and filling its column need a LanceDB Enterprise
deployment, so `test_function_enterprise` runs only when
LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the
Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and
LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The
walkthrough calls its table `function_demo` and its Function `double`; the test
creates both under names unique to its run, never replaces an existing object,
and drops only what it created.
The other tests run locally and check the client side of the same code: what
`@udf` sends for registration, how a registered version binds a column, where a
local connection stops, and when the Enterprise test connects. They do not run
a Function remotely.
"""
import json
import os
import sys
import uuid
from pathlib import Path
import lancedb
import pyarrow as pa
import pytest
from lancedb import col
from lancedb.functions import FunctionApplication, FunctionVersion
# --8<-- [start:function_define]
from lancedb import udf
@udf
def double(value: float) -> float:
return value * 2
# --8<-- [end:function_define]
# The definition the walkthrough registers. The Enterprise test registers a copy
# of it under a name that only its run uses.
WALKTHROUGH_DOUBLE = double
def describe(value: float):
return {"half": value / 2, "note": None}
# A Function version as the service returns it after registration: the SDK's
# shared golden reply, with the name and signature of the definition above.
SERVICE_REPLY = (
Path(__file__).parents[3]
/ "rust"
/ "lancedb"
/ "tests"
/ "fixtures"
/ "first_class_functions"
/ "v1"
/ "remote_function_job.json"
)
def registered_version(definition) -> FunctionVersion:
request = definition.registration_request
reply = json.loads(SERVICE_REPLY.read_text())["result"]
reply["name"] = request.name
reply["signature"] = json.loads(request.signature.to_canonical_json())
return FunctionVersion.from_json(json.dumps(reply))
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
# The connection settings alone never send this test to a deployment.
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
def require_deployment():
if os.environ.get(OPT_IN) != "1":
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
if missing:
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
def test_function_runs_locally_as_python():
# --8<-- [start:function_local_call]
print(double(1.5))
# 3.0
# --8<-- [end:function_local_call]
assert double(1.5) == 3.0
def test_function_registration_request():
request = double.registration_request
assert request.name == "double"
assert [
(parameter.name, parameter.arrow_type, parameter.nullable)
for parameter in request.signature.inputs
] == [("value", "float64", False)]
output = request.signature.output
assert (output.kind, output.arrow_type, output.nullable) == (
"scalar",
"float64",
False,
)
version = request.runtime.python_version
assert version == f"{sys.version_info.major}.{sys.version_info.minor}"
assert request.runtime.environment.packages == ()
def test_function_output_nullability():
inputs = pa.schema([pa.field("value", pa.float64(), nullable=False)])
# A named-struct output may declare nullable fields.
fields = [
pa.field("half", pa.float64(), nullable=False),
pa.field("note", pa.string(), nullable=True),
]
struct = udf(input_schema=inputs, output_schema=pa.schema(fields))(describe)
output = struct.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.nullable) for field in output.fields] == [
("half", False),
("note", True),
]
# A scalar output, and a struct declared as one nullable field, may not be.
for nullable in (
pa.field("result", pa.float64(), nullable=True),
pa.field("result", pa.struct(fields), nullable=True),
):
with pytest.raises(ValueError, match="non-nullable"):
udf(input_schema=inputs, output_schema=nullable)(describe)
def test_function_registration_is_remote(tmp_path):
db = lancedb.connect(tmp_path)
with pytest.raises(NotImplementedError):
db.create_function(double)
def test_function_version_binds_a_column(tmp_path):
version = registered_version(double)
application = version(value=col("value"))
assert isinstance(application, FunctionApplication)
assert application.function.name == "double"
assert application.function.version == version.version
assert [
(binding.parameter, binding.kind, binding.value["path"])
for binding in application.inputs
] == [("value", "column", "value")]
with pytest.raises(TypeError, match="direct col"):
version(value=1.5)
with pytest.raises(TypeError, match="missing"):
version(x=col("value"))
# The binding is accepted by add_columns, which a local table refuses:
# a Function column needs LanceDB Cloud or Enterprise.
table = lancedb.connect(tmp_path).create_table("t", [{"value": 1.0}])
with pytest.raises(NotImplementedError, match="Cloud and Enterprise"):
table.add_columns({"doubled": application})
def test_function_enterprise(monkeypatch):
require_deployment()
# --8<-- [start:function_connect]
import os
import lancedb
db = lancedb.connect(
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
# --8<-- [end:function_connect]
# In a shared database `function_demo` or `double` may already exist, so
# this run creates both under names only it uses, and drops only those.
run = uuid.uuid4().hex[:12]
table_name = f"function_demo_{run}"
double = udf(name=f"double_{run}")(WALKTHROUGH_DOUBLE.__wrapped__)
create_table = db.create_table
def create_run_table(name, *args, **kwargs):
assert name == "function_demo"
return create_table(table_name, *args, **kwargs)
monkeypatch.setattr(db, "create_table", create_run_table)
table = None
version = None
try:
# --8<-- [start:function_create_table]
import pyarrow as pa
schema = pa.schema([pa.field("value", pa.float64(), nullable=False)])
table = db.create_table(
"function_demo",
[{"value": 1.0}, {"value": 2.0}, {"value": 3.0}],
schema=schema,
)
# --8<-- [end:function_create_table]
# --8<-- [start:function_register]
job = db.create_function_async(double)
version = job.wait()
# --8<-- [end:function_register]
assert version.name == f"double_{run}"
# --8<-- [start:function_apply]
from lancedb import col
table.add_columns({"doubled": version(value=col("value"))})
# --8<-- [end:function_apply]
# --8<-- [start:function_refresh]
refresh = table.refresh_column_async("doubled")
result = refresh.wait()
print(result.rows_assigned)
# 3
# --8<-- [end:function_refresh]
assert result.rows_assigned == 3
assert result.rows_failed == 0
# --8<-- [start:function_inspect]
rows = table.search().select(["value", "doubled"]).to_list()
print(sorted((row["value"], row["doubled"]) for row in rows))
# [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]
# --8<-- [end:function_inspect]
assert sorted((row["value"], row["doubled"]) for row in rows) == [
(1.0, 2.0),
(2.0, 4.0),
(3.0, 6.0),
]
finally:
if table is not None:
db.drop_table(table_name)
if version is not None:
db.drop_function(version.name, version=version.version)
class ReachedConnection(Exception):
pass
SETTINGS = {
"LANCEDB_URI": "db://docs-test",
"LANCEDB_HOST_OVERRIDE": "https://deployment.invalid",
"LANCEDB_API_KEY": "docs-test-key",
}
@pytest.fixture
def connect_attempts(monkeypatch):
"""`lancedb.connect`, replaced so that it records each call and stops."""
attempts = []
def connect(**kwargs):
attempts.append(kwargs)
raise ReachedConnection
monkeypatch.setattr(lancedb, "connect", connect)
for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"):
monkeypatch.delenv(name, raising=False)
return attempts
def enterprise_outcome(monkeypatch):
try:
test_function_enterprise(monkeypatch)
except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome:
return outcome
@pytest.mark.parametrize(
"opt_in",
[None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"],
)
def test_function_enterprise_needs_the_exact_opt_in(
monkeypatch, connect_attempts, opt_in
):
for name, value in SETTINGS.items():
monkeypatch.setenv(name, value)
if opt_in is not None:
monkeypatch.setenv(OPT_IN, opt_in)
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, pytest.skip.Exception), outcome
assert str(outcome).startswith(f"set {OPT_IN}=1 ")
assert connect_attempts == []
@pytest.mark.parametrize("blank", ["unset", "empty"])
@pytest.mark.parametrize(
"missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)]
)
def test_function_enterprise_fails_when_asked_without_settings(
monkeypatch, connect_attempts, missing, blank
):
monkeypatch.setenv(OPT_IN, "1")
for name, value in SETTINGS.items():
if name not in missing:
monkeypatch.setenv(name, value)
elif blank == "empty":
monkeypatch.setenv(name, "")
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, pytest.fail.Exception), outcome
assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}"
assert connect_attempts == []
@pytest.mark.parametrize("region", [None, "eu-west-1"])
def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region):
monkeypatch.setenv(OPT_IN, "1")
for name, value in SETTINGS.items():
monkeypatch.setenv(name, value)
if region:
monkeypatch.setenv("LANCEDB_REGION", region)
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, ReachedConnection), outcome
assert connect_attempts == [
{
"uri": "db://docs-test",
"api_key": "docs-test-key",
"region": region or "us-east-1",
"host_override": "https://deployment.invalid",
}
]
docs/web-tests/py/test_indexing.py
+757
View File
@@ -0,0 +1,757 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import random
import string
import uuid
import numpy as np
import pyarrow as pa
import pytest
def _make_vector_rows(count: int, dim: int, column: str = "vector"):
rows = [
{column: np.random.random(dim).astype(np.float32).tolist(), "id": i}
for i in range(count)
]
assert len(rows) == count
assert all(len(row[column]) == dim for row in rows)
return rows
def test_vector_index_configure_ivf(tmp_db):
table = tmp_db.create_table(
"vector_index_configure_ivf",
_make_vector_rows(512, 4),
mode="overwrite",
)
# --8<-- [start:vector_index_configure_ivf]
table.create_index(metric="l2", num_partitions=16, num_sub_vectors=4)
# --8<-- [end:vector_index_configure_ivf]
assert table.list_indices()
def test_vector_index_setup(tmp_db):
tmp_db.create_table(
"vector-index-tbl",
_make_vector_rows(8, 4),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_setup]
table_name = "vector-index-tbl"
table = db.open_table(table_name)
# --8<-- [end:vector_index_setup]
assert table.name == table_name
def test_vector_index_build_ivf(tmp_db):
table = tmp_db.create_table(
"vector-index-build-ivf",
_make_vector_rows(512, 4, column="keywords_embeddings"),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_build_ivf]
table_name = "vector-index-build-ivf"
table = db.open_table(table_name)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [end:vector_index_build_ivf]
assert table.list_indices()
def test_vector_index_nested_field(tmp_db):
dim = 2
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
data = [
{
"id": i,
"image": {"embedding": np.random.random(dim).astype(np.float32).tolist()},
}
for i in range(512)
]
table = tmp_db.create_table(
"vector_index_nested_field", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:vector_index_nested_field]
# The vector column `embedding` is nested inside the `image` struct.
# Pass its full dotted path as `vector_column_name`; the same path is used
# at query time and is what `list_indices()` reports under `columns`.
table.create_index(
vector_column_name="image.embedding",
num_partitions=1,
num_sub_vectors=1,
name="image_embedding_idx",
)
results = (
table.search([0.0, 1.0], vector_column_name="image.embedding")
.limit(1)
.to_list()
)
# --8<-- [end:vector_index_nested_field]
assert table.index_stats("image_embedding_idx")
assert len(results) == 1
@pytest.mark.asyncio
async def test_vector_index_async_config(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:vector_index_async_config]
import lancedb
import numpy as np
from lancedb.index import IvfPq
async def main():
data = [
{"id": i, "vector": np.random.random(8).astype(np.float32).tolist()}
for i in range(512)
]
db = await lancedb.connect_async("ex_lancedb")
table = await db.create_table(
"vector_index_async", data=data, mode="overwrite"
)
await table.create_index(
"vector",
config=IvfPq(
distance_type="cosine",
num_partitions=16,
num_sub_vectors=4,
),
)
return await table.list_indices()
# --8<-- [end:vector_index_async_config]
assert await main()
def test_vector_index_query_ivf(tmp_db):
dim = 1536
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_query_ivf", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_query_ivf]
tbl = table
tbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(
10
).to_pandas()
# --8<-- [end:vector_index_query_ivf]
df = (
table.search(np.random.random((1536)))
.limit(2)
.nprobes(20)
.refine_factor(10)
.to_pandas()
)
assert len(df) == 2
def test_vector_index_nprobes(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_nprobes", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_nprobes]
# Always scan 10 partitions; scan up to 50 only if the initial pass
# returns fewer than `limit` results (common with narrow filters).
(
table.search(np.random.random(128))
.minimum_nprobes(10)
.maximum_nprobes(50)
.where("id > 100")
.limit(5)
.to_pandas()
)
# --8<-- [end:vector_index_nprobes]
def test_vector_index_distance_range(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_distance_range", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_distance_range]
# Only return results whose distance falls within [0.0, 0.5).
# Useful for near-duplicate detection or thresholded similarity search.
(
table.search(np.random.random(128))
.distance_range(lower_bound=0.0, upper_bound=0.5)
.limit(10)
.to_pandas()
)
# --8<-- [end:vector_index_distance_range]
def test_vector_index_bypass_recall(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_bypass_recall", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_bypass_recall]
query = np.random.random(128)
k = 10
# Ground truth: flat (exhaustive) scan, ignoring the ANN index.
truth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()["id"])
# ANN results with the current nprobes setting.
ann = set(table.search(query).nprobes(20).limit(k).to_pandas()["id"])
recall_at_k = len(truth & ann) / k
# --8<-- [end:vector_index_bypass_recall]
assert 0.0 <= recall_at_k <= 1.0
def test_vector_index_custom_name(tmp_db):
table = tmp_db.create_table(
"vector_index_custom_name",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
# --8<-- [start:vector_index_custom_name]
# Override the default `{column}_idx` convention by passing `name=...`.
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
name="my_custom_index",
)
table.wait_for_index(["my_custom_index"])
print(table.index_stats("my_custom_index"))
# --8<-- [end:vector_index_custom_name]
assert table.index_stats("my_custom_index")
def test_vector_index_hnsw(tmp_db):
table = tmp_db.create_table(
"vector_index_hnsw",
_make_vector_rows(64, 16),
mode="overwrite",
)
# --8<-- [start:vector_index_build_hnsw]
table.create_index(index_type="IVF_HNSW_SQ")
# --8<-- [end:vector_index_build_hnsw]
# --8<-- [start:vector_index_query_hnsw]
tbl = table
tbl.search(np.random.random((16))).limit(2).to_pandas()
# --8<-- [end:vector_index_query_hnsw]
df = table.search(np.random.random((16))).limit(2).to_pandas()
assert len(df) == 2
def test_vector_index_binary(tmp_db):
table_name = "hamming-index-tbl"
ndim = 256
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.uint8(), ndim // 8)),
]
)
# --8<-- [start:vector_index_binary_schema]
table = tmp_db.create_table(table_name, schema=schema, mode="overwrite")
# --8<-- [end:vector_index_binary_schema]
data = []
for i in range(64):
vector = np.random.randint(0, 2, size=ndim)
vector = np.packbits(vector)
data.append({"id": i, "vector": vector})
# --8<-- [start:vector_index_binary_add_data]
table.add(data)
# --8<-- [end:vector_index_binary_add_data]
# --8<-- [start:vector_index_binary_build_index]
table.create_index(
metric="hamming",
vector_column_name="vector",
index_type="IVF_FLAT",
)
# --8<-- [end:vector_index_binary_build_index]
# --8<-- [start:vector_index_binary_search]
query = np.random.randint(0, 2, size=ndim)
query = np.packbits(query)
df = table.search(query).metric("hamming").limit(10).to_pandas()
df.vector = df.vector.apply(np.unpackbits)
# --8<-- [end:vector_index_binary_search]
assert not df.empty
def test_quantization_custom_params(tmp_db):
table = tmp_db.create_table(
"quantization-custom-params",
_make_vector_rows(256, 64),
mode="overwrite",
)
# --8<-- [start:quantization_custom_params]
table.create_index(
index_type="IVF_RQ",
num_bits=2,
max_iterations=100,
sample_rate=512,
)
# --8<-- [end:quantization_custom_params]
assert table.list_indices()
def test_vector_index_check_status(tmp_db):
table = tmp_db.create_table(
"vector_index_check_status",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_check_status]
index_name = "keywords_embeddings_idx"
table.wait_for_index([index_name])
print(table.index_stats(index_name))
# --8<-- [end:vector_index_check_status]
assert table.index_stats(index_name)
def test_scalar_index_build(tmp_db):
table = tmp_db.create_table(
"scalar_index_build",
[
{"book_id": 1, "publisher": "A", "vector": [0.1, 0.2]},
{"book_id": 2, "publisher": "B", "vector": [0.2, 0.3]},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_build]
tbl = db.open_table("scalar_index_build")
tbl.create_scalar_index("book_id")
tbl.create_scalar_index("publisher", index_type="BITMAP")
# --8<-- [end:scalar_index_build]
assert tbl.list_indices()
def test_scalar_index_wait(tmp_db):
table = tmp_db.create_table(
"scalar_index_wait",
[{"label": "fiction"}],
mode="overwrite",
)
table.create_scalar_index("label")
# --8<-- [start:scalar_index_wait]
index_name = "label_idx"
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_wait]
assert table.list_indices()
def test_scalar_index_optimize(tmp_db):
table = tmp_db.create_table(
"scalar_index_optimize",
[{"vector": [7.0, 8.0], "book_id": 3}],
mode="overwrite",
)
# --8<-- [start:scalar_index_optimize]
table.add([{"vector": [7, 8], "book_id": 4}])
table.optimize()
# --8<-- [end:scalar_index_optimize]
result = table.search().where("book_id = 4").limit(10).to_pandas()
assert len(result) == 1
def test_scalar_index_filter(tmp_db):
table = tmp_db.create_table(
"books",
[
{"vector": [1.1, 1.2], "book_id": 1},
{"vector": [2.1, 2.2], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_filter]
table = db.open_table("books")
result = table.search().where("book_id = 2").limit(10).to_pandas()
# --8<-- [end:scalar_index_filter]
assert len(result) == 1
def test_scalar_index_prefilter(tmp_db):
table = tmp_db.create_table(
"book_with_embeddings",
[
{"vector": [1.2, 1.3], "book_id": 1},
{"vector": [4.2, 4.3], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_prefilter]
table = db.open_table("book_with_embeddings")
table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
# --8<-- [end:scalar_index_prefilter]
result = table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
assert len(result) == 2
def test_scalar_index_uuid(tmp_db):
# --8<-- [start:scalar_index_uuid_type]
import pyarrow as pa
# --8<-- [end:scalar_index_uuid_type]
# --8<-- [start:scalar_index_uuid_data]
def generate_random_names():
base_names = ["Alice", "Bob", "Carla", "David", "Eve", "Frank", "Grace"]
letter = random.choice(string.ascii_uppercase)
return f"{random.choice(base_names)} {letter}."
def generate_uuids(num_items):
return [uuid.uuid4().bytes for _ in range(num_items)]
# Generate some UUIDs and random names
n = 7
uuids = generate_uuids(n)
names = [generate_random_names() for _ in range(n)]
# --8<-- [end:scalar_index_uuid_data]
db = tmp_db
# --8<-- [start:scalar_index_uuid_table]
table_name = "index-on-uuid"
uuid_array = pa.array(uuids, pa.uuid())
name_array = pa.array(names, pa.string())
schema = pa.schema(
[
pa.field("id", pa.uuid()),
pa.field("name", pa.string()),
]
)
data_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)
table = db.create_table(table_name, data=data_table, mode="overwrite")
# --8<-- [end:scalar_index_uuid_table]
# --8<-- [start:scalar_index_uuid_wait]
index_name = "id_idx"
table.create_scalar_index("id")
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_uuid_wait]
# --8<-- [start:scalar_index_uuid_upsert]
new_users = [
{"id": uuid.uuid4().bytes, "name": "Hannah D."},
{"id": uuid.uuid4().bytes, "name": "Ian B."},
]
# Insert or update using the UUID index
table.merge_insert(
"id"
).when_matched_update_all().when_not_matched_insert_all().execute(new_users)
# --8<-- [end:scalar_index_uuid_upsert]
assert table.list_indices()
result = table.search().limit(100).to_pandas()
assert len(result) == n + len(new_users)
@pytest.mark.asyncio
async def test_scalar_index_nested_fields(mem_db_async):
db = mem_db_async
# --8<-- [start:scalar_index_nested_fields]
import pyarrow as pa
from lancedb.index import BTree
metadata_type = pa.struct(
[
pa.field("user_id", pa.int32()),
pa.field("user.id", pa.int32()),
]
)
data = pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int32()),
pa.array(
[
{"user_id": 10, "user.id": 100},
{"user_id": 20, "user.id": 200},
{"user_id": 30, "user.id": 300},
],
type=metadata_type,
),
],
names=["user_id", "metadata"],
)
table = await db.create_table("nested_scalar_index", data)
# Index a nested struct field.
await table.create_index(
"metadata.user_id", config=BTree(), name="nested_user_id_idx"
)
# Escape literal dots inside a segment with backticks.
await table.create_index(
"metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
)
# `columns` is returned as the canonical path you passed in.
for index in await table.list_indices():
print(index.name, index.columns)
# nested_user_id_idx ['metadata.user_id']
# escaped_user_id_idx ['metadata.`user.id`']
# --8<-- [end:scalar_index_nested_fields]
index_columns = {index.name: index.columns for index in await table.list_indices()}
assert index_columns["nested_user_id_idx"] == ["metadata.user_id"]
assert index_columns["escaped_user_id_idx"] == ["metadata.`user.id`"]
def test_fts_index_create(tmp_db):
table = tmp_db.create_table(
"fts-index-create",
[{"text": "hello world", "vector": [0.1, 0.2]}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_create]
table_name = "fts-index-create"
table = db.open_table(table_name)
table.create_fts_index("text")
# --8<-- [end:fts_index_create]
assert table.list_indices()
def test_fts_index_wait(tmp_db):
table = tmp_db.create_table(
"fts-index-wait",
[{"text": "full text search"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_wait]
table_name = "fts-index-wait"
table = db.open_table(table_name)
table.create_fts_index("text")
index_name = "text_idx"
table.wait_for_index([index_name])
# --8<-- [end:fts_index_wait]
assert table.list_indices()
def test_fts_index_nested_field(tmp_db):
nested_schema = pa.struct([
pa.field("text", pa.string()),
pa.field("count", pa.int32()),
])
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("payload", nested_schema),
])
tmp_db.create_table(
"fts-index-nested",
pa.table(
{
"id": pa.array([1, 2], pa.int64()),
"payload": pa.array(
[
{"text": "Frodo was a happy puppy", "count": 1},
{"text": "puppy runs through the meadow", "count": 2},
],
type=nested_schema,
),
},
schema=schema,
),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_nested]
from lancedb.query import MatchQuery, PhraseQuery
table = db.open_table("fts-index-nested")
# Index a text leaf inside a struct column using a dotted path.
table.create_fts_index("payload.text", with_position=True)
# The same dotted path works in MatchQuery and PhraseQuery.
matches = (
table.search(MatchQuery("puppy", "payload.text")).limit(5).to_list()
)
phrases = (
table.search(PhraseQuery("puppy runs", "payload.text"))
.limit(5)
.to_list()
)
# --8<-- [end:fts_index_nested]
assert len(matches) > 0
assert all("puppy" in row["payload"]["text"] for row in matches)
assert len(phrases) > 0
assert all("puppy runs" in row["payload"]["text"] for row in phrases)
@pytest.mark.asyncio
async def test_fts_index_async(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:fts_index_async]
import asyncio
import lancedb
import polars as pl
from lancedb.index import FTS
data = pl.DataFrame(
{
"id": [1, 2],
"text": [
"His first language is spanish",
"Her first language is english",
],
}
)
async def main(data: pl.DataFrame):
uri = "ex_lancedb"
db = await lancedb.connect_async(uri)
tbl = await db.create_table("my_text", data=data, mode="overwrite")
await tbl.create_index("text", config=FTS(language="English"))
response = await tbl.search("spanish", query_type="fts")
result = await response.limit(1).to_polars()
print(result)
return result
if __name__ == "__main__":
asyncio.run(main(data))
# --8<-- [end:fts_index_async]
result = await main(data)
assert result.height == 1
def test_gpu_index_snippets(tmp_db, monkeypatch):
table = tmp_db.create_table(
"gpu_index",
_make_vector_rows(32, 8),
mode="overwrite",
)
calls = []
def fake_create_index(*args, **kwargs):
calls.append(kwargs)
return None
monkeypatch.setattr(table, "create_index", fake_create_index)
# --8<-- [start:gpu_index_cuda]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="cuda",
)
# --8<-- [end:gpu_index_cuda]
# --8<-- [start:gpu_index_mps]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="mps",
)
# --8<-- [end:gpu_index_mps]
assert calls[0]["accelerator"] == "cuda"
assert calls[1]["accelerator"] == "mps"
def test_reindexing_incremental(tmp_db):
table = tmp_db.create_table(
"reindexing_incremental",
[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:reindexing_incremental]
table = db.open_table("reindexing_incremental")
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:reindexing_incremental]
result = table.search().limit(10).to_pandas()
assert len(result) == 2
docs/web-tests/py/test_ingestion_examples.py
+739
View File
@@ -0,0 +1,739 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_create_table_from_dicts():
# --8<-- [start:create_table_from_dicts]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
data = [
{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
{"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1},
]
db.create_table("test_table", data, mode="overwrite")
tbl = db["test_table"]
tbl.head()
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_dicts]
assert tbl.count_rows() == 2
def test_create_table_conflict_handling():
# --8<-- [start:create_table_conflict_handling]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
db.create_table("conflict_table", data)
# Idempotent open: reuse the existing table if it exists.
# The provided data is ignored; the schema is validated against the
# existing table and a mismatch raises an error.
tbl = db.create_table("conflict_table", data, exist_ok=True)
# Overwrite: replace the contents with the provided data.
tbl = db.create_table("conflict_table", data, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_conflict_handling]
def test_create_table_custom_schema():
# --8<-- [start:create_table_custom_schema]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
custom_schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field("lat", pa.float32()),
pa.field("long", pa.float32()),
]
)
data = [
{"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7},
{"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1},
]
tbl = db.create_table(
"my_table_custom_schema", data, schema=custom_schema, mode="overwrite"
)
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_custom_schema]
assert tbl.count_rows() == 2
def test_create_table_from_pandas():
# --8<-- [start:create_table_from_pandas]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pandas as pd
data = pd.DataFrame(
{
"vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
"lat": [45.5, 40.1],
"long": [-122.7, -74.1],
}
)
db.create_table("my_table_pandas", data, mode="overwrite")
db["my_table_pandas"].head()
# --8<-- [end:create_table_from_pandas]
def test_create_table_from_polars():
# --8<-- [start:create_table_from_polars]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import polars as pl
data = pl.DataFrame(
{
"vector": [[3.1, 4.1], [5.9, 26.5]],
"item": ["foo", "bar"],
"price": [10.0, 20.0],
}
)
tbl = db.create_table("my_table_pl", data, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_polars]
assert tbl.count_rows() == 2
def test_create_table_from_arrow():
# --8<-- [start:create_table_from_arrow]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import numpy as np
import pyarrow as pa
dim = 16
total = 2
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float16(), dim)),
pa.field("text", pa.string()),
]
)
data = pa.Table.from_arrays(
[
pa.array(
[np.random.randn(dim).astype(np.float16) for _ in range(total)],
pa.list_(pa.float16(), dim),
),
pa.array(["foo", "bar"]),
],
["vector", "text"],
)
tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_arrow]
assert tbl.count_rows() == 2
def test_create_table_from_pydantic():
# --8<-- [start:create_table_from_pydantic]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
class Content(LanceModel):
movie_id: int
vector: Vector(128)
genres: str
title: str
imdb_id: int
@property
def imdb_url(self) -> str:
return f"https://www.imdb.com/title/tt{self.imdb_id}"
tbl = db.create_table("movielens_small", schema=Content, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_pydantic]
assert tbl.count_rows() == 0
def test_create_table_nested_schema():
# --8<-- [start:create_table_nested_schema]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
from pydantic import BaseModel
class Document(BaseModel):
content: str
source: str
class NestedSchema(LanceModel):
id: str
vector: Vector(1536)
document: Document
tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_nested_schema]
assert tbl.count_rows() == 0
def test_create_empty_table():
# --8<-- [start:create_empty_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("item", pa.string()),
pa.field("price", pa.float32()),
]
)
tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_empty_table]
assert tbl.count_rows() == 0
def test_create_empty_table_pydantic():
# --8<-- [start:create_empty_table_pydantic]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
class Item(LanceModel):
vector: Vector(2)
item: str
price: float
tbl = db.create_table(
"test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite"
)
print("Rows:", tbl.count_rows())
# --8<-- [end:create_empty_table_pydantic]
assert tbl.count_rows() == 0
def test_open_existing_table():
# --8<-- [start:open_existing_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
# Create a table first
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
db.create_table("test_table", data, mode="overwrite")
# List table names
print(db.list_tables().tables)
# Open existing table
tbl = db.open_table("test_table")
print("Rows:", tbl.count_rows())
# --8<-- [end:open_existing_table]
assert tbl.count_rows() == 1
def test_drop_table():
# --8<-- [start:drop_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
# Create a table first
data = [{"vector": [1.1, 1.2], "lat": 45.5}]
db.create_table("my_table", data, mode="overwrite")
# Drop the table
db.drop_table("my_table")
# --8<-- [end:drop_table]
def test_add_from_dataset():
# --8<-- [start:add_from_dataset]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
data_path = str(Path(uri) / "input.parquet")
pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path)
import pyarrow.dataset as ds
dataset = ds.dataset(data_path, format="parquet")
table = db.create_table("my_table", schema=dataset.schema, mode="overwrite")
table.add(dataset)
print("Rows:", table.count_rows())
# --8<-- [end:add_from_dataset]
assert table.count_rows() == 2
def test_create_table_from_iterator():
# --8<-- [start:create_table_from_iterator]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field("item", pa.utf8()),
pa.field("price", pa.float32()),
]
)
def make_batches():
for i in range(5):
yield pa.RecordBatch.from_arrays(
[
pa.array(
[[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
pa.list_(pa.float32(), 4),
),
pa.array(["foo", "bar"]),
pa.array([10.0, 20.0]),
],
["vector", "item", "price"],
)
table = db.create_table("batched_table", schema=schema)
table.add(make_batches())
print("Rows:", table.count_rows())
# --8<-- [end:create_table_from_iterator]
assert table.count_rows() == 10
def test_update_operation():
# --8<-- [start:update_operation]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
table.update(where="id = 2", values={"name": "Bobby"})
print("Rows:", table.count_rows())
# --8<-- [end:update_operation]
assert table.count_rows() == 2
assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby"
def test_update_using_sql():
# --8<-- [start:update_using_sql]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
print("Rows:", table.count_rows())
# --8<-- [end:update_using_sql]
assert table.count_rows() == 2
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
def test_merge_matched_update_only():
# --8<-- [start:merge_matched_update_only]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(table.merge_insert("id").when_matched_update_all().execute(incoming_users))
print("Rows:", table.count_rows())
# --8<-- [end:merge_matched_update_only]
assert table.count_rows() == 2
def test_insert_if_not_exists():
# --8<-- [start:insert_if_not_exists]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users))
print("Rows:", table.count_rows())
# --8<-- [end:insert_if_not_exists]
assert table.count_rows() == 3
def test_merge_update_insert():
# --8<-- [start:merge_update_insert]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_update_insert]
assert table.count_rows() == 3
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
def test_merge_delete_missing_by_source():
# --8<-- [start:merge_delete_missing_by_source]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2, 3],
"name": ["Alice", "Bob", "Charlie"],
"login_count": [10, 20, 5],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.when_not_matched_by_source_delete()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_delete_missing_by_source]
assert table.count_rows() == 2
assert table.count_rows("id = 1") == 0
def test_merge_partial_columns():
# --8<-- [start:merge_partial_columns]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_partial_columns]
assert table.count_rows() == 3
assert table.search().where("id = 3").to_list()[0]["login_count"] is None
def test_write_workflow():
# --8<-- [start:write_workflow]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"users",
[
{"id": 1, "name": "Alice", "login_count": 10},
{"id": 2, "name": "Bob", "login_count": 20},
],
)
table.add([{"id": 3, "name": "Charlie", "login_count": 5}])
table.update(where="id = 2", values={"name": "Bobby"})
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(
[
{"id": 3, "name": "Charlie", "login_count": 6},
{"id": 4, "name": "Dana", "login_count": 1},
]
)
)
print("Rows selected for deletion:", table.search().where("id = 4").to_list())
table.delete("id = 4")
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print(rows)
# --8<-- [end:write_workflow]
assert rows == [
{"id": 1, "name": "Alice", "login_count": 10},
{"id": 2, "name": "Bobby", "login_count": 21},
{"id": 3, "name": "Charlie", "login_count": 6},
]
def test_append_rows():
# --8<-- [start:append_rows]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}])
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print(rows)
# --8<-- [end:append_rows]
assert [row["id"] for row in rows] == [1, 2, 3]
def test_delete_rows():
# --8<-- [start:delete_rows]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"users",
[
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"},
{"id": 3, "name": "Charlie"},
],
)
predicate = "id = 3"
print("Selected:", table.search().where(predicate).to_list())
table.delete(predicate)
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print("Remaining:", rows)
# --8<-- [end:delete_rows]
assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
def test_add_progress():
# --8<-- [start:add_progress]
from tempfile import TemporaryDirectory
import lancedb
def report_progress(progress):
print(f"Rows written: {progress['output_rows']}")
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
table.add(
[{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}],
progress=report_progress,
)
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print("After insertion:", rows)
# --8<-- [end:add_progress]
assert rows == [
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"},
{"id": 3, "name": "Charlie"},
]
docs/web-tests/py/test_integrations.py
+1794
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/py/test_materialized_view.py
+79
View File
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/feature-engineering/materialized-views.mdx."""
def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:materialized_view_create]
import lancedb
db = lancedb.connect("./.lancedb")
table = db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view(
"adults",
"people",
select=["name", ("shout", "upper(name)")],
where="age >= 18",
)
print(view.table.count_rows())
# 1
# --8<-- [end:materialized_view_create]
assert view.table.count_rows() == 1
# --8<-- [start:materialized_view_refresh]
table.add([{"name": "bea", "age": 41}])
view.refresh()
print(view.table.count_rows())
# 2
# --8<-- [end:materialized_view_refresh]
assert view.table.count_rows() == 2
assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [
"ADA",
"BEA",
]
def test_materialized_view_with_no_data(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view(
"adults", "people", where="age >= 18", with_no_data=True
)
assert view.table.count_rows() == 0
assert view.refresh().rows_written == 1
assert view.table.count_rows() == 1
def test_materialized_view_refresh_to_source_version(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
table = db.create_table(
"people",
[{"name": "ada", "age": 36}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view("adults", "people", where="age >= 18")
first = table.version
table.add([{"name": "bea", "age": 41}])
view.refresh()
assert view.table.count_rows() == 2
result = view.refresh(source_version=first)
assert (result.source_version, result.rows_written) == (first, 1)
assert view.table.count_rows() == 1
docs/web-tests/py/test_multimodal.py
+232
View File
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pytest
try:
import lancedb
import numpy as np
import pyarrow as pa
import io
from PIL import Image
except ImportError:
pass
# --8<-- [start:multimodal_imports]
import lancedb
import pyarrow as pa
import pandas as pd
import numpy as np
import io
from PIL import Image
# --8<-- [end:multimodal_imports]
def test_multimodal_ingestion(db_path_factory):
# Ensure dependencies are available
pytest.importorskip("PIL")
pytest.importorskip("lancedb")
pytest.importorskip("numpy")
# --8<-- [start:create_dummy_data]
# Create some dummy images
def create_dummy_image(color):
img = Image.new('RGB', (100, 100), color=color)
buf = io.BytesIO()
img.save(buf, format='PNG')
return buf.getvalue()
# Create dataset with metadata, vectors, and image blobs
data = [
{
"id": 1,
"filename": "red_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('red'),
"label": "red"
},
{
"id": 2,
"filename": "blue_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('blue'),
"label": "blue"
}
]
# --8<-- [end:create_dummy_data]
# --8<-- [start:define_schema]
# Define schema explicitly to ensure image_blob is treated as binary
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field("filename", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs
pa.field("label", pa.string())
])
# --8<-- [end:define_schema]
db_uri = db_path_factory("multimodal_db")
db = lancedb.connect(db_uri)
# --8<-- [start:ingest_data]
tbl = db.create_table("images", data=data, schema=schema, mode="overwrite")
# --8<-- [end:ingest_data]
assert len(tbl) == 2
# --8<-- [start:search_data]
# Search for similar images
query_vector = np.random.rand(128).astype(np.float32)
results = tbl.search(query_vector).limit(1).to_pandas()
# --8<-- [end:search_data]
# --8<-- [start:process_results]
# Convert back to PIL Image
for _, row in results.iterrows():
image_bytes = row['image_blob']
image = Image.open(io.BytesIO(image_bytes))
print(f"Retrieved image: {row['filename']}, Size: {image.size}")
# You can now use 'image' with other libraries or display it
# --8<-- [end:process_results]
assert len(results) == 1
def test_blob_api_definition(db_path_factory):
# --8<-- [start:blob_api_schema]
import pyarrow as pa
# Define schema with Blob API metadata for lazy loading
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"} # Enable Blob API
),
])
# --8<-- [end:blob_api_schema]
# --8<-- [start:blob_api_ingest]
import lancedb
db = lancedb.connect(db_path_factory("blob_db"))
# Create sample data
data = [
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"}
]
# Create the table
tbl = db.create_table("videos", data=data, schema=schema)
# --8<-- [end:blob_api_ingest]
assert len(tbl) == 2
def test_blob_api_to_pandas(db_path_factory):
db = lancedb.connect(db_path_factory("blob_to_pandas_db"))
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
tbl = db.create_table(
"videos",
data=[
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"},
],
schema=schema,
mode="overwrite",
)
# --8<-- [start:blob_api_to_pandas]
# Default: blob columns come back lazily
df_lazy = tbl.to_pandas()
# Materialize blob bytes eagerly
df_bytes = tbl.to_pandas(blob_mode="bytes")
# Return descriptors instead of payloads
df_desc = tbl.to_pandas(blob_mode="descriptions")
# Forward extra kwargs to PyArrow's to_pandas
df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)
# --8<-- [end:blob_api_to_pandas]
assert len(df_lazy) == 2
assert isinstance(df_bytes["video"].iloc[0], bytes)
assert df_bytes["video"].tolist() == [
b"fake_video_bytes_1",
b"fake_video_bytes_2",
]
assert len(df_desc) == 2
assert len(df_typed) == 2
@pytest.mark.asyncio
async def test_query_to_pandas_kwargs(db_path_factory):
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
data = [
{
"id": i,
"vector": np.random.rand(128).astype(np.float32),
"video": f"fake_video_bytes_{i}".encode(),
}
for i in range(10)
]
db = lancedb.connect(db_path_factory("query_to_pandas_db"))
tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite")
async_db = await lancedb.connect_async(
str(db_path_factory("query_to_pandas_async_db"))
)
tbl_async = await async_db.create_table(
"search_demo", data=data, schema=schema, mode="overwrite"
)
query_vector = np.random.rand(128).astype(np.float32)
# --8<-- [start:query_to_pandas_kwargs]
# Plain scan query: blob_mode is supported end to end
df_lazy = (
tbl.search()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="lazy")
)
# Same call shape works on async query builders
df_bytes = await (
tbl_async.query()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="bytes")
)
# Vector / FTS / hybrid queries can't materialize blob columns,
# so omit them from the projection
df_vec = (
tbl.search(query_vector)
.limit(10)
.select(["id", "vector"])
.to_pandas(split_blocks=True, self_destruct=True)
)
# --8<-- [end:query_to_pandas_kwargs]
assert len(df_lazy) == 1
assert len(df_bytes) == 1
assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1"
assert len(df_vec) == 10
assert "video" not in df_vec.columns
docs/web-tests/py/test_provider_examples.py
+160
View File
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""The provider reranker examples write only to a directory they create.
Each example runs offline. A stand-in embedding function answers for the model
the example asks the registry for, a stand-in reranker replaces the provider
client, and outbound connections are refused, so no model is downloaded and no
provider is called. Sentinel tables in the owned home directory, where these
examples used to write, and in the temporary directory must be left unchanged.
"""
import importlib.util
import socket
import tempfile
from pathlib import Path
from types import SimpleNamespace
from typing import ClassVar
import lancedb
import lancedb.embeddings
import lancedb.rerankers
import pyarrow as pa
import pytest
from lancedb.embeddings import TextEmbeddingFunction, register
from lancedb.rerankers import Reranker
EXAMPLES = Path(__file__).with_name("provider_examples.py")
RERANKERS = {
"reranking_cohere_usage": "CohereReranker",
"reranking_jina_usage": "JinaReranker",
"reranking_watsonx_usage": "WatsonxReranker",
"reranking_openai_usage": "OpenaiReranker",
"reranking_voyageai_usage": "VoyageAIReranker",
}
@register("docs-offline-text")
class OfflineEmbedding(TextEmbeddingFunction):
"""Deterministic four-dimensional vectors, computed locally."""
def ndims(self):
return 4
def generate_embeddings(self, texts):
return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts]
class OfflineReranker(Reranker):
"""Scores every row 1.0 and records which searches it reranked."""
instances: ClassVar[list["OfflineReranker"]] = []
def __init__(self, **kwargs):
super().__init__()
self.kwargs = kwargs
self.searches = []
OfflineReranker.instances.append(self)
def _score(self, results, search):
self.searches.append(search)
drop = [c for c in ("_distance", "_score") if c in results.column_names]
results = results.drop_columns(drop)
scores = pa.array([1.0] * results.num_rows, pa.float32())
return results.append_column("_relevance_score", scores)
def rerank_vector(self, query, vector_results):
return self._score(vector_results, "vector")
def rerank_fts(self, query, fts_results):
return self._score(fts_results, "fts")
def rerank_hybrid(self, query, vector_results, fts_results):
merged = self.merge_results(vector_results, fts_results)
return self._score(merged, "hybrid")
def snapshot(connect, uri):
db = connect(str(uri))
tables = {}
for name in db.list_tables().tables:
table = db.open_table(name)
tables[name] = (table.version, table.schema, table.to_arrow().to_pylist())
return tables
def load_examples():
spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.fixture
def owned(tmp_path, monkeypatch):
home, temp_root = tmp_path / "home", tmp_path / "tmp"
home.mkdir()
temp_root.mkdir()
monkeypatch.setenv("HOME", str(home))
monkeypatch.setattr(tempfile, "tempdir", str(temp_root))
monkeypatch.chdir(tmp_path)
# Anything the examples resolve from "~" must land in the owned home.
assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve())
def refuse(*args, **kwargs):
raise OSError("network access is disabled in this test")
monkeypatch.setattr(socket.socket, "connect", refuse)
monkeypatch.setattr(socket, "create_connection", refuse)
real_connect = lancedb.connect
sentinels = [home / ".lancedb", temp_root / "existing"]
for uri in sentinels:
row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]}
real_connect(str(uri)).create_table("test", [row])
before = [snapshot(real_connect, uri) for uri in sentinels]
registry = lancedb.embeddings.get_registry()
monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding)
for name in RERANKERS.values():
monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker)
monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example")
OfflineReranker.instances.clear()
connected = []
def recording_connect(uri, *args, **kwargs):
connected.append(Path(uri).expanduser().resolve())
return real_connect(uri, *args, **kwargs)
monkeypatch.setattr(lancedb, "connect", recording_connect)
return SimpleNamespace(
home=home,
temp_root=temp_root,
sentinels=sentinels,
before=before,
connect=real_connect,
connected=connected,
)
@pytest.mark.parametrize("example", sorted(RERANKERS))
def test_reranker_example_writes_only_to_its_own_directory(owned, example):
existing = set(owned.temp_root.iterdir())
getattr(load_examples(), example)()
after = [snapshot(owned.connect, uri) for uri in owned.sentinels]
assert after == owned.before
created = set(owned.temp_root.iterdir()) - existing
assert len(created) == 1
(own_directory,) = created
assert len(owned.connected) == 1
assert owned.connected[0].is_relative_to(own_directory.resolve())
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
(reranker,) = OfflineReranker.instances
assert reranker.searches == ["vector", "fts", "hybrid"]
assert reranker.kwargs["api_key"] == "your-api-key"
assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id"
docs/web-tests/py/test_quickstart.py
+353
View File
@@ -0,0 +1,353 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
import shutil
import uuid
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
# What the quickstart's download step leaves in an application's directory.
IMAGE = (
Path(__file__).resolve().parents[2]
/ "web/static/assets/images/quickstart/sir-lancelot.jpg"
)
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
# The connection settings alone never send these tests to a deployment.
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
def deployment_settings():
if os.environ.get(OPT_IN) != "1":
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
if missing:
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
return dict(
uri=os.environ["LANCEDB_URI"],
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
class OwnedTables:
"""A deployment connection on which a test creates only tables of its own.
Each name gets a suffix unique to the test and is claimed only while no
table has it; the test drops what it claimed, whether or not it passed.
"""
def __init__(self, db):
self.db = db
self.suffix = uuid.uuid4().hex[:12]
self.claimed = []
def claim(self, name, existing, mode):
owned = f"{name}_{self.suffix}"
if owned in self.claimed:
return owned, mode
if owned in existing:
raise RuntimeError(f"{owned} already exists and is not this test's")
self.claimed.append(owned)
return owned, "create"
def table_names(self):
names, token = [], None
while True:
page = self.db.list_tables(page_token=token)
names += page.tables
if not (token := page.page_token):
return names
def create_table(self, name, *args, mode="create", **kwargs):
owned, mode = self.claim(name, self.table_names(), mode)
return self.db.create_table(owned, *args, mode=mode, **kwargs)
def drop_claimed(self):
existing = self.table_names()
for name in self.claimed:
if name in existing:
self.db.drop_table(name)
class AsyncOwnedTables(OwnedTables):
async def table_names(self):
names, token = [], None
while True:
page = await self.db.list_tables(page_token=token)
names += page.tables
if not (token := page.page_token):
return names
async def create_table(self, name, *args, mode="create", **kwargs):
owned, mode = self.claim(name, await self.table_names(), mode)
return await self.db.create_table(owned, *args, mode=mode, **kwargs)
async def drop_claimed(self):
existing = await self.table_names()
for name in self.claimed:
if name in existing:
await self.db.drop_table(name)
@pytest.fixture
def deployment():
owned = OwnedTables(lancedb.connect(**deployment_settings()))
yield owned
owned.drop_claimed()
@pytest.fixture(params=["local", "enterprise"])
def db(request, db_path_factory):
if request.param == "local":
yield lancedb.connect(db_path_factory("quickstart_db"))
else:
yield request.getfixturevalue("deployment")
@pytest_asyncio.fixture(params=["local", "enterprise"])
async def async_db(request, db_path_factory):
if request.param == "local":
yield await lancedb.connect_async(db_path_factory("quickstart_async_db"))
return
owned = AsyncOwnedTables(await lancedb.connect_async(**deployment_settings()))
yield owned
await owned.drop_claimed()
def create_characters(db):
# --8<-- [start:quickstart_data]
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
]
# --8<-- [end:quickstart_data]
# --8<-- [start:quickstart_create_table]
table = db.create_table("characters", data=data, mode="overwrite")
# --8<-- [end:quickstart_create_table]
assert table.count_rows() == 3
# --8<-- [start:quickstart_add_data]
table.add(
[
{
"id": "4",
"name": "Morgana",
"role": "Sorceress",
"description": "Powerful sorceress of Avalon.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
"vector": [0.3, 0.9, 0.6, 0.8],
}
]
)
# --8<-- [end:quickstart_add_data]
assert table.count_rows() == 4
return table
def test_quickstart(db):
table = create_characters(db)
# --8<-- [start:quickstart_vector_search_1]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
result = (
table.search(query_vector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(result)
# --8<-- [end:quickstart_vector_search_1]
assert result["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_output_pandas]
# Ensure you run `pip install pandas` beforehand
result = table.search(query_vector).limit(2).to_pandas()
print(result)
# --8<-- [end:quickstart_output_pandas]
assert result["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_curate_with_metadata]
curated = (
table.search(query_vector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(curated)
# --8<-- [end:quickstart_curate_with_metadata]
assert curated["name"].to_list() == ["Merlin", "Morgana"]
curated = table.search(query_vector).where("stats.magic < 4").limit(2).to_list()
assert [row["name"] for row in curated] == ["King Arthur", "Sir Lancelot"]
@pytest.mark.asyncio
async def test_quickstart_async_api(async_db):
data = [
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
}
]
# --8<-- [start:quickstart_create_table_async]
async_table = await async_db.create_table(
"characters",
data=data,
mode="overwrite",
)
# --8<-- [end:quickstart_create_table_async]
# --8<-- [start:quickstart_vector_search_1_async]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
async_result = await (
await async_table.search(query_vector)
).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
print(async_result)
# --8<-- [end:quickstart_vector_search_1_async]
assert async_result["name"].to_list() == ["Merlin"]
def test_quickstart_derived_feature(db):
table = create_characters(db)
# --8<-- [start:quickstart_add_feature]
table.add_columns(
{
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:quickstart_add_feature]
# --8<-- [start:quickstart_query_feature]
features = table.search().select(["name", "role", "power_score"]).to_polars()
print(features)
# --8<-- [end:quickstart_query_feature]
assert sorted(zip(features["name"], features["power_score"])) == [
("King Arthur", 3.5),
("Merlin", 4.0),
("Morgana", 3.75),
("Sir Lancelot", 3.0),
]
def description_lengths(table):
rows = table.search().select(["description", "description_length"]).to_list()
return {row["description"]: row["description_length"] for row in rows}
def test_quickstart_computed_column(db_path_factory):
table = create_characters(lancedb.connect(db_path_factory("computed")))
# --8<-- [start:quickstart_computed_feature]
table.add_columns(computed={"description_length": "length(description)"})
table.refresh_column("description_length")
# --8<-- [end:quickstart_computed_feature]
lengths = description_lengths(table)
assert lengths == {text: len(text) for text in lengths}
def test_quickstart_computed_column_on_a_deployment(deployment):
table = create_characters(deployment)
# --8<-- [start:quickstart_computed_feature_enterprise]
table.add_columns(computed={"description_length": "length(description)"})
job = table.refresh_column_async("description_length")
job.wait()
# --8<-- [end:quickstart_computed_feature_enterprise]
lengths = description_lengths(table)
assert len(lengths) == 4
assert lengths == {text: len(text) for text in lengths}
with pytest.raises(Exception):
table.refresh_column("description_length")
def test_quickstart_multimodal(db, tmp_path, monkeypatch):
app = tmp_path / "app"
(app / "data").mkdir(parents=True)
shutil.copy(IMAGE, app / "data/sir-lancelot.jpg")
monkeypatch.chdir(app)
# --8<-- [start:quickstart_multimodal_bytes]
from pathlib import Path
image_bytes = Path("data/sir-lancelot.jpg").read_bytes()
multimodal_table = db.create_table(
"character_images",
data=[
{
"id": "lancelot",
"description": "Portrait of Sir Lancelot",
"image": image_bytes,
"vector": [0.9, 0.1, 0.5, 0.4],
}
],
mode="overwrite",
)
# --8<-- [end:quickstart_multimodal_bytes]
assert multimodal_table.count_rows() == 1
stored = multimodal_table.search().select(["image"]).to_list()[0]["image"]
assert stored == IMAGE.read_bytes()
def test_quickstart_training(db):
table = create_characters(db)
# --8<-- [start:quickstart_training]
import torch
from lancedb.permutation import Permutation
table.add_columns(
{
"magic": "cast(stats.magic as float)",
"strength": "cast(stats.strength as float)",
}
)
dataset = Permutation.identity(table).select_columns(["magic", "strength"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
# --8<-- [end:quickstart_training]
batches = list(dataloader)
assert len(batches) == 1
assert set(batches[0]) == {"magic", "strength"}
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
assert sorted(batches[0]["magic"].tolist()) == [1.0, 1.0, 5.0, 5.0]
docs/web-tests/py/test_read_consistency.py
+35
View File
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_read_consistency():
# --8<-- [start:read_consistency]
from datetime import timedelta
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-consistency-") as uri:
writer = lancedb.connect(uri)
table = writer.create_table("documents", [{"id": 1, "text": "first"}])
default_reader = lancedb.connect(uri).open_table("documents")
latest_reader = lancedb.connect(
uri, read_consistency_interval=timedelta(0)
).open_table("documents")
table.add([{"id": 2, "text": "second"}])
default_count = default_reader.count_rows()
latest_count = latest_reader.count_rows()
print(f"Default reader: {default_count} rows") # 1
print(f"Zero-interval reader: {latest_count} rows") # 2
default_reader.checkout_latest()
refreshed_count = default_reader.count_rows()
print(f"After explicit refresh: {refreshed_count} rows") # 2
# --8<-- [end:read_consistency]
assert default_count == 1
assert latest_count == 2
assert refreshed_count == 2
docs/web-tests/py/test_reranker_examples.py
+96
View File
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""The published reranker examples in test_integrations.py write only to a directory they create.
Each example runs as readers see it: its marked region is extracted the way the snippet
generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the
sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT
and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination
and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory,
where these examples used to write, and in the temporary directory must be left unchanged.
"""
import importlib.util
import runpy
import sys
from pathlib import Path
import lancedb.rerankers
import pytest
import test_provider_examples as provider
owned = provider.owned
SOURCE = Path(__file__).with_name("test_integrations.py")
GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py"
MODEL_RERANKERS = {
"reranking_answerdotai_usage": "AnswerdotaiRerankers",
"reranking_colbert_usage": "ColbertReranker",
"reranking_cross_encoder_usage": "CrossEncoderReranker",
}
LOCAL_RERANKERS = [
"reranking_linear_combination_usage",
"reranking_mrr_usage",
"reranking_rrf_usage",
]
def load_generator():
if "mdx_snippets_gen" not in sys.modules:
spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module # its dataclasses resolve annotations by module
spec.loader.exec_module(module)
return sys.modules["mdx_snippets_gen"]
def reader_code(name):
"""The example exactly as the snippet generator publishes it."""
generator = load_generator()
start, end = generator.markers_for_lang("py")
lines = SOURCE.read_text().splitlines(keepends=True)
snippet = generator.parse_snippets(lines, start, end)[name]
return generator.snippet_text(generator.dedent(snippet))
def run_example(owned, tmp_path, name):
script = tmp_path / "scripts" / f"{name}.py"
script.parent.mkdir(exist_ok=True)
script.write_text(reader_code(name))
existing = set(owned.temp_root.iterdir())
namespace = runpy.run_path(str(script))
after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels]
assert after == owned.before
created = set(owned.temp_root.iterdir()) - existing
assert len(created) == 1
(own_directory,) = created
assert len(owned.connected) == 1
assert owned.connected[0].is_relative_to(own_directory.resolve())
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
return namespace
@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS))
def test_model_reranker_example_writes_only_to_its_own_directory(
owned, monkeypatch, tmp_path, name
):
monkeypatch.setattr(
lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker
)
namespace = run_example(owned, tmp_path, name)
(reranker,) = provider.OfflineReranker.instances
assert reranker.searches == ["vector", "fts", "hybrid"]
assert namespace["result"]
@pytest.mark.parametrize("name", LOCAL_RERANKERS)
def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name):
namespace = run_example(owned, tmp_path, name)
assert namespace["result"]
assert all("_relevance_score" in row for row in namespace["result"])
if name == "reranking_mrr_usage":
assert namespace["combined"].num_rows > 0
docs/web-tests/py/test_search.py
+730
View File
@@ -0,0 +1,730 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:import-lancedb]
import lancedb
# --8<-- [end:import-lancedb]
# --8<-- [start:import-numpy]
from lancedb.query import BoostQuery, MatchQuery
import numpy as np
import pyarrow as pa
# --8<-- [end:import-numpy]
# --8<-- [start:import-datetime]
from datetime import datetime
# --8<-- [end:import-datetime]
# --8<-- [start:import-lancedb-pydantic]
from lancedb.pydantic import Vector, LanceModel
# --8<-- [end:import-lancedb-pydantic]
# --8<-- [start:import-pydantic-base-model]
from pydantic import BaseModel
# --8<-- [end:import-pydantic-base-model]
# --8<-- [start:import-lancedb-fts]
from lancedb.index import FTS
# --8<-- [end:import-lancedb-fts]
# --8<-- [start:import-os]
import os
# --8<-- [end:import-os]
# --8<-- [start:import-embeddings]
from lancedb.embeddings import get_registry
# --8<-- [end:import-embeddings]
import pytest
# --8<-- [start:class-definition]
class Metadata(BaseModel):
source: str
timestamp: datetime
class Document(BaseModel):
content: str
meta: Metadata
class LanceSchema(LanceModel):
id: str
vector: Vector(1536)
payload: Document
# --8<-- [end:class-definition]
def test_vector_search():
# --8<-- [start:exhaustive_search]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
tbl = db.create_table("vector_search", data=data, mode="overwrite")
tbl.search(np.random.random((1536))).limit(10).to_list()
# --8<-- [end:exhaustive_search]
# --8<-- [start:exhaustive_search_cosine]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:exhaustive_search_cosine]
# --8<-- [start:create_table_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
# Synchronous client
tbl = db.create_table("documents", data=data, mode="overwrite")
# --8<-- [end:create_table_with_nested_schema]
# --8<-- [start:search_result_as_pyarrow]
tbl.search(np.random.randn(1536)).to_arrow()
# --8<-- [end:search_result_as_pyarrow]
# --8<-- [start:search_result_as_pandas]
tbl.search(np.random.randn(1536)).to_pandas()
# --8<-- [end:search_result_as_pandas]
# --8<-- [start:search_result_as_pandas_flatten_true]
tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
# --8<-- [end:search_result_as_pandas_flatten_true]
# --8<-- [start:search_result_as_pandas_flatten_1]
tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
# --8<-- [end:search_result_as_pandas_flatten_1]
# --8<-- [start:search_result_as_list]
tbl.search(np.random.randn(1536)).to_list()
# --8<-- [end:search_result_as_list]
# --8<-- [start:search_result_as_pydantic]
tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
# --8<-- [end:search_result_as_pydantic]
@pytest.mark.asyncio
async def test_vector_search_async():
# --8<-- [start:exhaustive_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
async_tbl = await async_db.create_table(
"vector_search_async", data=data, mode="overwrite"
)
(await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list())
# --8<-- [end:exhaustive_search_async]
# --8<-- [start:exhaustive_search_async_cosine]
(
await (await async_tbl.search(np.random.random((1536))))
.distance_type("cosine")
.limit(10)
.to_list()
)
# --8<-- [end:exhaustive_search_async_cosine]
# --8<-- [start:create_table_async_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
async_tbl = await async_db.create_table(
"documents_async", data=data, mode="overwrite"
)
# --8<-- [end:create_table_async_with_nested_schema]
# --8<-- [start:search_result_async_as_pyarrow]
await (await async_tbl.search(np.random.randn(1536))).to_arrow()
# --8<-- [end:search_result_async_as_pyarrow]
# --8<-- [start:search_result_async_as_pandas]
await (await async_tbl.search(np.random.randn(1536))).to_pandas()
# --8<-- [end:search_result_async_as_pandas]
# --8<-- [start:search_result_async_as_list]
await (await async_tbl.search(np.random.randn(1536))).to_list()
# --8<-- [end:search_result_async_as_list]
def test_fts_fuzzy_query():
uri = "data/fuzzy-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_fuzzy",
data=pa.table(
{
"text": [
"fa",
"fo", # spellchecker:disable-line
"fob",
"focus",
"foo",
"food",
"foul",
]
}
),
mode="overwrite",
)
table.create_fts_index("text", replace=True)
results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
assert len(results) == 4
assert set(results["text"].to_list()) == {
"foo",
"fo", # 1 deletion # spellchecker:disable-line
"fob", # 1 substitution
"food", # 1 insertion
}
def test_fts_boost_query():
uri = "data/boost-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_boost",
data=pa.table(
{
"title": [
"The Hidden Gems of Travel",
"Exploring Nature's Wonders",
"Cultural Treasures Unveiled",
"The Nightlife Chronicles",
"Scenic Escapes and Challenges",
],
"desc": [
"A vibrant city with occasional traffic jams.",
"Beautiful landscapes but overpriced tourist spots.",
"Rich cultural heritage but humid summers.",
"Bustling nightlife but noisy streets.",
"Scenic views but limited public transport options.",
],
}
),
mode="overwrite",
)
table.create_fts_index("desc", replace=True)
results = table.search(
BoostQuery(
MatchQuery("beautiful, cultural, nightlife", "desc"),
MatchQuery("bad traffic jams, overpriced", "desc"),
),
).to_pandas()
# we will hit 3 results because the positive query has 3 hits
assert len(results) == 3
# the one containing "overpriced" will be negatively boosted,
# so it will be the last one
assert (
results["desc"].to_list()[2]
== "Beautiful landscapes but overpriced tourist spots."
)
def test_fts_native():
# --8<-- [start:basic_fts]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
table.create_fts_index("text")
table.search("puppy").limit(10).select(["text"]).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts]
# --8<-- [start:fts_config_stem]
table.create_fts_index("text", tokenizer_name="en_stem", replace=True)
# --8<-- [end:fts_config_stem]
# --8<-- [start:fts_config_folding]
table.create_fts_index(
"text",
language="French",
stem=True,
ascii_folding=True,
replace=True,
)
# --8<-- [end:fts_config_folding]
# --8<-- [start:fts_prefiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=True).to_list()
# --8<-- [end:fts_prefiltering]
# --8<-- [start:fts_postfiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
# --8<-- [end:fts_postfiltering]
# --8<-- [start:fts_with_position]
table.create_fts_index("text", with_position=True, replace=True)
# --8<-- [end:fts_with_position]
# --8<-- [start:fts_incremental_index]
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:fts_incremental_index]
@pytest.mark.asyncio
async def test_fts_native_async():
# --8<-- [start:basic_fts_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
async_tbl = await async_db.create_table(
"my_table_fts_async",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
# async API uses our native FTS algorithm
await async_tbl.create_index("text", config=FTS())
await (await async_tbl.search("puppy")).select(["text"]).limit(10).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts_async]
# --8<-- [start:fts_config_stem_async]
await async_tbl.create_index(
"text", config=FTS(language="English", stem=True, remove_stop_words=True)
)
# --8<-- [end:fts_config_stem_async]
# --8<-- [start:fts_config_folding_async]
await async_tbl.create_index(
"text", config=FTS(language="French", stem=True, ascii_folding=True)
)
# --8<-- [end:fts_config_folding_async]
# --8<-- [start:fts_prefiltering_async]
await (await async_tbl.search("puppy")).limit(10).where("text='foo'").to_list()
# --8<-- [end:fts_prefiltering_async]
# --8<-- [start:fts_postfiltering_async]
await (
(await async_tbl.search("puppy"))
.limit(10)
.where("text='foo'")
.postfilter()
.to_list()
)
# --8<-- [end:fts_postfiltering_async]
# --8<-- [start:fts_with_position_async]
await async_tbl.create_index("text", config=FTS(with_position=True))
# --8<-- [end:fts_with_position_async]
# --8<-- [start:fts_incremental_index_async]
await async_tbl.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
await async_tbl.optimize()
# --8<-- [end:fts_incremental_index_async]
def _vectors(n, dim, seed=0, column="vector"):
rng = np.random.default_rng(seed)
return [
{column: rng.random(dim).astype("float32"), "id": i} for i in range(n)
]
def _build_indexed_table(db, name, dim=128, n=512):
tbl = db.create_table(name, _vectors(n, dim), mode="overwrite")
tbl.create_index(metric="cosine", num_partitions=4, num_sub_vectors=8)
return tbl
def test_vs_distance_metric_and_brute_force(tmp_db):
tbl = tmp_db.create_table("vs_metric", _vectors(64, 1536), mode="overwrite")
# --8<-- [start:configure_distance_metric]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:configure_distance_metric]
# --8<-- [start:brute_force_search]
tbl.search(np.random.random((1536))).limit(3).to_list()
# --8<-- [end:brute_force_search]
def test_vs_select_vector_column(tmp_db):
db = tmp_db
# --8<-- [start:select_vector_column]
import pyarrow as pa
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
])
table = db.create_table(
"nested",
data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}],
schema=schema,
)
# Inferred: the only vector leaf is `image.embedding`.
table.search([0.0, 1.0]).limit(1).to_list()
# Explicit: required when more than one vector column matches.
table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list()
# --8<-- [end:select_vector_column]
def test_vs_index_nested_column(tmp_db):
dim = 16
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
rng = np.random.default_rng(0)
data = [
{"id": i, "image": {"embedding": rng.random(dim).astype("float32").tolist()}}
for i in range(512)
]
table = tmp_db.create_table(
"nested_index", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:index_nested_column]
table.create_index(vector_column_name="image.embedding")
# --8<-- [end:index_nested_column]
assert table.list_indices()
def test_vs_indexed_queries(tmp_db):
table = _build_indexed_table(tmp_db, "vs_indexed", dim=128, n=512)
embedding = np.random.random(128)
# --8<-- [start:exact_vs_approximate_distances]
# Indexed ANN search without refinement (fast, approximate `_distance`)
fast_results = (
table.search(embedding)
.limit(10)
.to_pandas()
)
# Recompute distances on full vectors for reranked candidates
exact_distance_results = (
table.search(embedding)
.limit(10)
.refine_factor(1)
.to_pandas()
)
# Rerank a larger candidate set for better recall (higher latency)
higher_recall_results = (
table.search(embedding)
.limit(10)
.refine_factor(20)
.to_pandas()
)
# --8<-- [end:exact_vs_approximate_distances]
# --8<-- [start:bypass_vector_index]
table.search(embedding).bypass_vector_index().limit(5).to_pandas()
# --8<-- [end:bypass_vector_index]
assert len(fast_results) == 10
assert len(exact_distance_results) == 10
assert len(higher_recall_results) == 10
def test_vs_fast_search(tmp_db, monkeypatch):
table = _build_indexed_table(tmp_db, "vs_fast", dim=128, n=512)
embedding = np.random.random(128)
# `fast_search` is an Enterprise/async query flag; strip it so the snippet
# runs against a local OSS table without changing what readers see.
real_search = table.search
monkeypatch.setattr(
table,
"search",
lambda *a, **k: real_search(
*a, **{key: val for key, val in k.items() if key != "fast_search"}
),
)
# --8<-- [start:fast_search]
table.search(embedding, fast_search=True).limit(5).to_pandas()
# --8<-- [end:fast_search]
def test_vs_distance_range(tmp_db):
tbl = tmp_db.create_table("vs_distance_range", _vectors(256, 256), mode="overwrite")
# --8<-- [start:search_distance_range]
query = np.random.random(256)
# Search for the vectors within the range of [0.1, 0.5)
tbl.search(query).distance_range(0.1, 0.5).to_arrow()
# Search for the vectors with the distance less than 0.5
tbl.search(query).distance_range(upper_bound=0.5).to_arrow()
# Search for the vectors with the distance greater or equal to 0.1
tbl.search(query).distance_range(lower_bound=0.1).to_arrow()
# --8<-- [end:search_distance_range]
def test_vs_multivector_search(tmp_db):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
]
)
rng = np.random.default_rng(0)
data = [{"id": i, "vector": rng.random(size=(2, 256)).tolist()} for i in range(64)]
tbl = tmp_db.create_table(
"vs_multivector", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:multivector_search]
query_multi = np.random.random(size=(2, 256))
results_multi = tbl.search(query_multi).limit(5).to_pandas()
# --8<-- [end:multivector_search]
assert len(results_multi) <= 5
def test_vs_binary_search(tmp_db):
db = tmp_db
# --8<-- [start:search_binary_vectors]
import numpy as np
import pyarrow as pa
schema = pa.schema(
[
pa.field("id", pa.int64()),
# for dim=256, lance stores every 8 bits in a byte
# so the vector field should be a list of 256 / 8 = 32 bytes
pa.field("vector", pa.list_(pa.uint8(), 32)),
]
)
tbl = db.create_table("my_binary_vectors", schema=schema)
data = []
for i in range(1024):
vector = np.random.randint(0, 2, size=256)
# pack the binary vector into bytes to save space
packed_vector = np.packbits(vector)
data.append(
{
"id": i,
"vector": packed_vector,
}
)
tbl.add(data)
query = np.random.randint(0, 2, size=256)
packed_query = np.packbits(query)
tbl.search(packed_query).distance_type("hamming").to_arrow()
# --8<-- [end:search_binary_vectors]
def test_vs_enterprise_filtering(tmp_db, monkeypatch):
import sys
import types
dim = 384
rng = np.random.default_rng(0)
rows = [
{
"vector": rng.random(dim).astype("float32"),
"text": f"story {i}",
"keywords": f"kw{i}",
"label": i % 4,
}
for i in range(50)
]
tmp_db.create_table("lancedb-enterprise-quickstart", data=rows, mode="overwrite")
db = tmp_db
# Mock the Hugging Face dataset loader to avoid network downloads.
class _FakeDataset:
def __init__(self, n):
self._emb = [rng.random(dim).astype("float32") for _ in range(n)]
self._kw = [f"kw{i}" for i in range(n)]
def __getitem__(self, key):
if isinstance(key, str):
return {"keywords_embeddings": self._emb, "keywords": self._kw}[key]
return {"keywords": self._kw[key], "keywords_embeddings": self._emb[key]}
fake_datasets = types.ModuleType("datasets")
fake_datasets.load_dataset = lambda *a, **k: _FakeDataset(10)
monkeypatch.setitem(sys.modules, "datasets", fake_datasets)
# --8<-- [start:vector_search_prefilter]
from datasets import load_dataset
# Load query vector from dataset
query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]")
print(f"Query keywords: {query_dataset[0]['keywords']}")
query_embed = query_dataset["keywords_embeddings"][0]
# Open table and perform search
table_name = "lancedb-enterprise-quickstart"
table = db.open_table(table_name)
# Vector search with filters (pre-filtering is the default)
search_results = (
table.search(query_embed)
.where("label > 2")
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Search results (with pre-filtering):")
print(search_results)
# --8<-- [end:vector_search_prefilter]
# --8<-- [start:vector_search_postfilter]
results_post_filtered = (
table.search(query_embed)
.where("label > 1", prefilter=False)
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Vector search results with post-filter:")
print(results_post_filtered)
# --8<-- [end:vector_search_postfilter]
# --8<-- [start:batch_search]
# Load a batch of query embeddings
query_dataset = load_dataset(
"sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]"
)
query_embeds = query_dataset["keywords_embeddings"]
batch_results = table.search(query_embeds).limit(5).to_pandas()
print(batch_results)
# --8<-- [end:batch_search]
@pytest.mark.skip()
def test_hybrid_search():
# --8<-- [start:import-openai]
import openai
# --8<-- [end:import-openai]
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search]
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table("documents", schema=Documents)
# ingest docs with auto-vectorization
table.add(data)
# Create a fts index before the hybrid search
table.create_fts_index("text")
# hybrid search with default re-ranker
table.search("flower moon", query_type="hybrid").to_pandas()
# --8<-- [end:basic_hybrid_search]
# --8<-- [start:hybrid_search_pass_vector_text]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
(
table.search(query_type="hybrid")
.vector(vector_query)
.text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text]
@pytest.mark.skip
async def test_hybrid_search_async():
import openai
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
async_tbl = await async_db.create_table("documents_async", schema=Documents)
# ingest docs with auto-vectorization
await async_tbl.add(data)
# Create a fts index before the hybrid search
await async_tbl.create_index("text", config=FTS())
text_query = "flower moon"
# hybrid search with default re-ranker
await (await async_tbl.search("flower moon", query_type="hybrid")).to_pandas()
# --8<-- [end:basic_hybrid_search_async]
# --8<-- [start:hybrid_search_pass_vector_text_async]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
await (
async_tbl.query()
.nearest_to(vector_query)
.nearest_to_text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text_async]
docs/web-tests/py/test_search_examples.py
+228
View File
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_vector_search():
# --8<-- [start:vector_search]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
rows = (
table.search([1.0, 0.0])
.distance_type("l2")
.select(["id", "text", "_distance"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:vector_search]
assert [row["id"] for row in rows] == [1, 2]
assert rows[0]["_distance"] == 0.0
def test_text_search():
# --8<-- [start:text_search]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.index import FTS
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
},
],
)
table.create_index("text", config=FTS())
rows = (
table.search("puppy", query_type="fts", fts_columns="text")
.select(["id", "text", "_score"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:text_search]
assert [row["id"] for row in rows] == [1]
assert rows[0]["_score"] > 0
def test_hybrid_search():
# --8<-- [start:hybrid_search]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.index import FTS
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
from lancedb.rerankers import RRFReranker
table.create_index("text", config=FTS())
rows = (
table.search(
query_type="hybrid", vector_column_name="vector", fts_columns="text"
)
.vector([1.0, 0.0])
.text("puppy")
.rerank(RRFReranker())
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:hybrid_search]
assert [row["id"] for row in rows] == [1, 2]
assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"]
def test_filter_search():
# --8<-- [start:filter_search]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
rows = (
table.search([0.0, 1.0])
.where("category = 'animals'", prefilter=True)
.select(["id", "text", "category", "_distance"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:filter_search]
assert {row["id"] for row in rows} == {1, 2}
assert all(row["category"] == "animals" for row in rows)
def test_query_plan():
# --8<-- [start:query_plan]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2)
print("Plan:", query.explain_plan())
print("Measured execution:", query.analyze_plan())
rows = query.to_list()
print(rows)
# --8<-- [end:query_plan]
assert [row["id"] for row in rows] == [1, 2]

Some files were not shown because too many files have changed in this diff Show More