rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-19 02:02:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-2325-1

Browse Source
This commit is contained in:
Gatefixer
2026-10-16 18:58:03 +00:00
parent 7ea251b3d0 fe34eb3549
commit c91149c56b
593 changed files with 85326 additions and 17533 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.11"
current_version = "0.41.0-beta.7"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/actions/setup-protoc/action.yml
+80
View File
@@ -0,0 +1,80 @@
name: Setup protoc
description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves
inputs:
install:
description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there."
required: false
default: "true"
outputs:
version:
description: "The pinned protoc version, e.g. 36.2"
value: ${{ steps.pin.outputs.version }}
linux-x86_64-sha256:
description: "SHA-256 of the protoc linux-x86_64 zip"
value: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
linux-aarch_64-sha256:
description: "SHA-256 of the protoc linux-aarch_64 zip"
value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
runs:
using: composite
steps:
# protobuf does not publish checksum files for protoc. These are the digests
# GitHub records for the release assets; when bumping, copy them from
# `gh api repos/protocolbuffers/protobuf/releases/tags/v<version>`.
- name: Set protoc pin
id: pin
shell: bash
run: |
{
echo "version=36.2"
echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278"
echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a"
echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b"
echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3"
} >> "$GITHUB_OUTPUT"
- name: Install protoc
if: ${{ inputs.install == 'true' }}
shell: bash
env:
PROTOC_VERSION: ${{ steps.pin.outputs.version }}
RUNNER_OS_NAME: ${{ runner.os }}
RUNNER_ARCH_NAME: ${{ runner.arch }}
LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }}
WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }}
run: |
set -euo pipefail
case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in
Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;;
Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;;
macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;;
Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;;
*) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;;
esac
zip="$RUNNER_TEMP/protoc.zip"
dest="$RUNNER_TEMP/protoc"
curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip"
# Hash stdin rather than a path: Windows paths contain backslashes,
# which make sha256sum escape its output.
if [ "$RUNNER_OS_NAME" = "macOS" ]; then
actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1)
else
actual=$(sha256sum < "$zip" | cut -d' ' -f1)
fi
if [ "$actual" != "$sha256" ]; then
echo "protoc checksum mismatch: expected $sha256, got $actual" >&2
exit 1
fi
if [ "$RUNNER_OS_NAME" = "Windows" ]; then
# Git Bash on Windows runners does not reliably ship unzip.
7z x -y -o"$dest" "$zip" > /dev/null
else
unzip -q -o "$zip" -d "$dest"
fi
rm "$zip"
echo "$dest/bin" >> "$GITHUB_PATH"
"$dest/bin/protoc" --version
.github/release_notes.json
+5 -1
View File
@@ -6,6 +6,10 @@
"title": "## 🏆 Highlights",
"labels": ["highlight"]
},
{
"title": "## 🚨 Critical Fixes",
"labels": ["critical-fix"]
},
{
"title": "## 🛠 Breaking Changes",
"labels": ["breaking-change"]
@@ -38,4 +42,4 @@
"labels": ["ci"]
}
]
}
}
.github/workflows/build_linux_wheel/action.yml
+11 -4
View File
@@ -29,6 +29,11 @@ inputs:
runs:
using: "composite"
steps:
- name: Get protoc version
id: protoc
uses: ./.github/actions/setup-protoc
with:
install: "false"
- name: CONFIRM ARM BUILD
shell: bash
env:
@@ -54,13 +59,14 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
target: x86_64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
@@ -71,14 +77,15 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
target: aarch64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
yum install -y clang
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
.github/workflows/build_mac_wheel/action.yml
+1 -4
View File
@@ -12,10 +12,7 @@ inputs:
runs:
using: "composite"
steps:
- name: Install macos dependency
shell: bash
run: |
brew install protobuf
- uses: ./.github/actions/setup-protoc
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
.github/workflows/build_windows_wheel/action.yml
+1 -9
View File
@@ -12,15 +12,7 @@ inputs:
runs:
using: "composite"
steps:
- name: Install Protoc v21.12
working-directory: C:\
run: |
New-Item -Path 'C:\protoc' -ItemType Directory
Set-Location C:\protoc
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
7z x protoc.zip
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
shell: powershell
- uses: ./.github/actions/setup-protoc
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
.github/workflows/license-header-check.yml
+1
View File
@@ -9,6 +9,7 @@ on:
- python/**
- nodejs/**
- java/**
- docs/web-tests/**
- .github/workflows/license-header-check.yml
permissions:
.github/workflows/nodejs.yml
+65
View File
@@ -164,6 +164,71 @@ jobs:
python ci/mock_openai.py &
cd nodejs/examples
node_modules/.bin/jest --testEnvironment jest-environment-node-single-context --verbose
arrow-compat:
# The main jobs resolve `apache-arrow` to the version pinned in
# devDependencies (the newest supported release). This job re-runs the
# suite with the library itself running on each older Arrow release the
# peer range still accepts, which is what a consumer on that version gets.
name: Linux (Arrow ${{ matrix.arrow-version }})
timeout-minutes: 30
strategy:
matrix:
arrow-version: [ "19.0.1", "20.0.0" ]
runs-on: "ubuntu-22.04"
defaults:
run:
shell: bash
working-directory: nodejs
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- uses: pnpm/action-setup@v6
with:
version: 11.1.1
- uses: actions/setup-node@v6
with:
node-version: 24
cache: 'pnpm'
cache-dependency-path: nodejs/pnpm-lock.yaml
- uses: Swatinem/rust-cache@v2
with:
# Same crate, profile and target as the `linux` job above, so reuse
# its cache: rust-cache keys on the job id by default, and this job
# would otherwise start cold on every run until it had saved its own.
shared-key: linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y protobuf-compiler libssl-dev
- name: Build
run: |
pnpm install --frozen-lockfile
# No `--` separator: pnpm forwards it literally, which would
# make napi-rs treat `--profile ci` as a cargo passthrough arg.
pnpm build:debug --profile ci
- name: Resolve apache-arrow to ${{ matrix.arrow-version }}
run: |
# The older releases are already installed as the `apache-arrow-NN`
# test aliases. With pnpm's hoisted linker each alias is a complete,
# self-contained directory, so swap it in for the library's own
# `apache-arrow` instead of running a second install, which has been
# seen to hang after printing "Done" in CI. A copy, not a symlink: the
# cross-version tests rely on the alias being a separate module
# instance from the one the library loads.
major="$(echo '${{ matrix.arrow-version }}' | cut -d. -f1)"
mv node_modules/apache-arrow node_modules/apache-arrow.pinned
cp -R "node_modules/apache-arrow-${major}" node_modules/apache-arrow
# `require("apache-arrow/package.json")` is blocked by the package's exports map.
node -p "'apache-arrow resolved to ' + JSON.parse(require('fs').readFileSync('node_modules/apache-arrow/package.json')).version"
pnpm tsc
- name: Test
env:
# Newer @smithy/core uses dynamic ESM imports.
NODE_OPTIONS: "--experimental-vm-modules"
run: node_modules/.bin/jest --verbose
macos:
timeout-minutes: 30
# macos-15 ships a newer linker; the older macos-14 linker fails to insert
.github/workflows/pypi-publish.yml
+1
View File
@@ -11,6 +11,7 @@ on:
- .github/workflows/build_linux_wheel/action.yml
- .github/workflows/build_mac_wheel/action.yml
- .github/workflows/build_windows_wheel/action.yml
- .github/actions/setup-protoc/action.yml
- Cargo.toml # Change in dependency frequently breaks builds
- Cargo.lock
.github/workflows/python.yml
+1
View File
@@ -15,6 +15,7 @@ on:
- .github/workflows/build_linux_wheel/**
- .github/workflows/build_mac_wheel/**
- .github/workflows/build_windows_wheel/**
- .github/actions/setup-protoc/**
- .github/workflows/run_tests/**
permissions:
.github/workflows/rust.yml
+4 -35
View File
@@ -285,7 +285,7 @@ jobs:
runs-on: ubuntu-24.04
strategy:
matrix:
msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
msrv: ["1.94.0"] # This should match up with rust-version in Cargo.toml
env:
# Need up-to-date compilers for kernels
CC: clang-18
@@ -308,40 +308,9 @@ jobs:
# unreadable outside their own branch anyway, since GitHub scopes
# caches to the creating ref.
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Downgrade dependencies
# These packages have newer requirements for MSRV
run: |
cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
cargo update -p aws-sdk-dynamodb --precise 1.68.0
cargo update -p aws-config --precise 1.6.0
cargo update -p aws-sdk-kms --precise 1.63.0
cargo update -p aws-sdk-s3 --precise 1.79.0
cargo update -p aws-sdk-sso --precise 1.62.0
cargo update -p aws-sdk-ssooidc --precise 1.63.0
cargo update -p aws-sdk-sts --precise 1.63.0
# aws-runtime/sigv4/credential-types/types and the aws-smithy-*
# crates bumped their MSRV to 1.91.1 in late 2026; pin to the last
# 1.91.0-compatible versions. The order matters — each downgrade
# only succeeds once everything that still pins it at a higher
# version has itself been downgraded.
cargo update -p aws-runtime --precise 1.5.12
cargo update -p aws-types --precise 1.3.9
cargo update -p aws-sigv4 --precise 1.3.5
cargo update -p aws-credential-types --precise 1.2.8
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
cargo update -p aws-smithy-checksums --precise 0.63.13
cargo update -p aws-smithy-runtime --precise 1.9.3
cargo update -p aws-smithy-http --precise 0.62.6
cargo update -p aws-smithy-eventstream --precise 0.60.14
cargo update -p aws-smithy-http-client --precise 1.1.3
cargo update -p aws-smithy-observability --precise 0.1.4
cargo update -p aws-smithy-query --precise 0.60.8
cargo update -p aws-smithy-runtime-api --precise 1.9.3
cargo update -p aws-smithy-async --precise 1.2.7
cargo update -p aws-smithy-types --precise 1.3.6
cargo update -p aws-smithy-xml --precise 0.60.11
cargo update -p home --precise 0.5.9
- name: Downgrade dependencies
# aws-smithy-types 1.6.1+ requires rustc 1.94.1
run: cargo update -p aws-smithy-types --precise 1.5.0
- name: cargo +${{ matrix.msrv }} check
env:
RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
.typos.toml
+15 -1
View File
@@ -1,5 +1,15 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
extend-ignore-re = [
"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$",
# Conventional import aliases in the docs examples: `import fiftyone as fo`
# and `import torchvision.io as tio`.
"\\bas (fo|tio)\\b",
"\\btio\\.",
# The substring query in the n-gram full-text search example.
'search\("nce"',
# Garbled OCR output quoted verbatim in the VLM fine-tuning tutorial.
"permission nstallation",
]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
@@ -12,6 +22,10 @@ mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
# Superlinked's self-hosted inference engine (SIE), its `sie-lancedb` package,
# and the `sie` embedding function it registers, in
# docs/web/integrations/embedding/superlinked.mdx.
SIE = "SIE"
[files]
extend-exclude = [
CONTRIBUTING.md
+82 -1
View File
@@ -71,7 +71,88 @@ are wrappers around the Rust library.
* `nodejs`: Typescript package source code
* `node`: **Deprecated** Typescript package source code
* `java`: Java package source code
* `docs`: Documentation source code
* `docs/src`: SDK reference, built with mkdocs
* `docs/web`: the open-source pages of [docs.lancedb.com](https://docs.lancedb.com)
* `docs/web-tests`: the tests the documentation's code examples are extracted from
## Documentation
The open-source pages of [docs.lancedb.com](https://docs.lancedb.com) live in
`docs/web`, so a change to behaviour and the change to the page describing it
belong in the same pull request.
Preview them locally — this serves `docs/web` on its own, with no other checkout:
```bash
npm i -g mint # https://mintlify.com/docs/installation
cd docs/web && mint dev
```
This repository holds a page at every path the site publishes, so the whole
navigation resolves locally. Enterprise pages exist here as open-source pages --
what the capability is, and what the embedded form does instead. A separate
private repository supplies the fuller version of each, which replaces the page
at the same path when the published site is assembled.
Four things to know before editing:
**Code examples are not written into the pages.** They live in real tests under
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
imported by the pages. Edit the test, then regenerate from the repository root:
```bash
uv run docs/web-tests/mdx_snippets_gen.py -s docs/web-tests/py -s docs/web-tests/ts -s docs/web-tests/rs -o docs/web/snippets
```
Review the diff before committing: a few modules currently regenerate with
unrelated changes, so commit only the snippets for the examples you changed.
Run the Python, TypeScript and Rust examples in `docs/web-tests/{py,ts,rs}`
locally before submitting the change. These examples are not yet wired into
pull-request CI; that integration remains follow-up work.
**Every heading carries an explicit `{#anchor}`.** Those anchors are how the
Enterprise pages attach their additions to the right section, so they are
assigned once and never regenerated. Leave an existing anchor alone even when
you reword the heading above it; only new headings need a new one.
**A page can declare where it applies.** Its frontmatter says whether LanceDB
OSS and LanceDB Enterprise support the feature: use only `available` or
`unavailable`, and one sentence that explains its scope. Availability describes
product support, not whether a particular deployment has enabled or configured
the feature. A supported feature stays `available` when it requires setup, a
supported SDK or server version, or has differences between offerings. State
those requirements in the summary or the relevant section, including unsupported
subfeatures. Do not use `varies`, infer support from an unknown status, or mark an
absent capability available. Narrow a page to the capability it actually covers
when necessary. For example:
```yaml
availability:
oss: unavailable
enterprise: available
summary: >-
Requests to a deployment carry an API key or an OAuth token. Embedded
LanceDB has no server to sign in to.
```
The assembled site renders this as a label above the page's content and as a
sidebar tag when only one offering has the feature. Do not also open the page
with a badge of its own, and keep a qualification that only one section needs
in that section's text. A `mint dev` preview of `docs/web` alone renders
neither the labels nor the tags.
**A feature can be documented before its release.** List a draft page in
`docs/web/.mintignore`, keep it out of the navigation in `docs.json`, and do not
link to it from published pages. Wrap a draft section in an MDX comment,
`{/* ... */}`, whose first line names the release, such as
`Draft for the Feature X release.` Keep draft table rows and list items in a
comment below the table or list, since a comment between rows or items breaks
it. Drafts are hidden from readers but stay in the source, and availability does
not change what is published. At release, remove the comments and the
`.mintignore` entries, and restore the navigation and links. See
[Writing docs before release](https://github.com/lancedb/docs#writing-docs-before-release)
in the documentation repository's README for previews and release checks.
## Release process
Cargo.lock
Generated
+186 -287
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+21 -17
View File
@@ -10,23 +10,23 @@ repository = "https://github.com/lancedb/lancedb"
description = "Serverless, low-latency vector database for AI applications"
keywords = ["lancedb", "lance", "database", "vector", "search"]
categories = ["database-implementations"]
rust-version = "1.91.0"
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -72,13 +72,17 @@ oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = "1"
serde_json = { version = "1", features = ["preserve_order"] }
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[patch.crates-io]
# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision.
object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" }
[profile.ci]
debug = "line-tables-only"
inherits = "dev"
README.md
+2 -4
View File
@@ -1,8 +1,6 @@
<a href="https://cloud.lancedb.com" target="_blank">
<img src="https://github.com/user-attachments/assets/92dad0a2-2a37-4ce1-b783-0d1b4f30a00c" alt="LanceDB Cloud Public Beta" width="100%" style="max-width: 100%;">
</a>
<div align="center">
[![Reverie](docs/src/assets/reverie-banner.png)](https://www.reveriesummit.com/?utm_medium=social&utm_source=github&utm_campaign=reverie-2026)
[![LanceDB](docs/src/assets/hero-header.png)](https://lancedb.com)
[![Website](https://img.shields.io/badge/-Website-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://lancedb.com/)
[![Blog](https://img.shields.io/badge/Blog-100000?style=for-the-badge&labelColor=645cfb&color=645cfb)](https://blog.lancedb.com/)
@@ -13,7 +11,7 @@
<img src="docs/src/assets/lancedb.png" alt="LanceDB" width="50%">
# **The Multimodal AI Lakehouse**
# **The Multimodal Lakehouse for AI**
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
deny.toml
+23
View File
@@ -78,6 +78,27 @@ ignore = [
{ id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
{ id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
# rustls 0.23.45: four advisories fixed in 0.23.46 (Debug impls leaking
# TLS secrets, ECH rejection buffering, Brotli certificate decompression
# allocation, ServerHello key-exchange downgrade). 0.23.46 was published
# 2026-10-14 and the dependency age policy requires releases to be 48
# hours old, so the upgrade cannot land yet. Remove these entries and run
# `cargo update -p rustls` once the window has passed. Mirrors lance #9985.
# https://rustsec.org/advisories/RUSTSEC-2026-0346
# https://rustsec.org/advisories/RUSTSEC-2026-0347
# https://rustsec.org/advisories/RUSTSEC-2026-0348
# https://rustsec.org/advisories/RUSTSEC-2026-0349
{ id = "RUSTSEC-2026-0346", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
{ id = "RUSTSEC-2026-0347", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
{ id = "RUSTSEC-2026-0348", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
{ id = "RUSTSEC-2026-0349", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
# unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera
# → serde_yaml_ng. No maintained upgrade exists; clearing it requires
# lindera to move off serde_yaml_ng.
# https://rustsec.org/advisories/RUSTSEC-2024-0449
{ id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" },
# rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
# rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
# we actively use is upgraded to 0.103.13 which contains the fix.
@@ -197,4 +218,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
# releases are cut to crates.io. Allow that specific host.
allow-git = [
"https://github.com/lance-format/lance",
# Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml).
"https://github.com/apache/arrow-rs-object-store",
]
docs/README.md
+7 -4
View File
@@ -15,18 +15,21 @@ unreleased features.
### Preview the docs
Run these commands from the LanceDB repository root. The subshell changes into
`docs` to load `mkdocs.yml`, while leaving your shell in the repository root.
```shell
cd docs
mkdocs serve
(cd docs && PYTHONPATH=. mkdocs serve)
```
If you want to just generate the HTML files:
```shell
PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
(cd docs && PYTHONPATH=. mkdocs build)
```
If successful, you should see a `docs/site` directory that you can verify locally.
If successful, you should see a `docs/site` directory in the repository root that
you can verify locally.
## Adding examples
docs/src/assets/hero-header.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 MiB

After

Width:  |  Height:  |  Size: 1.3 MiB

docs/src/assets/reverie-banner.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 722 KiB

docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.11</version>
<version>0.41.0-beta.7</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+9 -7
View File
@@ -350,7 +350,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -417,25 +417,27 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
***
### useLsm()
### useMemWal()
```ts
useLsm(enable): this
useMemWal(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
the SSTables), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
@@ -446,7 +448,7 @@ reads the base table.
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
`useMemWal(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
@@ -455,7 +457,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useLsm`
`StandardQueryBase.useMemWal`
***
docs/src/js/classes/BlobFile.md
+101 -5
View File
@@ -10,16 +10,52 @@ A lazy handle to blob bytes. Create one with [Table.fetchBlobFiles](Table.md#fet
## Methods
### close()
```ts
close(): Promise<void>
```
Releases the handle. Reads after `close()` fail. Calling it again does
nothing.
#### Returns
`Promise`&lt;`void`&gt;
***
### isClosed()
```ts
isClosed(): boolean
```
Returns true after [BlobFile.close](BlobFile.md#close).
#### Returns
`boolean`
***
### read()
```ts
read(): Promise<Buffer>
read(maxBytes?, options?): Promise<Buffer>
```
Reads from the cursor to the end and advances the cursor.
Reads from the cursor and advances the cursor.
A second call returns an empty buffer. [BlobFile.readRange](BlobFile.md#readrange) does
not move the cursor.
Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
otherwise. Returns an empty buffer at the end of the blob.
[BlobFile.readRange](BlobFile.md#readrange) does not move the cursor.
#### Parameters
* **maxBytes?**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
@@ -30,7 +66,10 @@ not move the cursor.
### readRange()
```ts
readRange(start, end): Promise<Buffer>
readRange(
start,
end,
options?): Promise<Buffer>
```
Reads the half-open byte range `[start, end)`.
@@ -43,12 +82,55 @@ Fails when `end` is past the blob size. Does not move the cursor.
* **end**: `bigint`
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`&gt;
***
### readRanges()
```ts
readRanges(ranges, options?): Promise<Buffer[]>
```
Reads several half-open byte ranges. Returns one buffer per range, in
the order given.
Fails when any `end` is past the blob size. Does not move the cursor.
#### Parameters
* **ranges**: [`BlobRange`](../type-aliases/BlobRange.md)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;`Buffer`[]&gt;
***
### seek()
```ts
seek(position): Promise<void>
```
Moves the cursor to `position`, in bytes from the start of the blob.
#### Parameters
* **position**: `bigint`
#### Returns
`Promise`&lt;`void`&gt;
***
### size()
```ts
@@ -60,3 +142,17 @@ Returns the blob size in bytes.
#### Returns
`bigint`
***
### tell()
```ts
tell(): Promise<bigint>
```
Returns the cursor position in bytes.
#### Returns
`Promise`&lt;`bigint`&gt;
docs/src/js/classes/Catalog.md
+15 -4
View File
@@ -89,19 +89,30 @@ Drop an empty database. The server rejects nonempty databases.
### listDatabases()
```ts
listDatabases(options): Promise<ListDatabasesResponse>
listDatabases(options): DatabaseNames
```
List one page of databases; pass pageToken from a response for the next page.
Iterate lazily over all database names using `for await...of`.
pageToken resumes from a saved token; omitted starts at the beginning.
pageLimit limits each REST response, not the total; omitted uses the server default.
Request errors are raised during iteration and terminate the iterator.
#### Parameters
* **options** = `{}`
* **options.limit?**: `number`
* **options.pageLimit?**: `number`
* **options.pageToken?**: `string`
#### Returns
`Promise`&lt;[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)&gt;
[`DatabaseNames`](DatabaseNames.md)
#### Example
```ts
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/classes/Connection.md
+24 -12
View File
@@ -183,10 +183,6 @@ Define a materialized view named `name` over the table `source`.
The view is populated before creation returns. Set `withNoData` to create
only its definition and empty backing table. The view is a normal table:
it can be queried, indexed and searched, and it appears in `tableNames`.
The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists.
#### Parameters
@@ -615,30 +611,42 @@ Return true if the connection has not been closed
### listJobs()
```ts
abstract listJobs(): Promise<JobInfo[]>
abstract listJobs(options?): Listing<JobInfo>
```
List server-side jobs across the database's tables.
Iterate lazily over server-side jobs across the database's tables.
Use `for await...of`. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;[`JobInfo`](../interfaces/JobInfo.md)[]&gt;
[`Listing`](Listing.md)&lt;[`JobInfo`](../interfaces/JobInfo.md)&gt;
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
abstract listMaterializedViews(options?): Listing<string>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
Iterate lazily with `for await...of`. Local listings inspect table schemas
one page at a time. Options control page size and the starting token.
See [Listing](Listing.md) for pagination state and error behavior.
#### Parameters
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
@@ -739,20 +747,24 @@ A page of table names and an
### listViews()
```ts
abstract listViews(namespacePath?): Promise<string[]>
abstract listViews(namespacePath?, options?): Listing<string>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
Iterate lazily with `for await...of`. See [Listing](Listing.md) for pagination
state and error behavior; options control page size and the starting token.
#### Parameters
* **namespacePath?**: `string`[]
* **options?**: [`ListingOptions`](../interfaces/ListingOptions.md)
#### Returns
`Promise`&lt;`string`[]&gt;
[`Listing`](Listing.md)&lt;`string`&gt;
***
docs/src/js/classes/DatabaseNames.md
+79
View File
@@ -0,0 +1,79 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DatabaseNames
# Class: DatabaseNames
A lazy async iterator of database names with pagination state.
Returned by [Catalog.listDatabases](Catalog.md#listdatabases).
## Implements
- `AsyncIterableIterator`&lt;`string`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<string>
```
#### Returns
`AsyncIterableIterator`&lt;`string`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<string, any>>
```
Fetch the next name, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`string`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of names available without another REST request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next REST request. Initially this is the supplied starting token.
Returns undefined after the final page is fetched, even if names remain cached.
Drain the cache before saving a token to avoid skipping names when resuming.
A failed request terminates iteration and retains its token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/Listing.md
+91
View File
@@ -0,0 +1,91 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / Listing
# Class: Listing&lt;T&gt;
A lazy async iterator over database resources with pagination state.
Requests occur during iteration; request errors terminate the iterator.
## Example
```ts
for await (const name of connection.listViews([], { pageLimit: 20 })) {
console.log(name);
}
```
## Type Parameters
• **T**
## Implements
- `AsyncIterableIterator`&lt;`T`&gt;
## Methods
### \[asyncIterator\]()
```ts
asyncIterator: AsyncIterableIterator<T>
```
#### Returns
`AsyncIterableIterator`&lt;`T`&gt;
#### Implementation of
`AsyncIterableIterator.[asyncIterator]`
***
### next()
```ts
next(): Promise<IteratorResult<T, any>>
```
Fetch the next item, requesting another page only when needed.
#### Returns
`Promise`&lt;`IteratorResult`&lt;`T`, `any`&gt;&gt;
#### Implementation of
`AsyncIterableIterator.next`
***
### numPageResults()
```ts
numPageResults(): number
```
Number of cached items available without another request.
#### Returns
`number`
***
### pageToken()
```ts
pageToken(): undefined | string
```
Token for the next request. Initially this is the supplied starting token.
Returns undefined after fetching the final page, even with cached items.
Drain the cache before saving a token to avoid skipping items on resumption.
Failed requests terminate iteration and retain their token for a new iterator.
#### Returns
`undefined` \| `string`
docs/src/js/classes/MaterializedView.md
+3 -10
View File
@@ -65,21 +65,14 @@ refresh(options?): Promise<RefreshMaterializedViewResult>
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
Every refresh rebuilds the complete view. For local views,
`sourceVersion` refreshes to that source version instead of the latest;
remote SQL refreshes do not support source-version pinning.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
docs/src/js/classes/MergeInsertBuilder.md
+6 -6
View File
@@ -76,15 +76,15 @@ the query optimizer chooses a suboptimal path.
***
### useLsm()
### useMemWal()
```ts
useLsm(enable): MergeInsertBuilder
useMemWal(enable): MergeInsertBuilder
```
Control MemWAL routing for this merge.
By default (unset), a `mergeInsert` on a table with an LSM write spec is
By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
routed through Lance's MemWAL shard writer, and a table without one uses the
standard path.
@@ -92,7 +92,7 @@ standard path.
* **enable**: `boolean`
`true` forces MemWAL routing and errors if the table has no
LSM write spec. `false` forces the standard write path even when a spec is set.
MemWAL write spec. `false` forces the standard write path even when a spec is set.
#### Returns
@@ -108,12 +108,12 @@ validateSingleShard(validateSingleShard): MergeInsertBuilder
Controls how an LSM merge checks that its input targets a single shard.
When a table has an LSM write spec, every row in a `mergeInsert` call must
When a table has a MemWAL write spec, every row in a `mergeInsert` call must
route to the same shard. When `true` (the default), every row is inspected
to verify this. When `false`, only the first row is inspected and the
shard it routes to is used for the whole input — a faster path for callers
that have already pre-sharded their input. Has no effect on tables without
an LSM write spec.
a MemWAL write spec.
#### Parameters
docs/src/js/classes/Query.md
+9 -7
View File
@@ -424,7 +424,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -491,25 +491,27 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
***
### useLsm()
### useMemWal()
```ts
useLsm(enable): this
useMemWal(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
the SSTables), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
@@ -520,7 +522,7 @@ reads the base table.
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
`useMemWal(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
@@ -529,7 +531,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useLsm`
`StandardQueryBase.useMemWal`
***
docs/src/js/classes/QueryBase.md
+3 -1
View File
@@ -193,7 +193,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -252,6 +252,8 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
***
### withRowId()
docs/src/js/classes/Table.md
+134 -118
View File
@@ -143,7 +143,8 @@ containing the new version number of the table after altering the columns.
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns, including nested dotted paths.
Blob v2 columns addressable by row ID, including struct-nested dotted paths.
Blobs inside lists are omitted because a row ID does not identify an element.
#### Returns
@@ -227,17 +228,17 @@ version of the table.
***
### checkpointLsm()
### checkpointMemWal()
```ts
abstract checkpointLsm(): Promise<void>
abstract checkpointMemWal(): Promise<void>
```
Converge this table's LSM write path into its base table.
Converge this table's MemWAL write path into its base table.
Seals once, then triggers compaction and polls until the L0 that existed
Freezes once, then triggers compaction and polls until the SSTables that existed
at the start is gone. The target set is fixed at the start, so
generations created *during* the checkpoint are ignored — that is what
SSTables created *during* the checkpoint are ignored — that is what
lets it terminate under write load, and what makes it best-effort: it
converges the fresh tier as of some instant. Idempotent, abandonable at
any point, and safe to run on a cadence.
@@ -253,9 +254,9 @@ is merging. The caller owns the deadline.
#### Example
```ts
const before = await table.getLsmStats();
await table.checkpointLsm();
const after = await table.getLsmStats();
const before = await table.getMemWalStats();
await table.checkpointMemWal();
const after = await table.getMemWalStats();
```
***
@@ -278,15 +279,15 @@ Any attempt to use the table after it is closed will result in an error.
***
### closeLsmWriters()
### closeMemWalWriters()
```ts
abstract closeLsmWriters(): Promise<void>
abstract closeMemWalWriters(): Promise<void>
```
Drain and close any cached MemWAL shard writers held for this table.
When an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) is installed, `mergeInsert` opens MemWAL
When an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) is installed, `mergeInsert` opens MemWAL
shard writers and caches them for reuse across calls. This closes them,
flushing pending data; writers reopen lazily on the next `mergeInsert`.
It is a no-op when no writers are cached.
@@ -297,17 +298,17 @@ It is a no-op when no writers are cached.
***
### compactLsm()
### compactMemWal()
```ts
abstract compactLsm(): Promise<void>
abstract compactMemWal(): Promise<void>
```
Trigger a background L0 → base compaction pass per bucket.
Trigger a background SSTable compaction pass per table shard.
Returns once the passes are *dispatched*, not once they finish — watch
[Table#getLsmStats](Table.md#getlsmstats) for progress, or use
[Table#checkpointLsm](Table.md#checkpointlsm) to wait for convergence.
[Table#getMemWalStats](Table.md#getmemwalstats) for progress, or use
[Table#checkpointMemWal](Table.md#checkpointmemwal) to wait for convergence.
#### Returns
@@ -448,6 +449,23 @@ containing the new version number of the table
***
### disableMemWal()
```ts
abstract disableMemWal(): Promise<void>
```
Remove the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) from this table, reverting to the standard
`mergeInsert` write path.
Errors if no spec is currently set.
#### Returns
`Promise`&lt;`void`&gt;
***
### display()
```ts
@@ -513,10 +531,61 @@ Drop an index from the table.
***
### enableMemWal()
```ts
abstract enableMemWal(spec): Promise<void>
```
Install an [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) on this table, selecting Lance's MemWAL
LSM-style write path for future `mergeInsert` calls.
`MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
- `"bucket"` — hash-bucket writes by the single-column unenforced primary
key (`column` and `numBuckets` required).
- `"identity"` — shard by the raw value of a scalar `column`.
- `"unsharded"` — route every write to a single shard.
All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)
The sharding spec to install.
#### Returns
`Promise`&lt;`void`&gt;
#### Example
```ts
await table.setUnenforcedPrimaryKey("id");
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 16,
maintainedIndexes: ["id_idx"],
});
```
***
### fetchBlobFiles()
```ts
abstract fetchBlobFiles(column, rowIds): Promise<(null | BlobFile)[]>
abstract fetchBlobFiles(
column,
rowIds,
options?): Promise<(null | BlobFile)[]>
```
Opens lazy blob handles for `column` at the given row IDs using the
@@ -524,6 +593,8 @@ table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
One missing or deleted row ID rejects the entire batch, including on remote
tables; no partial handles are returned.
#### Parameters
@@ -531,6 +602,8 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| [`BlobFile`](BlobFile.md))[]&gt;
@@ -540,7 +613,10 @@ See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
### fetchBlobs()
```ts
abstract fetchBlobs(column, rowIds): Promise<(null | Buffer)[]>
abstract fetchBlobs(
column,
rowIds,
options?): Promise<(null | Buffer)[]>
```
Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
@@ -548,6 +624,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
One missing or deleted row ID rejects the entire batch, including on remote
tables; no partial results are returned.
Blobs inside lists cannot be fetched by row ID.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
@@ -559,21 +639,23 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
* **rowIds**: readonly (`number` \| `bigint`)[]
* **options?**: [`BlobReadOptions`](../type-aliases/BlobReadOptions.md)
#### Returns
`Promise`&lt;(`null` \| `Buffer`)[]&gt;
***
### flushLsm()
### flushMemWal()
```ts
abstract flushLsm(): Promise<void>
abstract flushMemWal(): Promise<void>
```
Seal every bucket's active memtable into a new L0 generation.
Freeze every table shard's active memtable into a new SSTable.
Returns once the seal is committed. Sealing an empty memtable is a no-op,
Returns once the freeze is committed. Freezing an empty memtable is a no-op,
so this is safe to call repeatedly.
#### Returns
@@ -617,49 +699,48 @@ const { records, truncated } = await table.functionErrors({ column: "embedding"
***
### getLsmStats()
### getMemWalSpec()
```ts
abstract getLsmStats(includeGenerationRows?): Promise<undefined | LsmStats>
abstract getMemWalSpec(): Promise<undefined | MemWalWriteSpec>
```
Read live per-bucket LSM state.
Read the [MemWalWriteSpec](../interfaces/MemWalWriteSpec.md) currently installed on this table.
Answers "how far behind is my fresh tier", "which bucket is hot", and
Resolves to `undefined` when the MemWAL write path is not enabled (no
spec has been set, or it was removed with [Table#disableMemWal](Table.md#disablememwal)).
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
`Promise`&lt;`undefined` \| [`MemWalWriteSpec`](../interfaces/MemWalWriteSpec.md)&gt;
***
### getMemWalStats()
```ts
abstract getMemWalStats(includeSstableRows?): Promise<undefined | MemWalStats>
```
Read live per-table-shard MemWAL state.
Answers "how far behind is my fresh tier", "which table shard is hot", and
"why is my fresh-tier vector search brute-force". Mutates no table state.
Resolves to `undefined` only when the LSM write path is not enabled.
Resolves to `undefined` only when the MemWAL write path is not enabled.
#### Parameters
* **includeGenerationRows?**: `boolean`
Also count rows per L0 generation.
* **includeSstableRows?**: `boolean`
Also count rows per SSTable.
Off by default because each count opens an uncached Lance dataset.
#### Returns
`Promise`&lt;`undefined` \| [`LsmStats`](../interfaces/LsmStats.md)&gt;
***
### getLsmWriteSpec()
```ts
abstract getLsmWriteSpec(): Promise<undefined | LsmWriteSpec>
```
Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table.
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
#### Returns
`Promise`&lt;`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)&gt;
`Promise`&lt;`undefined` \| [`MemWalStats`](../interfaces/MemWalStats.md)&gt;
***
@@ -1077,54 +1158,6 @@ of the given query
***
### setLsmWriteSpec()
```ts
abstract setLsmWriteSpec(spec): Promise<void>
```
Install an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) on this table, selecting Lance's MemWAL
LSM-style write path for future `mergeInsert` calls.
`LsmWriteSpec` chooses one of three sharding strategies via `specType`:
- `"bucket"` — hash-bucket writes by the single-column unenforced primary
key (`column` and `numBuckets` required).
- `"identity"` — shard by the raw value of a scalar `column`.
- `"unsharded"` — route every write to a single shard.
All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
The sharding spec to install.
#### Returns
`Promise`&lt;`void`&gt;
#### Example
```ts
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({
specType: "bucket",
column: "id",
numBuckets: 16,
maintainedIndexes: ["id_idx"],
});
```
***
### setUnenforcedPrimaryKey()
```ts
@@ -1279,23 +1312,6 @@ same tokenizer model files must also exist locally.
***
### unsetLsmWriteSpec()
```ts
abstract unsetLsmWriteSpec(): Promise<void>
```
Remove the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) from this table, reverting to the standard
`mergeInsert` write path.
Errors if no spec is currently set.
#### Returns
`Promise`&lt;`void`&gt;
***
### update()
#### update(opts)
@@ -1371,7 +1387,7 @@ repeatedly calling this method.
##### Parameters
* **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **updates**: `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
the
columns to update
docs/src/js/classes/TakeQuery.md
+5 -3
View File
@@ -200,7 +200,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -267,16 +267,18 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
***
### useLsm()
### useMemWal()
```ts
useLsm(enable): this
useMemWal(enable): this
```
Control MemWAL read routing for this take query.
docs/src/js/classes/VectorQuery.md
+48 -13
View File
@@ -447,6 +447,8 @@ nprobes(nprobes): VectorQuery
Set the number of partitions to search (probe)
The number of probes must be greater than 0.
This argument is only used when the vector column has an IVF PQ index.
If there is no index then this value is ignored.
@@ -458,17 +460,16 @@ exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
For more fine grained control over behavior when you have a very narrow filter
you can use `minimumNprobes` and `maximumNprobes`. This method sets both
the minimum and maximum to the same value.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
#### Parameters
@@ -645,6 +646,38 @@ rerank(reranker): VectorQuery
***
### searchEffort()
```ts
searchEffort(searchEffort): VectorQuery
```
Set the initial IVF search effort to a finite value in [0, 1].
The default, null, uses Auto (internally 0.5). Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The MemWAL LSM
scanner does not yet support this option.
#### Parameters
* **searchEffort**: `null` \| `number`
#### Returns
[`VectorQuery`](VectorQuery.md)
#### Example
```ts
table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
```
***
### select()
```ts
@@ -673,7 +706,7 @@ input to this method would be:
#### Parameters
* **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
* **columns**: `string` \| `string`[] \| `Map`&lt;`string`, `string`&gt; \| `Record`&lt;`string`, `string`&gt;
#### Returns
@@ -740,25 +773,27 @@ Collect the results as an Arrow
ArrowTable.
Empty results retain the query's output schema.
#### Inherited from
`StandardQueryBase.toArrow`
***
### useLsm()
### useMemWal()
```ts
useLsm(enable): this
useMemWal(enable): this
```
Control MemWAL read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
[Table#enableMemWal](Table.md#enablememwal)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
the SSTables), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
@@ -769,7 +804,7 @@ reads the base table.
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
`useMemWal(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
#### Returns
@@ -778,7 +813,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useLsm`
`StandardQueryBase.useMemWal`
***
docs/src/js/functions/blob.md
+2
View File
@@ -14,6 +14,8 @@ Declares a `lance.blob.v2` column.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
Blobs inside lists can be written and queried as descriptors, but cannot
currently be fetched because a row ID does not identify a list element.
## Parameters
docs/src/js/functions/connectCatalog.md
+3 -1
View File
@@ -28,5 +28,7 @@ headers; opened database connections inherit authentication and client options.
```ts
const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
const db = await catalog.createDatabase("analytics", { existOk: true });
const page = await catalog.listDatabases({ limit: 20 });
for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
console.log(name);
}
```
docs/src/js/globals.md
+13 -5
View File
@@ -27,9 +27,11 @@
- [Branches](classes/Branches.md)
- [Catalog](classes/Catalog.md)
- [Connection](classes/Connection.md)
- [DatabaseNames](classes/DatabaseNames.md)
- [HeaderProvider](classes/HeaderProvider.md)
- [Index](classes/Index.md)
- [Job](classes/Job.md)
- [Listing](classes/Listing.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
@@ -64,7 +66,6 @@
- [BranchDiff](interfaces/BranchDiff.md)
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
@@ -96,7 +97,6 @@
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
- [GenerationStats](interfaces/GenerationStats.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
- [IndexConfig](interfaces/IndexConfig.md)
@@ -108,14 +108,14 @@
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemWalStats](interfaces/MemWalStats.md)
- [MemWalWriteSpec](interfaces/MemWalWriteSpec.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeResult](interfaces/MergeResult.md)
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
@@ -138,7 +138,9 @@
- [SplitHashOptions](interfaces/SplitHashOptions.md)
- [SplitRandomOptions](interfaces/SplitRandomOptions.md)
- [SplitSequentialOptions](interfaces/SplitSequentialOptions.md)
- [SsTableStats](interfaces/SsTableStats.md)
- [TableNamesOptions](interfaces/TableNamesOptions.md)
- [TableShardStats](interfaces/TableShardStats.md)
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
@@ -157,9 +159,15 @@
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
- [BlobUri](type-aliases/BlobUri.md)
- [Data](type-aliases/Data.md)
- [DataLike](type-aliases/DataLike.md)
- [DataTypeLike](type-aliases/DataTypeLike.md)
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
docs/src/js/interfaces/CherryPickResult.md
+3 -3
View File
@@ -38,9 +38,9 @@ preview: CherryPickPreview;
```ts
status:
| "failed"
| "unknown"
| "ready"
| "failed"
| "notImplemented"
| "cherryPicked";
| "cherryPicked"
| "unknown";
```
docs/src/js/interfaces/ClientConfig.md
+12
View File
@@ -8,6 +8,18 @@
## Properties
### blobRequestConcurrency?
```ts
optional blobRequestConcurrency: number;
```
Maximum number of concurrent blob HTTP requests across all tables and
blob handles on this connection. Defaults to 8. Can also be set with
`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
***
### extraHeaders?
```ts
docs/src/js/interfaces/DropNamespaceOptions.md
+1 -1
View File
@@ -21,7 +21,7 @@ Refuse to drop if non-empty (restrict) or drop recursively (cascade).
### mode?
```ts
optional mode: "fail" | "skip";
optional mode: "skip" | "fail";
```
Whether to skip if the namespace doesn't exist, or fail.
docs/src/js/interfaces/FtsOptions.md
+2
View File
@@ -37,6 +37,8 @@ The following tokenizers are available:
"raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
"code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
"icu" - ICU dictionary-based word segmentation.
"icu/split" - ICU segmentation with simple-style delimiter splitting.
docs/src/js/interfaces/GenerationStats.md
-40
View File
@@ -1,40 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / GenerationStats
# Interface: GenerationStats
One flushed L0 generation.
## Properties
### bytes
```ts
bytes: number;
```
On-disk size of the generation.
***
### generation
```ts
generation: number;
```
The generation number. Increases as memtables are sealed into L0.
***
### rows?
```ts
optional rows: number;
```
Present only when `includeGenerationRows` was requested. Off by default
because each count opens an uncached Lance dataset.
docs/src/js/interfaces/ListDatabasesResponse.md
-23
View File
@@ -1,23 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListDatabasesResponse
# Interface: ListDatabasesResponse
## Properties
### databases
```ts
databases: string[];
```
***
### pageToken?
```ts
optional pageToken: string;
```
docs/src/js/interfaces/ListTablesOptions.md
+1
View File
@@ -15,6 +15,7 @@ optional limit: number;
```
An upper bound on how many tables to return.
Defaults to 100. Zero returns an empty page without a continuation token.
A page may hold fewer than this and still not be the last one, so keep
going while the response carries a page token rather than while pages are
docs/src/js/interfaces/ListingOptions.md
+29
View File
@@ -0,0 +1,29 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListingOptions
# Interface: ListingOptions
Pagination controls for lazy resource listings.
## Properties
### pageLimit?
```ts
optional pageLimit: number;
```
Maximum results per request, not a limit on the whole listing.
***
### pageToken?
```ts
optional pageToken: string;
```
Starting continuation token; omitted starts at the beginning.
docs/src/js/interfaces/LsmStats.md
-22
View File
@@ -1,22 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / LsmStats
# Interface: LsmStats
Live per-bucket LSM state, as returned by `Table#getLsmStats`.
Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
the caller's to compute.
## Properties
### buckets
```ts
buckets: BucketStats[];
```
One entry per bucket backing this table.
docs/src/js/interfaces/MemWalStats.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MemWalStats
# Interface: MemWalStats
Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
the caller's to compute.
## Properties
### tableShards
```ts
tableShards: TableShardStats[];
```
One entry per table shard backing this table.
docs/src/js/interfaces/LsmWriteSpec.md → docs/src/js/interfaces/MemWalWriteSpec.md
+4 -5
View File
@@ -2,9 +2,9 @@
***
[@lancedb/lancedb](../globals.md) / LsmWriteSpec
[@lancedb/lancedb](../globals.md) / MemWalWriteSpec
# Interface: LsmWriteSpec
# Interface: MemWalWriteSpec
Specification selecting Lance's MemWAL LSM-style write path for
`mergeInsert`.
@@ -34,9 +34,8 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
Indexes the MemWAL keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/interfaces/MemtableStats.md
+1 -1
View File
@@ -36,7 +36,7 @@ Estimated in-memory size.
generation: number;
```
The generation this memtable will become once sealed.
The generation this memtable will become once frozen.
***
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+1 -1
View File
@@ -14,7 +14,7 @@
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
How the view was brought up to date: "rebuild".
***
docs/src/js/interfaces/SsTableStats.md
+40
View File
@@ -0,0 +1,40 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SsTableStats
# Interface: SsTableStats
One SSTable.
## Properties
### bytes
```ts
bytes: number;
```
On-disk size of the SSTable.
***
### generation
```ts
generation: number;
```
The generation number. Increases as memtables are frozen into SSTables.
***
### rows?
```ts
optional rows: number;
```
Present only when `includeSstableRows` was requested. Off by default
because each count opens an uncached Lance dataset.
docs/src/js/interfaces/TableNamesOptions.md
+1 -1
View File
@@ -19,7 +19,7 @@ instead.
optional limit: number;
```
An optional limit to the number of results to return.
The maximum number of names to return. Omitted returns all names; zero returns none.
***
docs/src/js/interfaces/BucketStats.md → docs/src/js/interfaces/TableShardStats.md
+17 -17
View File
@@ -2,12 +2,12 @@
***
[@lancedb/lancedb](../globals.md) / BucketStats
[@lancedb/lancedb](../globals.md) / TableShardStats
# Interface: BucketStats
# Interface: TableShardStats
Live state of one bucket. A table is N buckets on one node; flattening to a
single number hides the one hot bucket that is usually why someone opened
Live state of one table shard. A table is N table shards on one node; flattening to a
single number hides the one hot table shard that is usually why someone opened
this endpoint.
## Properties
@@ -18,7 +18,7 @@ this endpoint.
compacting: boolean;
```
Whether a pass owns this bucket's compaction latch right now. Says *a*
Whether a pass owns this table shard's compaction latch right now. Says *a*
driver is running, not *whose*, and the latch is held from dispatch —
including while the pass queues for a pod-wide compactor permit. Read it
as "do not pile on", never as "mine is progressing".
@@ -35,16 +35,6 @@ The generation the active memtable will become.
***
### generations
```ts
generations: GenerationStats[];
```
Flushed L0 generations not yet merged into the base table.
***
### manifestVersion
```ts
@@ -61,7 +51,7 @@ Version of the shard manifest these numbers were read from.
optional memtables: MemtableStats[];
```
Oldest first, active last. Absent for a `"Sealed"` bucket, whose
Oldest first, active last. Absent for a `"Sealed"` table shard, whose
in-memory state is torn down.
***
@@ -82,7 +72,17 @@ WAL position replay resumes from.
shardId: string;
```
The shard this bucket writes.
The id of this table shard.
***
### sstables
```ts
sstables: SsTableStats[];
```
SSTables not yet merged into the base table.
***
docs/src/js/interfaces/TableStatistics.md
+17
View File
@@ -18,6 +18,23 @@ Statistics on table fragments
***
### numDeletedRows?
```ts
optional numDeletedRows: number;
```
The number of rows marked as deleted across all fragments of the table
These rows are not included in `numRows`, but still occupy space on disk
until the table is compacted, so a large value here indicates that the
table should be optimized. Fragments in which every row was deleted are
dropped outright, so their rows are not counted here.
Absent (`undefined`) when the backend does not report deletion counts.
***
### numIndices
```ts
docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md
+2 -2
View File
@@ -52,7 +52,7 @@ new EmbeddingFunction<T, M>(): EmbeddingFunction<T, M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -63,7 +63,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
***
docs/src/js/namespaces/embedding/classes/TextEmbeddingFunction.md
+2 -2
View File
@@ -37,7 +37,7 @@ new TextEmbeddingFunction<M>(): TextEmbeddingFunction<M>
### computeQueryEmbeddings()
```ts
computeQueryEmbeddings(data): Promise<number[] | Uint8Array | Float32Array | Float64Array>
computeQueryEmbeddings(data): Promise<Float32Array | Float64Array | Uint8Array | number[]>
```
Compute the embeddings for a single query
@@ -48,7 +48,7 @@ Compute the embeddings for a single query
#### Returns
`Promise`&lt;`number`[] \| `Uint8Array` \| `Float32Array` \| `Float64Array`&gt;
`Promise`&lt;`Float32Array` \| `Float64Array` \| `Uint8Array` \| `number`[]&gt;
#### Overrides
docs/src/js/type-aliases/BaseTokenizer.md
+1
View File
@@ -12,6 +12,7 @@ type BaseTokenizer:
| "whitespace"
| "raw"
| "ngram"
| "code"
| "icu"
| "icu/split"
| `jieba/${string}`
docs/src/js/type-aliases/BlobData.md
+16
View File
@@ -0,0 +1,16 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobData
# Type Alias: BlobData
```ts
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
```
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
docs/src/js/type-aliases/BlobInput.md
+36
View File
@@ -0,0 +1,36 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobInput
# Type Alias: BlobInput
```ts
type BlobInput:
| BlobData
| BlobUri
| object
| object
| null
| undefined;
```
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
## Example
```ts
import { pathToFileURL } from "node:url";
import type { BlobInput } from "@lancedb/lancedb";
const rows: { id: bigint; image: BlobInput }[] = [
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
{ id: 2n, image: pathToFileURL("/data/cat.png") },
{ id: 3n, image: { data: new Blob(["hello"]) } },
];
await table.add(rows);
```
docs/src/js/type-aliases/BlobRange.md
+27
View File
@@ -0,0 +1,27 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobRange
# Type Alias: BlobRange
```ts
type BlobRange: object;
```
A half-open byte range `[start, end)` for [BlobFile.readRanges](../classes/BlobFile.md#readranges).
## Type declaration
### end
```ts
end: bigint;
```
### start
```ts
start: bigint;
```
docs/src/js/type-aliases/BlobReadOptions.md
+24
View File
@@ -0,0 +1,24 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobReadOptions
# Type Alias: BlobReadOptions
```ts
type BlobReadOptions: object;
```
Options for blob reads.
## Type declaration
### signal?
```ts
optional signal: AbortSignal;
```
Cancels the read. The call rejects with `signal.reason`, and the native
read stops, including in-flight requests to a remote table.
docs/src/js/type-aliases/BlobUri.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobUri
# Type Alias: BlobUri
```ts
type BlobUri: string | URL;
```
A URI accepted for a blob value. A `URL` is stored as its `href`.
docs/src/js/type-aliases/DataTypeLike.md
+18
View File
@@ -0,0 +1,18 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / DataTypeLike
# Type Alias: DataTypeLike
```ts
type DataTypeLike: DataType | object;
```
A `DataType` from any copy or version of apache-arrow.
Arrow 21 brands its classes with `unique symbol` properties, so a type
object from a second copy of the library no longer satisfies the `DataType`
type of this one even though it is structurally identical. Inputs that only
need to be sanitized accept this looser shape instead.
docs/src/python/python.md
+197 -29
View File
@@ -26,6 +26,14 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.db.DBConnection
Resource listings (`list_secrets`, `list_views`, `list_jobs`, `list_functions`, and
`list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to
collect synchronous results, or `[name async for name in db.list_views()]` with
an asynchronous connection. `page_limit` controls each request, and `page_token`
resumes a listing. Drain the iterator's cached results before saving its token.
::: lancedb.listing.Listing
::: lancedb.Session
## Catalogs (Synchronous)
@@ -37,7 +45,171 @@ are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.catalog.Catalog
::: lancedb.catalog.ListDatabasesResponse
::: lancedb.catalog.DatabaseNames
## Authorization
Some catalogs support the authorization API. This is an API that can be used to
bind access control lists (ACLs) to individual catalog objects.
Each ACL contains a subject, an object, and privilege.
### Subjects
There are three kinds of subjects: principals, groups, and roles.
A principal uniquely identifies each user accessing the system. Each LanceDB
Enterprise API key corresponds to a unique principal, and each OpenID user has
a unique principal.
All principals have an immutable ID which never changes, and is safe to include
in log files. For example, the API key
`sk_AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPQ====` maps to the
principal ID `pk_aaaqeayeaudaocajbifqydiob4`. APIs such as `list_acls` will
return the principal ID, never the API key.
OpenID principals also typically have a human-readable name associated with
them. For example, a principal might be named `bob@example.com` and might have
a principal ID of `8a004400-9443-4fab-9997-b9372a7ca8fd`. Whenever
possible, APIs will return the principal name for convenience. However,
principal names are mutable and can be changed by the identity provider.
Here is an example of getting information about an API key:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
principal = catalog.authz.get_principal(Subject.principal_api_key(api_key))
print(f"Found principal {principal.principal_id}")
```
OpenID principals can be members of groups. Like principals, groups also have
IDs and human-readable names. It is not possible to modify group membership via
this API; it is managed by the OpenID identity provider.
Here is an example of getting information about a group based on its name:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
group = catalog.authz.get_group(Subject.group_name("Engineering"))
print(f"Found group {group.group_id} with name {group.group_name}")
```
Roles can contain either principals or groups. Unlike groups, they are managed
by the catalog, not the identity provider. Roles can also contain API key
principals.
Here is an example of managing the `reader` role:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
catalog.authz.add_role_binding(subject=Subject.principal_name("Alice"), role="reader")
```
It may take a few milliseconds for changes to role bindings to become visible
to all clients.
More subject types may be added in the future.
### Objects
An object is any resource in the system that we can attach authorization rules to.
The Python builders and Rust client share resource validation and wire encoding.
Methods also accept literal strings such as `table:tenant/db:public$events`;
malformed names for known resource types raise `ValueError` locally before a
request is sent. Unknown resource types are preserved for compatibility with
newer servers.
Object types include:
The System object, a singleton which represents the entire system. You can
refer to it as `Object.system()`.
Database objects which represents specific databases. For example,
`Object.database("analytics")` represents the "analytics" database.
Table objects which represent specific tables. For example,
`Object.table(database="analytics", name="events")` represents the "events"
table inside the "analytics" database. Tables are in the default namespace (which
is named "public") unless a specific namespace is given.
Similarly, View objects represent specific views. For example,
`Object.view(database="analytics", name="recent_events")` represents the
"recent_events" view inside the "analytics" database. Views are in the default
namespace (which is named "public") unless a specific namespace is given.
Secret objects identify named credentials, for example,
`Object.secret(database="analytics", name="api-key")`. Function objects identify
registered functions, for example,
`Object.function(database="analytics", name="caption")`. Both default to the
`public` namespace. Pass `namespace_path=["platform", "ml"]` to select a nested
namespace. These objects identify resources for ACL operations; a secret object
contains the credential's name, not its value.
More object types may be added in the future.
### Access control lists
An access control list attaches a specific set of privileges to a specific
(subject, object) pair. So, for example, we might have an access control list
that gives `bob@example.com` the ability to read the foo table, that looks like
this:
```python
from lancedb.authz import Object, Subject, Privilege
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
catalog.authz.add_acl(
object=Object.table(database="analytics", name="foo"),
subject=Subject.principal_name("bob@example.com"),
privilege=Privilege.SELECT,
)
```
Reading a table generally requires USAGE on its database and namespace, plus
SELECT on the table. Each object has at most one owner.
OWNERSHIP is a special privilege which grants all the other privileges. By
default, the principal who created an object owns it. Only one subject can own
an object; granting OWNERSHIP to a new subject removes it from the previous
owner. OWNERSHIP cannot be deleted.
To delegate secret or function creation, grant `CREATE_SECRET` or
`CREATE_FUNCTION` on the containing namespace. This preserves namespace ownership:
```python
for privilege in [Privilege.CREATE_SECRET, Privilege.CREATE_FUNCTION]:
catalog.authz.add_acl(
object=Object.namespace(database="analytics", namespace_path=["public"]),
subject=Subject.role("developer"),
privilege=privilege,
)
```
Both privileges also accept their string names in synchronous and asynchronous
ACL methods.
By default, the `list_acls` function lists all the access control entries in
the system. By adding a function parameter, the output is filtered to just ACLs
matching that parameter. For example, this code lists all the ACLs for the
"reader" role:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
print(entry)
```
Or, when using the async API:
```python
catalog = await lancedb.connect_catalog_async(endpoint, api_key=api_key)
async for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
print(entry)
```
## Remote SQL
@@ -77,6 +249,15 @@ print(query.describe().status)
for batch in query.reader():
print(batch.num_rows)
# Values bind to $1 / $name placeholders and travel as Arrow, so a float32
# stays a float32 and a vector stays a compact fixed-size list:
import numpy as np
reader = db.execute_query(
"SELECT id FROM docs ORDER BY distance(vector, $vector) LIMIT $k",
parameters={"vector": np.random.rand(768).astype(np.float32), "k": 10},
)
# The async connection exposes the same lifecycle without blocking:
# async_db = await lancedb.connect_async(
# "db://analytics",
@@ -118,7 +299,7 @@ listing a storage directory.
::: lancedb.table.Branches
::: lancedb.LsmWriteSpec
::: lancedb.MemWalWriteSpec
## Functions and Jobs
@@ -194,39 +375,14 @@ listing a storage directory.
::: lancedb.sql.QueryDescription
::: lancedb.sql.QueryParameters
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
::: lancedb.vector_dedup
::: lancedb.materialized_view.VectorDedupSource
For an existing indexed source, declare and materialize a dedup result without
writing SQL. Creation captures the source version once and waits for the complete
result; remote connections use the service's MV jobs.
```python
from lancedb import vector_dedup
view = db.create_materialized_view(
"images_clean",
vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
```
To submit without waiting, call `db.create_materialized_view_async(...)` with the
same source expression and wait on the returned job. To declare only, pass
`with_no_data=True`, then use `view.refresh_async().wait()` when ready. Pass
`dataset_version=...` to `vector_dedup` for an explicit snapshot. The default
policy retains direct representatives (A-B-C without A-C keeps A and C), preserves
the source, and materializes its original columns. Only within-segment/partition
pairs are covered, using the index's distance representation.
## Views
::: lancedb.view.ViewDescription
@@ -351,6 +507,12 @@ pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
still work. Queries return descriptors. Call
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
One missing or deleted row ID rejects the entire blob-read batch; the APIs do
not return partial results.
Blobs inside lists can be written and queried as descriptors. They are omitted
from `blob_columns()` because row ids do not identify individual list elements,
so the fetch APIs cannot read their bytes yet.
::: lancedb.blob
@@ -368,6 +530,8 @@ still work. Queries return descriptors. Call
## Exceptions
::: lancedb.remote.errors.HttpError
::: lancedb.exceptions.MissingValueError
::: lancedb.exceptions.MissingColumnError
@@ -442,6 +606,8 @@ latency on your workload before opting in.
::: lancedb.catalog.AsyncCatalog
::: lancedb.catalog.AsyncDatabaseNames
Connections represent a connection to a LanceDb database and
can be used to create, list, or open tables.
@@ -449,6 +615,8 @@ can be used to create, list, or open tables.
::: lancedb.db.AsyncConnection
::: lancedb.listing.AsyncListing
## Namespaces (Asynchronous)
::: lancedb.connect_namespace_async
docs/web-tests/camelot.json
+66
View File
@@ -0,0 +1,66 @@
[
{
"id": 1,
"name": "King Arthur",
"role": "King of Camelot",
"description": "The legendary ruler of Camelot, wielder of Excalibur, and leader of the Knights of the Round Table.",
"vector": [0.72, -0.28, 0.60, 0.86],
"stats": { "strength": 2, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 2,
"name": "Merlin",
"role": "Wizard and Advisor",
"description": "A powerful wizard and prophet who mentors Arthur and shapes the destiny of Camelot through magic and foresight.",
"vector": [0.05, 0.88, 0.62, 0.85],
"stats": { "strength": 2, "courage": 4, "magic": 5, "wisdom": 5 }
},
{
"id": 3,
"name": "Queen Guinevere",
"role": "Queen of Camelot",
"description": "Arthur's queen, admired for her grace and diplomacy, whose romances and loyalties influence Camelot's fate.",
"vector": [0.22, -0.22, 0.42, 0.82],
"stats": { "strength": 1, "courage": 3, "magic": 1, "wisdom": 4 }
},
{
"id": 4,
"name": "Sir Lancelot",
"role": "Knight of the Round Table",
"description": "Arthur's most skilled knight, famed for unmatched combat prowess and his tragic love for Queen Guinevere.",
"vector": [0.86, -0.35, 0.38, 0.55],
"stats": { "strength": 5, "courage": 5, "magic": 1, "wisdom": 3 }
},
{
"id": 5,
"name": "Sir Gawain",
"role": "Knight of the Round Table",
"description": "A noble and honorable knight known for his courtesy and his encounter with the Green Knight.",
"vector": [0.82, -0.32, 0.52, 0.60],
"stats": { "strength": 4, "courage": 5, "magic": 1, "wisdom": 4 }
},
{
"id": 6,
"name": "Sir Galahad",
"role": "Knight of the Round Table",
"description": "The purest and most virtuous knight, chosen to achieve the Holy Grail due to his unwavering spiritual purity.",
"vector": [0.80, -0.20, 0.70, 0.78],
"stats": { "strength": 4, "courage": 5, "magic": 2, "wisdom": 5 }
},
{
"id": 7,
"name": "Sir Percival",
"role": "Knight of the Round Table",
"description": "A loyal and innocent knight whose bravery and sincerity make him one of the key seekers of the Holy Grail.",
"vector": [0.78, -0.36, 0.48, 0.52],
"stats": { "strength": 4, "courage": 4, "magic": 1, "wisdom": 3 }
},
{
"id": 8,
"name": "Mordred",
"role": "Traitor Knight",
"description": "Arthur's treacherous son or nephew who ultimately rebels against him, leading to Camelot's downfall.",
"vector": [0.68, -0.30, -0.65, 0.20],
"stats": { "strength": 4, "courage": 2, "magic": 1, "wisdom": 2 }
}
]
docs/web-tests/mdx_snippets_gen.py
+362
View File
@@ -0,0 +1,362 @@
"""
Generate MDX-ready snippet modules that export raw string constants.
The generator groups snippets by the originating test file name rather than by
language. For a source file like `tests/py/test_basic_usage.py`, the snippets are
written under `docs/snippets/basic.mdx`. Each exported constant is prefixed with
a language identifier (Py|Ts|Rs) followed by the snippet name converted to
TitleCase so that docs authors can selectively assemble `<CodeGroup>` blocks.
"""
from __future__ import annotations
import argparse
import json
import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, Iterator, List, Mapping, MutableMapping, Tuple
# Supported languages and relevant metadata.
LANG_ORDER = ("py", "ts", "rs")
LANG_PREFIX = {"py": "Py", "ts": "Ts", "rs": "Rs"}
LANG_EXTENSIONS = {
".py": "py",
".ts": "ts",
".tsx": "ts",
".rs": "rs",
}
LANG_MARKERS = {"py": ["#"], "ts": ["//"], "rs": ["//"]}
# Match each export block; we rely on the blank line we emit after each export to
# avoid accidentally truncating on semicolons inside the JSON string literal.
EXPORT_RE = re.compile(r"export const (\w+)\s*=\s*(.+?);\s*\n(?:\s*\n|$)", re.DOTALL)
DEFAULT_SOURCE_DIRS = (
Path("tests") / "py",
Path("tests") / "ts",
Path("tests") / "rs",
)
@dataclass(frozen=True)
class SnippetRecord:
lang: str
snippet_name: str
export_name: str
text: str
source_rel: str
def build_markers(
comment_markers: Iterable[str],
) -> Tuple[re.Pattern[str], re.Pattern[str]]:
escaped = "|".join(re.escape(c) for c in comment_markers)
start_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[start:([^\]]+)\]")
end_re = re.compile(rf"^\s*(?:{escaped})\s*--8<--\s*\[end:([^\]]+)\]")
return start_re, end_re
def parse_snippets(
lines: List[str], start_re: re.Pattern[str], end_re: re.Pattern[str]
) -> Dict[str, List[str]]:
snippets: Dict[str, List[str]] = {}
stack: List[Tuple[str, int]] = []
for idx, line in enumerate(lines):
m_start = start_re.match(line)
if m_start:
name = m_start.group(1).strip()
stack.append((name, idx))
continue
m_end = end_re.match(line)
if m_end:
name = m_end.group(1).strip()
if not stack:
raise ValueError(
f"End marker for '{name}' found at line {idx + 1} without matching start"
)
start_name, start_idx = stack.pop()
if start_name != name:
raise ValueError(
f"Mismatched markers: start '{start_name}' at line {start_idx + 1}, "
f"end '{name}' at line {idx + 1}"
)
snippets[name] = lines[start_idx + 1 : idx]
continue
if stack:
open_names = ", ".join(f"{n}@{i + 1}" for n, i in stack)
raise ValueError(f"Unclosed snippet markers for: {open_names}")
return snippets
def dedent(lines: List[str]) -> List[str]:
min_indent = None
for ln in lines:
if ln.strip() == "":
continue
expanded = ln.expandtabs(4)
indent = len(expanded) - len(expanded.lstrip(" "))
if min_indent is None or indent < min_indent:
min_indent = indent
if not lines:
return []
if min_indent in (None, 0):
return [ln.rstrip("\n") for ln in lines]
dedented: List[str] = []
for ln in lines:
expanded = ln.expandtabs(4)
if expanded.strip() == "":
dedented.append(expanded.rstrip("\n"))
else:
dedented.append(expanded[min_indent:].rstrip("\n"))
return dedented
def ensure_dir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def detect_language(file_path: Path) -> str:
lang = LANG_EXTENSIONS.get(file_path.suffix.lower())
if not lang:
raise ValueError(f"Unsupported file extension for {file_path}")
return lang
def markers_for_lang(lang: str) -> Tuple[re.Pattern[str], re.Pattern[str]]:
markers = LANG_MARKERS.get(lang)
if not markers:
raise ValueError(f"Unsupported language for markers: {lang}")
return build_markers(markers)
def normalize_target_name(file_path: Path) -> str:
stem = file_path.stem
if stem.startswith("test_"):
stem = stem[len("test_") :]
if stem.endswith("_test"):
stem = stem[: -len("_test")]
if stem.endswith(".test"):
stem = stem[: -len(".test")]
if stem.endswith(".spec"):
stem = stem[: -len(".spec")]
if stem == "":
stem = file_path.stem
return stem.replace(" ", "_")
def to_title_case(name: str) -> str:
parts = re.split(r"[^0-9a-zA-Z]+", name)
filtered = [p for p in parts if p]
return "".join(p[:1].upper() + p[1:].lower() for p in filtered) or "Snippet"
def format_export_name(lang: str, snippet_name: str) -> str:
prefix = LANG_PREFIX[lang]
return f"{prefix}{to_title_case(snippet_name)}"
def snippet_text(lines: List[str]) -> str:
if not lines:
return ""
joined = "\n".join(lines).rstrip("\n")
return (joined + "\n") if joined else ""
def to_js_literal(text: str) -> str:
return json.dumps(text, ensure_ascii=False)
def split_export_prefix(export_name: str) -> Tuple[str | None, str]:
for lang, prefix in LANG_PREFIX.items():
if export_name.startswith(prefix):
suffix = export_name[len(prefix) :]
return lang, suffix or export_name
return None, export_name
def parse_existing_module(path: Path) -> Mapping[str, Mapping[str, SnippetRecord]]:
try:
text = path.read_text(encoding="utf-8")
except FileNotFoundError:
return {}
existing: Dict[str, Dict[str, SnippetRecord]] = {}
for match in EXPORT_RE.finditer(text):
export_name, literal = match.group(1), match.group(2).strip()
lang, snippet_suffix = split_export_prefix(export_name)
if not lang:
continue
try:
decoded = json.loads(literal)
except Exception:
decoded = literal.strip().strip(";")
record = SnippetRecord(
lang=lang,
snippet_name=snippet_suffix or export_name,
export_name=export_name,
text=decoded,
source_rel="(existing module)",
)
existing.setdefault(lang, {})[record.snippet_name] = record
return existing
def iter_source_files(source_dirs: Iterable[Path]) -> Iterator[Path]:
for root in source_dirs:
for path in root.rglob("*"):
if path.is_file() and path.suffix.lower() in LANG_EXTENSIONS:
yield path
def write_if_changed(path: Path, content: str) -> bool:
if path.exists():
try:
current = path.read_text(encoding="utf-8")
if current == content:
return False
except Exception:
pass
ensure_dir(path.parent)
path.write_text(content, encoding="utf-8")
return True
def collect_snippets(
source_dirs: Iterable[Path],
) -> Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]]:
result: Dict[str, Dict[str, Dict[str, SnippetRecord]]] = {}
for file_path in iter_source_files(source_dirs):
lang = detect_language(file_path)
start_re, end_re = markers_for_lang(lang)
try:
text = file_path.read_text(encoding="utf-8")
except Exception as exc: # pragma: no cover
raise RuntimeError(f"Failed to read {file_path}: {exc}") from exc
lines = text.splitlines(keepends=True)
snippets = parse_snippets(lines, start_re, end_re)
if not snippets:
continue
target = normalize_target_name(file_path)
lang_map = result.setdefault(target, {}).setdefault(lang, {})
for snippet_name, content_lines in snippets.items():
if snippet_name in lang_map:
raise ValueError(
f"Duplicate snippet '{snippet_name}' in {file_path} and {lang_map[snippet_name].source_rel}"
)
dedented = dedent(content_lines)
export_name = format_export_name(lang, snippet_name)
source_rel = os.path.relpath(file_path, Path.cwd())
lang_map[snippet_name] = SnippetRecord(
lang=lang,
snippet_name=snippet_name,
export_name=export_name,
text=snippet_text(dedented),
source_rel=source_rel,
)
return result
def render_module(
target: str, lang_map: Mapping[str, Mapping[str, SnippetRecord]]
) -> str:
parts: List[str] = []
parts.append(
"{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}\n\n"
)
for lang in LANG_ORDER:
snippets = lang_map.get(lang)
if not snippets:
continue
for snippet_name in sorted(snippets.keys()):
record = snippets[snippet_name]
literal = to_js_literal(record.text)
parts.append(f"export const {record.export_name} = {literal};\n\n")
return "".join(parts)
def generate_modules(
snippets_by_target: Mapping[str, Mapping[str, Mapping[str, SnippetRecord]]],
output_root: Path,
) -> None:
modules_written = 0
total_exports = 0
for target in sorted(snippets_by_target.keys()):
module_path = output_root / f"{target}.mdx"
lang_map = snippets_by_target[target]
existing_langs = parse_existing_module(module_path)
for lang, snippets in existing_langs.items():
if lang in lang_map:
continue
print(module_path, " -- ", lang)
merged_lang = lang_map.setdefault(lang, {})
existing_export_names = {rec.export_name for rec in merged_lang.values()}
for snippet_name, record in snippets.items():
# If an export with the same name already exists (e.g., regenerated),
# prefer the newly generated snippet and skip the old one to avoid duplicates.
if record.export_name in existing_export_names:
continue
merged_lang.setdefault(snippet_name, record)
total_exports += sum(len(snippets) for snippets in lang_map.values())
module_content = render_module(target, lang_map)
if write_if_changed(module_path, module_content):
modules_written += 1
print(
f"\nGenerated {total_exports} snippets\n---"
)
def resolve_source_dirs(args_dirs: List[str] | None) -> List[Path]:
if args_dirs:
dirs = [Path(p) for p in args_dirs]
else:
dirs = list(DEFAULT_SOURCE_DIRS)
missing = [str(p) for p in dirs if not p.exists()]
if missing:
raise FileNotFoundError(f"Source directories not found: {', '.join(missing)}")
return dirs
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate MDX snippet modules grouped by test filename."
)
parser.add_argument(
"-s",
"--source-dir",
action="append",
help="Path(s) to language test directories (default: tests/py, ts, rs)",
)
parser.add_argument(
"-o",
"--output-dir",
default=str(Path("docs") / "snippets"),
help="Directory where snippet modules are written (default: docs/snippets)",
)
args = parser.parse_args()
source_dirs = resolve_source_dirs(args.source_dir)
snippets = collect_snippets(source_dirs)
if not snippets:
print("No snippets found.")
return 0
output_root = Path(args.output_dir)
generate_modules(snippets, output_root)
return 0
if __name__ == "__main__":
raise SystemExit(main())
docs/web-tests/py/conftest.py
+56
View File
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import shutil
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
from lancedb.db import AsyncConnection, DBConnection
class DatabasePathFactory:
"""Create per-test database directories and ensure they get removed."""
def __init__(self, base_dir: Path) -> None:
self._base_dir = base_dir
self._created: list[Path] = []
def __call__(self, name: str) -> Path:
safe_name = name.replace("/", "_")
path = self._base_dir / safe_name
if path.exists():
shutil.rmtree(path, ignore_errors=True)
path.mkdir(parents=True, exist_ok=True)
self._created.append(path)
return path
def cleanup(self) -> None:
for path in reversed(self._created):
shutil.rmtree(path, ignore_errors=True)
self._created.clear()
@pytest.fixture
def db_path_factory(tmp_path: Path):
factory = DatabasePathFactory(tmp_path)
yield factory
factory.cleanup()
@pytest.fixture
def mem_db() -> DBConnection:
return lancedb.connect("memory://")
@pytest.fixture
def tmp_db(db_path_factory) -> DBConnection:
"""Create a temporary database connection for testing."""
db_path = db_path_factory("tmp_db")
return lancedb.connect(str(db_path))
@pytest_asyncio.fixture
async def mem_db_async() -> AsyncConnection:
return await lancedb.connect_async("memory://")
docs/web-tests/py/provider_examples.py
+355
View File
@@ -0,0 +1,355 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Reader examples with explicit placeholders.
These functions are documentation source. test_provider_examples.py runs the
reranker examples offline, with stand-ins for the model and the provider, to
check that each one writes only to its own new directory. Corresponding opt-in
integration tests remain in test_integrations.py and retain real credential
loading. Replace placeholders before running these examples.
"""
def embedding_ibm_usage():
# --8<-- [start:embedding_ibm_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.pydantic import LanceModel, Vector
registry = EmbeddingFunctionRegistry.get_instance()
registry.set_var("watsonx-api-key", "your-api-key")
watsonx_embed = registry.get("watsonx").create(
name="ibm/slate-125m-english-rtrvr",
api_key="$var:watsonx-api-key",
project_id="your-project-id",
)
class TextModel(LanceModel):
text: str = watsonx_embed.SourceField()
vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo"))
tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
tbl.add(data)
rs = tbl.search("hello").limit(1).to_pandas()
print(rs.head())
# --8<-- [end:embedding_ibm_usage]
def embedding_jina_text():
# --8<-- [start:embedding_jina_text]
import os
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.pydantic import LanceModel, Vector
os.environ["JINA_API_KEY"] = "your-api-key"
jina_embed = (
EmbeddingFunctionRegistry.get_instance()
.get("jina")
.create(name="jina-embeddings-v2-base-en")
)
class TextModel(LanceModel):
text: str = jina_embed.SourceField()
vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
data = [{"text": "hello world"}, {"text": "goodbye world"}]
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text"))
tbl = db.create_table("test", schema=TextModel, mode="overwrite")
tbl.add(data)
# --8<-- [end:embedding_jina_text]
def embedding_jina_multimodal():
# --8<-- [start:embedding_jina_multimodal]
import os
import tempfile
from pathlib import Path
import lancedb
import pandas as pd
import requests
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
os.environ["JINA_API_KEY"] = "your-api-key"
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images"))
func = get_registry().get("jina").create()
class Images(LanceModel):
label: str
image_uri: str = func.SourceField()
image_bytes: bytes = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
vec_from_bytes: Vector(func.ndims()) = func.VectorField()
table = db.create_table("images", schema=Images)
labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
uris = [
"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
]
image_bytes = [requests.get(uri).content for uri in uris]
table.add(
pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
)
# --8<-- [end:embedding_jina_multimodal]
def reranking_cohere_usage():
# --8<-- [start:reranking_cohere_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import CohereReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = CohereReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_cohere_usage]
def reranking_jina_usage():
# --8<-- [start:reranking_jina_usage]
import os
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import JinaReranker
os.environ["JINA_API_KEY"] = "your-api-key"
embedder = get_registry().get("jina").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = JinaReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_jina_usage]
def reranking_watsonx_usage():
# --8<-- [start:reranking_watsonx_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import WatsonxReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
# Supply your API key and project ID (or use space_id instead of project_id).
reranker = WatsonxReranker(
api_key="your-api-key",
project_id="your-project-id",
)
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_watsonx_usage]
def reranking_openai_usage():
# --8<-- [start:reranking_openai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import OpenaiReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = OpenaiReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_openai_usage]
def reranking_voyageai_usage():
# --8<-- [start:reranking_voyageai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import VoyageAIReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_voyageai_usage]
docs/web-tests/py/test_basic_usage.py
+205
View File
@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:basic_imports]
import json
import lancedb
import pandas as pd
import polars as pl
import pyarrow as pa
# --8<-- [end:basic_imports]
import pytest
data_path = "tests/camelot.json"
def test_basic_usage(db_path_factory):
uri = "./basic_usage_db"
uri = db_path_factory("basic_usage_db")
db = lancedb.connect(uri)
# --8<-- [start:data_load]
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [end:data_load]
# --8<-- [start:basic_create_table]
table = db.create_table("camelot", data=data, mode="overwrite")
# --8<-- [end:basic_create_table]
assert len(table) == 8
# --8<-- [start:basic_open_table]
table = db.open_table("camelot")
# --8<-- [end:basic_open_table]
# --8<-- [start:basic_create_table_pandas]
pandas_df = pd.DataFrame(data)
table_pd = db.create_table("camelot_pd", data=pandas_df, mode="overwrite")
# --8<-- [end:basic_create_table_pandas]
assert len(table_pd) == 8
db.drop_table("camelot_pd")
# --8<-- [start:basic_create_table_polars]
polars_df = pl.DataFrame(data)
table_pl = db.create_table("camelot_pl", data=polars_df, mode="overwrite")
# --8<-- [end:basic_create_table_polars]
assert len(table_pl) == 8
db.drop_table("camelot_pl")
# --8<-- [start:basic_create_empty_table]
schema = pa.schema(
[
pa.field("id", pa.uint16()),
pa.field("name", pa.string()),
pa.field("role", pa.string()),
pa.field("description", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field(
"stats",
pa.struct(
[
pa.field("strength", pa.int8()),
pa.field("courage", pa.int8()),
pa.field("magic", pa.int8()),
pa.field("wisdom", pa.int8()),
]
),
),
]
)
db.create_table("camelot_pa", schema=schema, mode="overwrite")
# --8<-- [end:basic_create_empty_table]
assert "camelot_pa" in db.list_tables().tables
db.drop_table("camelot_pa")
# --8<-- [start:basic_add_data]
magical_characters = [
{
"id": 9,
"name": "Morgan le Fay",
"role": "Sorceress",
"description": "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.",
"vector": [0.10, 0.84, 0.25, 0.70],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 4 }
},
{
"id": 10,
"name": "The Lady of the Lake",
"role": "Mystical Guardian",
"description": "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.",
"vector": [0.00, 0.90, 0.58, 0.88],
"stats": { "strength": 2, "courage": 3, "magic": 5, "wisdom": 5 }
}
]
table.add(magical_characters)
# --8<-- [end:basic_add_data]
assert len(table) == 10
# --8<-- [start:basic_vector_search]
query_vector = [0.03, 0.85, 0.61, 0.90]
result = table.search(query_vector).limit(5).to_polars()
print(result)
# --8<-- [end:basic_vector_search]
# --8<-- [start:basic_add_columns]
table.add_columns(
{
"power": "cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:basic_add_columns]
assert "power" in table.schema.names
# Run examples to illustrate search
# --8<-- [start:basic_vector_search_q1]
# Who are the characters similar to "wizard"?
query_vector_1 = [0.03, 0.85, 0.61, 0.90]
r1 = (
table.search(query_vector_1)
.limit(5)
.select(["name", "role", "description"])
.to_polars()
)
print(r1)
# --8<-- [end:basic_vector_search_q1]
# --8<-- [start:basic_vector_search_q2]
# Who are the characters with high magic stats?
query_vector_2 = [0.03, 0.85, 0.61, 0.90]
r2 = (
table.search(query_vector_2)
.where("stats.magic > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r2)
# --8<-- [end:basic_vector_search_q2]
# --8<-- [start:basic_vector_search_q3]
# Who are the strongest characters?
r3 = (
table.search()
.where("stats.strength > 3")
.select(["name", "role", "description"])
.limit(5)
.to_polars()
)
print(r3)
# --8<-- [end:basic_vector_search_q3]
# --8<-- [start:basic_vector_search_q4]
# Who are the strongest characters?
r4 = (
table.search()
.select(["name", "role", "description", "power"])
.to_polars()
)
print(r4)
# --8<-- [end:basic_vector_search_q4]
# --8<-- [start:basic_sort_polars]
# Sort Polars DataFrame by power in descending order
print(r4.sort("power", descending=True).limit(5))
# --8<-- [end:basic_sort_polars]
sorted_power = r4.sort("power", descending=True)["power"].to_list()
assert sorted_power == sorted(sorted_power, reverse=True)
assert sorted_power[0] == 4.0
# --8<-- [start:basic_drop_columns]
table.drop_columns(["power"])
# --8<-- [end:basic_drop_columns]
# --8<-- [start:basic_delete_rows]
table.delete('role = "Traitor Knight"')
# --8<-- [end:basic_delete_rows]
assert len(table) == 9
# --8<-- [start:basic_drop_table]
db.drop_table("camelot")
# --8<-- [end:basic_drop_table]
assert "camelot" not in db.list_tables().tables
@pytest.mark.asyncio
async def test_basic_usage_async_api(db_path_factory):
uri = db_path_factory("basic_usage_async_db")
with open(data_path, "r") as f:
data = json.load(f)
# --8<-- [start:basic_async_api]
import lancedb
async_db = await lancedb.connect_async(uri)
async_table = await async_db.create_table(
"camelot_async",
data=data,
mode="overwrite",
)
query_vector = [0.03, 0.85, 0.61, 0.90]
async_results = await (
await async_table.search(query_vector)
).limit(5).select(["name", "role", "description"]).to_polars()
print(async_results)
# --8<-- [end:basic_async_api]
assert async_results.height >= 1
docs/web-tests/py/test_build_with_ai_agents.py
+109
View File
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:camelot_schema]
from lancedb.pydantic import LanceModel
from pydantic import ConfigDict
class Stats(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
strength: int
courage: int
magic: int
wisdom: int
class Character(LanceModel):
model_config = ConfigDict(strict=True, extra="forbid")
id: int
name: str
role: str
description: str
stats: Stats
image_filename: str
image: bytes
# --8<-- [end:camelot_schema]
# --8<-- [start:camelot_batches]
import json
from collections.abc import Iterator
from pathlib import Path
def validated_batches(
input_path: Path, batch_size: int
) -> Iterator[list[dict]]:
raw_records = json.loads(input_path.read_text())
asset_root = input_path.parent.parent
batch: list[dict] = []
for raw in raw_records:
payload = dict(raw)
image_path = asset_root / payload.pop("img")
payload["image_filename"] = image_path.name
payload["image"] = image_path.read_bytes()
character = Character.model_validate(payload)
batch.append(character.model_dump(mode="python"))
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch
# --8<-- [end:camelot_batches]
# --8<-- [start:camelot_oss_ingestion]
import lancedb
def ingest_oss(
input_path: Path,
uri: str = "data/camelot.lancedb",
table_name: str = "camelot_multimodal",
batch_size: int = 64,
):
db = lancedb.connect(uri)
if table_name in db.list_tables():
raise ValueError(
f"Table {table_name!r} already exists. Choose a fresh table name."
)
table = db.create_table(table_name, schema=Character)
for batch in validated_batches(input_path, batch_size):
table.add(batch)
table.optimize()
return table
if __name__ == "__main__":
ingest_oss(Path("data/camelot.json"))
# --8<-- [end:camelot_oss_ingestion]
def test_camelot_oss_ingestion(tmp_path):
source = Path(
"docs/web/static/assets/tutorials/build-with-ai-agents/camelot/data/camelot.json"
)
table = ingest_oss(
source,
uri=str(tmp_path / "camelot.lancedb"),
table_name="camelot",
)
rows = (
table.search()
.select(["id", "name", "image", "image_filename"])
.limit(8)
.to_list()
)
assert len(rows) == 8
assert all(row["image"] for row in rows)
assert all(row["image_filename"].endswith(".jpg") for row in rows)
docs/web-tests/py/test_connection.py
+210
View File
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from pathlib import Path
import shutil
import pytest
def test_connection():
# --8<-- [start:connect]
import lancedb
uri = "ex_lancedb"
db = lancedb.connect(uri)
# --8<-- [end:connect]
assert db is not None
shutil.rmtree(uri, ignore_errors=True)
assert not Path(uri).exists()
async def connect_async_example():
# --8<-- [start:connect_async]
import lancedb
uri = "ex_lancedb"
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_async]
return async_db
def connect_enterprise_quickstart_config():
# --8<-- [start:connect_enterprise_quickstart]
import os
import lancedb
db = lancedb.connect(
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"], # https://your-endpoint
)
# --8<-- [end:connect_enterprise_quickstart]
return db
def test_connect_enterprise_quickstart(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
monkeypatch.delenv("LANCEDB_REGION", raising=False)
monkeypatch.setenv("LANCEDB_HOST_OVERRIDE", "https://your-endpoint")
assert connect_enterprise_quickstart_config() is not None
assert captured == {
"uri": "db://your-database",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-endpoint",
}
def test_connect_enterprise_quickstart_requires_an_endpoint(monkeypatch):
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
monkeypatch.delenv("LANCEDB_HOST_OVERRIDE", raising=False)
with pytest.raises(KeyError, match="LANCEDB_HOST_OVERRIDE"):
connect_enterprise_quickstart_config()
def connect_object_storage_config():
# --8<-- [start:connect_object_storage]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
db = lancedb.connect(uri)
# --8<-- [end:connect_object_storage]
return db
def namespace_table_ops_example():
# --8<-- [start:namespace_table_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
# Create namespace tree: prod/search
db.create_namespace(["prod"], mode="exist_ok")
db.create_namespace(["prod", "search"], mode="exist_ok")
db.create_namespace(["prod", "recommendations"], mode="exist_ok")
db.create_table(
"user",
data=[{"id": 1, "vector": [0.1, 0.2], "name": "alice"}],
namespace_path=["prod", "search"],
mode="create", # use "overwrite" only if you want to replace existing table
)
db.create_table(
"user",
data=[{"id": 2, "vector": [0.3, 0.4], "name": "bob"}],
namespace_path=["prod", "recommendations"],
mode="create", # use "overwrite" only if you want to replace existing table
)
# Verify: listings return a response with the names and a page token
print(db.list_namespaces().namespaces) # ['prod']
print(db.list_namespaces(namespace_path=["prod"]).namespaces)
# ['recommendations', 'search']
print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user']
print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user']
# --8<-- [end:namespace_table_ops]
def test_namespace_table_ops(tmp_path, monkeypatch, capsys):
monkeypatch.chdir(tmp_path)
namespace_table_ops_example()
assert capsys.readouterr().out.splitlines() == [
"['prod']",
"['recommendations', 'search']",
"['user']",
"['user']",
]
def namespace_admin_ops_example():
# --8<-- [start:namespace_admin_ops]
import lancedb
db = lancedb.connect_namespace("dir", {"root": "./local_lancedb"})
namespace = ["prod", "search"]
db.create_namespace(["prod"])
db.create_namespace(["prod", "search"])
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
print(f"Child namespaces under ['prod']: {child_namespaces}")
# Child namespaces under ['prod']: ['search']
metadata = db.describe_namespace(["prod", "search"])
print(f"Metadata for namespace {namespace}: {metadata}")
# Metadata for namespace ['prod', 'search']: properties=None
db.drop_namespace(["prod", "search"], mode="skip")
db.drop_namespace(["prod"], mode="skip")
# --8<-- [end:namespace_admin_ops]
return db, child_namespaces, metadata
def test_namespace_admin_ops(tmp_path, monkeypatch, capsys):
monkeypatch.chdir(tmp_path)
db, child_namespaces, metadata = namespace_admin_ops_example()
assert child_namespaces == ["search"]
assert capsys.readouterr().out.splitlines()[0] == (
"Child namespaces under ['prod']: ['search']"
)
assert db.list_namespaces().namespaces == []
async def connect_object_storage_config_async():
# --8<-- [start:connect_object_storage_async]
import lancedb
uri = "s3://your-bucket/path"
# You can also use "gs://your-bucket/path" or "az://your-container/path".
async_db = await lancedb.connect_async(uri)
# --8<-- [end:connect_object_storage_async]
return async_db
def test_connect_enterprise_placeholders(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
# --8<-- [start:connect_enterprise_placeholders]
import lancedb
db = lancedb.connect(
uri="db://your-database",
api_key="your-api-key",
region="us-east-1",
host_override="https://your-endpoint",
)
# --8<-- [end:connect_enterprise_placeholders]
assert db is not None
assert captured == {
"uri": "db://your-database",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-endpoint",
}
docs/web-tests/py/test_curation_sampling.py
+27
View File
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_sample_dataset():
# --8<-- [start:sample_dataset]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.permutation import Permutation, permutation_builder
with TemporaryDirectory(prefix="lancedb-sampling-") as uri:
db = lancedb.connect(uri)
table = db.create_table("records", [{"id": i} for i in range(10)])
splits = (
permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()
)
sample = Permutation.from_tables(table, splits, split=0).with_format("arrow")
selected_ids = []
for batch in sample.iter(batch_size=3):
selected_ids.extend(batch.column("id").to_pylist())
print(f"Selected {len(selected_ids)} rows: {selected_ids}")
# --8<-- [end:sample_dataset]
assert len(selected_ids) == 3
assert len(set(selected_ids)) == 3
assert all(0 <= value < 10 for value in selected_ids)
docs/web-tests/py/test_doc_function_examples.py
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from lancedb.index import FTS
def test_python_boolean_query_operators(tmp_db):
table = tmp_db.create_table(
"boolean_docs",
[
{"id": 1, "text": "puppy"},
{"id": 2, "text": "merrily"},
{"id": 3, "text": "puppy merrily"},
{"id": 4, "text": "other"},
],
)
table.create_index("text", config=FTS())
# --8<-- [start:fts_boolean_queries]
from lancedb.query import MatchQuery
intersection = MatchQuery("puppy", "text") & MatchQuery("merrily", "text")
union = MatchQuery("puppy", "text") | MatchQuery("merrily", "text")
# --8<-- [end:fts_boolean_queries]
def ids(query):
return set(table.search(query, query_type="fts").to_arrow()["id"].to_pylist())
assert ids(intersection) == {3}
assert ids(union) == {1, 2, 3}
a = MatchQuery("puppy", "text")
b = MatchQuery("merrily", "text")
assert ids(a and b) == {2, 3}
assert ids(a or b) == {1, 3}
docs/web-tests/py/test_doc_storage_examples.py
+84
View File
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import numpy as np
import pyarrow as pa
import pytest
def test_rebuild_vector_index(tmp_db):
rng = np.random.default_rng(7)
table = tmp_db.create_table(
"rebuild_index",
[{"id": i, "vector": v.tolist()} for i, v in enumerate(rng.random((512, 4)))],
)
table.create_index(
"vector", config=lancedb.index.IvfFlat(distance_type="l2", num_partitions=4)
)
original_index = table.list_indices()[0]
table.add(
[
{"id": 512 + i, "vector": v.tolist()}
for i, v in enumerate(rng.random((64, 4)) + 5)
]
)
assert table.index_stats("vector_idx").num_unindexed_rows == 64
# --8<-- [start:storage_rebuild_vector_index]
from lancedb.index import IvfFlat
table.create_index(
"vector",
config=IvfFlat(distance_type="l2", num_partitions=4),
replace=True,
)
# --8<-- [end:storage_rebuild_vector_index]
indices = table.list_indices()
assert len(indices) == 1
assert indices[0].name == original_index.name
assert indices[0].index_uuid != original_index.index_uuid
stats = table.index_stats("vector_idx")
assert stats.num_indexed_rows == 576
assert stats.num_unindexed_rows == 0
def test_local_shallow_clone_pins_source_version(tmp_path):
db = lancedb.connect(str(tmp_path))
source = db.create_table("source", [{"id": 1}])
clone = db.clone_table("snapshot", source_uri=str(tmp_path / "source.lance"))
source.add([{"id": 2}])
assert source.count_rows() == 2
assert clone.to_arrow().to_pylist() == [{"id": 1}]
def _reader():
return pa.RecordBatchReader.from_batches(
pa.schema([("id", pa.int64())]),
[pa.record_batch([[1, 2]], names=["id"]), pa.record_batch([[3]], names=["id"])],
)
@pytest.mark.parametrize("parallelism", [1, 2])
def test_streaming_write_parallelism(tmp_db, parallelism):
table = tmp_db.create_table("streaming", [{"id": 0}])
progress = []
table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
assert sorted(table.to_arrow()["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
@pytest.mark.asyncio
@pytest.mark.parametrize("parallelism", [1, 2])
async def test_async_streaming_write_parallelism(mem_db_async, parallelism):
table = await mem_db_async.create_table("streaming", [{"id": 0}])
progress = []
await table.add(_reader(), write_parallelism=parallelism, progress=progress.append)
data = await table.to_arrow()
assert sorted(data["id"].to_pylist()) == [0, 1, 2, 3]
assert progress[-1]["done"]
assert progress[-1]["output_rows"] == 3
assert {p["total_tasks"] for p in progress} == {parallelism}
docs/web-tests/py/test_embedding.py
+141
View File
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
# --8<-- [start:imports]
from lancedb.pydantic import LanceModel, Vector
from lancedb.embeddings import get_registry
# --8<-- [end:imports]
import pytest
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_create_embedding_function():
# --8<-- [start:create_embedding_function]
func = get_registry().get("openai").create(
name="text-embedding-3-small",
max_retries=7,
)
# --8<-- [end:create_embedding_function]
assert func is not None
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_openai():
# --8<-- [start:openai_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = table.search(query).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
@pytest.mark.asyncio
async def test_embeddings_openai_async():
uri = "memory://"
# --8<-- [start:async_openai_embeddings]
db = await lancedb.connect_async(uri)
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = await db.create_table("words", schema=Words, mode="overwrite")
await table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query = "greetings"
actual = await (await table.search(query)).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:async_openai_embeddings]
@pytest.mark.skip(reason="OpenAI is not available in the test environment")
def test_embeddings_manual_query():
# --8<-- [start:manual_query_embeddings]
db = lancedb.connect("/tmp/db")
func = get_registry().get("openai").create(name="text-embedding-ada-002")
class Words(LanceModel):
text: str = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
table = db.create_table("words", schema=Words, mode="overwrite")
table.add([{"text": "hello world"}, {"text": "goodbye world"}])
query_vector = func.generate_embeddings(["greetings"])[0]
# --8<-- [start:manual_query_search]
# query_vector is assumed to already be generated by your embedding function
actual = table.search(query_vector).limit(1).to_pydantic(Words)[0]
print(actual.text)
# --8<-- [end:manual_query_search]
# --8<-- [end:manual_query_embeddings]
def test_custom_embedding_function():
# --8<-- [start:embedding_function]
from functools import cached_property
from lancedb.embeddings import TextEmbeddingFunction, register
class MyEmbeddingModel:
def __init__(self, model_name: str):
self.model_name = model_name
def encode(self, texts: list[str]) -> list[list[float]]:
return [[1.0, 2.0, 3.0] for _ in texts]
@register("my-embedder")
class MyTextEmbedder(TextEmbeddingFunction):
model_name: str = "my-model"
def generate_embeddings(self, texts: list[str]) -> list[list[float]]:
# Your embedding logic here
return self._model.encode(texts)
def ndims(self) -> int:
# Return the dimensionality of the embeddings
return len(self.generate_embeddings(["test"])[0])
@cached_property
def _model(self) -> MyEmbeddingModel:
# Initialize your model once
return MyEmbeddingModel(self.model_name)
# --8<-- [end:embedding_function]
def test_embeddings_secret():
# --8<-- [start:register_secret]
registry = get_registry()
registry.set_var("api_key", "sk-...")
func = registry.get("openai").create(api_key="$var:api_key")
# --8<-- [end:register_secret]
try:
import torch
except ImportError:
pytest.skip("torch not installed")
# --8<-- [start:register_device]
import torch
registry = get_registry()
if torch.cuda.is_available():
registry.set_var("device", "cuda")
func = registry.get("huggingface").create(device="$var:device:cpu")
# --8<-- [end:register_device]
assert func.device == "cuda" if torch.cuda.is_available() else "cpu"
docs/web-tests/py/test_function.py
+335
View File
@@ -0,0 +1,335 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/management/function.mdx.
Registering a Function and filling its column need a LanceDB Enterprise
deployment, so `test_function_enterprise` runs only when
LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the
Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and
LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The
walkthrough calls its table `function_demo` and its Function `double`; the test
creates both under names unique to its run, never replaces an existing object,
and drops only what it created.
The other tests run locally and check the client side of the same code: what
`@udf` sends for registration, how a registered version binds a column, where a
local connection stops, and when the Enterprise test connects. They do not run
a Function remotely.
"""
import json
import os
import sys
import uuid
from pathlib import Path
import lancedb
import pyarrow as pa
import pytest
from lancedb import col
from lancedb.functions import FunctionApplication, FunctionVersion
# --8<-- [start:function_define]
from lancedb import udf
@udf
def double(value: float) -> float:
return value * 2
# --8<-- [end:function_define]
# The definition the walkthrough registers. The Enterprise test registers a copy
# of it under a name that only its run uses.
WALKTHROUGH_DOUBLE = double
def describe(value: float):
return {"half": value / 2, "note": None}
# A Function version as the service returns it after registration: the SDK's
# shared golden reply, with the name and signature of the definition above.
SERVICE_REPLY = (
Path(__file__).parents[3]
/ "rust"
/ "lancedb"
/ "tests"
/ "fixtures"
/ "first_class_functions"
/ "v1"
/ "remote_function_job.json"
)
def registered_version(definition) -> FunctionVersion:
request = definition.registration_request
reply = json.loads(SERVICE_REPLY.read_text())["result"]
reply["name"] = request.name
reply["signature"] = json.loads(request.signature.to_canonical_json())
return FunctionVersion.from_json(json.dumps(reply))
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
# The connection settings alone never send this test to a deployment.
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
def require_deployment():
if os.environ.get(OPT_IN) != "1":
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
if missing:
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
def test_function_runs_locally_as_python():
# --8<-- [start:function_local_call]
print(double(1.5))
# 3.0
# --8<-- [end:function_local_call]
assert double(1.5) == 3.0
def test_function_registration_request():
request = double.registration_request
assert request.name == "double"
assert [
(parameter.name, parameter.arrow_type, parameter.nullable)
for parameter in request.signature.inputs
] == [("value", "float64", False)]
output = request.signature.output
assert (output.kind, output.arrow_type, output.nullable) == (
"scalar",
"float64",
False,
)
version = request.runtime.python_version
assert version == f"{sys.version_info.major}.{sys.version_info.minor}"
assert request.runtime.environment.packages == ()
def test_function_output_nullability():
inputs = pa.schema([pa.field("value", pa.float64(), nullable=False)])
# A named-struct output may declare nullable fields.
fields = [
pa.field("half", pa.float64(), nullable=False),
pa.field("note", pa.string(), nullable=True),
]
struct = udf(input_schema=inputs, output_schema=pa.schema(fields))(describe)
output = struct.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.nullable) for field in output.fields] == [
("half", False),
("note", True),
]
# A scalar output, and a struct declared as one nullable field, may not be.
for nullable in (
pa.field("result", pa.float64(), nullable=True),
pa.field("result", pa.struct(fields), nullable=True),
):
with pytest.raises(ValueError, match="non-nullable"):
udf(input_schema=inputs, output_schema=nullable)(describe)
def test_function_registration_is_remote(tmp_path):
db = lancedb.connect(tmp_path)
with pytest.raises(NotImplementedError):
db.create_function(double)
def test_function_version_binds_a_column(tmp_path):
version = registered_version(double)
application = version(value=col("value"))
assert isinstance(application, FunctionApplication)
assert application.function.name == "double"
assert application.function.version == version.version
assert [
(binding.parameter, binding.kind, binding.value["path"])
for binding in application.inputs
] == [("value", "column", "value")]
with pytest.raises(TypeError, match="direct col"):
version(value=1.5)
with pytest.raises(TypeError, match="missing"):
version(x=col("value"))
# The binding is accepted by add_columns, which a local table refuses:
# a Function column needs LanceDB Cloud or Enterprise.
table = lancedb.connect(tmp_path).create_table("t", [{"value": 1.0}])
with pytest.raises(NotImplementedError, match="Cloud and Enterprise"):
table.add_columns({"doubled": application})
def test_function_enterprise(monkeypatch):
require_deployment()
# --8<-- [start:function_connect]
import os
import lancedb
db = lancedb.connect(
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
# --8<-- [end:function_connect]
# In a shared database `function_demo` or `double` may already exist, so
# this run creates both under names only it uses, and drops only those.
run = uuid.uuid4().hex[:12]
table_name = f"function_demo_{run}"
double = udf(name=f"double_{run}")(WALKTHROUGH_DOUBLE.__wrapped__)
create_table = db.create_table
def create_run_table(name, *args, **kwargs):
assert name == "function_demo"
return create_table(table_name, *args, **kwargs)
monkeypatch.setattr(db, "create_table", create_run_table)
table = None
version = None
try:
# --8<-- [start:function_create_table]
import pyarrow as pa
schema = pa.schema([pa.field("value", pa.float64(), nullable=False)])
table = db.create_table(
"function_demo",
[{"value": 1.0}, {"value": 2.0}, {"value": 3.0}],
schema=schema,
)
# --8<-- [end:function_create_table]
# --8<-- [start:function_register]
job = db.create_function_async(double)
version = job.wait()
# --8<-- [end:function_register]
assert version.name == f"double_{run}"
# --8<-- [start:function_apply]
from lancedb import col
table.add_columns({"doubled": version(value=col("value"))})
# --8<-- [end:function_apply]
# --8<-- [start:function_refresh]
refresh = table.refresh_column_async("doubled")
result = refresh.wait()
print(result.rows_assigned)
# 3
# --8<-- [end:function_refresh]
assert result.rows_assigned == 3
assert result.rows_failed == 0
# --8<-- [start:function_inspect]
rows = table.search().select(["value", "doubled"]).to_list()
print(sorted((row["value"], row["doubled"]) for row in rows))
# [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]
# --8<-- [end:function_inspect]
assert sorted((row["value"], row["doubled"]) for row in rows) == [
(1.0, 2.0),
(2.0, 4.0),
(3.0, 6.0),
]
finally:
if table is not None:
db.drop_table(table_name)
if version is not None:
db.drop_function(version.name, version=version.version)
class ReachedConnection(Exception):
pass
SETTINGS = {
"LANCEDB_URI": "db://docs-test",
"LANCEDB_HOST_OVERRIDE": "https://deployment.invalid",
"LANCEDB_API_KEY": "docs-test-key",
}
@pytest.fixture
def connect_attempts(monkeypatch):
"""`lancedb.connect`, replaced so that it records each call and stops."""
attempts = []
def connect(**kwargs):
attempts.append(kwargs)
raise ReachedConnection
monkeypatch.setattr(lancedb, "connect", connect)
for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"):
monkeypatch.delenv(name, raising=False)
return attempts
def enterprise_outcome(monkeypatch):
try:
test_function_enterprise(monkeypatch)
except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome:
return outcome
@pytest.mark.parametrize(
"opt_in",
[None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"],
)
def test_function_enterprise_needs_the_exact_opt_in(
monkeypatch, connect_attempts, opt_in
):
for name, value in SETTINGS.items():
monkeypatch.setenv(name, value)
if opt_in is not None:
monkeypatch.setenv(OPT_IN, opt_in)
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, pytest.skip.Exception), outcome
assert str(outcome).startswith(f"set {OPT_IN}=1 ")
assert connect_attempts == []
@pytest.mark.parametrize("blank", ["unset", "empty"])
@pytest.mark.parametrize(
"missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)]
)
def test_function_enterprise_fails_when_asked_without_settings(
monkeypatch, connect_attempts, missing, blank
):
monkeypatch.setenv(OPT_IN, "1")
for name, value in SETTINGS.items():
if name not in missing:
monkeypatch.setenv(name, value)
elif blank == "empty":
monkeypatch.setenv(name, "")
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, pytest.fail.Exception), outcome
assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}"
assert connect_attempts == []
@pytest.mark.parametrize("region", [None, "eu-west-1"])
def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region):
monkeypatch.setenv(OPT_IN, "1")
for name, value in SETTINGS.items():
monkeypatch.setenv(name, value)
if region:
monkeypatch.setenv("LANCEDB_REGION", region)
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, ReachedConnection), outcome
assert connect_attempts == [
{
"uri": "db://docs-test",
"api_key": "docs-test-key",
"region": region or "us-east-1",
"host_override": "https://deployment.invalid",
}
]
docs/web-tests/py/test_indexing.py
+757
View File
@@ -0,0 +1,757 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import random
import string
import uuid
import numpy as np
import pyarrow as pa
import pytest
def _make_vector_rows(count: int, dim: int, column: str = "vector"):
rows = [
{column: np.random.random(dim).astype(np.float32).tolist(), "id": i}
for i in range(count)
]
assert len(rows) == count
assert all(len(row[column]) == dim for row in rows)
return rows
def test_vector_index_configure_ivf(tmp_db):
table = tmp_db.create_table(
"vector_index_configure_ivf",
_make_vector_rows(512, 4),
mode="overwrite",
)
# --8<-- [start:vector_index_configure_ivf]
table.create_index(metric="l2", num_partitions=16, num_sub_vectors=4)
# --8<-- [end:vector_index_configure_ivf]
assert table.list_indices()
def test_vector_index_setup(tmp_db):
tmp_db.create_table(
"vector-index-tbl",
_make_vector_rows(8, 4),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_setup]
table_name = "vector-index-tbl"
table = db.open_table(table_name)
# --8<-- [end:vector_index_setup]
assert table.name == table_name
def test_vector_index_build_ivf(tmp_db):
table = tmp_db.create_table(
"vector-index-build-ivf",
_make_vector_rows(512, 4, column="keywords_embeddings"),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:vector_index_build_ivf]
table_name = "vector-index-build-ivf"
table = db.open_table(table_name)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [end:vector_index_build_ivf]
assert table.list_indices()
def test_vector_index_nested_field(tmp_db):
dim = 2
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
data = [
{
"id": i,
"image": {"embedding": np.random.random(dim).astype(np.float32).tolist()},
}
for i in range(512)
]
table = tmp_db.create_table(
"vector_index_nested_field", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:vector_index_nested_field]
# The vector column `embedding` is nested inside the `image` struct.
# Pass its full dotted path as `vector_column_name`; the same path is used
# at query time and is what `list_indices()` reports under `columns`.
table.create_index(
vector_column_name="image.embedding",
num_partitions=1,
num_sub_vectors=1,
name="image_embedding_idx",
)
results = (
table.search([0.0, 1.0], vector_column_name="image.embedding")
.limit(1)
.to_list()
)
# --8<-- [end:vector_index_nested_field]
assert table.index_stats("image_embedding_idx")
assert len(results) == 1
@pytest.mark.asyncio
async def test_vector_index_async_config(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:vector_index_async_config]
import lancedb
import numpy as np
from lancedb.index import IvfPq
async def main():
data = [
{"id": i, "vector": np.random.random(8).astype(np.float32).tolist()}
for i in range(512)
]
db = await lancedb.connect_async("ex_lancedb")
table = await db.create_table(
"vector_index_async", data=data, mode="overwrite"
)
await table.create_index(
"vector",
config=IvfPq(
distance_type="cosine",
num_partitions=16,
num_sub_vectors=4,
),
)
return await table.list_indices()
# --8<-- [end:vector_index_async_config]
assert await main()
def test_vector_index_query_ivf(tmp_db):
dim = 1536
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_query_ivf", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_query_ivf]
tbl = table
tbl.search(np.random.random((1536))).limit(2).nprobes(20).refine_factor(
10
).to_pandas()
# --8<-- [end:vector_index_query_ivf]
df = (
table.search(np.random.random((1536)))
.limit(2)
.nprobes(20)
.refine_factor(10)
.to_pandas()
)
assert len(df) == 2
def test_vector_index_nprobes(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(512)
]
table = tmp_db.create_table("vector_index_nprobes", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_nprobes]
# Always scan 10 partitions; scan up to 50 only if the initial pass
# returns fewer than `limit` results (common with narrow filters).
(
table.search(np.random.random(128))
.minimum_nprobes(10)
.maximum_nprobes(50)
.where("id > 100")
.limit(5)
.to_pandas()
)
# --8<-- [end:vector_index_nprobes]
def test_vector_index_distance_range(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_distance_range", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_distance_range]
# Only return results whose distance falls within [0.0, 0.5).
# Useful for near-duplicate detection or thresholded similarity search.
(
table.search(np.random.random(128))
.distance_range(lower_bound=0.0, upper_bound=0.5)
.limit(10)
.to_pandas()
)
# --8<-- [end:vector_index_distance_range]
def test_vector_index_bypass_recall(tmp_db):
dim = 128
data = [
{"id": i, "keywords_embeddings": np.random.random(dim).tolist()}
for i in range(256)
]
table = tmp_db.create_table("vector_index_bypass_recall", data, mode="overwrite")
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_bypass_recall]
query = np.random.random(128)
k = 10
# Ground truth: flat (exhaustive) scan, ignoring the ANN index.
truth = set(table.search(query).bypass_vector_index().limit(k).to_pandas()["id"])
# ANN results with the current nprobes setting.
ann = set(table.search(query).nprobes(20).limit(k).to_pandas()["id"])
recall_at_k = len(truth & ann) / k
# --8<-- [end:vector_index_bypass_recall]
assert 0.0 <= recall_at_k <= 1.0
def test_vector_index_custom_name(tmp_db):
table = tmp_db.create_table(
"vector_index_custom_name",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
# --8<-- [start:vector_index_custom_name]
# Override the default `{column}_idx` convention by passing `name=...`.
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
name="my_custom_index",
)
table.wait_for_index(["my_custom_index"])
print(table.index_stats("my_custom_index"))
# --8<-- [end:vector_index_custom_name]
assert table.index_stats("my_custom_index")
def test_vector_index_hnsw(tmp_db):
table = tmp_db.create_table(
"vector_index_hnsw",
_make_vector_rows(64, 16),
mode="overwrite",
)
# --8<-- [start:vector_index_build_hnsw]
table.create_index(index_type="IVF_HNSW_SQ")
# --8<-- [end:vector_index_build_hnsw]
# --8<-- [start:vector_index_query_hnsw]
tbl = table
tbl.search(np.random.random((16))).limit(2).to_pandas()
# --8<-- [end:vector_index_query_hnsw]
df = table.search(np.random.random((16))).limit(2).to_pandas()
assert len(df) == 2
def test_vector_index_binary(tmp_db):
table_name = "hamming-index-tbl"
ndim = 256
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.uint8(), ndim // 8)),
]
)
# --8<-- [start:vector_index_binary_schema]
table = tmp_db.create_table(table_name, schema=schema, mode="overwrite")
# --8<-- [end:vector_index_binary_schema]
data = []
for i in range(64):
vector = np.random.randint(0, 2, size=ndim)
vector = np.packbits(vector)
data.append({"id": i, "vector": vector})
# --8<-- [start:vector_index_binary_add_data]
table.add(data)
# --8<-- [end:vector_index_binary_add_data]
# --8<-- [start:vector_index_binary_build_index]
table.create_index(
metric="hamming",
vector_column_name="vector",
index_type="IVF_FLAT",
)
# --8<-- [end:vector_index_binary_build_index]
# --8<-- [start:vector_index_binary_search]
query = np.random.randint(0, 2, size=ndim)
query = np.packbits(query)
df = table.search(query).metric("hamming").limit(10).to_pandas()
df.vector = df.vector.apply(np.unpackbits)
# --8<-- [end:vector_index_binary_search]
assert not df.empty
def test_quantization_custom_params(tmp_db):
table = tmp_db.create_table(
"quantization-custom-params",
_make_vector_rows(256, 64),
mode="overwrite",
)
# --8<-- [start:quantization_custom_params]
table.create_index(
index_type="IVF_RQ",
num_bits=2,
max_iterations=100,
sample_rate=512,
)
# --8<-- [end:quantization_custom_params]
assert table.list_indices()
def test_vector_index_check_status(tmp_db):
table = tmp_db.create_table(
"vector_index_check_status",
_make_vector_rows(512, 8, column="keywords_embeddings"),
mode="overwrite",
)
table.create_index(
metric="cosine",
vector_column_name="keywords_embeddings",
)
# --8<-- [start:vector_index_check_status]
index_name = "keywords_embeddings_idx"
table.wait_for_index([index_name])
print(table.index_stats(index_name))
# --8<-- [end:vector_index_check_status]
assert table.index_stats(index_name)
def test_scalar_index_build(tmp_db):
table = tmp_db.create_table(
"scalar_index_build",
[
{"book_id": 1, "publisher": "A", "vector": [0.1, 0.2]},
{"book_id": 2, "publisher": "B", "vector": [0.2, 0.3]},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_build]
tbl = db.open_table("scalar_index_build")
tbl.create_scalar_index("book_id")
tbl.create_scalar_index("publisher", index_type="BITMAP")
# --8<-- [end:scalar_index_build]
assert tbl.list_indices()
def test_scalar_index_wait(tmp_db):
table = tmp_db.create_table(
"scalar_index_wait",
[{"label": "fiction"}],
mode="overwrite",
)
table.create_scalar_index("label")
# --8<-- [start:scalar_index_wait]
index_name = "label_idx"
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_wait]
assert table.list_indices()
def test_scalar_index_optimize(tmp_db):
table = tmp_db.create_table(
"scalar_index_optimize",
[{"vector": [7.0, 8.0], "book_id": 3}],
mode="overwrite",
)
# --8<-- [start:scalar_index_optimize]
table.add([{"vector": [7, 8], "book_id": 4}])
table.optimize()
# --8<-- [end:scalar_index_optimize]
result = table.search().where("book_id = 4").limit(10).to_pandas()
assert len(result) == 1
def test_scalar_index_filter(tmp_db):
table = tmp_db.create_table(
"books",
[
{"vector": [1.1, 1.2], "book_id": 1},
{"vector": [2.1, 2.2], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_filter]
table = db.open_table("books")
result = table.search().where("book_id = 2").limit(10).to_pandas()
# --8<-- [end:scalar_index_filter]
assert len(result) == 1
def test_scalar_index_prefilter(tmp_db):
table = tmp_db.create_table(
"book_with_embeddings",
[
{"vector": [1.2, 1.3], "book_id": 1},
{"vector": [4.2, 4.3], "book_id": 2},
],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:scalar_index_prefilter]
table = db.open_table("book_with_embeddings")
table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
# --8<-- [end:scalar_index_prefilter]
result = table.search([1.2] * 2).where("book_id != 3").limit(10).to_pandas()
assert len(result) == 2
def test_scalar_index_uuid(tmp_db):
# --8<-- [start:scalar_index_uuid_type]
import pyarrow as pa
# --8<-- [end:scalar_index_uuid_type]
# --8<-- [start:scalar_index_uuid_data]
def generate_random_names():
base_names = ["Alice", "Bob", "Carla", "David", "Eve", "Frank", "Grace"]
letter = random.choice(string.ascii_uppercase)
return f"{random.choice(base_names)} {letter}."
def generate_uuids(num_items):
return [uuid.uuid4().bytes for _ in range(num_items)]
# Generate some UUIDs and random names
n = 7
uuids = generate_uuids(n)
names = [generate_random_names() for _ in range(n)]
# --8<-- [end:scalar_index_uuid_data]
db = tmp_db
# --8<-- [start:scalar_index_uuid_table]
table_name = "index-on-uuid"
uuid_array = pa.array(uuids, pa.uuid())
name_array = pa.array(names, pa.string())
schema = pa.schema(
[
pa.field("id", pa.uuid()),
pa.field("name", pa.string()),
]
)
data_table = pa.Table.from_arrays([uuid_array, name_array], schema=schema)
table = db.create_table(table_name, data=data_table, mode="overwrite")
# --8<-- [end:scalar_index_uuid_table]
# --8<-- [start:scalar_index_uuid_wait]
index_name = "id_idx"
table.create_scalar_index("id")
table.wait_for_index([index_name])
# --8<-- [end:scalar_index_uuid_wait]
# --8<-- [start:scalar_index_uuid_upsert]
new_users = [
{"id": uuid.uuid4().bytes, "name": "Hannah D."},
{"id": uuid.uuid4().bytes, "name": "Ian B."},
]
# Insert or update using the UUID index
table.merge_insert(
"id"
).when_matched_update_all().when_not_matched_insert_all().execute(new_users)
# --8<-- [end:scalar_index_uuid_upsert]
assert table.list_indices()
result = table.search().limit(100).to_pandas()
assert len(result) == n + len(new_users)
@pytest.mark.asyncio
async def test_scalar_index_nested_fields(mem_db_async):
db = mem_db_async
# --8<-- [start:scalar_index_nested_fields]
import pyarrow as pa
from lancedb.index import BTree
metadata_type = pa.struct(
[
pa.field("user_id", pa.int32()),
pa.field("user.id", pa.int32()),
]
)
data = pa.Table.from_arrays(
[
pa.array([1, 2, 3], type=pa.int32()),
pa.array(
[
{"user_id": 10, "user.id": 100},
{"user_id": 20, "user.id": 200},
{"user_id": 30, "user.id": 300},
],
type=metadata_type,
),
],
names=["user_id", "metadata"],
)
table = await db.create_table("nested_scalar_index", data)
# Index a nested struct field.
await table.create_index(
"metadata.user_id", config=BTree(), name="nested_user_id_idx"
)
# Escape literal dots inside a segment with backticks.
await table.create_index(
"metadata.`user.id`", config=BTree(), name="escaped_user_id_idx"
)
# `columns` is returned as the canonical path you passed in.
for index in await table.list_indices():
print(index.name, index.columns)
# nested_user_id_idx ['metadata.user_id']
# escaped_user_id_idx ['metadata.`user.id`']
# --8<-- [end:scalar_index_nested_fields]
index_columns = {index.name: index.columns for index in await table.list_indices()}
assert index_columns["nested_user_id_idx"] == ["metadata.user_id"]
assert index_columns["escaped_user_id_idx"] == ["metadata.`user.id`"]
def test_fts_index_create(tmp_db):
table = tmp_db.create_table(
"fts-index-create",
[{"text": "hello world", "vector": [0.1, 0.2]}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_create]
table_name = "fts-index-create"
table = db.open_table(table_name)
table.create_fts_index("text")
# --8<-- [end:fts_index_create]
assert table.list_indices()
def test_fts_index_wait(tmp_db):
table = tmp_db.create_table(
"fts-index-wait",
[{"text": "full text search"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_wait]
table_name = "fts-index-wait"
table = db.open_table(table_name)
table.create_fts_index("text")
index_name = "text_idx"
table.wait_for_index([index_name])
# --8<-- [end:fts_index_wait]
assert table.list_indices()
def test_fts_index_nested_field(tmp_db):
nested_schema = pa.struct([
pa.field("text", pa.string()),
pa.field("count", pa.int32()),
])
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("payload", nested_schema),
])
tmp_db.create_table(
"fts-index-nested",
pa.table(
{
"id": pa.array([1, 2], pa.int64()),
"payload": pa.array(
[
{"text": "Frodo was a happy puppy", "count": 1},
{"text": "puppy runs through the meadow", "count": 2},
],
type=nested_schema,
),
},
schema=schema,
),
mode="overwrite",
)
db = tmp_db
# --8<-- [start:fts_index_nested]
from lancedb.query import MatchQuery, PhraseQuery
table = db.open_table("fts-index-nested")
# Index a text leaf inside a struct column using a dotted path.
table.create_fts_index("payload.text", with_position=True)
# The same dotted path works in MatchQuery and PhraseQuery.
matches = (
table.search(MatchQuery("puppy", "payload.text")).limit(5).to_list()
)
phrases = (
table.search(PhraseQuery("puppy runs", "payload.text"))
.limit(5)
.to_list()
)
# --8<-- [end:fts_index_nested]
assert len(matches) > 0
assert all("puppy" in row["payload"]["text"] for row in matches)
assert len(phrases) > 0
assert all("puppy runs" in row["payload"]["text"] for row in phrases)
@pytest.mark.asyncio
async def test_fts_index_async(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:fts_index_async]
import asyncio
import lancedb
import polars as pl
from lancedb.index import FTS
data = pl.DataFrame(
{
"id": [1, 2],
"text": [
"His first language is spanish",
"Her first language is english",
],
}
)
async def main(data: pl.DataFrame):
uri = "ex_lancedb"
db = await lancedb.connect_async(uri)
tbl = await db.create_table("my_text", data=data, mode="overwrite")
await tbl.create_index("text", config=FTS(language="English"))
response = await tbl.search("spanish", query_type="fts")
result = await response.limit(1).to_polars()
print(result)
return result
if __name__ == "__main__":
asyncio.run(main(data))
# --8<-- [end:fts_index_async]
result = await main(data)
assert result.height == 1
def test_gpu_index_snippets(tmp_db, monkeypatch):
table = tmp_db.create_table(
"gpu_index",
_make_vector_rows(32, 8),
mode="overwrite",
)
calls = []
def fake_create_index(*args, **kwargs):
calls.append(kwargs)
return None
monkeypatch.setattr(table, "create_index", fake_create_index)
# --8<-- [start:gpu_index_cuda]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="cuda",
)
# --8<-- [end:gpu_index_cuda]
# --8<-- [start:gpu_index_mps]
table.create_index(
num_partitions=256,
num_sub_vectors=96,
accelerator="mps",
)
# --8<-- [end:gpu_index_mps]
assert calls[0]["accelerator"] == "cuda"
assert calls[1]["accelerator"] == "mps"
def test_reindexing_incremental(tmp_db):
table = tmp_db.create_table(
"reindexing_incremental",
[{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}],
mode="overwrite",
)
db = tmp_db
# --8<-- [start:reindexing_incremental]
table = db.open_table("reindexing_incremental")
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:reindexing_incremental]
result = table.search().limit(10).to_pandas()
assert len(result) == 2
docs/web-tests/py/test_ingestion_examples.py
+739
View File
@@ -0,0 +1,739 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_create_table_from_dicts():
# --8<-- [start:create_table_from_dicts]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
data = [
{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
{"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1},
]
db.create_table("test_table", data, mode="overwrite")
tbl = db["test_table"]
tbl.head()
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_dicts]
assert tbl.count_rows() == 2
def test_create_table_conflict_handling():
# --8<-- [start:create_table_conflict_handling]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
db.create_table("conflict_table", data)
# Idempotent open: reuse the existing table if it exists.
# The provided data is ignored; the schema is validated against the
# existing table and a mismatch raises an error.
tbl = db.create_table("conflict_table", data, exist_ok=True)
# Overwrite: replace the contents with the provided data.
tbl = db.create_table("conflict_table", data, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_conflict_handling]
def test_create_table_custom_schema():
# --8<-- [start:create_table_custom_schema]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
custom_schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field("lat", pa.float32()),
pa.field("long", pa.float32()),
]
)
data = [
{"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7},
{"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1},
]
tbl = db.create_table(
"my_table_custom_schema", data, schema=custom_schema, mode="overwrite"
)
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_custom_schema]
assert tbl.count_rows() == 2
def test_create_table_from_pandas():
# --8<-- [start:create_table_from_pandas]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pandas as pd
data = pd.DataFrame(
{
"vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
"lat": [45.5, 40.1],
"long": [-122.7, -74.1],
}
)
db.create_table("my_table_pandas", data, mode="overwrite")
db["my_table_pandas"].head()
# --8<-- [end:create_table_from_pandas]
def test_create_table_from_polars():
# --8<-- [start:create_table_from_polars]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import polars as pl
data = pl.DataFrame(
{
"vector": [[3.1, 4.1], [5.9, 26.5]],
"item": ["foo", "bar"],
"price": [10.0, 20.0],
}
)
tbl = db.create_table("my_table_pl", data, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_polars]
assert tbl.count_rows() == 2
def test_create_table_from_arrow():
# --8<-- [start:create_table_from_arrow]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import numpy as np
import pyarrow as pa
dim = 16
total = 2
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float16(), dim)),
pa.field("text", pa.string()),
]
)
data = pa.Table.from_arrays(
[
pa.array(
[np.random.randn(dim).astype(np.float16) for _ in range(total)],
pa.list_(pa.float16(), dim),
),
pa.array(["foo", "bar"]),
],
["vector", "text"],
)
tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_arrow]
assert tbl.count_rows() == 2
def test_create_table_from_pydantic():
# --8<-- [start:create_table_from_pydantic]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
class Content(LanceModel):
movie_id: int
vector: Vector(128)
genres: str
title: str
imdb_id: int
@property
def imdb_url(self) -> str:
return f"https://www.imdb.com/title/tt{self.imdb_id}"
tbl = db.create_table("movielens_small", schema=Content, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_pydantic]
assert tbl.count_rows() == 0
def test_create_table_nested_schema():
# --8<-- [start:create_table_nested_schema]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
from pydantic import BaseModel
class Document(BaseModel):
content: str
source: str
class NestedSchema(LanceModel):
id: str
vector: Vector(1536)
document: Document
tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_nested_schema]
assert tbl.count_rows() == 0
def test_create_empty_table():
# --8<-- [start:create_empty_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("item", pa.string()),
pa.field("price", pa.float32()),
]
)
tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_empty_table]
assert tbl.count_rows() == 0
def test_create_empty_table_pydantic():
# --8<-- [start:create_empty_table_pydantic]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
class Item(LanceModel):
vector: Vector(2)
item: str
price: float
tbl = db.create_table(
"test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite"
)
print("Rows:", tbl.count_rows())
# --8<-- [end:create_empty_table_pydantic]
assert tbl.count_rows() == 0
def test_open_existing_table():
# --8<-- [start:open_existing_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
# Create a table first
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
db.create_table("test_table", data, mode="overwrite")
# List table names
print(db.list_tables().tables)
# Open existing table
tbl = db.open_table("test_table")
print("Rows:", tbl.count_rows())
# --8<-- [end:open_existing_table]
assert tbl.count_rows() == 1
def test_drop_table():
# --8<-- [start:drop_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
# Create a table first
data = [{"vector": [1.1, 1.2], "lat": 45.5}]
db.create_table("my_table", data, mode="overwrite")
# Drop the table
db.drop_table("my_table")
# --8<-- [end:drop_table]
def test_add_from_dataset():
# --8<-- [start:add_from_dataset]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
data_path = str(Path(uri) / "input.parquet")
pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path)
import pyarrow.dataset as ds
dataset = ds.dataset(data_path, format="parquet")
table = db.create_table("my_table", schema=dataset.schema, mode="overwrite")
table.add(dataset)
print("Rows:", table.count_rows())
# --8<-- [end:add_from_dataset]
assert table.count_rows() == 2
def test_create_table_from_iterator():
# --8<-- [start:create_table_from_iterator]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field("item", pa.utf8()),
pa.field("price", pa.float32()),
]
)
def make_batches():
for i in range(5):
yield pa.RecordBatch.from_arrays(
[
pa.array(
[[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
pa.list_(pa.float32(), 4),
),
pa.array(["foo", "bar"]),
pa.array([10.0, 20.0]),
],
["vector", "item", "price"],
)
table = db.create_table("batched_table", schema=schema)
table.add(make_batches())
print("Rows:", table.count_rows())
# --8<-- [end:create_table_from_iterator]
assert table.count_rows() == 10
def test_update_operation():
# --8<-- [start:update_operation]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
table.update(where="id = 2", values={"name": "Bobby"})
print("Rows:", table.count_rows())
# --8<-- [end:update_operation]
assert table.count_rows() == 2
assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby"
def test_update_using_sql():
# --8<-- [start:update_using_sql]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
print("Rows:", table.count_rows())
# --8<-- [end:update_using_sql]
assert table.count_rows() == 2
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
def test_merge_matched_update_only():
# --8<-- [start:merge_matched_update_only]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(table.merge_insert("id").when_matched_update_all().execute(incoming_users))
print("Rows:", table.count_rows())
# --8<-- [end:merge_matched_update_only]
assert table.count_rows() == 2
def test_insert_if_not_exists():
# --8<-- [start:insert_if_not_exists]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users))
print("Rows:", table.count_rows())
# --8<-- [end:insert_if_not_exists]
assert table.count_rows() == 3
def test_merge_update_insert():
# --8<-- [start:merge_update_insert]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_update_insert]
assert table.count_rows() == 3
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
def test_merge_delete_missing_by_source():
# --8<-- [start:merge_delete_missing_by_source]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2, 3],
"name": ["Alice", "Bob", "Charlie"],
"login_count": [10, 20, 5],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.when_not_matched_by_source_delete()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_delete_missing_by_source]
assert table.count_rows() == 2
assert table.count_rows("id = 1") == 0
def test_merge_partial_columns():
# --8<-- [start:merge_partial_columns]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_partial_columns]
assert table.count_rows() == 3
assert table.search().where("id = 3").to_list()[0]["login_count"] is None
def test_write_workflow():
# --8<-- [start:write_workflow]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"users",
[
{"id": 1, "name": "Alice", "login_count": 10},
{"id": 2, "name": "Bob", "login_count": 20},
],
)
table.add([{"id": 3, "name": "Charlie", "login_count": 5}])
table.update(where="id = 2", values={"name": "Bobby"})
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(
[
{"id": 3, "name": "Charlie", "login_count": 6},
{"id": 4, "name": "Dana", "login_count": 1},
]
)
)
print("Rows selected for deletion:", table.search().where("id = 4").to_list())
table.delete("id = 4")
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print(rows)
# --8<-- [end:write_workflow]
assert rows == [
{"id": 1, "name": "Alice", "login_count": 10},
{"id": 2, "name": "Bobby", "login_count": 21},
{"id": 3, "name": "Charlie", "login_count": 6},
]
def test_append_rows():
# --8<-- [start:append_rows]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}])
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print(rows)
# --8<-- [end:append_rows]
assert [row["id"] for row in rows] == [1, 2, 3]
def test_delete_rows():
# --8<-- [start:delete_rows]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"users",
[
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"},
{"id": 3, "name": "Charlie"},
],
)
predicate = "id = 3"
print("Selected:", table.search().where(predicate).to_list())
table.delete(predicate)
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print("Remaining:", rows)
# --8<-- [end:delete_rows]
assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
def test_add_progress():
# --8<-- [start:add_progress]
from tempfile import TemporaryDirectory
import lancedb
def report_progress(progress):
print(f"Rows written: {progress['output_rows']}")
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
table.add(
[{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}],
progress=report_progress,
)
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print("After insertion:", rows)
# --8<-- [end:add_progress]
assert rows == [
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"},
{"id": 3, "name": "Charlie"},
]
docs/web-tests/py/test_integrations.py
+1794
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/py/test_materialized_view.py
+79
View File
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Examples for docs/web/feature-engineering/materialized-views.mdx."""
def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
# --8<-- [start:materialized_view_create]
import lancedb
db = lancedb.connect("./.lancedb")
table = db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view(
"adults",
"people",
select=["name", ("shout", "upper(name)")],
where="age >= 18",
)
print(view.table.count_rows())
# 1
# --8<-- [end:materialized_view_create]
assert view.table.count_rows() == 1
# --8<-- [start:materialized_view_refresh]
table.add([{"name": "bea", "age": 41}])
view.refresh()
print(view.table.count_rows())
# 2
# --8<-- [end:materialized_view_refresh]
assert view.table.count_rows() == 2
assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [
"ADA",
"BEA",
]
def test_materialized_view_with_no_data(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view(
"adults", "people", where="age >= 18", with_no_data=True
)
assert view.table.count_rows() == 0
assert view.refresh().rows_written == 1
assert view.table.count_rows() == 1
def test_materialized_view_refresh_to_source_version(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
table = db.create_table(
"people",
[{"name": "ada", "age": 36}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view("adults", "people", where="age >= 18")
first = table.version
table.add([{"name": "bea", "age": 41}])
view.refresh()
assert view.table.count_rows() == 2
result = view.refresh(source_version=first)
assert (result.source_version, result.rows_written) == (first, 1)
assert view.table.count_rows() == 1
docs/web-tests/py/test_multimodal.py
+232
View File
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pytest
try:
import lancedb
import numpy as np
import pyarrow as pa
import io
from PIL import Image
except ImportError:
pass
# --8<-- [start:multimodal_imports]
import lancedb
import pyarrow as pa
import pandas as pd
import numpy as np
import io
from PIL import Image
# --8<-- [end:multimodal_imports]
def test_multimodal_ingestion(db_path_factory):
# Ensure dependencies are available
pytest.importorskip("PIL")
pytest.importorskip("lancedb")
pytest.importorskip("numpy")
# --8<-- [start:create_dummy_data]
# Create some dummy images
def create_dummy_image(color):
img = Image.new('RGB', (100, 100), color=color)
buf = io.BytesIO()
img.save(buf, format='PNG')
return buf.getvalue()
# Create dataset with metadata, vectors, and image blobs
data = [
{
"id": 1,
"filename": "red_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('red'),
"label": "red"
},
{
"id": 2,
"filename": "blue_square.png",
"vector": np.random.rand(128).astype(np.float32),
"image_blob": create_dummy_image('blue'),
"label": "blue"
}
]
# --8<-- [end:create_dummy_data]
# --8<-- [start:define_schema]
# Define schema explicitly to ensure image_blob is treated as binary
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field("filename", pa.string()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field("image_blob", pa.binary()), # Important: Use pa.binary() for blobs
pa.field("label", pa.string())
])
# --8<-- [end:define_schema]
db_uri = db_path_factory("multimodal_db")
db = lancedb.connect(db_uri)
# --8<-- [start:ingest_data]
tbl = db.create_table("images", data=data, schema=schema, mode="overwrite")
# --8<-- [end:ingest_data]
assert len(tbl) == 2
# --8<-- [start:search_data]
# Search for similar images
query_vector = np.random.rand(128).astype(np.float32)
results = tbl.search(query_vector).limit(1).to_pandas()
# --8<-- [end:search_data]
# --8<-- [start:process_results]
# Convert back to PIL Image
for _, row in results.iterrows():
image_bytes = row['image_blob']
image = Image.open(io.BytesIO(image_bytes))
print(f"Retrieved image: {row['filename']}, Size: {image.size}")
# You can now use 'image' with other libraries or display it
# --8<-- [end:process_results]
assert len(results) == 1
def test_blob_api_definition(db_path_factory):
# --8<-- [start:blob_api_schema]
import pyarrow as pa
# Define schema with Blob API metadata for lazy loading
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"} # Enable Blob API
),
])
# --8<-- [end:blob_api_schema]
# --8<-- [start:blob_api_ingest]
import lancedb
db = lancedb.connect(db_path_factory("blob_db"))
# Create sample data
data = [
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"}
]
# Create the table
tbl = db.create_table("videos", data=data, schema=schema)
# --8<-- [end:blob_api_ingest]
assert len(tbl) == 2
def test_blob_api_to_pandas(db_path_factory):
db = lancedb.connect(db_path_factory("blob_to_pandas_db"))
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
tbl = db.create_table(
"videos",
data=[
{"id": 1, "video": b"fake_video_bytes_1"},
{"id": 2, "video": b"fake_video_bytes_2"},
],
schema=schema,
mode="overwrite",
)
# --8<-- [start:blob_api_to_pandas]
# Default: blob columns come back lazily
df_lazy = tbl.to_pandas()
# Materialize blob bytes eagerly
df_bytes = tbl.to_pandas(blob_mode="bytes")
# Return descriptors instead of payloads
df_desc = tbl.to_pandas(blob_mode="descriptions")
# Forward extra kwargs to PyArrow's to_pandas
df_typed = tbl.to_pandas(split_blocks=True, self_destruct=True)
# --8<-- [end:blob_api_to_pandas]
assert len(df_lazy) == 2
assert isinstance(df_bytes["video"].iloc[0], bytes)
assert df_bytes["video"].tolist() == [
b"fake_video_bytes_1",
b"fake_video_bytes_2",
]
assert len(df_desc) == 2
assert len(df_typed) == 2
@pytest.mark.asyncio
async def test_query_to_pandas_kwargs(db_path_factory):
schema = pa.schema([
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 128)),
pa.field(
"video",
pa.large_binary(),
metadata={"lance-encoding:blob": "true"},
),
])
data = [
{
"id": i,
"vector": np.random.rand(128).astype(np.float32),
"video": f"fake_video_bytes_{i}".encode(),
}
for i in range(10)
]
db = lancedb.connect(db_path_factory("query_to_pandas_db"))
tbl = db.create_table("search_demo", data=data, schema=schema, mode="overwrite")
async_db = await lancedb.connect_async(
str(db_path_factory("query_to_pandas_async_db"))
)
tbl_async = await async_db.create_table(
"search_demo", data=data, schema=schema, mode="overwrite"
)
query_vector = np.random.rand(128).astype(np.float32)
# --8<-- [start:query_to_pandas_kwargs]
# Plain scan query: blob_mode is supported end to end
df_lazy = (
tbl.search()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="lazy")
)
# Same call shape works on async query builders
df_bytes = await (
tbl_async.query()
.where("id = 1")
.select(["id", "video"])
.to_pandas(blob_mode="bytes")
)
# Vector / FTS / hybrid queries can't materialize blob columns,
# so omit them from the projection
df_vec = (
tbl.search(query_vector)
.limit(10)
.select(["id", "vector"])
.to_pandas(split_blocks=True, self_destruct=True)
)
# --8<-- [end:query_to_pandas_kwargs]
assert len(df_lazy) == 1
assert len(df_bytes) == 1
assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1"
assert len(df_vec) == 10
assert "video" not in df_vec.columns
docs/web-tests/py/test_provider_examples.py
+160
View File
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""The provider reranker examples write only to a directory they create.
Each example runs offline. A stand-in embedding function answers for the model
the example asks the registry for, a stand-in reranker replaces the provider
client, and outbound connections are refused, so no model is downloaded and no
provider is called. Sentinel tables in the owned home directory, where these
examples used to write, and in the temporary directory must be left unchanged.
"""
import importlib.util
import socket
import tempfile
from pathlib import Path
from types import SimpleNamespace
from typing import ClassVar
import lancedb
import lancedb.embeddings
import lancedb.rerankers
import pyarrow as pa
import pytest
from lancedb.embeddings import TextEmbeddingFunction, register
from lancedb.rerankers import Reranker
EXAMPLES = Path(__file__).with_name("provider_examples.py")
RERANKERS = {
"reranking_cohere_usage": "CohereReranker",
"reranking_jina_usage": "JinaReranker",
"reranking_watsonx_usage": "WatsonxReranker",
"reranking_openai_usage": "OpenaiReranker",
"reranking_voyageai_usage": "VoyageAIReranker",
}
@register("docs-offline-text")
class OfflineEmbedding(TextEmbeddingFunction):
"""Deterministic four-dimensional vectors, computed locally."""
def ndims(self):
return 4
def generate_embeddings(self, texts):
return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts]
class OfflineReranker(Reranker):
"""Scores every row 1.0 and records which searches it reranked."""
instances: ClassVar[list["OfflineReranker"]] = []
def __init__(self, **kwargs):
super().__init__()
self.kwargs = kwargs
self.searches = []
OfflineReranker.instances.append(self)
def _score(self, results, search):
self.searches.append(search)
drop = [c for c in ("_distance", "_score") if c in results.column_names]
results = results.drop_columns(drop)
scores = pa.array([1.0] * results.num_rows, pa.float32())
return results.append_column("_relevance_score", scores)
def rerank_vector(self, query, vector_results):
return self._score(vector_results, "vector")
def rerank_fts(self, query, fts_results):
return self._score(fts_results, "fts")
def rerank_hybrid(self, query, vector_results, fts_results):
merged = self.merge_results(vector_results, fts_results)
return self._score(merged, "hybrid")
def snapshot(connect, uri):
db = connect(str(uri))
tables = {}
for name in db.list_tables().tables:
table = db.open_table(name)
tables[name] = (table.version, table.schema, table.to_arrow().to_pylist())
return tables
def load_examples():
spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.fixture
def owned(tmp_path, monkeypatch):
home, temp_root = tmp_path / "home", tmp_path / "tmp"
home.mkdir()
temp_root.mkdir()
monkeypatch.setenv("HOME", str(home))
monkeypatch.setattr(tempfile, "tempdir", str(temp_root))
monkeypatch.chdir(tmp_path)
# Anything the examples resolve from "~" must land in the owned home.
assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve())
def refuse(*args, **kwargs):
raise OSError("network access is disabled in this test")
monkeypatch.setattr(socket.socket, "connect", refuse)
monkeypatch.setattr(socket, "create_connection", refuse)
real_connect = lancedb.connect
sentinels = [home / ".lancedb", temp_root / "existing"]
for uri in sentinels:
row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]}
real_connect(str(uri)).create_table("test", [row])
before = [snapshot(real_connect, uri) for uri in sentinels]
registry = lancedb.embeddings.get_registry()
monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding)
for name in RERANKERS.values():
monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker)
monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example")
OfflineReranker.instances.clear()
connected = []
def recording_connect(uri, *args, **kwargs):
connected.append(Path(uri).expanduser().resolve())
return real_connect(uri, *args, **kwargs)
monkeypatch.setattr(lancedb, "connect", recording_connect)
return SimpleNamespace(
home=home,
temp_root=temp_root,
sentinels=sentinels,
before=before,
connect=real_connect,
connected=connected,
)
@pytest.mark.parametrize("example", sorted(RERANKERS))
def test_reranker_example_writes_only_to_its_own_directory(owned, example):
existing = set(owned.temp_root.iterdir())
getattr(load_examples(), example)()
after = [snapshot(owned.connect, uri) for uri in owned.sentinels]
assert after == owned.before
created = set(owned.temp_root.iterdir()) - existing
assert len(created) == 1
(own_directory,) = created
assert len(owned.connected) == 1
assert owned.connected[0].is_relative_to(own_directory.resolve())
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
(reranker,) = OfflineReranker.instances
assert reranker.searches == ["vector", "fts", "hybrid"]
assert reranker.kwargs["api_key"] == "your-api-key"
assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id"
docs/web-tests/py/test_quickstart.py
+353
View File
@@ -0,0 +1,353 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
import shutil
import uuid
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
# What the quickstart's download step leaves in an application's directory.
IMAGE = (
Path(__file__).resolve().parents[2]
/ "web/static/assets/images/quickstart/sir-lancelot.jpg"
)
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
# The connection settings alone never send these tests to a deployment.
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
def deployment_settings():
if os.environ.get(OPT_IN) != "1":
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
if missing:
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
return dict(
uri=os.environ["LANCEDB_URI"],
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
class OwnedTables:
"""A deployment connection on which a test creates only tables of its own.
Each name gets a suffix unique to the test and is claimed only while no
table has it; the test drops what it claimed, whether or not it passed.
"""
def __init__(self, db):
self.db = db
self.suffix = uuid.uuid4().hex[:12]
self.claimed = []
def claim(self, name, existing, mode):
owned = f"{name}_{self.suffix}"
if owned in self.claimed:
return owned, mode
if owned in existing:
raise RuntimeError(f"{owned} already exists and is not this test's")
self.claimed.append(owned)
return owned, "create"
def table_names(self):
names, token = [], None
while True:
page = self.db.list_tables(page_token=token)
names += page.tables
if not (token := page.page_token):
return names
def create_table(self, name, *args, mode="create", **kwargs):
owned, mode = self.claim(name, self.table_names(), mode)
return self.db.create_table(owned, *args, mode=mode, **kwargs)
def drop_claimed(self):
existing = self.table_names()
for name in self.claimed:
if name in existing:
self.db.drop_table(name)
class AsyncOwnedTables(OwnedTables):
async def table_names(self):
names, token = [], None
while True:
page = await self.db.list_tables(page_token=token)
names += page.tables
if not (token := page.page_token):
return names
async def create_table(self, name, *args, mode="create", **kwargs):
owned, mode = self.claim(name, await self.table_names(), mode)
return await self.db.create_table(owned, *args, mode=mode, **kwargs)
async def drop_claimed(self):
existing = await self.table_names()
for name in self.claimed:
if name in existing:
await self.db.drop_table(name)
@pytest.fixture
def deployment():
owned = OwnedTables(lancedb.connect(**deployment_settings()))
yield owned
owned.drop_claimed()
@pytest.fixture(params=["local", "enterprise"])
def db(request, db_path_factory):
if request.param == "local":
yield lancedb.connect(db_path_factory("quickstart_db"))
else:
yield request.getfixturevalue("deployment")
@pytest_asyncio.fixture(params=["local", "enterprise"])
async def async_db(request, db_path_factory):
if request.param == "local":
yield await lancedb.connect_async(db_path_factory("quickstart_async_db"))
return
owned = AsyncOwnedTables(await lancedb.connect_async(**deployment_settings()))
yield owned
await owned.drop_claimed()
def create_characters(db):
# --8<-- [start:quickstart_data]
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
]
# --8<-- [end:quickstart_data]
# --8<-- [start:quickstart_create_table]
table = db.create_table("characters", data=data, mode="overwrite")
# --8<-- [end:quickstart_create_table]
assert table.count_rows() == 3
# --8<-- [start:quickstart_add_data]
table.add(
[
{
"id": "4",
"name": "Morgana",
"role": "Sorceress",
"description": "Powerful sorceress of Avalon.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
"vector": [0.3, 0.9, 0.6, 0.8],
}
]
)
# --8<-- [end:quickstart_add_data]
assert table.count_rows() == 4
return table
def test_quickstart(db):
table = create_characters(db)
# --8<-- [start:quickstart_vector_search_1]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
result = (
table.search(query_vector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(result)
# --8<-- [end:quickstart_vector_search_1]
assert result["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_output_pandas]
# Ensure you run `pip install pandas` beforehand
result = table.search(query_vector).limit(2).to_pandas()
print(result)
# --8<-- [end:quickstart_output_pandas]
assert result["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_curate_with_metadata]
curated = (
table.search(query_vector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.to_polars()
)
print(curated)
# --8<-- [end:quickstart_curate_with_metadata]
assert curated["name"].to_list() == ["Merlin", "Morgana"]
curated = table.search(query_vector).where("stats.magic < 4").limit(2).to_list()
assert [row["name"] for row in curated] == ["King Arthur", "Sir Lancelot"]
@pytest.mark.asyncio
async def test_quickstart_async_api(async_db):
data = [
{
"id": "2",
"name": "Merlin",
"role": "Wizard",
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
}
]
# --8<-- [start:quickstart_create_table_async]
async_table = await async_db.create_table(
"characters",
data=data,
mode="overwrite",
)
# --8<-- [end:quickstart_create_table_async]
# --8<-- [start:quickstart_vector_search_1_async]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
# Ensure you run `pip install polars` beforehand
async_result = await (
await async_table.search(query_vector)
).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
print(async_result)
# --8<-- [end:quickstart_vector_search_1_async]
assert async_result["name"].to_list() == ["Merlin"]
def test_quickstart_derived_feature(db):
table = create_characters(db)
# --8<-- [start:quickstart_add_feature]
table.add_columns(
{
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:quickstart_add_feature]
# --8<-- [start:quickstart_query_feature]
features = table.search().select(["name", "role", "power_score"]).to_polars()
print(features)
# --8<-- [end:quickstart_query_feature]
assert sorted(zip(features["name"], features["power_score"])) == [
("King Arthur", 3.5),
("Merlin", 4.0),
("Morgana", 3.75),
("Sir Lancelot", 3.0),
]
def description_lengths(table):
rows = table.search().select(["description", "description_length"]).to_list()
return {row["description"]: row["description_length"] for row in rows}
def test_quickstart_computed_column(db_path_factory):
table = create_characters(lancedb.connect(db_path_factory("computed")))
# --8<-- [start:quickstart_computed_feature]
table.add_columns(computed={"description_length": "length(description)"})
table.refresh_column("description_length")
# --8<-- [end:quickstart_computed_feature]
lengths = description_lengths(table)
assert lengths == {text: len(text) for text in lengths}
def test_quickstart_computed_column_on_a_deployment(deployment):
table = create_characters(deployment)
# --8<-- [start:quickstart_computed_feature_enterprise]
table.add_columns(computed={"description_length": "length(description)"})
job = table.refresh_column_async("description_length")
job.wait()
# --8<-- [end:quickstart_computed_feature_enterprise]
lengths = description_lengths(table)
assert len(lengths) == 4
assert lengths == {text: len(text) for text in lengths}
with pytest.raises(Exception):
table.refresh_column("description_length")
def test_quickstart_multimodal(db, tmp_path, monkeypatch):
app = tmp_path / "app"
(app / "data").mkdir(parents=True)
shutil.copy(IMAGE, app / "data/sir-lancelot.jpg")
monkeypatch.chdir(app)
# --8<-- [start:quickstart_multimodal_bytes]
from pathlib import Path
image_bytes = Path("data/sir-lancelot.jpg").read_bytes()
multimodal_table = db.create_table(
"character_images",
data=[
{
"id": "lancelot",
"description": "Portrait of Sir Lancelot",
"image": image_bytes,
"vector": [0.9, 0.1, 0.5, 0.4],
}
],
mode="overwrite",
)
# --8<-- [end:quickstart_multimodal_bytes]
assert multimodal_table.count_rows() == 1
stored = multimodal_table.search().select(["image"]).to_list()[0]["image"]
assert stored == IMAGE.read_bytes()
def test_quickstart_training(db):
table = create_characters(db)
# --8<-- [start:quickstart_training]
import torch
from lancedb.permutation import Permutation
table.add_columns(
{
"magic": "cast(stats.magic as float)",
"strength": "cast(stats.strength as float)",
}
)
dataset = Permutation.identity(table).select_columns(["magic", "strength"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
# --8<-- [end:quickstart_training]
batches = list(dataloader)
assert len(batches) == 1
assert set(batches[0]) == {"magic", "strength"}
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
assert sorted(batches[0]["magic"].tolist()) == [1.0, 1.0, 5.0, 5.0]
docs/web-tests/py/test_read_consistency.py
+35
View File
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_read_consistency():
# --8<-- [start:read_consistency]
from datetime import timedelta
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-consistency-") as uri:
writer = lancedb.connect(uri)
table = writer.create_table("documents", [{"id": 1, "text": "first"}])
default_reader = lancedb.connect(uri).open_table("documents")
latest_reader = lancedb.connect(
uri, read_consistency_interval=timedelta(0)
).open_table("documents")
table.add([{"id": 2, "text": "second"}])
default_count = default_reader.count_rows()
latest_count = latest_reader.count_rows()
print(f"Default reader: {default_count} rows") # 1
print(f"Zero-interval reader: {latest_count} rows") # 2
default_reader.checkout_latest()
refreshed_count = default_reader.count_rows()
print(f"After explicit refresh: {refreshed_count} rows") # 2
# --8<-- [end:read_consistency]
assert default_count == 1
assert latest_count == 2
assert refreshed_count == 2
docs/web-tests/py/test_reranker_examples.py
+96
View File
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""The published reranker examples in test_integrations.py write only to a directory they create.
Each example runs as readers see it: its marked region is extracted the way the snippet
generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the
sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT
and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination
and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory,
where these examples used to write, and in the temporary directory must be left unchanged.
"""
import importlib.util
import runpy
import sys
from pathlib import Path
import lancedb.rerankers
import pytest
import test_provider_examples as provider
owned = provider.owned
SOURCE = Path(__file__).with_name("test_integrations.py")
GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py"
MODEL_RERANKERS = {
"reranking_answerdotai_usage": "AnswerdotaiRerankers",
"reranking_colbert_usage": "ColbertReranker",
"reranking_cross_encoder_usage": "CrossEncoderReranker",
}
LOCAL_RERANKERS = [
"reranking_linear_combination_usage",
"reranking_mrr_usage",
"reranking_rrf_usage",
]
def load_generator():
if "mdx_snippets_gen" not in sys.modules:
spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module # its dataclasses resolve annotations by module
spec.loader.exec_module(module)
return sys.modules["mdx_snippets_gen"]
def reader_code(name):
"""The example exactly as the snippet generator publishes it."""
generator = load_generator()
start, end = generator.markers_for_lang("py")
lines = SOURCE.read_text().splitlines(keepends=True)
snippet = generator.parse_snippets(lines, start, end)[name]
return generator.snippet_text(generator.dedent(snippet))
def run_example(owned, tmp_path, name):
script = tmp_path / "scripts" / f"{name}.py"
script.parent.mkdir(exist_ok=True)
script.write_text(reader_code(name))
existing = set(owned.temp_root.iterdir())
namespace = runpy.run_path(str(script))
after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels]
assert after == owned.before
created = set(owned.temp_root.iterdir()) - existing
assert len(created) == 1
(own_directory,) = created
assert len(owned.connected) == 1
assert owned.connected[0].is_relative_to(own_directory.resolve())
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
return namespace
@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS))
def test_model_reranker_example_writes_only_to_its_own_directory(
owned, monkeypatch, tmp_path, name
):
monkeypatch.setattr(
lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker
)
namespace = run_example(owned, tmp_path, name)
(reranker,) = provider.OfflineReranker.instances
assert reranker.searches == ["vector", "fts", "hybrid"]
assert namespace["result"]
@pytest.mark.parametrize("name", LOCAL_RERANKERS)
def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name):
namespace = run_example(owned, tmp_path, name)
assert namespace["result"]
assert all("_relevance_score" in row for row in namespace["result"])
if name == "reranking_mrr_usage":
assert namespace["combined"].num_rows > 0
docs/web-tests/py/test_search.py
+730
View File
@@ -0,0 +1,730 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
# --8<-- [start:import-lancedb]
import lancedb
# --8<-- [end:import-lancedb]
# --8<-- [start:import-numpy]
from lancedb.query import BoostQuery, MatchQuery
import numpy as np
import pyarrow as pa
# --8<-- [end:import-numpy]
# --8<-- [start:import-datetime]
from datetime import datetime
# --8<-- [end:import-datetime]
# --8<-- [start:import-lancedb-pydantic]
from lancedb.pydantic import Vector, LanceModel
# --8<-- [end:import-lancedb-pydantic]
# --8<-- [start:import-pydantic-base-model]
from pydantic import BaseModel
# --8<-- [end:import-pydantic-base-model]
# --8<-- [start:import-lancedb-fts]
from lancedb.index import FTS
# --8<-- [end:import-lancedb-fts]
# --8<-- [start:import-os]
import os
# --8<-- [end:import-os]
# --8<-- [start:import-embeddings]
from lancedb.embeddings import get_registry
# --8<-- [end:import-embeddings]
import pytest
# --8<-- [start:class-definition]
class Metadata(BaseModel):
source: str
timestamp: datetime
class Document(BaseModel):
content: str
meta: Metadata
class LanceSchema(LanceModel):
id: str
vector: Vector(1536)
payload: Document
# --8<-- [end:class-definition]
def test_vector_search():
# --8<-- [start:exhaustive_search]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
tbl = db.create_table("vector_search", data=data, mode="overwrite")
tbl.search(np.random.random((1536))).limit(10).to_list()
# --8<-- [end:exhaustive_search]
# --8<-- [start:exhaustive_search_cosine]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:exhaustive_search_cosine]
# --8<-- [start:create_table_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
# Synchronous client
tbl = db.create_table("documents", data=data, mode="overwrite")
# --8<-- [end:create_table_with_nested_schema]
# --8<-- [start:search_result_as_pyarrow]
tbl.search(np.random.randn(1536)).to_arrow()
# --8<-- [end:search_result_as_pyarrow]
# --8<-- [start:search_result_as_pandas]
tbl.search(np.random.randn(1536)).to_pandas()
# --8<-- [end:search_result_as_pandas]
# --8<-- [start:search_result_as_pandas_flatten_true]
tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
# --8<-- [end:search_result_as_pandas_flatten_true]
# --8<-- [start:search_result_as_pandas_flatten_1]
tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
# --8<-- [end:search_result_as_pandas_flatten_1]
# --8<-- [start:search_result_as_list]
tbl.search(np.random.randn(1536)).to_list()
# --8<-- [end:search_result_as_list]
# --8<-- [start:search_result_as_pydantic]
tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
# --8<-- [end:search_result_as_pydantic]
@pytest.mark.asyncio
async def test_vector_search_async():
# --8<-- [start:exhaustive_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"vector": row, "item": f"item {i}"}
for i, row in enumerate(np.random.random((10_000, 1536)).astype("float32"))
]
async_tbl = await async_db.create_table(
"vector_search_async", data=data, mode="overwrite"
)
(await (await async_tbl.search(np.random.random((1536)))).limit(10).to_list())
# --8<-- [end:exhaustive_search_async]
# --8<-- [start:exhaustive_search_async_cosine]
(
await (await async_tbl.search(np.random.random((1536))))
.distance_type("cosine")
.limit(10)
.to_list()
)
# --8<-- [end:exhaustive_search_async_cosine]
# --8<-- [start:create_table_async_with_nested_schema]
# Let's add 100 sample rows to our dataset
data = [
LanceSchema(
id=f"id{i}",
vector=np.random.randn(1536),
payload=Document(
content=f"document{i}",
meta=Metadata(source=f"source{i % 10}", timestamp=datetime.now()),
),
)
for i in range(100)
]
async_tbl = await async_db.create_table(
"documents_async", data=data, mode="overwrite"
)
# --8<-- [end:create_table_async_with_nested_schema]
# --8<-- [start:search_result_async_as_pyarrow]
await (await async_tbl.search(np.random.randn(1536))).to_arrow()
# --8<-- [end:search_result_async_as_pyarrow]
# --8<-- [start:search_result_async_as_pandas]
await (await async_tbl.search(np.random.randn(1536))).to_pandas()
# --8<-- [end:search_result_async_as_pandas]
# --8<-- [start:search_result_async_as_list]
await (await async_tbl.search(np.random.randn(1536))).to_list()
# --8<-- [end:search_result_async_as_list]
def test_fts_fuzzy_query():
uri = "data/fuzzy-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_fuzzy",
data=pa.table(
{
"text": [
"fa",
"fo", # spellchecker:disable-line
"fob",
"focus",
"foo",
"food",
"foul",
]
}
),
mode="overwrite",
)
table.create_fts_index("text", replace=True)
results = table.search(MatchQuery("foo", "text", fuzziness=1)).to_pandas()
assert len(results) == 4
assert set(results["text"].to_list()) == {
"foo",
"fo", # 1 deletion # spellchecker:disable-line
"fob", # 1 substitution
"food", # 1 insertion
}
def test_fts_boost_query():
uri = "data/boost-example"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts_boost",
data=pa.table(
{
"title": [
"The Hidden Gems of Travel",
"Exploring Nature's Wonders",
"Cultural Treasures Unveiled",
"The Nightlife Chronicles",
"Scenic Escapes and Challenges",
],
"desc": [
"A vibrant city with occasional traffic jams.",
"Beautiful landscapes but overpriced tourist spots.",
"Rich cultural heritage but humid summers.",
"Bustling nightlife but noisy streets.",
"Scenic views but limited public transport options.",
],
}
),
mode="overwrite",
)
table.create_fts_index("desc", replace=True)
results = table.search(
BoostQuery(
MatchQuery("beautiful, cultural, nightlife", "desc"),
MatchQuery("bad traffic jams, overpriced", "desc"),
),
).to_pandas()
# we will hit 3 results because the positive query has 3 hits
assert len(results) == 3
# the one containing "overpriced" will be negatively boosted,
# so it will be the last one
assert (
results["desc"].to_list()[2]
== "Beautiful landscapes but overpriced tourist spots."
)
def test_fts_native():
# --8<-- [start:basic_fts]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table(
"my_table_fts",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
table.create_fts_index("text")
table.search("puppy").limit(10).select(["text"]).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts]
# --8<-- [start:fts_config_stem]
table.create_fts_index("text", tokenizer_name="en_stem", replace=True)
# --8<-- [end:fts_config_stem]
# --8<-- [start:fts_config_folding]
table.create_fts_index(
"text",
language="French",
stem=True,
ascii_folding=True,
replace=True,
)
# --8<-- [end:fts_config_folding]
# --8<-- [start:fts_prefiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=True).to_list()
# --8<-- [end:fts_prefiltering]
# --8<-- [start:fts_postfiltering]
table.search("puppy").limit(10).where("text='foo'", prefilter=False).to_list()
# --8<-- [end:fts_postfiltering]
# --8<-- [start:fts_with_position]
table.create_fts_index("text", with_position=True, replace=True)
# --8<-- [end:fts_with_position]
# --8<-- [start:fts_incremental_index]
table.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
table.optimize()
# --8<-- [end:fts_incremental_index]
@pytest.mark.asyncio
async def test_fts_native_async():
# --8<-- [start:basic_fts_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
async_tbl = await async_db.create_table(
"my_table_fts_async",
data=[
{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"},
{"vector": [5.9, 26.5], "text": "There are several kittens playing"},
],
mode="overwrite",
)
# async API uses our native FTS algorithm
await async_tbl.create_index("text", config=FTS())
await (await async_tbl.search("puppy")).select(["text"]).limit(10).to_list()
# [{'text': 'Frodo was a happy puppy', '_score': 0.6931471824645996}]
# ...
# --8<-- [end:basic_fts_async]
# --8<-- [start:fts_config_stem_async]
await async_tbl.create_index(
"text", config=FTS(language="English", stem=True, remove_stop_words=True)
)
# --8<-- [end:fts_config_stem_async]
# --8<-- [start:fts_config_folding_async]
await async_tbl.create_index(
"text", config=FTS(language="French", stem=True, ascii_folding=True)
)
# --8<-- [end:fts_config_folding_async]
# --8<-- [start:fts_prefiltering_async]
await (await async_tbl.search("puppy")).limit(10).where("text='foo'").to_list()
# --8<-- [end:fts_prefiltering_async]
# --8<-- [start:fts_postfiltering_async]
await (
(await async_tbl.search("puppy"))
.limit(10)
.where("text='foo'")
.postfilter()
.to_list()
)
# --8<-- [end:fts_postfiltering_async]
# --8<-- [start:fts_with_position_async]
await async_tbl.create_index("text", config=FTS(with_position=True))
# --8<-- [end:fts_with_position_async]
# --8<-- [start:fts_incremental_index_async]
await async_tbl.add([{"vector": [3.1, 4.1], "text": "Frodo was a happy puppy"}])
await async_tbl.optimize()
# --8<-- [end:fts_incremental_index_async]
def _vectors(n, dim, seed=0, column="vector"):
rng = np.random.default_rng(seed)
return [
{column: rng.random(dim).astype("float32"), "id": i} for i in range(n)
]
def _build_indexed_table(db, name, dim=128, n=512):
tbl = db.create_table(name, _vectors(n, dim), mode="overwrite")
tbl.create_index(metric="cosine", num_partitions=4, num_sub_vectors=8)
return tbl
def test_vs_distance_metric_and_brute_force(tmp_db):
tbl = tmp_db.create_table("vs_metric", _vectors(64, 1536), mode="overwrite")
# --8<-- [start:configure_distance_metric]
tbl.search(np.random.random((1536))).distance_type("cosine").limit(10).to_list()
# --8<-- [end:configure_distance_metric]
# --8<-- [start:brute_force_search]
tbl.search(np.random.random((1536))).limit(3).to_list()
# --8<-- [end:brute_force_search]
def test_vs_select_vector_column(tmp_db):
db = tmp_db
# --8<-- [start:select_vector_column]
import pyarrow as pa
schema = pa.schema([
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), 2))]),
),
])
table = db.create_table(
"nested",
data=[{"id": 0, "image": {"embedding": [0.0, 1.0]}}],
schema=schema,
)
# Inferred: the only vector leaf is `image.embedding`.
table.search([0.0, 1.0]).limit(1).to_list()
# Explicit: required when more than one vector column matches.
table.search([0.0, 1.0], vector_column_name="image.embedding").limit(1).to_list()
# --8<-- [end:select_vector_column]
def test_vs_index_nested_column(tmp_db):
dim = 16
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"image",
pa.struct([pa.field("embedding", pa.list_(pa.float32(), dim))]),
),
]
)
rng = np.random.default_rng(0)
data = [
{"id": i, "image": {"embedding": rng.random(dim).astype("float32").tolist()}}
for i in range(512)
]
table = tmp_db.create_table(
"nested_index", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:index_nested_column]
table.create_index(vector_column_name="image.embedding")
# --8<-- [end:index_nested_column]
assert table.list_indices()
def test_vs_indexed_queries(tmp_db):
table = _build_indexed_table(tmp_db, "vs_indexed", dim=128, n=512)
embedding = np.random.random(128)
# --8<-- [start:exact_vs_approximate_distances]
# Indexed ANN search without refinement (fast, approximate `_distance`)
fast_results = (
table.search(embedding)
.limit(10)
.to_pandas()
)
# Recompute distances on full vectors for reranked candidates
exact_distance_results = (
table.search(embedding)
.limit(10)
.refine_factor(1)
.to_pandas()
)
# Rerank a larger candidate set for better recall (higher latency)
higher_recall_results = (
table.search(embedding)
.limit(10)
.refine_factor(20)
.to_pandas()
)
# --8<-- [end:exact_vs_approximate_distances]
# --8<-- [start:bypass_vector_index]
table.search(embedding).bypass_vector_index().limit(5).to_pandas()
# --8<-- [end:bypass_vector_index]
assert len(fast_results) == 10
assert len(exact_distance_results) == 10
assert len(higher_recall_results) == 10
def test_vs_fast_search(tmp_db, monkeypatch):
table = _build_indexed_table(tmp_db, "vs_fast", dim=128, n=512)
embedding = np.random.random(128)
# `fast_search` is an Enterprise/async query flag; strip it so the snippet
# runs against a local OSS table without changing what readers see.
real_search = table.search
monkeypatch.setattr(
table,
"search",
lambda *a, **k: real_search(
*a, **{key: val for key, val in k.items() if key != "fast_search"}
),
)
# --8<-- [start:fast_search]
table.search(embedding, fast_search=True).limit(5).to_pandas()
# --8<-- [end:fast_search]
def test_vs_distance_range(tmp_db):
tbl = tmp_db.create_table("vs_distance_range", _vectors(256, 256), mode="overwrite")
# --8<-- [start:search_distance_range]
query = np.random.random(256)
# Search for the vectors within the range of [0.1, 0.5)
tbl.search(query).distance_range(0.1, 0.5).to_arrow()
# Search for the vectors with the distance less than 0.5
tbl.search(query).distance_range(upper_bound=0.5).to_arrow()
# Search for the vectors with the distance greater or equal to 0.1
tbl.search(query).distance_range(lower_bound=0.1).to_arrow()
# --8<-- [end:search_distance_range]
def test_vs_multivector_search(tmp_db):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.list_(pa.float32(), 256))),
]
)
rng = np.random.default_rng(0)
data = [{"id": i, "vector": rng.random(size=(2, 256)).tolist()} for i in range(64)]
tbl = tmp_db.create_table(
"vs_multivector", data=data, schema=schema, mode="overwrite"
)
# --8<-- [start:multivector_search]
query_multi = np.random.random(size=(2, 256))
results_multi = tbl.search(query_multi).limit(5).to_pandas()
# --8<-- [end:multivector_search]
assert len(results_multi) <= 5
def test_vs_binary_search(tmp_db):
db = tmp_db
# --8<-- [start:search_binary_vectors]
import numpy as np
import pyarrow as pa
schema = pa.schema(
[
pa.field("id", pa.int64()),
# for dim=256, lance stores every 8 bits in a byte
# so the vector field should be a list of 256 / 8 = 32 bytes
pa.field("vector", pa.list_(pa.uint8(), 32)),
]
)
tbl = db.create_table("my_binary_vectors", schema=schema)
data = []
for i in range(1024):
vector = np.random.randint(0, 2, size=256)
# pack the binary vector into bytes to save space
packed_vector = np.packbits(vector)
data.append(
{
"id": i,
"vector": packed_vector,
}
)
tbl.add(data)
query = np.random.randint(0, 2, size=256)
packed_query = np.packbits(query)
tbl.search(packed_query).distance_type("hamming").to_arrow()
# --8<-- [end:search_binary_vectors]
def test_vs_enterprise_filtering(tmp_db, monkeypatch):
import sys
import types
dim = 384
rng = np.random.default_rng(0)
rows = [
{
"vector": rng.random(dim).astype("float32"),
"text": f"story {i}",
"keywords": f"kw{i}",
"label": i % 4,
}
for i in range(50)
]
tmp_db.create_table("lancedb-enterprise-quickstart", data=rows, mode="overwrite")
db = tmp_db
# Mock the Hugging Face dataset loader to avoid network downloads.
class _FakeDataset:
def __init__(self, n):
self._emb = [rng.random(dim).astype("float32") for _ in range(n)]
self._kw = [f"kw{i}" for i in range(n)]
def __getitem__(self, key):
if isinstance(key, str):
return {"keywords_embeddings": self._emb, "keywords": self._kw}[key]
return {"keywords": self._kw[key], "keywords_embeddings": self._emb[key]}
fake_datasets = types.ModuleType("datasets")
fake_datasets.load_dataset = lambda *a, **k: _FakeDataset(10)
monkeypatch.setitem(sys.modules, "datasets", fake_datasets)
# --8<-- [start:vector_search_prefilter]
from datasets import load_dataset
# Load query vector from dataset
query_dataset = load_dataset("sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5001]")
print(f"Query keywords: {query_dataset[0]['keywords']}")
query_embed = query_dataset["keywords_embeddings"][0]
# Open table and perform search
table_name = "lancedb-enterprise-quickstart"
table = db.open_table(table_name)
# Vector search with filters (pre-filtering is the default)
search_results = (
table.search(query_embed)
.where("label > 2")
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Search results (with pre-filtering):")
print(search_results)
# --8<-- [end:vector_search_prefilter]
# --8<-- [start:vector_search_postfilter]
results_post_filtered = (
table.search(query_embed)
.where("label > 1", prefilter=False)
.select(["text", "keywords", "label"])
.limit(5)
.to_pandas()
)
print("Vector search results with post-filter:")
print(results_post_filtered)
# --8<-- [end:vector_search_postfilter]
# --8<-- [start:batch_search]
# Load a batch of query embeddings
query_dataset = load_dataset(
"sunhaozhepy/ag_news_sbert_keywords_embeddings", split="test[5000:5005]"
)
query_embeds = query_dataset["keywords_embeddings"]
batch_results = table.search(query_embeds).limit(5).to_pandas()
print(batch_results)
# --8<-- [end:batch_search]
@pytest.mark.skip()
def test_hybrid_search():
# --8<-- [start:import-openai]
import openai
# --8<-- [end:import-openai]
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search]
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
uri = "data/sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table("documents", schema=Documents)
# ingest docs with auto-vectorization
table.add(data)
# Create a fts index before the hybrid search
table.create_fts_index("text")
# hybrid search with default re-ranker
table.search("flower moon", query_type="hybrid").to_pandas()
# --8<-- [end:basic_hybrid_search]
# --8<-- [start:hybrid_search_pass_vector_text]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
(
table.search(query_type="hybrid")
.vector(vector_query)
.text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text]
@pytest.mark.skip
async def test_hybrid_search_async():
import openai
# --8<-- [start:openai-embeddings]
# Ingest embedding function in LanceDB table
# Configuring the environment variable OPENAI_API_KEY
if "OPENAI_API_KEY" not in os.environ:
# OR set the key here as a variable
openai.api_key = "sk-..."
embeddings = get_registry().get("openai").create()
# --8<-- [end:openai-embeddings]
# --8<-- [start:class-Documents]
class Documents(LanceModel):
vector: Vector(embeddings.ndims()) = embeddings.VectorField()
text: str = embeddings.SourceField()
# --8<-- [end:class-Documents]
# --8<-- [start:basic_hybrid_search_async]
uri = "data/sample-lancedb"
async_db = await lancedb.connect_async(uri)
data = [
{"text": "rebel spaceships striking from a hidden base"},
{"text": "have won their first victory against the evil Galactic Empire"},
{"text": "during the battle rebel spies managed to steal secret plans"},
{"text": "to the Empire's ultimate weapon the Death Star"},
]
async_tbl = await async_db.create_table("documents_async", schema=Documents)
# ingest docs with auto-vectorization
await async_tbl.add(data)
# Create a fts index before the hybrid search
await async_tbl.create_index("text", config=FTS())
text_query = "flower moon"
# hybrid search with default re-ranker
await (await async_tbl.search("flower moon", query_type="hybrid")).to_pandas()
# --8<-- [end:basic_hybrid_search_async]
# --8<-- [start:hybrid_search_pass_vector_text_async]
vector_query = [0.1, 0.2, 0.3, 0.4, 0.5]
text_query = "flower moon"
await (
async_tbl.query()
.nearest_to(vector_query)
.nearest_to_text(text_query)
.limit(5)
.to_pandas()
)
# --8<-- [end:hybrid_search_pass_vector_text_async]
docs/web-tests/py/test_search_examples.py
+228
View File
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_vector_search():
# --8<-- [start:vector_search]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
rows = (
table.search([1.0, 0.0])
.distance_type("l2")
.select(["id", "text", "_distance"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:vector_search]
assert [row["id"] for row in rows] == [1, 2]
assert rows[0]["_distance"] == 0.0
def test_text_search():
# --8<-- [start:text_search]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.index import FTS
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
},
],
)
table.create_index("text", config=FTS())
rows = (
table.search("puppy", query_type="fts", fts_columns="text")
.select(["id", "text", "_score"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:text_search]
assert [row["id"] for row in rows] == [1]
assert rows[0]["_score"] > 0
def test_hybrid_search():
# --8<-- [start:hybrid_search]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.index import FTS
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
from lancedb.rerankers import RRFReranker
table.create_index("text", config=FTS())
rows = (
table.search(
query_type="hybrid", vector_column_name="vector", fts_columns="text"
)
.vector([1.0, 0.0])
.text("puppy")
.rerank(RRFReranker())
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:hybrid_search]
assert [row["id"] for row in rows] == [1, 2]
assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"]
def test_filter_search():
# --8<-- [start:filter_search]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
rows = (
table.search([0.0, 1.0])
.where("category = 'animals'", prefilter=True)
.select(["id", "text", "category", "_distance"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:filter_search]
assert {row["id"] for row in rows} == {1, 2}
assert all(row["category"] == "animals" for row in rows)
def test_query_plan():
# --8<-- [start:query_plan]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2)
print("Plan:", query.explain_plan())
print("Measured execution:", query.analyze_plan())
rows = query.to_list()
print(rows)
# --8<-- [end:query_plan]
assert [row["id"] for row in rows] == [1, 2]
docs/web-tests/py/test_storage.py
+162
View File
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
class DummyTable:
def __init__(self, name: str, storage_options: dict | None = None):
self.name = name
self.storage_options = storage_options
class DummyConnection:
def __init__(self, uri: str, options: dict):
self.uri = uri
self.options = options
self.created_tables: list[DummyTable] = []
def create_table(
self,
name: str,
data,
storage_options: dict | None = None,
):
table = DummyTable(name, storage_options=storage_options)
self.created_tables.append(table)
return table
@pytest.fixture
def fake_connect(monkeypatch):
calls: list[DummyConnection] = []
def _fake_connect(uri: str, **kwargs):
conn = DummyConnection(uri, kwargs)
calls.append(conn)
return conn
monkeypatch.setattr(lancedb, "connect", _fake_connect)
return calls
def test_storage_snippets(fake_connect):
# --8<-- [start:storage_connect_s3]
db = lancedb.connect("s3://bucket/path")
# --8<-- [end:storage_connect_s3]
# --8<-- [start:storage_connect_gcs]
db = lancedb.connect("gs://bucket/path")
# --8<-- [end:storage_connect_gcs]
# --8<-- [start:storage_connect_azure]
db = lancedb.connect("az://bucket/path")
# --8<-- [end:storage_connect_azure]
# --8<-- [start:storage_connect_timeout]
db = lancedb.connect(
"s3://bucket/path",
storage_options={"timeout": "60s"},
)
# --8<-- [end:storage_connect_timeout]
# --8<-- [start:storage_table_timeout]
table = db.create_table(
"table",
[{"a": 1, "b": 2}],
storage_options={"timeout": "60s"},
)
# --8<-- [end:storage_table_timeout]
# --8<-- [start:storage_s3_ddb]
db = lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
)
# --8<-- [end:storage_s3_ddb]
# --8<-- [start:storage_s3_ddb_local]
db = lancedb.connect(
"s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
storage_options={
"endpoint": "http://localhost:4566",
"dynamodb_endpoint": "http://localhost:4566",
"allow_http": "true",
},
)
# --8<-- [end:storage_s3_ddb_local]
# --8<-- [start:storage_s3_sse_kms]
db = lancedb.connect(
"s3://bucket/path",
storage_options={
"aws_server_side_encryption": "aws:kms",
"aws_sse_kms_key_id": "<your-kms-key-id-or-arn>",
},
)
# --8<-- [end:storage_s3_sse_kms]
# --8<-- [start:storage_azure_sas]
db = lancedb.connect(
"az://my-container/my-database",
storage_options={
"azure_storage_account_name": "some-account",
"azure_storage_sas_token": "<sas-token>",
},
)
# --8<-- [end:storage_azure_sas]
# --8<-- [start:storage_s3_minio]
db = lancedb.connect(
"s3://bucket/path",
storage_options={
"region": "us-east-1",
"endpoint": "http://minio:9000",
},
)
# --8<-- [end:storage_s3_minio]
# --8<-- [start:storage_s3_express]
db = lancedb.connect(
"s3://my-bucket--use1-az4--x-s3/path",
storage_options={
"region": "us-east-1",
"s3_express": "true",
},
)
# --8<-- [end:storage_s3_express]
# --8<-- [start:storage_gcs_service_account]
db = lancedb.connect(
"gs://my-bucket/my-database",
storage_options={
"service_account": "path/to/service-account.json",
},
)
# --8<-- [end:storage_gcs_service_account]
# --8<-- [start:storage_azure_account]
db = lancedb.connect(
"az://my-container/my-database",
storage_options={
"account_name": "some-account",
"account_key": "some-key",
},
)
# --8<-- [end:storage_azure_account]
# --8<-- [start:storage_tigris_connect]
db = lancedb.connect(
"s3://your-bucket/path",
storage_options={
"endpoint": "https://t3.storage.dev",
"region": "auto",
},
)
# --8<-- [end:storage_tigris_connect]
assert len(fake_connect) == 13
assert all(
conn.uri.startswith(("s3://", "gs://", "az://", "s3+ddb://"))
for conn in fake_connect
)
docs/web-tests/py/test_tables.py
+1536
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/rs/Cargo.lock
Generated
+7503
View File
File diff suppressed because it is too large Load Diff
docs/web-tests/rs/Cargo.toml
+76
View File
@@ -0,0 +1,76 @@
[package]
name = "rs"
version = "0.1.0"
edition = "2024"
[dependencies]
arrow-array = "58.3.0"
arrow-json = "58.3.0"
arrow-schema = "58.3.0"
futures = "0.3.31"
futures-util = "0.3.31"
lance-namespace = "8.0.0"
lance-index = "=8.0.0"
lancedb = { version = "0.31.0", features = ["polars", "remote"] }
polars = ">=0.37,<0.40.0"
polars-arrow = ">=0.37,<0.40.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1.0.145"
tempfile = "3.23.0"
tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
[[example]]
name = "basic_usage"
path = "basic_usage.rs"
[[example]]
name = "connection"
path = "connection.rs"
[[example]]
name = "quickstart"
path = "quickstart.rs"
[[example]]
name = "tables"
path = "tables.rs"
[[example]]
name = "multimodal"
path = "multimodal.rs"
[[example]]
name = "search"
path = "search.rs"
[[example]]
name = "ingestion_examples"
path = "ingestion_examples.rs"
[[example]]
name = "ingestion_delete"
path = "ingestion_delete.rs"
[[example]]
name = "ingestion_progress"
path = "ingestion_progress.rs"
[[example]]
name = "search_vector_search"
path = "search_vector_search.rs"
[[example]]
name = "search_text_search"
path = "search_text_search.rs"
[[example]]
name = "search_hybrid_search"
path = "search_hybrid_search.rs"
[[example]]
name = "search_filter_search"
path = "search_filter_search.rs"
[[example]]
name = "search_query_plan"
path = "search_query_plan.rs"
docs/web-tests/rs/basic_usage.rs
+346
View File
@@ -0,0 +1,346 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use serde::Deserialize;
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
// --8<-- [start:basic_imports]
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int8Array, Int16Array, RecordBatch, RecordBatchIterator, StringArray,
StructArray,
};
use arrow_schema::{DataType, Field, FieldRef, Schema};
use futures_util::TryStreamExt;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, table::NewColumnTransform};
// --8<-- [end:basic_imports]
#[derive(Debug, Clone, Deserialize)]
struct Stats {
strength: i8,
courage: i8,
magic: i8,
wisdom: i8,
}
#[derive(Debug, Clone, Deserialize)]
struct Character {
id: i16,
name: String,
role: String,
description: String,
vector: [f32; 4],
stats: Stats,
}
fn camelot_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
Field::new("id", DataType::Int16, false),
Field::new("name", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("description", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
]))
}
fn characters_to_record_batch(schema: Arc<Schema>, characters: &[Character]) -> RecordBatch {
let ids = Int16Array::from_iter_values(characters.iter().map(|c| c.id));
let names = StringArray::from_iter_values(characters.iter().map(|c| c.name.as_str()));
let roles = StringArray::from_iter_values(characters.iter().map(|c| c.role.as_str()));
let descriptions =
StringArray::from_iter_values(characters.iter().map(|c| c.description.as_str()));
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
characters
.iter()
.map(|c| Some(c.vector.iter().copied().map(Some).collect::<Vec<_>>())),
4,
);
let strength = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.strength));
let courage = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.courage));
let magic = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.magic));
let wisdom = Int8Array::from_iter_values(characters.iter().map(|c| c.stats.wisdom));
let stats_fields: Vec<FieldRef> = vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
];
let stats = StructArray::new(
stats_fields.into(),
vec![
Arc::new(strength),
Arc::new(courage),
Arc::new(magic),
Arc::new(wisdom),
],
None,
);
RecordBatch::try_new(
schema,
vec![
Arc::new(ids),
Arc::new(names),
Arc::new(roles),
Arc::new(descriptions),
Arc::new(vectors),
Arc::new(stats),
],
)
.unwrap()
}
fn characters_to_reader(schema: Arc<Schema>, characters: &[Character]) -> BatchIter {
let batch = characters_to_record_batch(schema.clone(), characters);
Box::new(RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema))
}
fn camelot_json_path() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("camelot.json")
}
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().to_str().unwrap();
let db = connect(uri).execute().await.unwrap();
// --8<-- [start:data_load]
let data: Vec<Character> =
serde_json::from_str(&fs::read_to_string(camelot_json_path()).unwrap()).unwrap();
// --8<-- [end:data_load]
let schema = camelot_schema();
// --8<-- [start:basic_create_table]
let mut table = db
.create_table("camelot", characters_to_reader(schema.clone(), &data))
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:basic_create_table]
assert_eq!(table.count_rows(None).await.unwrap(), 8);
// --8<-- [start:basic_open_table]
table = db.open_table("camelot").execute().await.unwrap();
// --8<-- [end:basic_open_table]
// --8<-- [start:basic_create_empty_table]
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int16, false),
Field::new("name", DataType::Utf8, false),
Field::new("role", DataType::Utf8, false),
Field::new("description", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
false,
),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("courage", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("wisdom", DataType::Int8, false)),
])),
false,
),
]));
db.create_empty_table("camelot_empty", schema)
.mode(CreateTableMode::Overwrite)
.execute()
.await
.unwrap();
// --8<-- [end:basic_create_empty_table]
db.drop_table("camelot_empty", &[]).await.unwrap();
// --8<-- [start:basic_add_data]
let magical_characters = vec![
Character {
id: 9,
name: "Morgan le Fay".to_string(),
role: "Sorceress".to_string(),
description: "A powerful enchantress, Arthur's half-sister, and a complex figure who oscillates between aiding and opposing Camelot.".to_string(),
vector: [0.10, 0.84, 0.25, 0.70],
stats: Stats {
strength: 2,
courage: 3,
magic: 5,
wisdom: 4,
},
},
Character {
id: 10,
name: "The Lady of the Lake".to_string(),
role: "Mystical Guardian".to_string(),
description: "A mysterious supernatural figure associated with Avalon, known for giving Arthur the sword Excalibur.".to_string(),
vector: [0.00, 0.90, 0.58, 0.88],
stats: Stats {
strength: 2,
courage: 3,
magic: 5,
wisdom: 5,
},
},
];
table
.add(characters_to_reader(camelot_schema(), &magical_characters))
.execute()
.await
.unwrap();
// --8<-- [end:basic_add_data]
// --8<-- [start:basic_vector_search]
let query_vector = [0.03, 0.85, 0.61, 0.90];
let result = table
.query()
.nearest_to(&query_vector)
.unwrap()
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{result:?}");
// --8<-- [end:basic_vector_search]
// --8<-- [start:basic_add_columns]
table
.add_columns(
NewColumnTransform::SqlExpressions(vec![(
"power".to_string(),
"cast(((stats.strength + stats.courage + stats.magic + stats.wisdom) / 4.0) as float)"
.to_string(),
)]),
None,
)
.await
.unwrap();
// --8<-- [end:basic_add_columns]
// --8<-- [start:basic_vector_search_q1]
// Who are the characters similar to "wizard"?
let query_vector_1 = [0.03, 0.85, 0.61, 0.90];
let r1 = table
.query()
.nearest_to(&query_vector_1)
.unwrap()
.limit(5)
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r1:?}");
// --8<-- [end:basic_vector_search_q1]
// --8<-- [start:basic_vector_search_q2]
// Who are the characters similar to "wizard" with high magic stats?
let query_vector_2 = [0.03, 0.85, 0.61, 0.90];
let r2 = table
.query()
.nearest_to(&query_vector_2)
.unwrap()
.only_if("stats.magic > 3")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r2:?}");
// --8<-- [end:basic_vector_search_q2]
// --8<-- [start:basic_vector_search_q3]
// Who are the strongest characters?
let r3 = table
.query()
.only_if("stats.strength > 3")
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
]))
.limit(5)
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r3:?}");
// --8<-- [end:basic_vector_search_q3]
// --8<-- [start:basic_vector_search_q4]
// Who are the strongest characters?
let r4 = table
.query()
.select(Select::Columns(vec![
"name".to_string(),
"role".to_string(),
"description".to_string(),
"power".to_string(),
]))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
println!("{r4:?}");
// --8<-- [end:basic_vector_search_q4]
// --8<-- [start:basic_drop_columns]
table.drop_columns(&["power"]).await.unwrap();
// --8<-- [end:basic_drop_columns]
// --8<-- [start:basic_delete_rows]
table.delete("role = 'Traitor Knight'").await.unwrap();
// --8<-- [end:basic_delete_rows]
// --8<-- [start:basic_drop_table]
db.drop_table("camelot", &[]).await.unwrap();
// --8<-- [end:basic_drop_table]
}
docs/web-tests/rs/bedrock.rs
+92
View File
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{iter::once, sync::Arc};
use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use aws_config::Region;
use aws_sdk_bedrockruntime::Client;
use futures::StreamExt;
use lancedb::{
arrow::IntoArrow,
connect,
embeddings::{bedrock::BedrockEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let tempdir = tempfile::tempdir().unwrap();
let tempdir = tempdir.path().to_str().unwrap();
// create Bedrock embedding function
let region: String = "us-east-1".to_string();
let config = aws_config::defaults(aws_config::BehaviorVersion::latest())
.region(Region::new(region))
.load()
.await;
let embedding = Arc::new(BedrockEmbeddingFunction::new(
Client::new(&config), // AWS Region
));
let db = connect(tempdir).execute().await?;
db.embedding_registry()
.register("bedrock", embedding.clone())?;
let table = db
.create_table("vectors", make_data())
.add_embedding(EmbeddingDefinition::new(
"text",
"bedrock",
Some("embeddings"),
))?
.execute()
.await?;
// execute vector search
let query = Arc::new(StringArray::from_iter_values(once("something warm")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table
.vector_search(query_vector)?
.limit(1)
.execute()
.await?;
let rb = results.next().await.unwrap()?;
let out = rb
.column_by_name("text")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
let text = out.iter().next().unwrap().unwrap();
println!("Closest match: {}", text);
Ok(())
}
fn make_data() -> impl IntoArrow {
let schema = Schema::new(vec![
Field::new("id", DataType::Int32, true),
Field::new("text", DataType::Utf8, false),
Field::new("price", DataType::Float64, false),
]);
let id = Int32Array::from(vec![1, 2, 3, 4]);
let text = StringArray::from_iter_values(vec![
"Black T-Shirt",
"Leather Jacket",
"Winter Parka",
"Hooded Sweatshirt",
]);
let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
let schema = Arc::new(schema);
let rb = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(id), Arc::new(text), Arc::new(price)],
)
.unwrap();
Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
}
docs/web-tests/rs/connection.rs
+169
View File
@@ -0,0 +1,169 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::path::PathBuf;
use lancedb::{Connection, connect};
async fn connect_local() -> lancedb::Result<Connection> {
// --8<-- [start:connect]
let db = connect("ex_lancedb").execute().await?;
// --8<-- [end:connect]
Ok(db)
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let temp_dir = tempfile::tempdir()?;
std::env::set_current_dir(temp_dir.path())?;
let db = connect_local().await?;
db.create_empty_table(
"probe",
std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
"id",
arrow_schema::DataType::Int64,
false,
)])),
)
.execute()
.await?;
let reopened = connect(temp_dir.path().join("ex_lancedb").to_str().unwrap())
.execute()
.await?;
assert_eq!(reopened.table_names().execute().await?, vec!["probe"]);
// Connects to a deployment only when asked, and only lists its tables.
if std::env::var("LANCEDB_DOCS_TEST_DEPLOYMENT").as_deref() == Ok("1") {
let db = connect_enterprise_quickstart().await?;
println!("{:?}", db.table_names().execute().await?);
}
Ok(())
}
async fn connect_enterprise_quickstart() -> Result<Connection, Box<dyn std::error::Error>> {
// --8<-- [start:connect_enterprise_quickstart]
let uri = std::env::var("LANCEDB_URI")?; // db://your-database
let api_key = std::env::var("LANCEDB_API_KEY")?;
let region = std::env::var("LANCEDB_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let host_override = std::env::var("LANCEDB_HOST_OVERRIDE")?; // https://your-endpoint
let db = connect(&uri)
.api_key(&api_key)
.region(&region)
.host_override(&host_override)
.execute()
.await?;
// --8<-- [end:connect_enterprise_quickstart]
Ok(db)
}
// Compiled as documentation, never called by the example runner.
#[allow(dead_code)]
async fn connect_object_storage() -> lancedb::Result<Connection> {
// --8<-- [start:connect_object_storage]
let uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
let db = connect(uri).execute().await?;
// --8<-- [end:connect_object_storage]
Ok(db)
}
async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> {
// --8<-- [start:namespace_table_ops]
let conn = connect(uri).execute().await?;
let search_namespace = vec!["prod".to_string(), "search".to_string()];
let recommendations_namespace = vec!["prod".to_string(), "recommendations".to_string()];
let schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![
arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false),
]));
conn.create_empty_table("user", schema.clone())
.namespace(search_namespace.clone())
.execute()
.await?;
conn.create_empty_table("user", schema)
.namespace(recommendations_namespace.clone())
.execute()
.await?;
let search_table_names = conn
.table_names()
.namespace(search_namespace)
.execute()
.await?;
let recommendation_table_names = conn
.table_names()
.namespace(recommendations_namespace)
.execute()
.await?;
println!("{search_table_names:?}"); // ["user"]
println!("{recommendation_table_names:?}"); // ["user"]
// --8<-- [end:namespace_table_ops]
Ok(())
}
async fn namespace_admin_ops_example() -> lancedb::Result<()> {
// --8<-- [start:namespace_admin_ops]
let mut properties = std::collections::HashMap::new();
properties.insert("root".to_string(), "./local_lancedb".to_string());
let db = lancedb::connect_namespace("dir", properties).execute().await?;
let namespace = vec!["prod".to_string(), "search".to_string()];
db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
db.create_namespace(lance_namespace::models::CreateNamespaceRequest {
id: Some(namespace.clone()),
..Default::default()
})
.await?;
let child_namespaces = db
.list_namespaces(lance_namespace::models::ListNamespacesRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
println!(
"Child namespaces under {:?}: {:?}",
namespace, child_namespaces
);
// Child namespaces under ["prod", "search"]: ["search"]
db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
id: Some(namespace.clone()),
..Default::default()
})
.await?;
db.drop_namespace(lance_namespace::models::DropNamespaceRequest {
id: Some(vec!["prod".to_string()]),
..Default::default()
})
.await?;
// --8<-- [end:namespace_admin_ops]
Ok(())
}
#[allow(dead_code)]
fn repo_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}
// Compiled as documentation, never called by the example runner.
#[allow(dead_code)]
async fn connect_enterprise_placeholders() -> Result<Connection, Box<dyn std::error::Error>> {
// --8<-- [start:connect_enterprise_placeholders]
let db = connect("db://your-database")
.api_key("your-api-key")
.region("us-east-1")
.host_override("https://your-endpoint")
.execute()
.await?;
// --8<-- [end:connect_enterprise_placeholders]
Ok(db)
}
docs/web-tests/rs/embedding.rs
+182
View File
@@ -0,0 +1,182 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:openai_embeddings]
use std::{iter::once, sync::Arc};
use arrow_array::{record_batch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = "your-openai-api-key".to_string();
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
db.embedding_registry().register("openai", embedding.clone())?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
let table = db
.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
.execute()
.await?;
table
.add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
.execute()
.await?;
let query = Arc::new(StringArray::from_iter_values(once("test example")));
let query_vector = embedding.compute_query_embeddings(query)?;
let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
while let Some(batch) = results.next().await {
println!("{:?}", batch?);
}
Ok(())
}
// --8<-- [end:openai_embeddings]
// --8<-- [start:create_embedding_function]
use std::sync::Arc;
use lancedb::embeddings::openai::OpenAIEmbeddingFunction;
let api_key = "your-openai-api-key".to_string();
let embedding = Arc::new(
OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small")
.expect("failed to create OpenAI embedding function"),
);
// --8<-- [end:create_embedding_function]
// --8<-- [start:manual_query_embeddings]
use std::{iter::once, sync::Arc};
use arrow_array::{record_batch, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::StreamExt;
use lancedb::{
connect,
embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
query::{ExecutableQuery, QueryBase},
Result,
};
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = "your-openai-api-key".to_string();
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
)?);
db.embedding_registry().register("openai", embedding.clone())?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
let table = db
.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new("text", "openai", Some("vector")))?
.execute()
.await?;
table
.add(record_batch!(("text", Utf8, ["This is a test.", "Another example."]))?)
.execute()
.await?;
// Manually generate embeddings for the query (Enterprise path)
let query = Arc::new(StringArray::from_iter_values(once("test example")));
let query_vector = embedding.compute_query_embeddings(query)?;
// --8<-- [start:manual_query_search]
// query_vector is assumed to already be generated by your embedding function
let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;
while let Some(batch) = results.next().await {
println!("{:?}", batch?);
}
// --8<-- [end:manual_query_search]
Ok(())
}
// --8<-- [end:manual_query_embeddings]
// --8<-- [start:embedding_function]
use std::{borrow::Cow, sync::Arc};
use arrow_array::{Array, FixedSizeListArray, Float32Array};
use arrow_schema::{DataType, Field, Schema};
use lancedb::{
connect,
embeddings::{EmbeddingDefinition, EmbeddingFunction},
Result,
};
#[derive(Debug, Clone)]
struct MyTextEmbedder {
dim: usize,
}
impl EmbeddingFunction for MyTextEmbedder {
fn name(&self) -> &str {
"my-embedder"
}
fn source_type(&self) -> Result<Cow<'_, DataType>> {
Ok(Cow::Owned(DataType::Utf8))
}
fn dest_type(&self) -> Result<Cow<'_, DataType>> {
Ok(Cow::Owned(DataType::new_fixed_size_list(
DataType::Float32,
self.dim as i32,
true,
)))
}
fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));
let field = Arc::new(Field::new("item", DataType::Float32, true));
Ok(Arc::new(FixedSizeListArray::new(
field,
self.dim as i32,
values,
None,
)))
}
fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
unimplemented!()
}
}
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
db.embedding_registry()
.register("my-embedder", Arc::new(MyTextEmbedder { dim: 3 }))?;
let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
db.create_empty_table("mytable", schema)
.add_embedding(EmbeddingDefinition::new(
"text",
"my-embedder",
Some("vector"),
))?
.execute()
.await?;
Ok(())
}
// --8<-- [end:embedding_function]
docs/web-tests/rs/full_text_search.rs
+103
View File
@@ -0,0 +1,103 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lance_index::scalar::FullTextSearchQuery;
use lancedb::connection::Connection;
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, Result, Table};
use rand::random;
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
search_index(&tbl).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("doc", DataType::Utf8, true),
]));
let words = random_word::all(random_word::Lang::En)
.iter()
.step_by(1024)
.take(500)
.copied()
.collect::<Vec<_>>();
let n_terms = 3;
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(StringArray::from_iter_values((0..TOTAL).map(|_| {
(0..n_terms)
.map(|_| words[random::<usize>() % words.len()])
.collect::<Vec<_>>()
.join(" ")
}))),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db.create_table("my_table", initial_data).execute().await?;
Ok(tbl)
}
async fn create_index(table: &Table) -> Result<()> {
table
.create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
.execute()
.await?;
Ok(())
}
async fn search_index(table: &Table) -> Result<()> {
let words = random_word::all(random_word::Lang::En)
.iter()
.step_by(1024)
.take(500)
.copied()
.collect::<Vec<_>>();
let query = words[0].to_owned();
println!("Searching for: {}", query);
let mut results = table
.query()
.full_text_search(FullTextSearchQuery::new(words[0].to_owned()))
.select(lancedb::query::Select::Columns(vec!["doc".to_owned()]))
.limit(10)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
Ok(())
}
docs/web-tests/rs/ingestion_delete.rs
+72
View File
@@ -0,0 +1,72 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example]
use std::sync::Arc;
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase};
fn make_users_reader(rows: &[(i64, &str)]) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table(
"users",
make_users_reader(&[(1, "Alice"), (2, "Bob"), (3, "Charlie")]),
)
.execute()
.await?;
let predicate = "id = 3";
let selected: Vec<RecordBatch> = table
.query()
.only_if(predicate)
.execute()
.await?
.try_collect()
.await?;
println!("Selected: {selected:?}");
table.delete(predicate).await?;
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
println!("Remaining: {rows:?}");
assert_eq!(table.count_rows(None).await?, 2);
assert_eq!(
table
.count_rows(Some("id = 1 AND name = 'Alice'".into()))
.await?,
1
);
assert_eq!(
table
.count_rows(Some("id = 2 AND name = 'Bob'".into()))
.await?,
1
);
assert_eq!(table.count_rows(Some("id = 3".into())).await?, 0);
Ok(())
}
// --8<-- [end:example]
docs/web-tests/rs/ingestion_examples.rs
+97
View File
@@ -0,0 +1,97 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:write_workflow]
use std::sync::Arc;
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase};
fn make_users_reader(rows: &[(i64, &str, i64)]) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, false),
Field::new("login_count", DataType::Int64, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table(
"users",
make_users_reader(&[(1, "Alice", 10), (2, "Bob", 20)]),
)
.execute()
.await?;
table
.add(make_users_reader(&[(3, "Charlie", 5)]))
.execute()
.await?;
table
.update()
.only_if("id = 2")
.column("name", "'Bobby'")
.execute()
.await?;
table
.update()
.only_if("id = 2")
.column("login_count", "login_count + 1")
.execute()
.await?;
let mut merge = table.merge_insert(&["id"]);
merge
.when_matched_update_all(None)
.when_not_matched_insert_all();
merge
.execute(make_users_reader(&[(3, "Charlie", 6), (4, "Dana", 1)]))
.await?;
let selected: Vec<RecordBatch> = table
.query()
.only_if("id = 4")
.execute()
.await?
.try_collect()
.await?;
println!("Rows selected for deletion: {selected:?}");
table.delete("id = 4").await?;
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
println!("Remaining rows: {rows:?}");
assert_eq!(table.count_rows(None).await?, 3);
assert_eq!(
table
.count_rows(Some(
"id = 2 AND name = 'Bobby' AND login_count = 21".into()
))
.await?,
1
);
assert_eq!(
table
.count_rows(Some("id = 3 AND login_count = 6".into()))
.await?,
1
);
Ok(())
}
// --8<-- [end:write_workflow]
docs/web-tests/rs/ingestion_progress.rs
+64
View File
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example]
use std::sync::Arc;
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::ExecutableQuery;
fn make_users_reader(rows: &[(i64, &str)]) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table("users", make_users_reader(&[(1, "Alice")]))
.execute()
.await?;
table
.add(make_users_reader(&[(2, "Bob"), (3, "Charlie")]))
.progress(|progress| println!("Rows written: {}", progress.output_rows()))
.execute()
.await?;
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
println!("After insertion: {rows:?}");
assert_eq!(table.count_rows(None).await?, 3);
assert_eq!(
table
.count_rows(Some("id = 1 AND name = 'Alice'".into()))
.await?,
1
);
assert_eq!(
table
.count_rows(Some("id = 2 AND name = 'Bob'".into()))
.await?,
1
);
assert_eq!(table.count_rows(Some("id = 3".into())).await?, 1);
Ok(())
}
// --8<-- [end:example]
docs/web-tests/rs/ivf_pq.rs
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! This example demonstrates setting advanced parameters when building an IVF PQ index
//!
//! Snippets from this example are used in the documentation on ANN indices.
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::connection::Connection;
use lancedb::index::vector::IvfPqIndexBuilder;
use lancedb::index::Index;
use lancedb::query::{ExecutableQuery, QueryBase};
use lancedb::{connect, DistanceType, Result, Table};
#[tokio::main]
async fn main() -> Result<()> {
if std::path::Path::new("data").exists() {
std::fs::remove_dir_all("data").unwrap();
}
let uri = "data/sample-lancedb";
let db = connect(uri).execute().await?;
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
search_index(&tbl).await?;
Ok(())
}
fn create_some_records() -> Result<Box<dyn RecordBatchReader + Send>> {
const TOTAL: usize = 1000;
const DIM: usize = 128;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
DIM as i32,
),
true,
),
]));
// Create a RecordBatch stream.
let batches = RecordBatchIterator::new(
vec![RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..TOTAL).map(|_| Some(vec![Some(1.0); DIM])),
DIM as i32,
),
),
],
)
.unwrap()]
.into_iter()
.map(Ok),
schema.clone(),
);
Ok(Box::new(batches))
}
async fn create_table(db: &Connection) -> Result<Table> {
let initial_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
let tbl = db
.create_table("my_table", initial_data)
.execute()
.await
.unwrap();
Ok(tbl)
}
async fn create_index(table: &Table) -> Result<()> {
// --8<-- [start:create_index]
// For this example, `table` is a lancedb::Table with a column named
// "vector" that is a vector column with dimension 128.
// By default, if the column "vector" appears to be a vector column,
// then an IVF_PQ index with reasonable defaults is created.
table
.create_index(&["vector"], Index::Auto)
.execute()
.await?;
// For advanced cases, it is also possible to specifically request an
// IVF_PQ index and provide custom parameters.
table
.create_index(
&["vector"],
Index::IvfPq(
// Here we specify advanced indexing parameters. In this case
// we are creating an index that my have better recall than the
// default but is also larger and slower.
IvfPqIndexBuilder::default()
// This overrides the default distance type of l2
.distance_type(DistanceType::Cosine)
// With 1000 rows this have been ~31 by default
.num_partitions(50)
// With dimension 128 this would have been 8 by default
.num_sub_vectors(16),
),
)
.execute()
.await?;
// --8<-- [end:create_index]
Ok(())
}
async fn search_index(table: &Table) -> Result<()> {
// --8<-- [start:search1]
let query_vector = [1.0; 128];
// By default the index will find the 10 closest results using default
// search parameters that give a reasonable tradeoff between accuracy
// and search latency
let mut results = table
.vector_search(&query_vector)?
// Note: you should always set the distance_type to match the value used
// to train the index
.distance_type(DistanceType::Cosine)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
// We can also provide custom search parameters. Here we perform a
// slower but more accurate search
let mut results = table
.vector_search(&query_vector)?
.distance_type(DistanceType::Cosine)
// Override the default of 10 to get more rows
.limit(15)
// Override the default of 20 to search more partitions
.nprobes(30)
// Override the default of None to apply a refine step
.refine_factor(1)
.execute()
.await?;
while let Some(batch) = results.try_next().await? {
println!("{:?}", batch);
}
Ok(())
// --8<-- [end:search1]
}

Some files were not shown because too many files have changed in this diff Show More