rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-23 12:12:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4435-1

Browse Source
This commit is contained in:
Gatefixer
2026-10-22 20:40:56 +00:00
parent 4503f0c2bc 88203d2515
commit f9d6589a93
453 changed files with 42568 additions and 8581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.41.0-beta.3"
current_version = "0.41.0-beta.13"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/actions/setup-protoc/action.yml
+80
View File
@@ -0,0 +1,80 @@
name: Setup protoc
description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves
inputs:
install:
description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there."
required: false
default: "true"
outputs:
version:
description: "The pinned protoc version, e.g. 36.2"
value: ${{ steps.pin.outputs.version }}
linux-x86_64-sha256:
description: "SHA-256 of the protoc linux-x86_64 zip"
value: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
linux-aarch_64-sha256:
description: "SHA-256 of the protoc linux-aarch_64 zip"
value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
runs:
using: composite
steps:
# protobuf does not publish checksum files for protoc. These are the digests
# GitHub records for the release assets; when bumping, copy them from
# `gh api repos/protocolbuffers/protobuf/releases/tags/v<version>`.
- name: Set protoc pin
id: pin
shell: bash
run: |
{
echo "version=36.2"
echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278"
echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a"
echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b"
echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3"
} >> "$GITHUB_OUTPUT"
- name: Install protoc
if: ${{ inputs.install == 'true' }}
shell: bash
env:
PROTOC_VERSION: ${{ steps.pin.outputs.version }}
RUNNER_OS_NAME: ${{ runner.os }}
RUNNER_ARCH_NAME: ${{ runner.arch }}
LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }}
WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }}
run: |
set -euo pipefail
case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in
Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;;
Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;;
macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;;
Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;;
*) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;;
esac
zip="$RUNNER_TEMP/protoc.zip"
dest="$RUNNER_TEMP/protoc"
curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip"
# Hash stdin rather than a path: Windows paths contain backslashes,
# which make sha256sum escape its output.
if [ "$RUNNER_OS_NAME" = "macOS" ]; then
actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1)
else
actual=$(sha256sum < "$zip" | cut -d' ' -f1)
fi
if [ "$actual" != "$sha256" ]; then
echo "protoc checksum mismatch: expected $sha256, got $actual" >&2
exit 1
fi
if [ "$RUNNER_OS_NAME" = "Windows" ]; then
# Git Bash on Windows runners does not reliably ship unzip.
7z x -y -o"$dest" "$zip" > /dev/null
else
unzip -q -o "$zip" -d "$dest"
fi
rm "$zip"
echo "$dest/bin" >> "$GITHUB_PATH"
"$dest/bin/protoc" --version
.github/workflows/build_linux_wheel/action.yml
+11 -4
View File
@@ -29,6 +29,11 @@ inputs:
runs:
using: "composite"
steps:
- name: Get protoc version
id: protoc
uses: ./.github/actions/setup-protoc
with:
install: "false"
- name: CONFIRM ARM BUILD
shell: bash
env:
@@ -54,13 +59,14 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
target: x86_64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
@@ -71,14 +77,15 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
target: aarch64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
yum install -y clang
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
.github/workflows/build_mac_wheel/action.yml
+1 -4
View File
@@ -12,10 +12,7 @@ inputs:
runs:
using: "composite"
steps:
- name: Install macos dependency
shell: bash
run: |
brew install protobuf
- uses: ./.github/actions/setup-protoc
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
.github/workflows/build_windows_wheel/action.yml
+1 -9
View File
@@ -12,15 +12,7 @@ inputs:
runs:
using: "composite"
steps:
- name: Install Protoc v21.12
working-directory: C:\
run: |
New-Item -Path 'C:\protoc' -ItemType Directory
Set-Location C:\protoc
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
7z x protoc.zip
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
shell: powershell
- uses: ./.github/actions/setup-protoc
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
.github/workflows/ci-scripts.yml
+4
View File
@@ -5,11 +5,15 @@ on:
branches:
- main
paths:
- ci/check_cargo_lock.py
- ci/check_sdist.py
- ci/set_lance_version.py
- ci/tests/**
- .github/workflows/ci-scripts.yml
pull_request:
paths:
- ci/check_cargo_lock.py
- ci/check_sdist.py
- ci/set_lance_version.py
- ci/tests/**
- .github/workflows/ci-scripts.yml
.github/workflows/docs-style.yml
+46
View File
@@ -0,0 +1,46 @@
name: Docs prose style
# Vale checks the mechanical rules in the LanceDB style guide
# (lancedb/docs, STYLE.md) with the package pinned in .vale.ini. Its findings
# are advisory annotations on the lines a pull request adds or changes and
# never fail the job; an install, sync or configuration error still does.
on:
pull_request:
paths:
- "docs/web/**/*.mdx"
- ".vale.ini"
permissions:
contents: read
pull-requests: read
env:
VALE_VERSION: 3.24.0
VALE_SHA256: 867534ddb678abca7f214bc4706ead0922bd5252ffe8035fc227013f66c8b214
jobs:
prose:
name: Lint changed prose
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- name: Install Vale
run: |
archive="vale_${VALE_VERSION}_Linux_64-bit.tar.gz"
curl --fail --location --silent --show-error --output "$RUNNER_TEMP/$archive" \
"https://github.com/vale-cli/vale/releases/download/v${VALE_VERSION}/$archive"
echo "${VALE_SHA256} $RUNNER_TEMP/$archive" | sha256sum --check
tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" vale
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
- name: Annotate changed lines
uses: vale-cli/vale-action@518a9136acc6e6668ce7c00d367051e0941e87ff # v3.0.0
with:
version: none
files: docs/web
reporter: github-pr-annotations
filter_mode: added
fail_on_error: false
.github/workflows/npm-publish.yml
+16 -16
View File
@@ -71,9 +71,9 @@ jobs:
# https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
pre_build: |-
set -e &&
apt-get update &&
apt-get install -y protobuf-compiler pkg-config &&
set -e
apt-get update
apt-get install -y protobuf-compiler pkg-config
# The base image (manylinux2014-cross) sets TARGET_CC to the old
# GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
# so it picks up GCC even though the napi-rs image sets CC=clang.
@@ -84,10 +84,10 @@ jobs:
host: ubuntu-2404-8x-x64
features: fp16kernels
pre_build: |-
set -e &&
sudo apt-get update &&
sudo apt-get install -y protobuf-compiler pkg-config &&
rustup target add x86_64-unknown-linux-musl &&
set -e
sudo apt-get update
sudo apt-get install -y protobuf-compiler pkg-config
rustup target add x86_64-unknown-linux-musl
export EXTRA_ARGS="-x"
- target: aarch64-unknown-linux-gnu
host: ubuntu-2404-8x-x64
@@ -103,13 +103,13 @@ jobs:
# linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313).
linker: /tmp/aarch64-lld-clang
pre_build: |-
set -e &&
apt-get update &&
apt-get install -y protobuf-compiler pkg-config &&
export TARGET_CC=clang TARGET_CXX=clang++ &&
set -e
apt-get update
apt-get install -y protobuf-compiler pkg-config
export TARGET_CC=clang TARGET_CXX=clang++
# The manylinux2014 sysroot has glibc 2.17 headers which lack
# AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
export CFLAGS="$CFLAGS -DAT_HWCAP2=26"
rustup target add aarch64-unknown-linux-gnu
# Not `&&`-chained: in dash, errexit does not fire for a
# non-final command in an `&&` list, so failures were ignored.
@@ -136,10 +136,10 @@ jobs:
lto: thin
codegen_units: 16
pre_build: |-
set -e &&
sudo apt-get update &&
sudo apt-get install -y protobuf-compiler &&
rustup target add aarch64-unknown-linux-musl &&
set -e
sudo apt-get update
sudo apt-get install -y protobuf-compiler
rustup target add aarch64-unknown-linux-musl
export EXTRA_ARGS="-x"
name: build - ${{ matrix.settings.target }}
runs-on: ${{ matrix.settings.host }}
.github/workflows/pypi-publish.yml
+102 -7
View File
@@ -11,6 +11,10 @@ on:
- .github/workflows/build_linux_wheel/action.yml
- .github/workflows/build_mac_wheel/action.yml
- .github/workflows/build_windows_wheel/action.yml
- .github/actions/setup-protoc/action.yml
- ci/check_cargo_lock.py
- ci/check_sdist.py
- python/pyproject.toml
- Cargo.toml # Change in dependency frequently breaks builds
- Cargo.lock
@@ -160,10 +164,91 @@ jobs:
name: wheels-windows
path: target/wheels/lancedb-*.whl
if-no-files-found: error
sdist:
name: Python sdist
timeout-minutes: 60
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Build sdist
uses: PyO3/maturin-action@v1
with:
maturin-version: "1.12.4"
command: sdist
working-directory: python
args: --out dist
# maturin ships the nodejs-only pins in Cargo.lock. Remove this step when
# the pinned maturin fixes https://github.com/PyO3/maturin/issues/2609.
- name: Sync Cargo.lock to the packaged workspace
shell: bash
run: |
# A repo lock in sync with every manifest means the prune only drops pins.
cargo metadata --locked --format-version 1 > /dev/null
mkdir "$RUNNER_TEMP/sdist-src"
tar -xzf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src"
lock=$(echo "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.lock)
cp "$lock" "$RUNNER_TEMP/Cargo.lock.before"
cargo metadata --format-version 1 \
--manifest-path "$RUNNER_TEMP"/sdist-src/lancedb-*/Cargo.toml > /dev/null
python ci/check_cargo_lock.py "$RUNNER_TEMP/Cargo.lock.before" "$lock"
tar -czf python/dist/lancedb-*.tar.gz -C "$RUNNER_TEMP/sdist-src" \
"$(basename python/dist/lancedb-*.tar.gz .tar.gz)"
- name: Install sdist check tools
shell: bash
run: python -m pip install twine==7.0.0 pydistcheck==0.11.3
- name: Check sdist size
shell: bash
run: |
pydistcheck --select distro-too-large-compressed \
--max-allowed-size-compressed 100M python/dist/lancedb-*.tar.gz
- name: Check sdist contents
shell: bash
run: python ci/check_sdist.py python/dist/lancedb-*.tar.gz
- name: Check sdist with twine
shell: bash
run: python -m twine check --strict python/dist/lancedb-*.tar.gz
- uses: ./.github/actions/setup-protoc
# Build the way `pip install lancedb` does when no wheel matches. The dev
# profile keeps the build within the runner's disk.
- name: Build wheel from sdist
shell: bash
env:
MATURIN_PEP517_ARGS: "--profile dev --locked"
CARGO_PROFILE_DEV_DEBUG: "0"
run: |
python -m pip wheel --no-deps --wheel-dir "$RUNNER_TEMP/sdist-wheel" \
python/dist/lancedb-*.tar.gz
- name: Smoke test wheel built from sdist
shell: bash
working-directory: ${{ runner.temp }}
run: |
python -m venv sdist-venv
sdist-venv/bin/pip install sdist-wheel/lancedb-*.whl
sdist-venv/bin/python -c '
import tempfile
import lancedb
db = lancedb.connect(tempfile.mkdtemp())
table = db.create_table("t", [{"vector": [1.0, 2.0], "id": 1}, {"vector": [3.0, 4.0], "id": 2}])
assert table.search([1.0, 2.0]).limit(1).to_list()[0]["id"] == 1
print("lancedb", lancedb.__version__, "built from sdist OK")
'
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/v')
with:
name: sdist
path: python/dist/lancedb-*.tar.gz
if-no-files-found: error
publish:
name: Publish wheels
name: Publish wheels and sdist
if: startsWith(github.ref, 'refs/tags/v')
needs: [linux, mac, windows]
needs: [linux, mac, windows, sdist]
runs-on: ubuntu-latest
permissions:
id-token: write
@@ -176,7 +261,12 @@ jobs:
pattern: wheels-*
path: target/wheels
merge-multiple: true
- name: List wheels
- name: Download sdist artifact
uses: actions/download-artifact@v8
with:
name: sdist
path: target/wheels
- name: List packages
run: ls -la target/wheels
- name: Choose repo
id: choose_repo
@@ -193,13 +283,18 @@ jobs:
run: |
shopt -s nullglob
WHEELS=(target/wheels/*.whl)
SDISTS=(target/wheels/*.tar.gz)
if [[ ${#WHEELS[@]} -eq 0 ]]; then
echo "No wheels found in target/wheels/" >&2
exit 1
fi
for WHEEL in "${WHEELS[@]}"; do
echo "Uploading $WHEEL to Fury"
curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
if [[ ${#SDISTS[@]} -ne 1 ]]; then
echo "Expected one sdist in target/wheels/, found ${#SDISTS[@]}" >&2
exit 1
fi
for PACKAGE in "${WHEELS[@]}" "${SDISTS[@]}"; do
echo "Uploading $PACKAGE to Fury"
curl -f -F package=@"$PACKAGE" "https://$FURY_TOKEN@push.fury.io/lancedb/"
done
# NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
# workflow file, not from inside a composite action. When called from a
@@ -214,7 +309,7 @@ jobs:
report-failure:
name: Report Workflow Failure
runs-on: ubuntu-latest
needs: [linux, mac, windows, publish]
needs: [linux, mac, windows, sdist, publish]
permissions:
contents: read
issues: write
.github/workflows/python.yml
+1
View File
@@ -15,6 +15,7 @@ on:
- .github/workflows/build_linux_wheel/**
- .github/workflows/build_mac_wheel/**
- .github/workflows/build_windows_wheel/**
- .github/actions/setup-protoc/**
- .github/workflows/run_tests/**
permissions:
.gitignore
+3
View File
@@ -40,3 +40,6 @@ dist
target
**/sccache.log
# Vale styles installed by `vale sync`
/.vale/
.vale.ini
+5
View File
@@ -0,0 +1,5 @@
# Prose style for the pages in docs/web: the shared LanceDB package, pinned by
# release. The rules and guide live in lancedb/docs (STYLE.md and vale/).
# Run `vale sync` once, then `vale docs/web/<page>.mdx`.
StylesPath = .vale/styles
Packages = https://github.com/lancedb/docs/releases/download/vale-lancedb-v0.1.0/LanceDB.zip
AGENTS.md
+2 -2
View File
@@ -102,8 +102,8 @@ TypeScript bindings changes:
1. Add napi-rs method binding on `Table` in `nodejs/src/table.rs`.
2. Run `pnpm build` to generate TypeScript definitions.
3. Add typescript method on abstract class `Table` in `nodejs/src/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/src/native_table.ts`.
3. Add typescript method on abstract class `Table` in `nodejs/lancedb/table.ts`.
4. Add concrete method on `LocalTable` class in `nodejs/lancedb/table.ts`.
* Note: despite the name, this class is also used for remote tables.
5. Add test in `nodejs/__test__/table.test.ts`.
6. Run `pnpm run docs` to generate TypeScript documentation.
CONTRIBUTING.md
+48 -1
View File
@@ -88,13 +88,22 @@ npm i -g mint # https://mintlify.com/docs/installation
cd docs/web && mint dev
```
Wording, names and punctuation follow the
[LanceDB style guide](https://github.com/lancedb/docs/blob/main/STYLE.md).
[Vale](https://vale.sh) checks part of it, and pull requests show its
suggestions on the lines they change. To check a page yourself:
```bash
vale sync && vale docs/web/indexing/vector-index.mdx
```
This repository holds a page at every path the site publishes, so the whole
navigation resolves locally. Enterprise pages exist here as open-source pages --
what the capability is, and what the embedded form does instead. A separate
private repository supplies the fuller version of each, which replaces the page
at the same path when the published site is assembled.
Two things to know before editing:
Four things to know before editing:
**Code examples are not written into the pages.** They live in real tests under
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
@@ -116,6 +125,44 @@ Enterprise pages attach their additions to the right section, so they are
assigned once and never regenerated. Leave an existing anchor alone even when
you reword the heading above it; only new headings need a new one.
**A page can declare where it applies.** Its frontmatter says whether LanceDB
OSS and LanceDB Enterprise support the feature: use only `available` or
`unavailable`, and one sentence that explains its scope. Availability describes
product support, not whether a particular deployment has enabled or configured
the feature. A supported feature stays `available` when it requires setup, a
supported SDK or server version, or has differences between offerings. State
those requirements in the summary or the relevant section, including unsupported
subfeatures. Do not use `varies`, infer support from an unknown status, or mark an
absent capability available. Narrow a page to the capability it actually covers
when necessary. For example:
```yaml
availability:
oss: unavailable
enterprise: available
summary: >-
Requests to a deployment carry an API key or an OAuth token. Embedded
LanceDB has no server to sign in to.
```
The assembled site renders this as a label above the page's content and as a
sidebar tag when only one offering has the feature. Do not also open the page
with a badge of its own, and keep a qualification that only one section needs
in that section's text. A `mint dev` preview of `docs/web` alone renders
neither the labels nor the tags.
**A feature can be documented before its release.** List a draft page in
`docs/web/.mintignore`, keep it out of the navigation in `docs.json`, and do not
link to it from published pages. Wrap a draft section in an MDX comment,
`{/* ... */}`, whose first line names the release, such as
`Draft for the Feature X release.` Keep draft table rows and list items in a
comment below the table or list, since a comment between rows or items breaks
it. Drafts are hidden from readers but stay in the source, and availability does
not change what is published. At release, remove the comments and the
`.mintignore` entries, and restore the navigation and links. See
[Writing docs before release](https://github.com/lancedb/docs#writing-docs-before-release)
in the documentation repository's README for previews and release checks.
## Release process
For information on the release process, see: [release_process.md](release_process.md)
Cargo.lock
Generated
+206 -245
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+25 -16
View File
@@ -13,20 +13,21 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-jsonb = { version = "=15.0.0-beta.1", tag = "v15.0.0-beta.1", git = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { version = "=15.0.0-beta.1", path = "python/vendor/lance-arrow" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
@@ -72,13 +73,21 @@ oauth2 = { version = "5.0", default-features = false }
regex = "1.10"
semver = "1.0.25"
serde = "1"
serde_json = "1"
serde_json = { version = "1", features = ["preserve_order"] }
tempfile = "3.5.0"
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-ring"] }
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
uuid = { version = "1.7.0", features = ["v4", "v7"] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
[patch."https://github.com/lance-format/lance.git"]
# Carry the offset-safe validity fix until it is available in a Lance release.
lance-arrow = { path = "python/vendor/lance-arrow" }
[patch.crates-io]
# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision.
object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" }
[profile.ci]
debug = "line-tables-only"
inherits = "dev"
ci/check_cargo_lock.py
+56
View File
@@ -0,0 +1,56 @@
"""Fail if pruning Cargo.lock did more than remove packages and the edges to them.
Usage: python ci/check_cargo_lock.py <lock-before> <lock-after>
"""
import sys
import tomllib
from pathlib import Path
def key(package: dict) -> tuple:
return tuple(package.get(k) for k in ("name", "version", "source", "checksum"))
def graph(lock: dict) -> dict[tuple, set[tuple]]:
"""Map each package to the packages its dependency strings resolve to."""
by_name = {}
for p in lock["package"]:
by_name.setdefault(p["name"], []).append(p)
def resolve(dep: str) -> tuple:
# Cargo writes "name", "name version" or "name version (source)".
name, *rest = dep.split(" ", 2)
(match,) = [
p
for p in by_name[name]
if (not rest or p["version"] == rest[0])
and (len(rest) < 2 or rest[1] == f"({p.get('source')})")
]
return key(match)
return {
key(p): {resolve(d) for d in p.get("dependencies", [])} for p in lock["package"]
}
def main() -> None:
before, after = (tomllib.loads(Path(p).read_text()) for p in sys.argv[1:])
errors = []
for field in (before.keys() | after.keys()) - {"package"}:
if before.get(field) != after.get(field):
errors.append(f"top-level {field!r} changed")
edges_before, edges_after = graph(before), graph(after)
removed = edges_before.keys() - edges_after.keys()
for package, deps in edges_after.items():
if package not in edges_before:
errors.append(f"added or changed {package}")
elif deps != edges_before[package] - removed:
errors.append(f"dependencies of {package[:2]} changed")
if errors:
sys.exit("error: the prune changed Cargo.lock:\n" + "\n".join(errors))
print(f"Cargo.lock prune only removed {len(removed)} packages")
if __name__ == "__main__":
main()
ci/check_sdist.py
+42
View File
@@ -0,0 +1,42 @@
"""Fail if the lancedb sdist lacks a git-tracked file of the Python package.
Usage: python ci/check_sdist.py <sdist> (run from the repository root)
"""
import subprocess
import sys
import tarfile
from pathlib import Path
# Where git tracks the package, and where maturin puts it under the sdist root.
TRACKED_PACKAGE = "python/python/lancedb"
SDIST_PACKAGE = "python/lancedb"
def main() -> None:
sdist = Path(sys.argv[1])
root = sdist.name.removesuffix(".tar.gz")
listing = subprocess.run(
["git", "ls-files", "-z", "--", TRACKED_PACKAGE],
check=True,
capture_output=True,
text=True,
).stdout
tracked = [p for p in listing.split("\0") if p]
if not tracked:
sys.exit(f"error: git tracks no files under {TRACKED_PACKAGE}")
with tarfile.open(sdist) as tar:
members = {m.name for m in tar if m.isfile()}
missing = [
p
for p in tracked
if f"{root}/{SDIST_PACKAGE}{p.removeprefix(TRACKED_PACKAGE)}" not in members
]
if missing:
sys.exit(f"error: {sdist.name} lacks tracked sources: {', '.join(missing)}")
print(f"{sdist.name} OK: all {len(tracked)} tracked sources present")
if __name__ == "__main__":
main()
ci/tests/test_check_cargo_lock.py
+97
View File
@@ -0,0 +1,97 @@
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / "ci" / "check_cargo_lock.py"
CRATES_IO = "registry+https://github.com/rust-lang/crates.io-index"
def package(name, version, deps=(), source=CRATES_IO, checksum="aa"):
text = f'[[package]]\nname = "{name}"\nversion = "{version}"\n'
if source:
text += f'source = "{source}"\nchecksum = "{checksum}"\n'
if deps:
text += f"dependencies = {list(deps)!r}\n".replace("'", '"')
return text
LIBC = package("libc", "0.2.1")
UNTRUSTED = package("untrusted", "0.9.0")
# The prune drops napi and untrusted 0.7.1, the edge from app to it, and the
# version qualifier on app's remaining untrusted edge.
BEFORE = "version = 4\n\n" + "".join(
[
package(
"app", "1.0.0", ["libc", "untrusted 0.7.1", "untrusted 0.9.0"], source=None
),
package("napi", "3.0.0", ["untrusted 0.7.1"]),
LIBC,
package("untrusted", "0.7.1"),
UNTRUSTED,
]
)
APP = package("app", "1.0.0", ["libc", "untrusted"], source=None)
class CheckCargoLockTest(unittest.TestCase):
def run_check(self, after, header="version = 4\n\n"):
with tempfile.TemporaryDirectory() as tmp:
before_path = Path(tmp, "before.lock")
after_path = Path(tmp, "after.lock")
before_path.write_text(BEFORE)
after_path.write_text(header + after)
return subprocess.run(
[sys.executable, SCRIPT, before_path, after_path],
capture_output=True,
text=True,
)
def test_prune_passes(self):
result = self.run_check(APP + LIBC + UNTRUSTED)
self.assertEqual(result.returncode, 0, result.stderr)
def test_added_entry_fails(self):
result = self.run_check(APP + LIBC + UNTRUSTED + package("ring", "0.17.0"))
self.assertNotEqual(result.returncode, 0)
self.assertIn("ring", result.stderr)
def test_changed_version_fails(self):
result = self.run_check(APP + package("libc", "0.2.2") + UNTRUSTED)
self.assertNotEqual(result.returncode, 0)
self.assertIn("0.2.2", result.stderr)
def test_changed_source_fails(self):
libc = package("libc", "0.2.1", source="git+https://example.com/libc")
result = self.run_check(APP + libc + UNTRUSTED)
self.assertNotEqual(result.returncode, 0)
self.assertIn("example.com", result.stderr)
def test_changed_checksum_fails(self):
libc = package("libc", "0.2.1", checksum="bb")
result = self.run_check(APP + libc + UNTRUSTED)
self.assertNotEqual(result.returncode, 0)
self.assertIn("'bb'", result.stderr)
def test_added_dependency_fails(self):
libc = package("libc", "0.2.1", ["untrusted"])
result = self.run_check(APP + libc + UNTRUSTED)
self.assertNotEqual(result.returncode, 0)
self.assertIn("dependencies of ('libc'", result.stderr)
def test_removed_dependency_fails(self):
app = package("app", "1.0.0", ["untrusted"], source=None)
result = self.run_check(app + LIBC + UNTRUSTED)
self.assertNotEqual(result.returncode, 0)
self.assertIn("dependencies of ('app'", result.stderr)
def test_changed_lock_version_fails(self):
result = self.run_check(APP + LIBC + UNTRUSTED, header="version = 3\n\n")
self.assertNotEqual(result.returncode, 0)
self.assertIn("top-level 'version'", result.stderr)
if __name__ == "__main__":
unittest.main()
ci/tests/test_check_sdist.py
+71
View File
@@ -0,0 +1,71 @@
import subprocess
import sys
import tarfile
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
SCRIPT = REPO_ROOT / "ci" / "check_sdist.py"
ROOT = "lancedb-1.0.0"
class CheckSdistTest(unittest.TestCase):
def run_check(self, tracked, shipped, links=(), root=ROOT):
"""Run the check in a git repo tracking `tracked`, on an sdist holding `shipped`
as files and `links` as symlinks, under the folder `root`."""
with tempfile.TemporaryDirectory() as tmp:
subprocess.run(["git", "init", "-q", tmp], check=True)
for name in tracked:
path = Path(tmp, "python/python/lancedb", name)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("x = 1\n")
if tracked:
subprocess.run(["git", "-C", tmp, "add", "."], check=True)
sdist = Path(tmp, f"{ROOT}.tar.gz")
with tarfile.open(sdist, "w:gz") as tar:
for name in shipped:
member = Path(tmp, "member")
member.write_text("x = 1\n")
tar.add(member, arcname=f"{root}/python/lancedb/{name}")
for name in links:
link = tarfile.TarInfo(f"{root}/python/lancedb/{name}")
link.type = tarfile.SYMTYPE
link.linkname = "__init__.py"
tar.addfile(link)
return subprocess.run(
[sys.executable, SCRIPT, sdist],
cwd=tmp,
capture_output=True,
text=True,
)
def test_complete_sdist_passes(self):
result = self.run_check(["__init__.py", "db.py"], ["__init__.py", "db.py"])
self.assertEqual(result.returncode, 0, result.stderr)
def test_missing_tracked_file_fails(self):
result = self.run_check(["__init__.py", "db.py"], ["__init__.py"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("python/python/lancedb/db.py", result.stderr)
def test_tracked_file_shipped_as_symlink_fails(self):
result = self.run_check(
["__init__.py", "db.py"], ["__init__.py"], links=["db.py"]
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("python/python/lancedb/db.py", result.stderr)
def test_files_outside_the_sdist_root_fail(self):
result = self.run_check(["__init__.py"], ["__init__.py"], root="other-1.0.0")
self.assertNotEqual(result.returncode, 0)
self.assertIn("python/python/lancedb/__init__.py", result.stderr)
def test_no_tracked_files_fails(self):
result = self.run_check([], ["__init__.py"])
self.assertNotEqual(result.returncode, 0)
self.assertIn("git tracks no files", result.stderr)
if __name__ == "__main__":
unittest.main()
deny.toml
+19 -21
View File
@@ -68,21 +68,11 @@ ignore = [
# https://rustsec.org/advisories/RUSTSEC-2025-0119
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
# rustls-webpki 0.101.7 (old major line): name-constraint checks for
# URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
# from aws-smithy-http-client. The 0.103 line we actively use is patched.
# Clearing the 0.101 copy requires the aws-sdk chain to migrate off
# rustls 0.21.
# https://rustsec.org/advisories/RUSTSEC-2026-0098
# https://rustsec.org/advisories/RUSTSEC-2026-0099
{ id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
{ id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
# rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
# rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
# we actively use is upgraded to 0.103.13 which contains the fix.
# https://rustsec.org/advisories/RUSTSEC-2026-0104
{ id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
# unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera
# → serde_yaml_ng. No maintained upgrade exists; clearing it requires
# lindera to move off serde_yaml_ng.
# https://rustsec.org/advisories/RUSTSEC-2024-0449
{ id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" },
# pyo3 advisories in the Python bindings; tracked pending a patched pyo3 release.
# https://rustsec.org/advisories/RUSTSEC-2026-0176
@@ -90,6 +80,18 @@ ignore = [
{ id = "RUSTSEC-2026-0176", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" },
{ id = "RUSTSEC-2026-0177", reason = "pyo3 in Python bindings; awaiting patched pyo3 release" },
# zip < 9.0.0: SUID/SGID bits, symlink targets and permissions applied through
# symlinks during archive extraction. Transitive via candle-core (optional
# local-embedding feature); every published candle release pins zip < 9
# (0.9.x/0.10.x ^7.2, 0.11.0 ^8.6), so no dependency bump clears it.
# LanceDB never extracts archives; the zip crate is only linked.
# https://rustsec.org/advisories/RUSTSEC-2026-0367
# https://rustsec.org/advisories/RUSTSEC-2026-0368
# https://rustsec.org/advisories/RUSTSEC-2026-0369
{ id = "RUSTSEC-2026-0367", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" },
{ id = "RUSTSEC-2026-0368", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" },
{ id = "RUSTSEC-2026-0369", reason = "zip via candle-core; candle pins zip < 9 and lancedb never extracts archives" },
# quick-xml < 0.41.0: quadratic runtime on duplicate attribute names (DoS).
# quick-xml < 0.41.0: unbounded namespace-declaration allocation in NsReader (DoS).
# Pulled in transitively by inferno (dev-only flame-graph dep), lance-namespace-impls
@@ -108,12 +110,6 @@ ignore = [
# compact_str/smol_str, so clearing this requires polars to migrate.
# https://rustsec.org/advisories/RUSTSEC-2026-0249
{ id = "RUSTSEC-2026-0249", reason = "smartstring unmaintained via polars; no fixed upstream release" },
# h2 0.3: empty DATA frames can be queued without limit. The patched
# h2 0.4 line is locked to 0.4.16, but no patched 0.3 release exists.
# The old copy is pulled in by aws-smithy's legacy hyper 0.14 client.
# https://rustsec.org/advisories/RUSTSEC-2026-0258
{ id = "RUSTSEC-2026-0258", reason = "h2 0.3 via legacy aws-smithy/hyper 0.14; no patched 0.3 release" },
]
# ---------------------------------------------------------------------------
@@ -197,4 +193,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
# releases are cut to crates.io. Allow that specific host.
allow-git = [
"https://github.com/lance-format/lance",
# Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml).
"https://github.com/apache/arrow-rs-object-store",
]
docs/README.md
+7 -4
View File
@@ -15,18 +15,21 @@ unreleased features.
### Preview the docs
Run these commands from the LanceDB repository root. The subshell changes into
`docs` to load `mkdocs.yml`, while leaving your shell in the repository root.
```shell
cd docs
mkdocs serve
(cd docs && PYTHONPATH=. mkdocs serve)
```
If you want to just generate the HTML files:
```shell
PYTHONPATH=. mkdocs build -f docs/mkdocs.yml
(cd docs && PYTHONPATH=. mkdocs build)
```
If successful, you should see a `docs/site` directory that you can verify locally.
If successful, you should see a `docs/site` directory in the repository root that
you can verify locally.
## Adding examples
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.41.0-beta.3</version>
<version>0.41.0-beta.13</version>
</dependency>
```
docs/src/js/classes/AutoQuery.md
+11 -11
View File
@@ -425,31 +425,31 @@ Empty results retain the query's output schema.
***
### useLsm()
### useRTE()
```ts
useLsm(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this query.
Control RTE read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
By default (unset), when the table carries an RTE write spec (see
[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
the SSTables), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
RTE write spec. `false` bypasses the RTE and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
hybrid search, `orderBy`). On an RTE table those shapes error unless
`useRTE(false)` is set, because a base-only read would silently exclude
un-compacted RTE data.
#### Returns
@@ -457,7 +457,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useLsm`
`StandardQueryBase.useRTE`
***
docs/src/js/classes/MergeInsertBuilder.md
+8 -8
View File
@@ -76,23 +76,23 @@ the query optimizer chooses a suboptimal path.
***
### useLsm()
### useRTE()
```ts
useLsm(enable): MergeInsertBuilder
useRTE(enable): MergeInsertBuilder
```
Control MemWAL routing for this merge.
Control RTE routing for this merge.
By default (unset), a `mergeInsert` on a table with an LSM write spec is
By default (unset), a `mergeInsert` on a table with an RTE write spec is
routed through Lance's MemWAL shard writer, and a table without one uses the
standard path.
#### Parameters
* **enable**: `boolean`
`true` forces MemWAL routing and errors if the table has no
LSM write spec. `false` forces the standard write path even when a spec is set.
`true` forces RTE routing and errors if the table has no
RTE write spec. `false` forces the standard write path even when a spec is set.
#### Returns
@@ -108,12 +108,12 @@ validateSingleShard(validateSingleShard): MergeInsertBuilder
Controls how an LSM merge checks that its input targets a single shard.
When a table has an LSM write spec, every row in a `mergeInsert` call must
When a table has an RTE write spec, every row in a `mergeInsert` call must
route to the same shard. When `true` (the default), every row is inspected
to verify this. When `false`, only the first row is inspected and the
shard it routes to is used for the whole input — a faster path for callers
that have already pre-sharded their input. Has no effect on tables without
an LSM write spec.
an RTE write spec.
#### Parameters
docs/src/js/classes/Query.md
+11 -11
View File
@@ -499,31 +499,31 @@ Empty results retain the query's output schema.
***
### useLsm()
### useRTE()
```ts
useLsm(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this query.
Control RTE read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
By default (unset), when the table carries an RTE write spec (see
[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
the SSTables), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
RTE write spec. `false` bypasses the RTE and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
hybrid search, `orderBy`). On an RTE table those shapes error unless
`useRTE(false)` is set, because a base-only read would silently exclude
un-compacted RTE data.
#### Returns
@@ -531,7 +531,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useLsm`
`StandardQueryBase.useRTE`
***
docs/src/js/classes/Table.md
+130 -119
View File
@@ -143,7 +143,8 @@ containing the new version number of the table after altering the columns.
abstract blobColumns(): Promise<string[]>
```
Blob v2 columns, including nested dotted paths.
Blob v2 columns addressable by row ID, including struct-nested dotted paths.
Blobs inside lists are omitted because a row ID does not identify an element.
#### Returns
@@ -227,17 +228,17 @@ version of the table.
***
### checkpointLsm()
### checkpointRTE()
```ts
abstract checkpointLsm(): Promise<void>
abstract checkpointRTE(): Promise<void>
```
Converge this table's LSM write path into its base table.
Converge this table's RTE write path into its base table.
Seals once, then triggers compaction and polls until the L0 that existed
Freezes once, then triggers compaction and polls until the SSTables that existed
at the start is gone. The target set is fixed at the start, so
generations created *during* the checkpoint are ignored — that is what
SSTables created *during* the checkpoint are ignored — that is what
lets it terminate under write load, and what makes it best-effort: it
converges the fresh tier as of some instant. Idempotent, abandonable at
any point, and safe to run on a cadence.
@@ -253,9 +254,9 @@ is merging. The caller owns the deadline.
#### Example
```ts
const before = await table.getLsmStats();
await table.checkpointLsm();
const after = await table.getLsmStats();
const before = await table.getRTEStats();
await table.checkpointRTE();
const after = await table.getRTEStats();
```
***
@@ -278,15 +279,15 @@ Any attempt to use the table after it is closed will result in an error.
***
### closeLsmWriters()
### closeRTEWriters()
```ts
abstract closeLsmWriters(): Promise<void>
abstract closeRTEWriters(): Promise<void>
```
Drain and close any cached MemWAL shard writers held for this table.
Drain and close any cached RTE shard writers held for this table.
When an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) is installed, `mergeInsert` opens MemWAL
When an [RTEWriteSpec](../interfaces/RTEWriteSpec.md) is installed, `mergeInsert` opens RTE
shard writers and caches them for reuse across calls. This closes them,
flushing pending data; writers reopen lazily on the next `mergeInsert`.
It is a no-op when no writers are cached.
@@ -297,17 +298,17 @@ It is a no-op when no writers are cached.
***
### compactLsm()
### compactRTE()
```ts
abstract compactLsm(): Promise<void>
abstract compactRTE(): Promise<void>
```
Trigger a background L0 → base compaction pass per bucket.
Trigger a background SSTable compaction pass per table shard.
Returns once the passes are *dispatched*, not once they finish — watch
[Table#getLsmStats](Table.md#getlsmstats) for progress, or use
[Table#checkpointLsm](Table.md#checkpointlsm) to wait for convergence.
[Table#getRTEStats](Table.md#getrtestats) for progress, or use
[Table#checkpointRTE](Table.md#checkpointrte) to wait for convergence.
#### Returns
@@ -448,6 +449,23 @@ containing the new version number of the table
***
### disableRTE()
```ts
abstract disableRTE(): Promise<void>
```
Remove the [RTEWriteSpec](../interfaces/RTEWriteSpec.md) from this table, reverting to the standard
`mergeInsert` write path.
Errors if no spec is currently set.
#### Returns
`Promise`&lt;`void`&gt;
***
### display()
```ts
@@ -513,6 +531,54 @@ Drop an index from the table.
***
### enableRTE()
```ts
abstract enableRTE(spec): Promise<void>
```
Install an [RTEWriteSpec](../interfaces/RTEWriteSpec.md) on this table, selecting Lance's MemWAL
LSM-style write path for future `mergeInsert` calls.
`RTEWriteSpec` chooses one of three sharding strategies via `specType`:
- `"bucket"` — hash-bucket writes by the single-column unenforced primary
key (`column` and `numBuckets` required).
- `"identity"` — shard by the raw value of a scalar `column`.
- `"unsharded"` — route every write to a single shard.
All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the RTE cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`RTEWriteSpec`](../interfaces/RTEWriteSpec.md)
The sharding spec to install.
#### Returns
`Promise`&lt;`void`&gt;
#### Example
```ts
await table.setUnenforcedPrimaryKey("id");
await table.enableRTE({
specType: "bucket",
column: "id",
numBuckets: 16,
maintainedIndexes: ["id_idx"],
});
```
***
### fetchBlobFiles()
```ts
@@ -527,6 +593,8 @@ table's current checkout.
Preserves input order, duplicates, and nulls. Use this for large payloads.
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
One missing or deleted row ID rejects the entire batch, including on remote
tables; no partial handles are returned.
#### Parameters
@@ -556,6 +624,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
Reads the table's current checkout. IDs from another version can fail after
compaction unless stable row ids are enabled. Results keep input order and
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
One missing or deleted row ID rejects the entire batch, including on remote
tables; no partial results are returned.
Blobs inside lists cannot be fetched by row ID.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
The client splits requests automatically and reads an individual larger
blob through the Range route. This method still materializes all bytes in
@@ -575,15 +647,15 @@ memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
***
### flushLsm()
### flushRTE()
```ts
abstract flushLsm(): Promise<void>
abstract flushRTE(): Promise<void>
```
Seal every bucket's active memtable into a new L0 generation.
Freeze every table shard's active memtable into a new SSTable.
Returns once the seal is committed. Sealing an empty memtable is a no-op,
Returns once the freeze is committed. Freezing an empty memtable is a no-op,
so this is safe to call repeatedly.
#### Returns
@@ -627,48 +699,48 @@ const { records, truncated } = await table.functionErrors({ column: "embedding"
***
### getLsmStats()
### getRTESpec()
```ts
abstract getLsmStats(includeGenerationRows?): Promise<undefined | LsmStats>
abstract getRTESpec(): Promise<undefined | RTEWriteSpec>
```
Read live per-bucket LSM state.
Read the [RTEWriteSpec](../interfaces/RTEWriteSpec.md) currently installed on this table.
Answers "how far behind is my fresh tier", "which bucket is hot", and
"why is my fresh-tier vector search brute-force". Mutates no table state.
Resolves to `undefined` only when the LSM write path is not enabled.
#### Parameters
* **includeGenerationRows?**: `boolean`
Also count rows per L0 generation.
Off by default because each count opens an uncached Lance dataset.
#### Returns
`Promise`&lt;`undefined` \| [`LsmStats`](../interfaces/LsmStats.md)&gt;
***
### getLsmWriteSpec()
```ts
abstract getLsmWriteSpec(): Promise<undefined | LsmWriteSpec>
```
Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table.
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
Resolves to `undefined` when the RTE write path is not enabled (no
spec has been set, or it was removed with [Table#disableRTE](Table.md#disablerte)).
The spec is the one installed, including its maintained-index selection:
an absent `maintainedIndexes` for every index the table has, an empty
array for none.
#### Returns
`Promise`&lt;`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)&gt;
`Promise`&lt;`undefined` \| [`RTEWriteSpec`](../interfaces/RTEWriteSpec.md)&gt;
***
### getRTEStats()
```ts
abstract getRTEStats(includeSstableRows?): Promise<undefined | RTEStats>
```
Read live per-table-shard RTE state.
Answers "how far behind is my fresh tier", "which table shard is hot", and
"why is my fresh-tier vector search brute-force". Mutates no table state.
Resolves to `undefined` only when the RTE write path is not enabled.
#### Parameters
* **includeSstableRows?**: `boolean`
Also count rows per SSTable.
Off by default because each count opens an uncached Lance dataset.
#### Returns
`Promise`&lt;`undefined` \| [`RTEStats`](../interfaces/RTEStats.md)&gt;
***
@@ -1086,54 +1158,6 @@ of the given query
***
### setLsmWriteSpec()
```ts
abstract setLsmWriteSpec(spec): Promise<void>
```
Install an [LsmWriteSpec](../interfaces/LsmWriteSpec.md) on this table, selecting Lance's MemWAL
LSM-style write path for future `mergeInsert` calls.
`LsmWriteSpec` chooses one of three sharding strategies via `specType`:
- `"bucket"` — hash-bucket writes by the single-column unenforced primary
key (`column` and `numBuckets` required).
- `"identity"` — shard by the raw value of a scalar `column`.
- `"unsharded"` — route every write to a single shard.
All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index the table has,
including ones created later, and skips a kind the MemWAL cannot maintain.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
The sharding spec to install.
#### Returns
`Promise`&lt;`void`&gt;
#### Example
```ts
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({
specType: "bucket",
column: "id",
numBuckets: 16,
maintainedIndexes: ["id_idx"],
});
```
***
### setUnenforcedPrimaryKey()
```ts
@@ -1274,7 +1298,11 @@ Specify exactly one of `column` or `indexName`.
Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
the client process from index metadata. For remote tables, this means the
same tokenizer model files must also exist locally.
same tokenizer model files must also exist locally (under
`LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
mismatch cannot be detected: the returned tokens may differ from what the
server's index produces. If the model files are missing, an error naming
the tokenizer is raised.
#### Parameters
@@ -1288,23 +1316,6 @@ same tokenizer model files must also exist locally.
***
### unsetLsmWriteSpec()
```ts
abstract unsetLsmWriteSpec(): Promise<void>
```
Remove the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) from this table, reverting to the standard
`mergeInsert` write path.
Errors if no spec is currently set.
#### Returns
`Promise`&lt;`void`&gt;
***
### update()
#### update(opts)
docs/src/js/classes/TakeQuery.md
+5 -5
View File
@@ -275,17 +275,17 @@ Empty results retain the query's output schema.
***
### useLsm()
### useRTE()
```ts
useLsm(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this take query.
Control RTE read routing for this take query.
`false` bypasses the MemWAL and reads the base table only — the escape hatch,
`false` bypasses the RTE and reads the base table only — the escape hatch,
since take-by-row-id/offset is not supported on the LSM scanner and, on a
MemWAL table, auto-routes to it and errors otherwise.
RTE table, auto-routes to it and errors otherwise.
#### Parameters
docs/src/js/classes/VectorQuery.md
+43 -11
View File
@@ -646,6 +646,38 @@ rerank(reranker): VectorQuery
***
### searchEffort()
```ts
searchEffort(searchEffort): VectorQuery
```
Set the initial IVF search effort to a finite value in [0, 1].
The default, null, uses Auto (internally 0.5). Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found.
Any explicit effort, including 0.5, overrides nprobes in either call order.
Explicit minimum and maximum bounds still apply. This is not a recall
guarantee and does not change refinement or HNSW ef. The RTE LSM
scanner does not yet support this option.
#### Parameters
* **searchEffort**: `null` \| `number`
#### Returns
[`VectorQuery`](VectorQuery.md)
#### Example
```ts
table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
```
***
### select()
```ts
@@ -749,31 +781,31 @@ Empty results retain the query's output schema.
***
### useLsm()
### useRTE()
```ts
useLsm(enable): this
useRTE(enable): this
```
Control MemWAL read routing for this query.
Control RTE read routing for this query.
By default (unset), when the table carries a MemWAL write spec (see
[Table#setLsmWriteSpec](Table.md#setlsmwritespec)), reads are routed through the LSM scanner so
By default (unset), when the table carries an RTE write spec (see
[Table#enableRTE](Table.md#enablerte)), reads are routed through the LSM scanner so
they also return data written via the `mergeInsert` LSM path that has not yet
been compacted into the base table (the active/frozen in-memory memtables and
the flushed generations), deduplicated by primary key; a table without a spec
the SSTables), deduplicated by primary key; a table without a spec
reads the base table.
#### Parameters
* **enable**: `boolean`
`true` forces the LSM scanner and errors if the table has no
MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
RTE write spec. `false` bypasses the RTE and reads the base table only,
even when a spec is present.
Note: the LSM scanner does not support every query shape (e.g. reranking,
hybrid search, `orderBy`). On a MemWAL table those shapes error unless
`useLsm(false)` is set, because a base-only read would silently exclude
un-compacted MemWAL data.
hybrid search, `orderBy`). On an RTE table those shapes error unless
`useRTE(false)` is set, because a base-only read would silently exclude
un-compacted RTE data.
#### Returns
@@ -781,7 +813,7 @@ reads the base table.
#### Inherited from
`StandardQueryBase.useLsm`
`StandardQueryBase.useRTE`
***
docs/src/js/functions/blob.md
+11
View File
@@ -11,9 +11,20 @@ function blob(name, options): Field
```
Declares a `lance.blob.v2` column.
Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
whole external files, and external byte ranges. Set both `position` and
positive `size` for a range, or omit both for a whole file.
Existing two-field columns remain writable. Appends and merge inserts fill
missing range fields with nulls for four-field targets and drop all-null
range fields for two-field targets. Non-null ranges require a four-field
target. Overwrite with Arrow input replaces the table's schema; row objects
are converted using the current table schema.
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
Blobs inside lists can be written and queried as descriptors, but cannot
currently be fetched because a row ID does not identify a list element.
## Parameters
docs/src/js/globals.md
+5 -4
View File
@@ -66,7 +66,6 @@
- [BranchDiff](interfaces/BranchDiff.md)
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CatalogOptions](interfaces/CatalogOptions.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
@@ -98,7 +97,6 @@
- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md)
- [FunctionErrors](interfaces/FunctionErrors.md)
- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md)
- [GenerationStats](interfaces/GenerationStats.md)
- [HnswPqOptions](interfaces/HnswPqOptions.md)
- [HnswSqOptions](interfaces/HnswSqOptions.md)
- [IndexConfig](interfaces/IndexConfig.md)
@@ -115,8 +113,6 @@
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [ListingOptions](interfaces/ListingOptions.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeResult](interfaces/MergeResult.md)
@@ -126,6 +122,8 @@
- [OptimizeOptions](interfaces/OptimizeOptions.md)
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RTEStats](interfaces/RTEStats.md)
- [RTEWriteSpec](interfaces/RTEWriteSpec.md)
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
- [RemovalStats](interfaces/RemovalStats.md)
@@ -140,7 +138,9 @@
- [SplitHashOptions](interfaces/SplitHashOptions.md)
- [SplitRandomOptions](interfaces/SplitRandomOptions.md)
- [SplitSequentialOptions](interfaces/SplitSequentialOptions.md)
- [SsTableStats](interfaces/SsTableStats.md)
- [TableNamesOptions](interfaces/TableNamesOptions.md)
- [TableShardStats](interfaces/TableShardStats.md)
- [TableStatistics](interfaces/TableStatistics.md)
- [TimeoutConfig](interfaces/TimeoutConfig.md)
- [TlsConfig](interfaces/TlsConfig.md)
@@ -161,6 +161,7 @@
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
- [BlobData](type-aliases/BlobData.md)
- [BlobInput](type-aliases/BlobInput.md)
- [BlobMode](type-aliases/BlobMode.md)
- [BlobOptions](type-aliases/BlobOptions.md)
- [BlobRange](type-aliases/BlobRange.md)
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
docs/src/js/interfaces/ClientConfig.md
+12
View File
@@ -8,6 +8,18 @@
## Properties
### blobRequestConcurrency?
```ts
optional blobRequestConcurrency: number;
```
Maximum number of concurrent blob HTTP requests across all tables and
blob handles on this connection. Defaults to 8. Can also be set with
`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
***
### extraHeaders?
```ts
docs/src/js/interfaces/CreateTableOptions.md
+4 -1
View File
@@ -65,7 +65,7 @@ then no error will be raised.
### mode
```ts
mode: "overwrite" | "create";
mode: "overwrite" | "create" | "exist_ok";
```
The mode to use when creating the table.
@@ -76,6 +76,9 @@ happen. Any provided data will be ignored.
If this is set to "overwrite" then any existing table will be replaced.
If this is set to "exist_ok" then an existing table is opened and any
provided data is ignored.
***
### schema?
docs/src/js/interfaces/GenerationStats.md
-40
View File
@@ -1,40 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / GenerationStats
# Interface: GenerationStats
One flushed L0 generation.
## Properties
### bytes
```ts
bytes: number;
```
On-disk size of the generation.
***
### generation
```ts
generation: number;
```
The generation number. Increases as memtables are sealed into L0.
***
### rows?
```ts
optional rows: number;
```
Present only when `includeGenerationRows` was requested. Off by default
because each count opens an uncached Lance dataset.
docs/src/js/interfaces/LsmStats.md
-22
View File
@@ -1,22 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / LsmStats
# Interface: LsmStats
Live per-bucket LSM state, as returned by `Table#getLsmStats`.
Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
the caller's to compute.
## Properties
### buckets
```ts
buckets: BucketStats[];
```
One entry per bucket backing this table.
docs/src/js/interfaces/MemtableStats.md
+1 -1
View File
@@ -36,7 +36,7 @@ Estimated in-memory size.
generation: number;
```
The generation this memtable will become once sealed.
The generation this memtable will become once frozen.
***
docs/src/js/interfaces/QueryExecutionOptions.md
+34
View File
@@ -10,6 +10,40 @@ Options that control the behavior of a particular query execution
## Properties
### blobMode?
```ts
optional blobMode: BlobMode;
```
How to return blob v2 columns. Applies to `toArrow()` and `toArray()`.
- `"descriptions"` (default): each blob is a descriptor. Read the bytes
with [Table.fetchBlobs](../classes/Table.md#fetchblobs) or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles).
- `"bytes"`: each blob column, including blobs nested in structs, holds
the bytes as `LargeBinary`. The query also reads `_rowid` to fetch them, and
leaves it out of the result unless [QueryBase.withRowId](../classes/QueryBase.md#withrowid) was
called. All the bytes are held in memory.
Selecting a parent struct or directly aliasing a blob or struct also returns
bytes. Blobs inside lists keep their descriptors. Arbitrary projection
expressions are not materialized as bytes.
The rows and the bytes come from the same table version. If the table
changes while the query runs, the query runs again, up to three times.
#### Example
```ts
const rows = await table
.query()
.select(["id", "image"])
.toArray({ blobMode: "bytes" });
const image: Uint8Array | null = rows[0].image;
```
***
### maxBatchLength?
```ts
docs/src/js/interfaces/RTEStats.md
+22
View File
@@ -0,0 +1,22 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / RTEStats
# Interface: RTEStats
Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
the caller's to compute.
## Properties
### tableShards
```ts
tableShards: TableShardStats[];
```
One entry per table shard backing this table.
docs/src/js/interfaces/LsmWriteSpec.md → docs/src/js/interfaces/RTEWriteSpec.md
+5 -5
View File
@@ -2,12 +2,12 @@
***
[@lancedb/lancedb](../globals.md) / LsmWriteSpec
[@lancedb/lancedb](../globals.md) / RTEWriteSpec
# Interface: LsmWriteSpec
# Interface: RTEWriteSpec
Specification selecting Lance's MemWAL LSM-style write path for
`mergeInsert`.
Specification enabling the realtime engine (RTE), which routes
`mergeInsert` through Lance's MemWAL LSM-style write path.
`specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
`column` and `numBuckets` are required; for `"identity"`, `column` is
@@ -34,7 +34,7 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit for every index the table has,
Indexes the RTE keeps up to date. Omit for every index the table has,
including ones created later. Pass `[]` for none.
***
docs/src/js/interfaces/SsTableStats.md
+40
View File
@@ -0,0 +1,40 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / SsTableStats
# Interface: SsTableStats
One SSTable.
## Properties
### bytes
```ts
bytes: number;
```
On-disk size of the SSTable.
***
### generation
```ts
generation: number;
```
The generation number. Increases as memtables are frozen into SSTables.
***
### rows?
```ts
optional rows: number;
```
Present only when `includeSstableRows` was requested. Off by default
because each count opens an uncached Lance dataset.
docs/src/js/interfaces/BucketStats.md → docs/src/js/interfaces/TableShardStats.md
+17 -17
View File
@@ -2,12 +2,12 @@
***
[@lancedb/lancedb](../globals.md) / BucketStats
[@lancedb/lancedb](../globals.md) / TableShardStats
# Interface: BucketStats
# Interface: TableShardStats
Live state of one bucket. A table is N buckets on one node; flattening to a
single number hides the one hot bucket that is usually why someone opened
Live state of one table shard. A table is N table shards on one node; flattening to a
single number hides the one hot table shard that is usually why someone opened
this endpoint.
## Properties
@@ -18,7 +18,7 @@ this endpoint.
compacting: boolean;
```
Whether a pass owns this bucket's compaction latch right now. Says *a*
Whether a pass owns this table shard's compaction latch right now. Says *a*
driver is running, not *whose*, and the latch is held from dispatch —
including while the pass queues for a pod-wide compactor permit. Read it
as "do not pile on", never as "mine is progressing".
@@ -35,16 +35,6 @@ The generation the active memtable will become.
***
### generations
```ts
generations: GenerationStats[];
```
Flushed L0 generations not yet merged into the base table.
***
### manifestVersion
```ts
@@ -61,7 +51,7 @@ Version of the shard manifest these numbers were read from.
optional memtables: MemtableStats[];
```
Oldest first, active last. Absent for a `"Sealed"` bucket, whose
Oldest first, active last. Absent for a `"Sealed"` table shard, whose
in-memory state is torn down.
***
@@ -82,7 +72,17 @@ WAL position replay resumes from.
shardId: string;
```
The shard this bucket writes.
The id of this table shard.
***
### sstables
```ts
sstables: SsTableStats[];
```
SSTables not yet merged into the base table.
***
docs/src/js/type-aliases/BlobInput.md
+4
View File
@@ -12,6 +12,7 @@ type BlobInput:
| BlobUri
| object
| object
| object
| null
| undefined;
```
@@ -20,6 +21,9 @@ A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md))
Either inline bytes, a URI pointing at external bytes, or a struct that sets
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
External byte ranges require both
`position` (non-negative) and `size` (positive). Use safe integer numbers or
`bigint` values for these unsigned 64-bit byte counts.
## Example
docs/src/js/type-aliases/BlobMode.md
+13
View File
@@ -0,0 +1,13 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / BlobMode
# Type Alias: BlobMode
```ts
type BlobMode: "descriptions" | "bytes";
```
How a query returns blob v2 columns. See [QueryExecutionOptions.blobMode](../interfaces/QueryExecutionOptions.md#blobmode).
docs/src/python/python.md
+188 -1
View File
@@ -48,6 +48,169 @@ are ordinary connections. Dropping a database requires it to be empty.
::: lancedb.catalog.DatabaseNames
## Authorization
Some catalogs support the authorization API. This is an API that can be used to
bind access control lists (ACLs) to individual catalog objects.
Each ACL contains a subject, an object, and privilege.
### Subjects
There are three kinds of subjects: principals, groups, and roles.
A principal uniquely identifies each user accessing the system. Each LanceDB
Enterprise API key corresponds to a unique principal, and each OpenID user has
a unique principal.
All principals have an immutable ID which never changes, and is safe to include
in log files. For example, the API key
`sk_AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPQ====` maps to the
principal ID `pk_aaaqeayeaudaocajbifqydiob4`. APIs such as `list_acls` will
return the principal ID, never the API key.
OpenID principals also typically have a human-readable name associated with
them. For example, a principal might be named `bob@example.com` and might have
a principal ID of `8a004400-9443-4fab-9997-b9372a7ca8fd`. Whenever
possible, APIs will return the principal name for convenience. However,
principal names are mutable and can be changed by the identity provider.
Here is an example of getting information about an API key:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
principal = catalog.authz.get_principal(Subject.principal_api_key(api_key))
print(f"Found principal {principal.principal_id}")
```
OpenID principals can be members of groups. Like principals, groups also have
IDs and human-readable names. It is not possible to modify group membership via
this API; it is managed by the OpenID identity provider.
Here is an example of getting information about a group based on its name:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
group = catalog.authz.get_group(Subject.group_name("Engineering"))
print(f"Found group {group.group_id} with name {group.group_name}")
```
Roles can contain either principals or groups. Unlike groups, they are managed
by the catalog, not the identity provider. Roles can also contain API key
principals.
Here is an example of managing the `reader` role:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
catalog.authz.add_role_binding(subject=Subject.principal_name("Alice"), role="reader")
```
It may take a few milliseconds for changes to role bindings to become visible
to all clients.
More subject types may be added in the future.
### Objects
An object is any resource in the system that we can attach authorization rules to.
The Python builders and Rust client share resource validation and wire encoding.
Methods also accept literal strings such as `table:tenant/db:public$events`;
malformed names for known resource types raise `ValueError` locally before a
request is sent. Unknown resource types are preserved for compatibility with
newer servers.
Object types include:
The System object, a singleton which represents the entire system. You can
refer to it as `Object.system()`.
Database objects which represents specific databases. For example,
`Object.database("analytics")` represents the "analytics" database.
Table objects which represent specific tables. For example,
`Object.table(database="analytics", name="events")` represents the "events"
table inside the "analytics" database. Tables are in the default namespace (which
is named "public") unless a specific namespace is given.
Similarly, View objects represent specific views. For example,
`Object.view(database="analytics", name="recent_events")` represents the
"recent_events" view inside the "analytics" database. Views are in the default
namespace (which is named "public") unless a specific namespace is given.
Secret objects identify named credentials, for example,
`Object.secret(database="analytics", name="api-key")`. Function objects identify
registered functions, for example,
`Object.function(database="analytics", name="caption")`. Both default to the
`public` namespace. Pass `namespace_path=["platform", "ml"]` to select a nested
namespace. These objects identify resources for ACL operations; a secret object
contains the credential's name, not its value.
More object types may be added in the future.
### Access control lists
An access control list attaches a specific set of privileges to a specific
(subject, object) pair. So, for example, we might have an access control list
that gives `bob@example.com` the ability to read the foo table, that looks like
this:
```python
from lancedb.authz import Object, Subject, Privilege
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
catalog.authz.add_acl(
object=Object.table(database="analytics", name="foo"),
subject=Subject.principal_name("bob@example.com"),
privilege=Privilege.SELECT,
)
```
Reading a table generally requires USAGE on its database and namespace, plus
SELECT on the table. Each object has at most one owner.
OWNERSHIP is a special privilege which grants all the other privileges. By
default, the principal who created an object owns it. Only one subject can own
an object; granting OWNERSHIP to a new subject removes it from the previous
owner. OWNERSHIP cannot be deleted.
To delegate secret or function creation, grant `CREATE_SECRET` or
`CREATE_FUNCTION` on the containing namespace. This preserves namespace ownership:
```python
for privilege in [Privilege.CREATE_SECRET, Privilege.CREATE_FUNCTION]:
catalog.authz.add_acl(
object=Object.namespace(database="analytics", namespace_path=["public"]),
subject=Subject.role("developer"),
privilege=privilege,
)
```
Both privileges also accept their string names in synchronous and asynchronous
ACL methods.
By default, the `list_acls` function lists all the access control entries in
the system. By adding a function parameter, the output is filtered to just ACLs
matching that parameter. For example, this code lists all the ACLs for the
"reader" role:
```python
from lancedb.authz import Subject
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
print(entry)
```
Or, when using the async API:
```python
catalog = await lancedb.connect_catalog_async(endpoint, api_key=api_key)
async for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
print(entry)
```
## Remote SQL
Submit SQL against a remote LanceDB database through the connection.
@@ -134,7 +297,7 @@ listing a storage directory.
::: lancedb.table.Branches
::: lancedb.LsmWriteSpec
::: lancedb.RTEWriteSpec
## Functions and Jobs
@@ -214,6 +377,22 @@ listing a storage directory.
## Materialized Views (Synchronous)
On Cloud and Enterprise, creation accepts `auto_refresh_enabled` and
`auto_refresh_interval_seconds`:
```python
view = db.create_materialized_view(
"recent_orders",
"orders",
auto_refresh_enabled=True,
auto_refresh_interval_seconds=300,
)
```
Omit `auto_refresh_interval_seconds` to let the server determine when to refresh.
These options also apply to asynchronous creation and creation jobs. Embedded
connections support manual refresh and reject automatic scheduling requests.
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
@@ -342,6 +521,12 @@ pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
still work. Queries return descriptors. Call
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
One missing or deleted row ID rejects the entire blob-read batch; the APIs do
not return partial results.
Blobs inside lists can be written and queried as descriptors. They are omitted
from `blob_columns()` because row ids do not identify individual list elements,
so the fetch APIs cannot read their bytes yet.
::: lancedb.blob
@@ -359,6 +544,8 @@ still work. Queries return descriptors. Call
## Exceptions
::: lancedb.remote.errors.HttpError
::: lancedb.exceptions.MissingValueError
::: lancedb.exceptions.MissingColumnError
docs/web-tests/py/provider_examples.py
+355
View File
@@ -0,0 +1,355 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Reader examples with explicit placeholders.
These functions are documentation source. test_provider_examples.py runs the
reranker examples offline, with stand-ins for the model and the provider, to
check that each one writes only to its own new directory. Corresponding opt-in
integration tests remain in test_integrations.py and retain real credential
loading. Replace placeholders before running these examples.
"""
def embedding_ibm_usage():
# --8<-- [start:embedding_ibm_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.pydantic import LanceModel, Vector
registry = EmbeddingFunctionRegistry.get_instance()
registry.set_var("watsonx-api-key", "your-api-key")
watsonx_embed = registry.get("watsonx").create(
name="ibm/slate-125m-english-rtrvr",
api_key="$var:watsonx-api-key",
project_id="your-project-id",
)
class TextModel(LanceModel):
text: str = watsonx_embed.SourceField()
vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo"))
tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
tbl.add(data)
rs = tbl.search("hello").limit(1).to_pandas()
print(rs.head())
# --8<-- [end:embedding_ibm_usage]
def embedding_jina_text():
# --8<-- [start:embedding_jina_text]
import os
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.pydantic import LanceModel, Vector
os.environ["JINA_API_KEY"] = "your-api-key"
jina_embed = (
EmbeddingFunctionRegistry.get_instance()
.get("jina")
.create(name="jina-embeddings-v2-base-en")
)
class TextModel(LanceModel):
text: str = jina_embed.SourceField()
vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
data = [{"text": "hello world"}, {"text": "goodbye world"}]
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text"))
tbl = db.create_table("test", schema=TextModel, mode="overwrite")
tbl.add(data)
# --8<-- [end:embedding_jina_text]
def embedding_jina_multimodal():
# --8<-- [start:embedding_jina_multimodal]
import os
import tempfile
from pathlib import Path
import lancedb
import pandas as pd
import requests
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
os.environ["JINA_API_KEY"] = "your-api-key"
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images"))
func = get_registry().get("jina").create()
class Images(LanceModel):
label: str
image_uri: str = func.SourceField()
image_bytes: bytes = func.SourceField()
vector: Vector(func.ndims()) = func.VectorField()
vec_from_bytes: Vector(func.ndims()) = func.VectorField()
table = db.create_table("images", schema=Images)
labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
uris = [
"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
]
image_bytes = [requests.get(uri).content for uri in uris]
table.add(
pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
)
# --8<-- [end:embedding_jina_multimodal]
def reranking_cohere_usage():
# --8<-- [start:reranking_cohere_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import CohereReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = CohereReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_cohere_usage]
def reranking_jina_usage():
# --8<-- [start:reranking_jina_usage]
import os
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import JinaReranker
os.environ["JINA_API_KEY"] = "your-api-key"
embedder = get_registry().get("jina").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = JinaReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_jina_usage]
def reranking_watsonx_usage():
# --8<-- [start:reranking_watsonx_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import WatsonxReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
# Supply your API key and project ID (or use space_id instead of project_id).
reranker = WatsonxReranker(
api_key="your-api-key",
project_id="your-project-id",
)
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_watsonx_usage]
def reranking_openai_usage():
# --8<-- [start:reranking_openai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import OpenaiReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = OpenaiReranker(api_key="your-api-key")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_openai_usage]
def reranking_voyageai_usage():
# --8<-- [start:reranking_voyageai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import VoyageAIReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
vector: Vector(embedder.ndims()) = embedder.VectorField()
data = [
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2")
# Run vector search with a reranker
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
print(vector_results)
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
fts_results = (
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
)
print(fts_results)
# Run hybrid search with a reranker
hybrid_results = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
print(hybrid_results)
# --8<-- [end:reranking_voyageai_usage]
docs/web-tests/py/test_connection.py
+84 -22
View File
@@ -4,6 +4,8 @@
from pathlib import Path
import shutil
import pytest
def test_connection():
# --8<-- [start:connect]
@@ -29,19 +31,16 @@ async def connect_async_example():
def connect_enterprise_quickstart_config():
# --8<-- [start:connect_enterprise_quickstart]
import os
import lancedb
# --8<-- [start:connect_enterprise_quickstart]
uri = "db://your-database-uri"
api_key = "your-api-key"
region = "us-east-1"
host_override = "https://your-enterprise-endpoint.com"
db = lancedb.connect(
uri=uri,
api_key=api_key,
region=region,
host_override=host_override,
uri=os.environ["LANCEDB_URI"], # db://your-database
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"], # https://your-endpoint
)
# --8<-- [end:connect_enterprise_quickstart]
return db
@@ -57,17 +56,29 @@ def test_connect_enterprise_quickstart(monkeypatch):
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
monkeypatch.delenv("LANCEDB_REGION", raising=False)
monkeypatch.setenv("LANCEDB_HOST_OVERRIDE", "https://your-endpoint")
db = connect_enterprise_quickstart_config()
assert db is not None
assert connect_enterprise_quickstart_config() is not None
assert captured == {
"uri": "db://your-database-uri",
"uri": "db://your-database",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-enterprise-endpoint.com",
"host_override": "https://your-endpoint",
}
def test_connect_enterprise_quickstart_requires_an_endpoint(monkeypatch):
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
monkeypatch.delenv("LANCEDB_HOST_OVERRIDE", raising=False)
with pytest.raises(KeyError, match="LANCEDB_HOST_OVERRIDE"):
connect_enterprise_quickstart_config()
def connect_object_storage_config():
# --8<-- [start:connect_object_storage]
import lancedb
@@ -105,14 +116,26 @@ def namespace_table_ops_example():
mode="create", # use "overwrite" only if you want to replace existing table
)
# Verify
print(db.list_namespaces()) # ['prod']
print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search']
print(db.list_tables(namespace_path=["prod", "search"])) # ['user']
print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user']
# Verify: listings return a response with the names and a page token
print(db.list_namespaces().namespaces) # ['prod']
print(db.list_namespaces(namespace_path=["prod"]).namespaces)
# ['recommendations', 'search']
print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user']
print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user']
# --8<-- [end:namespace_table_ops]
def test_namespace_table_ops(tmp_path, monkeypatch, capsys):
monkeypatch.chdir(tmp_path)
namespace_table_ops_example()
assert capsys.readouterr().out.splitlines() == [
"['prod']",
"['recommendations', 'search']",
"['user']",
"['user']",
]
def namespace_admin_ops_example():
# --8<-- [start:namespace_admin_ops]
import lancedb
@@ -124,8 +147,8 @@ def namespace_admin_ops_example():
db.create_namespace(["prod", "search"])
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
print(f"Child namespaces under {namespace}: {child_namespaces}")
# Child namespaces under ['prod', 'search']: ['search']
print(f"Child namespaces under ['prod']: {child_namespaces}")
# Child namespaces under ['prod']: ['search']
metadata = db.describe_namespace(["prod", "search"])
print(f"Metadata for namespace {namespace}: {metadata}")
@@ -134,7 +157,17 @@ def namespace_admin_ops_example():
db.drop_namespace(["prod", "search"], mode="skip")
db.drop_namespace(["prod"], mode="skip")
# --8<-- [end:namespace_admin_ops]
return child_namespaces, metadata
return db, child_namespaces, metadata
def test_namespace_admin_ops(tmp_path, monkeypatch, capsys):
monkeypatch.chdir(tmp_path)
db, child_namespaces, metadata = namespace_admin_ops_example()
assert child_namespaces == ["search"]
assert capsys.readouterr().out.splitlines()[0] == (
"Child namespaces under ['prod']: ['search']"
)
assert db.list_namespaces().namespaces == []
async def connect_object_storage_config_async():
# --8<-- [start:connect_object_storage_async]
@@ -146,3 +179,32 @@ async def connect_object_storage_config_async():
# --8<-- [end:connect_object_storage_async]
return async_db
def test_connect_enterprise_placeholders(monkeypatch):
import lancedb
captured = {}
def fake_connect(**kwargs):
captured.update(kwargs)
return object()
monkeypatch.setattr(lancedb, "connect", fake_connect)
# --8<-- [start:connect_enterprise_placeholders]
import lancedb
db = lancedb.connect(
uri="db://your-database",
api_key="your-api-key",
region="us-east-1",
host_override="https://your-endpoint",
)
# --8<-- [end:connect_enterprise_placeholders]
assert db is not None
assert captured == {
"uri": "db://your-database",
"api_key": "your-api-key",
"region": "us-east-1",
"host_override": "https://your-endpoint",
}
docs/web-tests/py/test_curation_sampling.py
+27
View File
@@ -0,0 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_sample_dataset():
# --8<-- [start:sample_dataset]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.permutation import Permutation, permutation_builder
with TemporaryDirectory(prefix="lancedb-sampling-") as uri:
db = lancedb.connect(uri)
table = db.create_table("records", [{"id": i} for i in range(10)])
splits = (
permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()
)
sample = Permutation.from_tables(table, splits, split=0).with_format("arrow")
selected_ids = []
for batch in sample.iter(batch_size=3):
selected_ids.extend(batch.column("id").to_pylist())
print(f"Selected {len(selected_ids)} rows: {selected_ids}")
# --8<-- [end:sample_dataset]
assert len(selected_ids) == 3
assert len(set(selected_ids)) == 3
assert all(0 <= value < 10 for value in selected_ids)
docs/web-tests/py/test_function.py
+110 -17
View File
@@ -4,16 +4,18 @@
"""Examples for docs/web/management/function.mdx.
Registering a Function and filling its column need a LanceDB Enterprise
deployment, so `test_function_enterprise` runs only when the connection
variables the Enterprise examples already use are set: LANCEDB_URI,
LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the
deployment needs one. The walkthrough calls its table `function_demo` and its
Function `double`; the test creates both under names unique to its run, never
replaces an existing object, and drops only what it created.
deployment, so `test_function_enterprise` runs only when
LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the
Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and
LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The
walkthrough calls its table `function_demo` and its Function `double`; the test
creates both under names unique to its run, never replaces an existing object,
and drops only what it created.
The other tests run locally and check the client side of the same code: what
`@udf` sends for registration, how a registered version binds a column, and
where a local connection stops. They do not run a Function remotely.
`@udf` sends for registration, how a registered version binds a column, where a
local connection stops, and when the Enterprise test connects. They do not run
a Function remotely.
"""
import json
@@ -70,12 +72,17 @@ def registered_version(definition) -> FunctionVersion:
return FunctionVersion.from_json(json.dumps(reply))
def require_env(var_name: str) -> str:
"""Skip the test unless the required environment variable is provided."""
value = os.environ.get(var_name)
if not value:
pytest.skip(f"Set {var_name} to run this Enterprise example")
return value
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
# The connection settings alone never send this test to a deployment.
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
def require_deployment():
if os.environ.get(OPT_IN) != "1":
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
if missing:
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
def test_function_runs_locally_as_python():
@@ -160,9 +167,7 @@ def test_function_version_binds_a_column(tmp_path):
def test_function_enterprise(monkeypatch):
require_env("LANCEDB_URI")
require_env("LANCEDB_API_KEY")
require_env("LANCEDB_HOST_OVERRIDE")
require_deployment()
# --8<-- [start:function_connect]
import os
@@ -240,3 +245,91 @@ def test_function_enterprise(monkeypatch):
db.drop_table(table_name)
if version is not None:
db.drop_function(version.name, version=version.version)
class ReachedConnection(Exception):
pass
SETTINGS = {
"LANCEDB_URI": "db://docs-test",
"LANCEDB_HOST_OVERRIDE": "https://deployment.invalid",
"LANCEDB_API_KEY": "docs-test-key",
}
@pytest.fixture
def connect_attempts(monkeypatch):
"""`lancedb.connect`, replaced so that it records each call and stops."""
attempts = []
def connect(**kwargs):
attempts.append(kwargs)
raise ReachedConnection
monkeypatch.setattr(lancedb, "connect", connect)
for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"):
monkeypatch.delenv(name, raising=False)
return attempts
def enterprise_outcome(monkeypatch):
try:
test_function_enterprise(monkeypatch)
except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome:
return outcome
@pytest.mark.parametrize(
"opt_in",
[None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"],
)
def test_function_enterprise_needs_the_exact_opt_in(
monkeypatch, connect_attempts, opt_in
):
for name, value in SETTINGS.items():
monkeypatch.setenv(name, value)
if opt_in is not None:
monkeypatch.setenv(OPT_IN, opt_in)
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, pytest.skip.Exception), outcome
assert str(outcome).startswith(f"set {OPT_IN}=1 ")
assert connect_attempts == []
@pytest.mark.parametrize("blank", ["unset", "empty"])
@pytest.mark.parametrize(
"missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)]
)
def test_function_enterprise_fails_when_asked_without_settings(
monkeypatch, connect_attempts, missing, blank
):
monkeypatch.setenv(OPT_IN, "1")
for name, value in SETTINGS.items():
if name not in missing:
monkeypatch.setenv(name, value)
elif blank == "empty":
monkeypatch.setenv(name, "")
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, pytest.fail.Exception), outcome
assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}"
assert connect_attempts == []
@pytest.mark.parametrize("region", [None, "eu-west-1"])
def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region):
monkeypatch.setenv(OPT_IN, "1")
for name, value in SETTINGS.items():
monkeypatch.setenv(name, value)
if region:
monkeypatch.setenv("LANCEDB_REGION", region)
outcome = enterprise_outcome(monkeypatch)
assert isinstance(outcome, ReachedConnection), outcome
assert connect_attempts == [
{
"uri": "db://docs-test",
"api_key": "docs-test-key",
"region": region or "us-east-1",
"host_override": "https://deployment.invalid",
}
]
docs/web-tests/py/test_ingestion_examples.py
+739
View File
@@ -0,0 +1,739 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_create_table_from_dicts():
# --8<-- [start:create_table_from_dicts]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
data = [
{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
{"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1},
]
db.create_table("test_table", data, mode="overwrite")
tbl = db["test_table"]
tbl.head()
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_dicts]
assert tbl.count_rows() == 2
def test_create_table_conflict_handling():
# --8<-- [start:create_table_conflict_handling]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
db.create_table("conflict_table", data)
# Idempotent open: reuse the existing table if it exists.
# The provided data is ignored; the schema is validated against the
# existing table and a mismatch raises an error.
tbl = db.create_table("conflict_table", data, exist_ok=True)
# Overwrite: replace the contents with the provided data.
tbl = db.create_table("conflict_table", data, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_conflict_handling]
def test_create_table_custom_schema():
# --8<-- [start:create_table_custom_schema]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
custom_schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field("lat", pa.float32()),
pa.field("long", pa.float32()),
]
)
data = [
{"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7},
{"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1},
]
tbl = db.create_table(
"my_table_custom_schema", data, schema=custom_schema, mode="overwrite"
)
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_custom_schema]
assert tbl.count_rows() == 2
def test_create_table_from_pandas():
# --8<-- [start:create_table_from_pandas]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pandas as pd
data = pd.DataFrame(
{
"vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
"lat": [45.5, 40.1],
"long": [-122.7, -74.1],
}
)
db.create_table("my_table_pandas", data, mode="overwrite")
db["my_table_pandas"].head()
# --8<-- [end:create_table_from_pandas]
def test_create_table_from_polars():
# --8<-- [start:create_table_from_polars]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import polars as pl
data = pl.DataFrame(
{
"vector": [[3.1, 4.1], [5.9, 26.5]],
"item": ["foo", "bar"],
"price": [10.0, 20.0],
}
)
tbl = db.create_table("my_table_pl", data, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_polars]
assert tbl.count_rows() == 2
def test_create_table_from_arrow():
# --8<-- [start:create_table_from_arrow]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import numpy as np
import pyarrow as pa
dim = 16
total = 2
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float16(), dim)),
pa.field("text", pa.string()),
]
)
data = pa.Table.from_arrays(
[
pa.array(
[np.random.randn(dim).astype(np.float16) for _ in range(total)],
pa.list_(pa.float16(), dim),
),
pa.array(["foo", "bar"]),
],
["vector", "text"],
)
tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_arrow]
assert tbl.count_rows() == 2
def test_create_table_from_pydantic():
# --8<-- [start:create_table_from_pydantic]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
class Content(LanceModel):
movie_id: int
vector: Vector(128)
genres: str
title: str
imdb_id: int
@property
def imdb_url(self) -> str:
return f"https://www.imdb.com/title/tt{self.imdb_id}"
tbl = db.create_table("movielens_small", schema=Content, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_from_pydantic]
assert tbl.count_rows() == 0
def test_create_table_nested_schema():
# --8<-- [start:create_table_nested_schema]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
from pydantic import BaseModel
class Document(BaseModel):
content: str
source: str
class NestedSchema(LanceModel):
id: str
vector: Vector(1536)
document: Document
tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_table_nested_schema]
assert tbl.count_rows() == 0
def test_create_empty_table():
# --8<-- [start:create_empty_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("item", pa.string()),
pa.field("price", pa.float32()),
]
)
tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite")
print("Rows:", tbl.count_rows())
# --8<-- [end:create_empty_table]
assert tbl.count_rows() == 0
def test_create_empty_table_pydantic():
# --8<-- [start:create_empty_table_pydantic]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from lancedb.pydantic import LanceModel, Vector
class Item(LanceModel):
vector: Vector(2)
item: str
price: float
tbl = db.create_table(
"test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite"
)
print("Rows:", tbl.count_rows())
# --8<-- [end:create_empty_table_pydantic]
assert tbl.count_rows() == 0
def test_open_existing_table():
# --8<-- [start:open_existing_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
# Create a table first
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
db.create_table("test_table", data, mode="overwrite")
# List table names
print(db.list_tables().tables)
# Open existing table
tbl = db.open_table("test_table")
print("Rows:", tbl.count_rows())
# --8<-- [end:open_existing_table]
assert tbl.count_rows() == 1
def test_drop_table():
# --8<-- [start:drop_table]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
# Create a table first
data = [{"vector": [1.1, 1.2], "lat": 45.5}]
db.create_table("my_table", data, mode="overwrite")
# Drop the table
db.drop_table("my_table")
# --8<-- [end:drop_table]
def test_add_from_dataset():
# --8<-- [start:add_from_dataset]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
from pathlib import Path
import pyarrow as pa
import pyarrow.parquet as pq
data_path = str(Path(uri) / "input.parquet")
pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path)
import pyarrow.dataset as ds
dataset = ds.dataset(data_path, format="parquet")
table = db.create_table("my_table", schema=dataset.schema, mode="overwrite")
table.add(dataset)
print("Rows:", table.count_rows())
# --8<-- [end:add_from_dataset]
assert table.count_rows() == 2
def test_create_table_from_iterator():
# --8<-- [start:create_table_from_iterator]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
schema = pa.schema(
[
pa.field("vector", pa.list_(pa.float32(), 4)),
pa.field("item", pa.utf8()),
pa.field("price", pa.float32()),
]
)
def make_batches():
for i in range(5):
yield pa.RecordBatch.from_arrays(
[
pa.array(
[[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
pa.list_(pa.float32(), 4),
),
pa.array(["foo", "bar"]),
pa.array([10.0, 20.0]),
],
["vector", "item", "price"],
)
table = db.create_table("batched_table", schema=schema)
table.add(make_batches())
print("Rows:", table.count_rows())
# --8<-- [end:create_table_from_iterator]
assert table.count_rows() == 10
def test_update_operation():
# --8<-- [start:update_operation]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
table.update(where="id = 2", values={"name": "Bobby"})
print("Rows:", table.count_rows())
# --8<-- [end:update_operation]
assert table.count_rows() == 2
assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby"
def test_update_using_sql():
# --8<-- [start:update_using_sql]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
print("Rows:", table.count_rows())
# --8<-- [end:update_using_sql]
assert table.count_rows() == 2
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
def test_merge_matched_update_only():
# --8<-- [start:merge_matched_update_only]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(table.merge_insert("id").when_matched_update_all().execute(incoming_users))
print("Rows:", table.count_rows())
# --8<-- [end:merge_matched_update_only]
assert table.count_rows() == 2
def test_insert_if_not_exists():
# --8<-- [start:insert_if_not_exists]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users))
print("Rows:", table.count_rows())
# --8<-- [end:insert_if_not_exists]
assert table.count_rows() == 3
def test_merge_update_insert():
# --8<-- [start:merge_update_insert]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_update_insert]
assert table.count_rows() == 3
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
def test_merge_delete_missing_by_source():
# --8<-- [start:merge_delete_missing_by_source]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2, 3],
"name": ["Alice", "Bob", "Charlie"],
"login_count": [10, 20, 5],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
"login_count": [21, 5],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.when_not_matched_by_source_delete()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_delete_missing_by_source]
assert table.count_rows() == 2
assert table.count_rows("id = 1") == 0
def test_merge_partial_columns():
# --8<-- [start:merge_partial_columns]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
import pyarrow as pa
table = db.create_table(
"users_example",
data=pa.table(
{
"id": [1, 2],
"name": ["Alice", "Bob"],
"login_count": [10, 20],
}
),
mode="overwrite",
)
incoming_users = pa.table(
{
"id": [2, 3],
"name": ["Bobby", "Charlie"],
}
)
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(incoming_users)
)
print("Rows:", table.count_rows())
# --8<-- [end:merge_partial_columns]
assert table.count_rows() == 3
assert table.search().where("id = 3").to_list()[0]["login_count"] is None
def test_write_workflow():
# --8<-- [start:write_workflow]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"users",
[
{"id": 1, "name": "Alice", "login_count": 10},
{"id": 2, "name": "Bob", "login_count": 20},
],
)
table.add([{"id": 3, "name": "Charlie", "login_count": 5}])
table.update(where="id = 2", values={"name": "Bobby"})
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
(
table.merge_insert("id")
.when_matched_update_all()
.when_not_matched_insert_all()
.execute(
[
{"id": 3, "name": "Charlie", "login_count": 6},
{"id": 4, "name": "Dana", "login_count": 1},
]
)
)
print("Rows selected for deletion:", table.search().where("id = 4").to_list())
table.delete("id = 4")
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print(rows)
# --8<-- [end:write_workflow]
assert rows == [
{"id": 1, "name": "Alice", "login_count": 10},
{"id": 2, "name": "Bobby", "login_count": 21},
{"id": 3, "name": "Charlie", "login_count": 6},
]
def test_append_rows():
# --8<-- [start:append_rows]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}])
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print(rows)
# --8<-- [end:append_rows]
assert [row["id"] for row in rows] == [1, 2, 3]
def test_delete_rows():
# --8<-- [start:delete_rows]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"users",
[
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"},
{"id": 3, "name": "Charlie"},
],
)
predicate = "id = 3"
print("Selected:", table.search().where(predicate).to_list())
table.delete(predicate)
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print("Remaining:", rows)
# --8<-- [end:delete_rows]
assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
def test_add_progress():
# --8<-- [start:add_progress]
from tempfile import TemporaryDirectory
import lancedb
def report_progress(progress):
print(f"Rows written: {progress['output_rows']}")
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
db = lancedb.connect(uri)
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
table.add(
[{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}],
progress=report_progress,
)
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
print("After insertion:", rows)
# --8<-- [end:add_progress]
assert rows == [
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"},
{"id": 3, "name": "Charlie"},
]
docs/web-tests/py/test_integrations.py
+51 -24
View File
@@ -740,13 +740,16 @@ def test_reranking_answerdotai_usage() -> None:
require_flag("RUN_RERANKER_SNIPPETS")
# --8<-- [start:reranking_answerdotai_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import AnswerdotaiRerankers
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect("~/.lancedb")
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "answerdotai-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
@@ -756,18 +759,20 @@ def test_reranking_answerdotai_usage() -> None:
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema, mode="overwrite")
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = AnswerdotaiRerankers()
# Run vector search with a reranker
result = tbl.search("hello").rerank(reranker=reranker).to_list()
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
# Run hybrid search with a reranker
tbl.create_fts_index("text", replace=True)
result = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
@@ -819,13 +824,16 @@ def test_reranking_colbert_usage() -> None:
require_flag("RUN_RERANKER_SNIPPETS")
# --8<-- [start:reranking_colbert_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import ColbertReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect("~/.lancedb")
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "colbert-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
@@ -835,18 +843,20 @@ def test_reranking_colbert_usage() -> None:
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema, mode="overwrite")
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = ColbertReranker()
# Run vector search with a reranker
result = tbl.search("hello").rerank(reranker=reranker).to_list()
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
# Run hybrid search with a reranker
tbl.create_fts_index("text", replace=True)
result = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
@@ -857,13 +867,16 @@ def test_reranking_cross_encoder_usage() -> None:
require_flag("RUN_RERANKER_SNIPPETS")
# --8<-- [start:reranking_cross_encoder_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import CrossEncoderReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect("~/.lancedb")
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cross-encoder-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
@@ -873,18 +886,20 @@ def test_reranking_cross_encoder_usage() -> None:
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema, mode="overwrite")
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = CrossEncoderReranker()
# Run vector search with a reranker
result = tbl.search("hello").rerank(reranker=reranker).to_list()
# Build the full-text index before either FTS or hybrid search.
tbl.create_fts_index("text", replace=True)
# Run FTS search with a reranker
result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
# Run hybrid search with a reranker
tbl.create_fts_index("text", replace=True)
result = (
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
)
@@ -936,13 +951,16 @@ def test_reranking_linear_combination_usage() -> None:
require_flag("RUN_RERANKER_SNIPPETS")
# --8<-- [start:reranking_linear_combination_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import LinearCombinationReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect("~/.lancedb")
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "linear-combination-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
@@ -952,7 +970,7 @@ def test_reranking_linear_combination_usage() -> None:
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema, mode="overwrite")
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = LinearCombinationReranker()
@@ -1007,13 +1025,16 @@ def test_reranking_rrf_usage() -> None:
require_flag("RUN_RERANKER_SNIPPETS")
# --8<-- [start:reranking_rrf_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import RRFReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect("~/.lancedb")
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "rrf-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
@@ -1023,7 +1044,7 @@ def test_reranking_rrf_usage() -> None:
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema, mode="overwrite")
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = RRFReranker()
@@ -1039,13 +1060,16 @@ def test_reranking_mrr_usage() -> None:
require_flag("RUN_RERANKER_SNIPPETS")
# --8<-- [start:reranking_mrr_usage]
import tempfile
from pathlib import Path
import lancedb
from lancedb.embeddings import get_registry
from lancedb.pydantic import LanceModel, Vector
from lancedb.rerankers import MRRReranker
embedder = get_registry().get("sentence-transformers").create()
db = lancedb.connect("~/.lancedb")
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "mrr-rerank"))
class Schema(LanceModel):
text: str = embedder.SourceField()
@@ -1055,7 +1079,7 @@ def test_reranking_mrr_usage() -> None:
{"text": "hello world"},
{"text": "goodbye world"},
]
tbl = db.create_table("test", schema=Schema, mode="overwrite")
tbl = db.create_table("test", schema=Schema)
tbl.add(data)
reranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)
@@ -1390,9 +1414,12 @@ def test_frameworks_pydantic_examples() -> None:
{"text": "goodbye world", "vector": [0.0, 1.0]},
]
)
results = table.search("hello world").limit(1).to_pydantic(LanceDocs)
results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)
print(results[0].text)
# hello world
# --8<-- [end:frameworks_pydantic_base_example]
assert isinstance(results[0], LanceDocs)
assert results[0].text == "hello world"
# Platform integrations
@@ -1686,7 +1713,7 @@ def test_platforms_pandas_examples() -> None:
# --8<-- [end:platforms_pandas_vector_search]
# --8<-- [start:platforms_pandas_async_example]
async def run_pandas_async_example() -> None:
async def run_pandas_async_example() -> pd.DataFrame:
async_db = await lancedb.connect_async(
str(Path(tempfile.mkdtemp()) / "pandas-async")
)
@@ -1699,16 +1726,16 @@ def test_platforms_pandas_examples() -> None:
async_table = await async_db.create_table(
"creatures_async", data=async_df, mode="overwrite"
)
async_results = await (
async_table.search([0.6, 0.4, 0.8])
.select(["text", "_distance"])
.limit(1)
.to_pandas()
)
# An async search returns its query builder from an await
query = await async_table.search([0.6, 0.4, 0.8])
async_results = await query.select(["text", "_distance"]).limit(1).to_pandas()
print(async_results)
return async_results
asyncio.run(run_pandas_async_example())
async_results = asyncio.run(run_pandas_async_example())
# --8<-- [end:platforms_pandas_async_example]
assert async_results["text"].tolist() == ["sage"]
assert async_results["_distance"].tolist() == [0.0]
def test_platforms_polars_examples() -> None:
docs/web-tests/py/test_materialized_view.py
+18 -8
View File
@@ -12,7 +12,9 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
db = lancedb.connect("./.lancedb")
table = db.create_table(
"people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view(
@@ -29,12 +31,11 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
# --8<-- [start:materialized_view_refresh]
table.add([{"name": "bea", "age": 41}])
result = view.refresh()
print(result.rows_written)
view.refresh()
print(view.table.count_rows())
# 2
# --8<-- [end:materialized_view_refresh]
assert result.mode == "rebuild"
assert result.rows_written == 2
assert view.table.count_rows() == 2
assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [
"ADA",
"BEA",
@@ -45,7 +46,11 @@ def test_materialized_view_with_no_data(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view(
"adults", "people", where="age >= 18", with_no_data=True
)
@@ -58,11 +63,16 @@ def test_materialized_view_refresh_to_source_version(tmp_path):
import lancedb
db = lancedb.connect(tmp_path)
table = db.create_table("people", [{"name": "ada", "age": 36}])
table = db.create_table(
"people",
[{"name": "ada", "age": 36}],
storage_options={"new_table_enable_stable_row_ids": "true"},
)
view = db.create_materialized_view("adults", "people", where="age >= 18")
first = table.version
table.add([{"name": "bea", "age": 41}])
assert view.refresh().rows_written == 2
view.refresh()
assert view.table.count_rows() == 2
result = view.refresh(source_version=first)
assert (result.source_version, result.rows_written) == (first, 1)
docs/web-tests/py/test_provider_examples.py
+160
View File
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""The provider reranker examples write only to a directory they create.
Each example runs offline. A stand-in embedding function answers for the model
the example asks the registry for, a stand-in reranker replaces the provider
client, and outbound connections are refused, so no model is downloaded and no
provider is called. Sentinel tables in the owned home directory, where these
examples used to write, and in the temporary directory must be left unchanged.
"""
import importlib.util
import socket
import tempfile
from pathlib import Path
from types import SimpleNamespace
from typing import ClassVar
import lancedb
import lancedb.embeddings
import lancedb.rerankers
import pyarrow as pa
import pytest
from lancedb.embeddings import TextEmbeddingFunction, register
from lancedb.rerankers import Reranker
EXAMPLES = Path(__file__).with_name("provider_examples.py")
RERANKERS = {
"reranking_cohere_usage": "CohereReranker",
"reranking_jina_usage": "JinaReranker",
"reranking_watsonx_usage": "WatsonxReranker",
"reranking_openai_usage": "OpenaiReranker",
"reranking_voyageai_usage": "VoyageAIReranker",
}
@register("docs-offline-text")
class OfflineEmbedding(TextEmbeddingFunction):
"""Deterministic four-dimensional vectors, computed locally."""
def ndims(self):
return 4
def generate_embeddings(self, texts):
return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts]
class OfflineReranker(Reranker):
"""Scores every row 1.0 and records which searches it reranked."""
instances: ClassVar[list["OfflineReranker"]] = []
def __init__(self, **kwargs):
super().__init__()
self.kwargs = kwargs
self.searches = []
OfflineReranker.instances.append(self)
def _score(self, results, search):
self.searches.append(search)
drop = [c for c in ("_distance", "_score") if c in results.column_names]
results = results.drop_columns(drop)
scores = pa.array([1.0] * results.num_rows, pa.float32())
return results.append_column("_relevance_score", scores)
def rerank_vector(self, query, vector_results):
return self._score(vector_results, "vector")
def rerank_fts(self, query, fts_results):
return self._score(fts_results, "fts")
def rerank_hybrid(self, query, vector_results, fts_results):
merged = self.merge_results(vector_results, fts_results)
return self._score(merged, "hybrid")
def snapshot(connect, uri):
db = connect(str(uri))
tables = {}
for name in db.list_tables().tables:
table = db.open_table(name)
tables[name] = (table.version, table.schema, table.to_arrow().to_pylist())
return tables
def load_examples():
spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.fixture
def owned(tmp_path, monkeypatch):
home, temp_root = tmp_path / "home", tmp_path / "tmp"
home.mkdir()
temp_root.mkdir()
monkeypatch.setenv("HOME", str(home))
monkeypatch.setattr(tempfile, "tempdir", str(temp_root))
monkeypatch.chdir(tmp_path)
# Anything the examples resolve from "~" must land in the owned home.
assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve())
def refuse(*args, **kwargs):
raise OSError("network access is disabled in this test")
monkeypatch.setattr(socket.socket, "connect", refuse)
monkeypatch.setattr(socket, "create_connection", refuse)
real_connect = lancedb.connect
sentinels = [home / ".lancedb", temp_root / "existing"]
for uri in sentinels:
row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]}
real_connect(str(uri)).create_table("test", [row])
before = [snapshot(real_connect, uri) for uri in sentinels]
registry = lancedb.embeddings.get_registry()
monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding)
for name in RERANKERS.values():
monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker)
monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example")
OfflineReranker.instances.clear()
connected = []
def recording_connect(uri, *args, **kwargs):
connected.append(Path(uri).expanduser().resolve())
return real_connect(uri, *args, **kwargs)
monkeypatch.setattr(lancedb, "connect", recording_connect)
return SimpleNamespace(
home=home,
temp_root=temp_root,
sentinels=sentinels,
before=before,
connect=real_connect,
connected=connected,
)
@pytest.mark.parametrize("example", sorted(RERANKERS))
def test_reranker_example_writes_only_to_its_own_directory(owned, example):
existing = set(owned.temp_root.iterdir())
getattr(load_examples(), example)()
after = [snapshot(owned.connect, uri) for uri in owned.sentinels]
assert after == owned.before
created = set(owned.temp_root.iterdir()) - existing
assert len(created) == 1
(own_directory,) = created
assert len(owned.connected) == 1
assert owned.connected[0].is_relative_to(own_directory.resolve())
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
(reranker,) = OfflineReranker.instances
assert reranker.searches == ["vector", "fts", "hybrid"]
assert reranker.kwargs["api_key"] == "your-api-key"
assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id"
docs/web-tests/py/test_quickstart.py
+255 -142
View File
@@ -1,14 +1,125 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
import shutil
import uuid
from pathlib import Path
import lancedb
import pytest
import pytest_asyncio
# What the quickstart's download step leaves in an application's directory.
IMAGE = (
Path(__file__).resolve().parents[2]
/ "web/static/assets/images/quickstart/sir-lancelot.jpg"
)
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
# The connection settings alone never send these tests to a deployment.
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
def test_quickstart(db_path_factory):
uri = db_path_factory("quickstart_db")
db = lancedb.connect(uri)
def deployment_settings():
if os.environ.get(OPT_IN) != "1":
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
if missing:
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
return dict(
uri=os.environ["LANCEDB_URI"],
api_key=os.environ["LANCEDB_API_KEY"],
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
)
class OwnedTables:
"""A deployment connection on which a test creates only tables of its own.
Each name gets a suffix unique to the test and is claimed only while no
table has it; the test drops what it claimed, whether or not it passed.
"""
def __init__(self, db):
self.db = db
self.suffix = uuid.uuid4().hex[:12]
self.claimed = []
def claim(self, name, existing, mode):
owned = f"{name}_{self.suffix}"
if owned in self.claimed:
return owned, mode
if owned in existing:
raise RuntimeError(f"{owned} already exists and is not this test's")
self.claimed.append(owned)
return owned, "create"
def table_names(self):
names, token = [], None
while True:
page = self.db.list_tables(page_token=token)
names += page.tables
if not (token := page.page_token):
return names
def create_table(self, name, *args, mode="create", **kwargs):
owned, mode = self.claim(name, self.table_names(), mode)
return self.db.create_table(owned, *args, mode=mode, **kwargs)
def drop_claimed(self):
existing = self.table_names()
for name in self.claimed:
if name in existing:
self.db.drop_table(name)
class AsyncOwnedTables(OwnedTables):
async def table_names(self):
names, token = [], None
while True:
page = await self.db.list_tables(page_token=token)
names += page.tables
if not (token := page.page_token):
return names
async def create_table(self, name, *args, mode="create", **kwargs):
owned, mode = self.claim(name, await self.table_names(), mode)
return await self.db.create_table(owned, *args, mode=mode, **kwargs)
async def drop_claimed(self):
existing = await self.table_names()
for name in self.claimed:
if name in existing:
await self.db.drop_table(name)
@pytest.fixture
def deployment():
owned = OwnedTables(lancedb.connect(**deployment_settings()))
yield owned
owned.drop_claimed()
@pytest.fixture(params=["local", "enterprise"])
def db(request, db_path_factory):
if request.param == "local":
yield lancedb.connect(db_path_factory("quickstart_db"))
else:
yield request.getfixturevalue("deployment")
@pytest_asyncio.fixture(params=["local", "enterprise"])
async def async_db(request, db_path_factory):
if request.param == "local":
yield await lancedb.connect_async(db_path_factory("quickstart_async_db"))
return
owned = AsyncOwnedTables(await lancedb.connect_async(**deployment_settings()))
yield owned
await owned.drop_claimed()
def create_characters(db):
# --8<-- [start:quickstart_data]
data = [
{
@@ -41,14 +152,29 @@ def test_quickstart(db_path_factory):
# --8<-- [start:quickstart_create_table]
table = db.create_table("characters", data=data, mode="overwrite")
# --8<-- [end:quickstart_create_table]
assert len(table) == 3
# Drop the table to test create without overwrite
db.drop_table("characters")
assert table.count_rows() == 3
# --8<-- [start:quickstart_add_data]
table.add(
[
{
"id": "4",
"name": "Morgana",
"role": "Sorceress",
"description": "Powerful sorceress of Avalon.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
"vector": [0.3, 0.9, 0.6, 0.8],
}
]
)
# --8<-- [end:quickstart_add_data]
assert table.count_rows() == 4
return table
def test_quickstart(db):
table = create_characters(db)
# --8<-- [start:quickstart_create_table_no_overwrite]
table = db.create_table("characters", data=data)
# --8<-- [end:quickstart_create_table_no_overwrite]
assert len(table) == 3
# --8<-- [start:quickstart_vector_search_1]
# Search for examples similar to a "wise magical advisor"
query_vector = [0.2, 0.8, 0.4, 0.9]
@@ -62,7 +188,14 @@ def test_quickstart(db_path_factory):
)
print(result)
# --8<-- [end:quickstart_vector_search_1]
assert result.head(1)["name"][0] == "Merlin"
assert result["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_output_pandas]
# Ensure you run `pip install pandas` beforehand
result = table.search(query_vector).limit(2).to_pandas()
print(result)
# --8<-- [end:quickstart_output_pandas]
assert result["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_curate_with_metadata]
curated = (
@@ -74,131 +207,15 @@ def test_quickstart(db_path_factory):
)
print(curated)
# --8<-- [end:quickstart_curate_with_metadata]
assert curated.head(1)["name"][0] == "Merlin"
assert curated["name"].to_list() == ["Merlin", "Morgana"]
# --8<-- [start:quickstart_output_pandas]
# Ensure you run `pip install pandas` beforehand
result = table.search(query_vector).limit(2).to_pandas()
print(result)
# --8<-- [end:quickstart_output_pandas]
assert result.iloc[0]["name"] == "Merlin"
# --8<-- [start:quickstart_add_feature]
table.add_columns(
{
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:quickstart_add_feature]
assert "power_score" in table.schema.names
# --8<-- [start:quickstart_query_feature]
features = table.search().select(["name", "role", "power_score"]).to_polars()
print(features)
# --8<-- [end:quickstart_query_feature]
assert "power_score" in features.columns
# --8<-- [start:quickstart_multimodal_bytes]
from pathlib import Path
image_path = Path("docs/web/static/assets/images/quickstart/sir-lancelot.jpg")
image_bytes = image_path.read_bytes()
multimodal_table = db.create_table(
"character_images",
data=[
{
"id": "lancelot",
"description": "Portrait of Sir Lancelot",
"image": image_bytes,
"vector": [0.9, 0.1, 0.5, 0.4],
}
],
mode="overwrite",
)
# --8<-- [end:quickstart_multimodal_bytes]
assert len(multimodal_table) == 1
# --8<-- [start:quickstart_open_table]
table = db.open_table("characters")
# --8<-- [end:quickstart_open_table]
# --8<-- [start:quickstart_add_data]
more_data = [
{
"id": "4",
"name": "Morgana",
"role": "Sorceress",
"description": "Powerful sorceress of Avalon.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
"vector": [0.3, 0.9, 0.6, 0.8],
"power_score": 3.75,
},
]
# Add data to table
table.add(more_data)
# --8<-- [end:quickstart_add_data]
assert len(table) == 4
# --8<-- [start:quickstart_vector_search_2]
# Search for examples similar to a "powerful sorceress"
query_vector = [0.3, 0.9, 0.6, 0.8]
results = table.search(query_vector).limit(2).to_polars()
print(results)
# --8<-- [end:quickstart_vector_search_2]
assert results.head(1)["name"][0] == "Morgana"
# --8<-- [start:quickstart_computed_feature]
table.add_columns(computed={"combat_score": "(stats.magic + stats.strength) / 2.0"})
table.refresh_column("combat_score")
# --8<-- [end:quickstart_computed_feature]
scores = (
table.search().select(["combat_score"]).to_arrow()["combat_score"].to_pylist()
)
assert scores == [2.5, 3.5, 3.0, 3.5]
# --8<-- [start:quickstart_training]
import torch
from lancedb.permutation import Permutation
dataset = Permutation.identity(table).select_columns(
["power_score", "combat_score"]
)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
# --8<-- [end:quickstart_training]
batches = list(dataloader)
assert len(batches) == 1
assert set(batches[0]) == {"power_score", "combat_score"}
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
assert sorted(batches[0]["combat_score"].tolist()) == sorted(scores)
# --8<-- [start:quickstart_training_projection]
dataset = Permutation.identity(table).select_columns(["power_score"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024)
# --8<-- [end:quickstart_training_projection]
batch = next(iter(dataloader))
assert set(batch) == {"power_score"}
assert batch["power_score"].tolist() == [3.5, 4.0, 3.0, 3.75]
curated = table.search(query_vector).where("stats.magic < 4").limit(2).to_list()
assert [row["name"] for row in curated] == ["King Arthur", "Sir Lancelot"]
@pytest.mark.asyncio
async def test_quickstart_async_api(db_path_factory):
db_uri = db_path_factory("quickstart_async_db")
import lancedb
async_db = await lancedb.connect_async(db_uri)
# --8<-- [start:quickstart_data_async]
async def test_quickstart_async_api(async_db):
data = [
{
"id": "1",
"name": "King Arthur",
"role": "King",
"description": "Leader of Camelot and wielder of Excalibur.",
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
"vector": [0.7, 0.1, 0.9, 0.7],
},
{
"id": "2",
"name": "Merlin",
@@ -206,17 +223,8 @@ async def test_quickstart_async_api(db_path_factory):
"description": "Advisor and prophet with deep magical knowledge.",
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
"vector": [0.2, 0.9, 0.4, 0.9],
},
{
"id": "3",
"name": "Sir Lancelot",
"role": "Knight",
"description": "Legendary knight known for courage and combat skill.",
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
"vector": [0.9, 0.1, 0.5, 0.4],
},
}
]
# --8<-- [end:quickstart_data_async]
# --8<-- [start:quickstart_create_table_async]
async_table = await async_db.create_table(
@@ -236,5 +244,110 @@ async def test_quickstart_async_api(db_path_factory):
).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
print(async_result)
# --8<-- [end:quickstart_vector_search_1_async]
assert async_result["name"].to_list() == ["Merlin"]
assert async_result.head(1)["name"][0] == "Merlin"
def test_quickstart_derived_feature(db):
table = create_characters(db)
# --8<-- [start:quickstart_add_feature]
table.add_columns(
{
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
}
)
# --8<-- [end:quickstart_add_feature]
# --8<-- [start:quickstart_query_feature]
features = table.search().select(["name", "role", "power_score"]).to_polars()
print(features)
# --8<-- [end:quickstart_query_feature]
assert sorted(zip(features["name"], features["power_score"])) == [
("King Arthur", 3.5),
("Merlin", 4.0),
("Morgana", 3.75),
("Sir Lancelot", 3.0),
]
def description_lengths(table):
rows = table.search().select(["description", "description_length"]).to_list()
return {row["description"]: row["description_length"] for row in rows}
def test_quickstart_computed_column(db_path_factory):
table = create_characters(lancedb.connect(db_path_factory("computed")))
# --8<-- [start:quickstart_computed_feature]
table.add_columns(computed={"description_length": "length(description)"})
table.refresh_column("description_length")
# --8<-- [end:quickstart_computed_feature]
lengths = description_lengths(table)
assert lengths == {text: len(text) for text in lengths}
def test_quickstart_computed_column_on_a_deployment(deployment):
table = create_characters(deployment)
# --8<-- [start:quickstart_computed_feature_enterprise]
table.add_columns(computed={"description_length": "length(description)"})
job = table.refresh_column_async("description_length")
job.wait()
# --8<-- [end:quickstart_computed_feature_enterprise]
lengths = description_lengths(table)
assert len(lengths) == 4
assert lengths == {text: len(text) for text in lengths}
with pytest.raises(Exception):
table.refresh_column("description_length")
def test_quickstart_multimodal(db, tmp_path, monkeypatch):
app = tmp_path / "app"
(app / "data").mkdir(parents=True)
shutil.copy(IMAGE, app / "data/sir-lancelot.jpg")
monkeypatch.chdir(app)
# --8<-- [start:quickstart_multimodal_bytes]
from pathlib import Path
image_bytes = Path("data/sir-lancelot.jpg").read_bytes()
multimodal_table = db.create_table(
"character_images",
data=[
{
"id": "lancelot",
"description": "Portrait of Sir Lancelot",
"image": image_bytes,
"vector": [0.9, 0.1, 0.5, 0.4],
}
],
mode="overwrite",
)
# --8<-- [end:quickstart_multimodal_bytes]
assert multimodal_table.count_rows() == 1
stored = multimodal_table.search().select(["image"]).to_list()[0]["image"]
assert stored == IMAGE.read_bytes()
def test_quickstart_training(db):
table = create_characters(db)
# --8<-- [start:quickstart_training]
import torch
from lancedb.permutation import Permutation
table.add_columns(
{
"magic": "cast(stats.magic as float)",
"strength": "cast(stats.strength as float)",
}
)
dataset = Permutation.identity(table).select_columns(["magic", "strength"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
# --8<-- [end:quickstart_training]
batches = list(dataloader)
assert len(batches) == 1
assert set(batches[0]) == {"magic", "strength"}
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
assert sorted(batches[0]["magic"].tolist()) == [1.0, 1.0, 5.0, 5.0]
docs/web-tests/py/test_read_consistency.py
+35
View File
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_read_consistency():
# --8<-- [start:read_consistency]
from datetime import timedelta
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-consistency-") as uri:
writer = lancedb.connect(uri)
table = writer.create_table("documents", [{"id": 1, "text": "first"}])
default_reader = lancedb.connect(uri).open_table("documents")
latest_reader = lancedb.connect(
uri, read_consistency_interval=timedelta(0)
).open_table("documents")
table.add([{"id": 2, "text": "second"}])
default_count = default_reader.count_rows()
latest_count = latest_reader.count_rows()
print(f"Default reader: {default_count} rows") # 1
print(f"Zero-interval reader: {latest_count} rows") # 2
default_reader.checkout_latest()
refreshed_count = default_reader.count_rows()
print(f"After explicit refresh: {refreshed_count} rows") # 2
# --8<-- [end:read_consistency]
assert default_count == 1
assert latest_count == 2
assert refreshed_count == 2
docs/web-tests/py/test_reranker_examples.py
+96
View File
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""The published reranker examples in test_integrations.py write only to a directory they create.
Each example runs as readers see it: its marked region is extracted the way the snippet
generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the
sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT
and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination
and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory,
where these examples used to write, and in the temporary directory must be left unchanged.
"""
import importlib.util
import runpy
import sys
from pathlib import Path
import lancedb.rerankers
import pytest
import test_provider_examples as provider
owned = provider.owned
SOURCE = Path(__file__).with_name("test_integrations.py")
GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py"
MODEL_RERANKERS = {
"reranking_answerdotai_usage": "AnswerdotaiRerankers",
"reranking_colbert_usage": "ColbertReranker",
"reranking_cross_encoder_usage": "CrossEncoderReranker",
}
LOCAL_RERANKERS = [
"reranking_linear_combination_usage",
"reranking_mrr_usage",
"reranking_rrf_usage",
]
def load_generator():
if "mdx_snippets_gen" not in sys.modules:
spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module # its dataclasses resolve annotations by module
spec.loader.exec_module(module)
return sys.modules["mdx_snippets_gen"]
def reader_code(name):
"""The example exactly as the snippet generator publishes it."""
generator = load_generator()
start, end = generator.markers_for_lang("py")
lines = SOURCE.read_text().splitlines(keepends=True)
snippet = generator.parse_snippets(lines, start, end)[name]
return generator.snippet_text(generator.dedent(snippet))
def run_example(owned, tmp_path, name):
script = tmp_path / "scripts" / f"{name}.py"
script.parent.mkdir(exist_ok=True)
script.write_text(reader_code(name))
existing = set(owned.temp_root.iterdir())
namespace = runpy.run_path(str(script))
after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels]
assert after == owned.before
created = set(owned.temp_root.iterdir()) - existing
assert len(created) == 1
(own_directory,) = created
assert len(owned.connected) == 1
assert owned.connected[0].is_relative_to(own_directory.resolve())
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
return namespace
@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS))
def test_model_reranker_example_writes_only_to_its_own_directory(
owned, monkeypatch, tmp_path, name
):
monkeypatch.setattr(
lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker
)
namespace = run_example(owned, tmp_path, name)
(reranker,) = provider.OfflineReranker.instances
assert reranker.searches == ["vector", "fts", "hybrid"]
assert namespace["result"]
@pytest.mark.parametrize("name", LOCAL_RERANKERS)
def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name):
namespace = run_example(owned, tmp_path, name)
assert namespace["result"]
assert all("_relevance_score" in row for row in namespace["result"])
if name == "reranking_mrr_usage":
assert namespace["combined"].num_rows > 0
docs/web-tests/py/test_search_examples.py
+228
View File
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
def test_vector_search():
# --8<-- [start:vector_search]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
rows = (
table.search([1.0, 0.0])
.distance_type("l2")
.select(["id", "text", "_distance"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:vector_search]
assert [row["id"] for row in rows] == [1, 2]
assert rows[0]["_distance"] == 0.0
def test_text_search():
# --8<-- [start:text_search]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.index import FTS
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
},
],
)
table.create_index("text", config=FTS())
rows = (
table.search("puppy", query_type="fts", fts_columns="text")
.select(["id", "text", "_score"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:text_search]
assert [row["id"] for row in rows] == [1]
assert rows[0]["_score"] > 0
def test_hybrid_search():
# --8<-- [start:hybrid_search]
from tempfile import TemporaryDirectory
import lancedb
from lancedb.index import FTS
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
from lancedb.rerankers import RRFReranker
table.create_index("text", config=FTS())
rows = (
table.search(
query_type="hybrid", vector_column_name="vector", fts_columns="text"
)
.vector([1.0, 0.0])
.text("puppy")
.rerank(RRFReranker())
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:hybrid_search]
assert [row["id"] for row in rows] == [1, 2]
assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"]
def test_filter_search():
# --8<-- [start:filter_search]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
rows = (
table.search([0.0, 1.0])
.where("category = 'animals'", prefilter=True)
.select(["id", "text", "category", "_distance"])
.limit(2)
.to_list()
)
print(rows)
# --8<-- [end:filter_search]
assert {row["id"] for row in rows} == {1, 2}
assert all(row["category"] == "animals" for row in rows)
def test_query_plan():
# --8<-- [start:query_plan]
from tempfile import TemporaryDirectory
import lancedb
with TemporaryDirectory(prefix="lancedb-search-") as uri:
db = lancedb.connect(uri)
table = db.create_table(
"documents",
[
{
"id": 1,
"text": "A puppy playing in the park",
"category": "animals",
"vector": [1.0, 0.0],
},
{
"id": 2,
"text": "A dog resting at home",
"category": "animals",
"vector": [0.9, 0.1],
},
{
"id": 3,
"text": "A guide to growing tomatoes",
"category": "gardening",
"vector": [0.0, 1.0],
},
],
)
query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2)
print("Plan:", query.explain_plan())
print("Measured execution:", query.analyze_plan())
rows = query.to_list()
print(rows)
# --8<-- [end:query_plan]
assert [row["id"] for row in rows] == [1, 2]
docs/web-tests/py/test_tables.py
+25 -35
View File
@@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db):
# ============================================================================
def _setup_schema_add_table(tmp_db, data=None):
def _setup_schema_add_table(db):
# --8<-- [start:schema_add_setup]
table_name = "schema_evolution_add_example"
if data is None:
data = [
{
"id": 1,
"name": "Laptop",
"price": 1200.00,
"vector": np.random.random(128).tolist(),
},
{
"id": 2,
"name": "Smartphone",
"price": 800.00,
"vector": np.random.random(128).tolist(),
},
{
"id": 3,
"name": "Headphones",
"price": 150.00,
"vector": np.random.random(128).tolist(),
},
]
table = tmp_db.create_table(table_name, data, mode="overwrite")
import numpy as np
data = [
{
"id": 1,
"name": "Laptop",
"price": 1200.00,
"vector": np.random.random(128).tolist(),
},
{
"id": 2,
"name": "Smartphone",
"price": 800.00,
"vector": np.random.random(128).tolist(),
},
{
"id": 3,
"name": "Headphones",
"price": 150.00,
"vector": np.random.random(128).tolist(),
},
]
table = db.create_table("schema_evolution_add_example", data, mode="overwrite")
# --8<-- [end:schema_add_setup]
return table
@@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db):
def test_add_columns_nullable(tmp_db):
table = _setup_schema_add_table(
tmp_db,
data=[
{
"id": 1,
"name": "Laptop",
"price": 1200.00,
"vector": np.random.random(128).tolist(),
}
],
)
table = _setup_schema_add_table(tmp_db)
# --8<-- [start:add_columns_nullable]
# Add a nullable timestamp column
docs/web-tests/rs/Cargo.lock
Generated
+248 -199
View File
@@ -452,6 +452,61 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "axum"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
dependencies = [
"async-trait",
"axum-core",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "backtrace"
version = "0.3.76"
@@ -556,6 +611,15 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bs58"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
dependencies = [
"tinyvec",
]
[[package]]
name = "bumpalo"
version = "3.19.0"
@@ -836,6 +900,16 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation"
version = "0.10.1"
@@ -1065,12 +1139,12 @@ dependencies = [
[[package]]
name = "darling"
version = "0.21.3"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0"
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
"darling_core 0.21.3",
"darling_macro 0.21.3",
"darling_core 0.23.0",
"darling_macro 0.23.0",
]
[[package]]
@@ -1089,11 +1163,10 @@ dependencies = [
[[package]]
name = "darling_core"
version = "0.21.3"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4"
checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
@@ -1114,11 +1187,11 @@ dependencies = [
[[package]]
name = "darling_macro"
version = "0.21.3"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core 0.21.3",
"darling_core 0.23.0",
"quote",
"syn 2.0.117",
]
@@ -1177,7 +1250,7 @@ dependencies = [
"log",
"object_store",
"parking_lot",
"rand 0.9.2",
"rand 0.9.5",
"regex",
"sqlparser 0.61.0",
"tempfile",
@@ -1292,7 +1365,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
"rand 0.9.2",
"rand 0.9.5",
"tokio",
"url",
]
@@ -1392,7 +1465,7 @@ dependencies = [
"log",
"object_store",
"parking_lot",
"rand 0.9.2",
"rand 0.9.5",
"tempfile",
"url",
]
@@ -1457,7 +1530,7 @@ dependencies = [
"md-5",
"memchr",
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
"regex",
"sha2",
"unicode-segmentation",
@@ -1761,26 +1834,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "deepsize"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
dependencies = [
"deepsize_derive",
]
[[package]]
name = "deepsize_derive"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "deranged"
version = "0.5.5"
@@ -2078,7 +2131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7af6e24ed12cf382082d5a7f365df2b8e3d6b1518f615d54c85165e9b9718250"
dependencies = [
"arrow-array",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -2408,6 +2461,12 @@ version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
[[package]]
name = "httpdate"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.3.0"
@@ -2428,6 +2487,7 @@ dependencies = [
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"pin-utils",
@@ -2472,9 +2532,11 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"socket2",
"system-configuration",
"tokio",
"tower-service",
"tracing",
"windows-registry",
]
[[package]]
@@ -2498,7 +2560,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.2",
"windows-core 0.61.2",
]
[[package]]
@@ -2877,7 +2939,7 @@ dependencies = [
"nom",
"num-traits",
"ordered-float",
"rand 0.9.2",
"rand 0.9.5",
"ryu",
"serde",
"serde_json",
@@ -2930,15 +2992,15 @@ dependencies = [
"half",
"humantime",
"itertools 0.13.0",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-arrow",
"lance-core",
"lance-datafusion",
"lance-encoding",
"lance-file",
"lance-index",
"lance-io",
"lance-linalg",
"lance-namespace 8.0.0",
"lance-namespace",
"lance-select",
"lance-table",
"lance-tokenizer",
@@ -2950,7 +3012,7 @@ dependencies = [
"prost",
"prost-build",
"prost-types",
"rand 0.9.2",
"rand 0.9.5",
"rayon",
"roaring",
"rustc-hash",
@@ -2966,28 +3028,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "lance-arrow"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ipc",
"arrow-ord",
"arrow-schema",
"arrow-select",
"bytes",
"futures",
"getrandom 0.2.16",
"half",
"jsonb",
"num-traits",
"rand 0.9.2",
]
[[package]]
name = "lance-arrow"
version = "8.0.0"
@@ -3007,7 +3047,7 @@ dependencies = [
"half",
"jsonb",
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -3048,41 +3088,6 @@ dependencies = [
"seq-macro",
]
[[package]]
name = "lance-core"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-schema",
"async-trait",
"byteorder",
"bytes",
"deepsize",
"futures",
"itertools 0.13.0",
"lance-arrow 7.0.0",
"libc",
"log",
"moka",
"num_cpus",
"object_store",
"pin-project",
"prost",
"rand 0.9.2",
"roaring",
"serde_json",
"snafu 0.9.1",
"tempfile",
"tokio",
"tokio-stream",
"tokio-util",
"tracing",
"url",
]
[[package]]
name = "lance-core"
version = "8.0.0"
@@ -3100,7 +3105,7 @@ dependencies = [
"datafusion-sql",
"futures",
"itertools 0.13.0",
"lance-arrow 8.0.0",
"lance-arrow",
"lance-derive",
"libc",
"libm",
@@ -3110,7 +3115,7 @@ dependencies = [
"object_store",
"pin-project",
"prost",
"rand 0.9.2",
"rand 0.9.5",
"roaring",
"serde_json",
"snafu 0.9.1",
@@ -3144,8 +3149,8 @@ dependencies = [
"datafusion-physical-expr",
"futures",
"jsonb",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-arrow",
"lance-core",
"lance-datagen",
"log",
"pin-project",
@@ -3169,7 +3174,7 @@ dependencies = [
"futures",
"half",
"hex",
"rand 0.9.2",
"rand 0.9.5",
"rand_distr 0.5.1",
"rand_xoshiro",
]
@@ -3206,15 +3211,15 @@ dependencies = [
"hex",
"hyperloglogplus",
"itertools 0.13.0",
"lance-arrow 8.0.0",
"lance-arrow",
"lance-bitpacking",
"lance-core 8.0.0",
"lance-core",
"log",
"lz4",
"num-traits",
"prost",
"prost-build",
"rand 0.9.2",
"rand 0.9.5",
"strum 0.26.3",
"tokio",
"tracing",
@@ -3240,8 +3245,8 @@ dependencies = [
"bytes",
"datafusion-common",
"futures",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-arrow",
"lance-core",
"lance-encoding",
"lance-io",
"log",
@@ -3286,9 +3291,9 @@ dependencies = [
"itertools 0.13.0",
"jieba-rs",
"jsonb",
"lance-arrow 8.0.0",
"lance-arrow",
"lance-arrow-stats",
"lance-core 8.0.0",
"lance-core",
"lance-datafusion",
"lance-datagen",
"lance-encoding",
@@ -3306,7 +3311,7 @@ dependencies = [
"prost",
"prost-build",
"prost-types",
"rand 0.9.2",
"rand 0.9.5",
"rand_distr 0.5.1",
"rangemap",
"rayon",
@@ -3343,16 +3348,16 @@ dependencies = [
"futures",
"http",
"io-uring",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-namespace 8.0.0",
"lance-arrow",
"lance-core",
"lance-namespace",
"log",
"moka",
"object_store",
"path_abs",
"pin-project",
"prost",
"rand 0.9.2",
"rand 0.9.5",
"serde",
"tempfile",
"tokio",
@@ -3371,24 +3376,10 @@ dependencies = [
"arrow-schema",
"cc",
"half",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-arrow",
"lance-core",
"num-traits",
"rand 0.9.2",
]
[[package]]
name = "lance-namespace"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2"
dependencies = [
"arrow",
"async-trait",
"bytes",
"lance-core 7.0.0",
"lance-namespace-reqwest-client 0.7.7",
"snafu 0.9.1",
"rand 0.9.5",
]
[[package]]
@@ -3400,8 +3391,8 @@ dependencies = [
"arrow",
"async-trait",
"bytes",
"lance-core 8.0.0",
"lance-namespace-reqwest-client 0.8.6",
"lance-core",
"lance-namespace-reqwest-client",
"snafu 0.9.1",
]
@@ -3415,42 +3406,33 @@ dependencies = [
"arrow-ipc",
"arrow-schema",
"async-trait",
"axum",
"bytes",
"datafusion-common",
"datafusion-physical-plan",
"futures",
"lance",
"lance-core 8.0.0",
"lance-core",
"lance-index",
"lance-io",
"lance-linalg",
"lance-namespace 8.0.0",
"lance-namespace",
"lance-table",
"log",
"object_store",
"rand 0.9.2",
"rand 0.9.5",
"reqwest",
"roaring",
"serde",
"serde_json",
"time",
"tokio",
"tower",
"tower-http 0.5.2",
"url",
"uuid",
]
[[package]]
name = "lance-namespace-reqwest-client"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99"
dependencies = [
"reqwest",
"serde",
"serde_json",
"serde_repr",
"serde_with",
"url",
]
[[package]]
name = "lance-namespace-reqwest-client"
version = "0.8.6"
@@ -3477,7 +3459,7 @@ dependencies = [
"byteorder",
"bytes",
"itertools 0.13.0",
"lance-core 8.0.0",
"lance-core",
"roaring",
"tracing",
]
@@ -3498,8 +3480,8 @@ dependencies = [
"bytes",
"chrono",
"futures",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-arrow",
"lance-core",
"lance-file",
"lance-io",
"lance-select",
@@ -3508,7 +3490,7 @@ dependencies = [
"prost",
"prost-build",
"prost-types",
"rand 0.9.2",
"rand 0.9.5",
"rangemap",
"roaring",
"semver",
@@ -3530,10 +3512,10 @@ dependencies = [
"arrow-array",
"arrow-schema",
"criterion",
"lance-arrow 8.0.0",
"lance-arrow",
"num-traits",
"pprof",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -3581,9 +3563,10 @@ dependencies = [
"datafusion-sql",
"futures",
"half",
"http",
"lance",
"lance-arrow 8.0.0",
"lance-core 8.0.0",
"lance-arrow",
"lance-core",
"lance-datafusion",
"lance-datagen",
"lance-encoding",
@@ -3591,7 +3574,7 @@ dependencies = [
"lance-index",
"lance-io",
"lance-linalg",
"lance-namespace 8.0.0",
"lance-namespace",
"lance-namespace-impls",
"lance-table",
"lance-testing",
@@ -3603,8 +3586,9 @@ dependencies = [
"pin-project",
"polars",
"polars-arrow",
"rand 0.9.2",
"rand 0.9.5",
"regex",
"reqwest",
"semver",
"serde",
"serde_json",
@@ -3613,6 +3597,7 @@ dependencies = [
"tempfile",
"tokio",
"url",
"urlencoding",
"uuid",
]
@@ -3859,6 +3844,12 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
@@ -4366,7 +4357,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared 0.11.3",
"rand 0.8.5",
"rand 0.8.8",
]
[[package]]
@@ -4584,7 +4575,7 @@ dependencies = [
"polars-error",
"polars-row",
"polars-utils",
"rand 0.8.5",
"rand 0.8.8",
"rand_distr 0.4.3",
"rayon",
"regex",
@@ -4783,7 +4774,7 @@ dependencies = [
"polars-error",
"polars-lazy",
"polars-plan",
"rand 0.8.5",
"rand 0.8.8",
"serde",
"serde_json",
"sqlparser 0.39.0",
@@ -5096,9 +5087,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.8.5"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
dependencies = [
"libc",
"rand_chacha 0.3.1",
@@ -5107,9 +5098,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.9.2"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.3",
@@ -5177,7 +5168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
"rand 0.8.8",
]
[[package]]
@@ -5187,7 +5178,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
dependencies = [
"num-traits",
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -5383,7 +5374,7 @@ dependencies = [
"tokio-rustls",
"tokio-util",
"tower",
"tower-http",
"tower-http 0.6.8",
"tower-service",
"url",
"wasm-bindgen",
@@ -5464,7 +5455,8 @@ dependencies = [
"arrow-schema",
"futures",
"futures-util",
"lance-namespace 7.0.0",
"lance-index",
"lance-namespace",
"lancedb",
"polars",
"polars-arrow",
@@ -5533,9 +5525,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.35"
version = "0.23.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
dependencies = [
"once_cell",
"ring",
@@ -5569,9 +5561,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.13"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"ring",
"rustls-pki-types",
@@ -5651,7 +5643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef"
dependencies = [
"bitflags 2.10.0",
"core-foundation",
"core-foundation 0.10.1",
"core-foundation-sys",
"libc",
"security-framework-sys",
@@ -5722,6 +5714,17 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_repr"
version = "0.1.20"
@@ -5747,15 +5750,17 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.16.1"
version = "3.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7"
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
dependencies = [
"base64 0.22.1",
"bs58",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.14.0",
"jiff",
"schemars 0.9.0",
"schemars 1.1.0",
"serde_core",
@@ -5766,11 +5771,11 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.16.1"
version = "3.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c"
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
dependencies = [
"darling 0.21.3",
"darling 0.23.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -6193,6 +6198,27 @@ dependencies = [
"windows 0.52.0",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 2.10.0",
"core-foundation 0.9.4",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "tagptr"
version = "0.2.0"
@@ -6434,6 +6460,24 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-http"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
dependencies = [
"bitflags 2.10.0",
"bytes",
"http",
"http-body",
"http-body-util",
"pin-project-lite",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -6477,6 +6521,7 @@ version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647"
dependencies = [
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -6544,7 +6589,7 @@ version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
dependencies = [
"rand 0.9.2",
"rand 0.9.5",
]
[[package]]
@@ -6631,6 +6676,12 @@ dependencies = [
"serde",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf8-ranges"
version = "1.0.5"
@@ -6921,19 +6972,6 @@ dependencies = [
"windows-strings 0.4.2",
]
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link 0.2.1",
"windows-result 0.4.1",
"windows-strings 0.5.1",
]
[[package]]
name = "windows-future"
version = "0.2.1"
@@ -6989,6 +7027,17 @@ dependencies = [
"windows-link 0.1.3",
]
[[package]]
name = "windows-registry"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
dependencies = [
"windows-link 0.2.1",
"windows-result 0.4.1",
"windows-strings 0.5.1",
]
[[package]]
name = "windows-result"
version = "0.3.4"
@@ -7310,9 +7359,9 @@ dependencies = [
[[package]]
name = "xxhash-rust"
version = "0.8.15"
version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
[[package]]
name = "yoke"
docs/web-tests/rs/Cargo.toml
+35 -2
View File
@@ -9,8 +9,9 @@ arrow-json = "58.3.0"
arrow-schema = "58.3.0"
futures = "0.3.31"
futures-util = "0.3.31"
lance-namespace = "7.0.0"
lancedb = { version = "0.31.0", features = ["polars"] }
lance-namespace = "8.0.0"
lance-index = "=8.0.0"
lancedb = { version = "0.31.0", features = ["polars", "remote"] }
polars = ">=0.37,<0.40.0"
polars-arrow = ">=0.37,<0.40.0"
serde = { version = "1", features = ["derive"] }
@@ -41,3 +42,35 @@ path = "multimodal.rs"
[[example]]
name = "search"
path = "search.rs"
[[example]]
name = "ingestion_examples"
path = "ingestion_examples.rs"
[[example]]
name = "ingestion_delete"
path = "ingestion_delete.rs"
[[example]]
name = "ingestion_progress"
path = "ingestion_progress.rs"
[[example]]
name = "search_vector_search"
path = "search_vector_search.rs"
[[example]]
name = "search_text_search"
path = "search_text_search.rs"
[[example]]
name = "search_hybrid_search"
path = "search_hybrid_search.rs"
[[example]]
name = "search_filter_search"
path = "search_filter_search.rs"
[[example]]
name = "search_query_plan"
path = "search_query_plan.rs"
docs/web-tests/rs/connection.rs
+66 -31
View File
@@ -3,48 +3,69 @@
use std::path::PathBuf;
use lancedb::connect;
use lancedb::{Connection, connect};
// --8<-- [start:connect]
async fn connect_example(uri: &str) {
let db = connect(uri).execute().await.unwrap();
let _ = db;
async fn connect_local() -> lancedb::Result<Connection> {
// --8<-- [start:connect]
let db = connect("ex_lancedb").execute().await?;
// --8<-- [end:connect]
Ok(db)
}
// --8<-- [end:connect]
#[tokio::main]
async fn main() {
let temp_dir = tempfile::tempdir().unwrap();
let uri = temp_dir.path().join("ex_lancedb");
connect_example(uri.to_str().unwrap()).await;
// Keep the enterprise snippet in this file, but don't run it in CI.
let _ = connect_enterprise_quickstart_config();
let _ = connect_object_storage_config();
}
fn connect_enterprise_quickstart_config() -> (String, String, String, String) {
// --8<-- [start:connect_enterprise_quickstart]
let uri = "db://your-database-uri";
let api_key = "your-api-key";
let region = "us-east-1";
let host_override = "https://your-enterprise-endpoint.com";
// --8<-- [end:connect_enterprise_quickstart]
(
uri.to_string(),
api_key.to_string(),
region.to_string(),
host_override.to_string(),
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let temp_dir = tempfile::tempdir()?;
std::env::set_current_dir(temp_dir.path())?;
let db = connect_local().await?;
db.create_empty_table(
"probe",
std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
"id",
arrow_schema::DataType::Int64,
false,
)])),
)
.execute()
.await?;
let reopened = connect(temp_dir.path().join("ex_lancedb").to_str().unwrap())
.execute()
.await?;
assert_eq!(reopened.table_names().execute().await?, vec!["probe"]);
// Connects to a deployment only when asked, and only lists its tables.
if std::env::var("LANCEDB_DOCS_TEST_DEPLOYMENT").as_deref() == Ok("1") {
let db = connect_enterprise_quickstart().await?;
println!("{:?}", db.table_names().execute().await?);
}
Ok(())
}
fn connect_object_storage_config() -> &'static str {
async fn connect_enterprise_quickstart() -> Result<Connection, Box<dyn std::error::Error>> {
// --8<-- [start:connect_enterprise_quickstart]
let uri = std::env::var("LANCEDB_URI")?; // db://your-database
let api_key = std::env::var("LANCEDB_API_KEY")?;
let region = std::env::var("LANCEDB_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let host_override = std::env::var("LANCEDB_HOST_OVERRIDE")?; // https://your-endpoint
let db = connect(&uri)
.api_key(&api_key)
.region(&region)
.host_override(&host_override)
.execute()
.await?;
// --8<-- [end:connect_enterprise_quickstart]
Ok(db)
}
// Compiled as documentation, never called by the example runner.
#[allow(dead_code)]
async fn connect_object_storage() -> lancedb::Result<Connection> {
// --8<-- [start:connect_object_storage]
let uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
let db = connect(uri).execute().await?;
// --8<-- [end:connect_object_storage]
uri
Ok(db)
}
async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> {
@@ -132,3 +153,17 @@ async fn namespace_admin_ops_example() -> lancedb::Result<()> {
fn repo_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}
// Compiled as documentation, never called by the example runner.
#[allow(dead_code)]
async fn connect_enterprise_placeholders() -> Result<Connection, Box<dyn std::error::Error>> {
// --8<-- [start:connect_enterprise_placeholders]
let db = connect("db://your-database")
.api_key("your-api-key")
.region("us-east-1")
.host_override("https://your-endpoint")
.execute()
.await?;
// --8<-- [end:connect_enterprise_placeholders]
Ok(db)
}
docs/web-tests/rs/embedding.rs
+3 -3
View File
@@ -17,7 +17,7 @@ use lancedb::{
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let api_key = "your-openai-api-key".to_string();
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
@@ -54,7 +54,7 @@ use std::sync::Arc;
use lancedb::embeddings::openai::OpenAIEmbeddingFunction;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let api_key = "your-openai-api-key".to_string();
let embedding = Arc::new(
OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small")
.expect("failed to create OpenAI embedding function"),
@@ -77,7 +77,7 @@ use lancedb::{
#[tokio::main]
async fn main() -> Result<()> {
let db = connect("./mydb").execute().await?;
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
let api_key = "your-openai-api-key".to_string();
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
api_key,
"text-embedding-3-large",
docs/web-tests/rs/ingestion_delete.rs
+72
View File
@@ -0,0 +1,72 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example]
use std::sync::Arc;
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase};
fn make_users_reader(rows: &[(i64, &str)]) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table(
"users",
make_users_reader(&[(1, "Alice"), (2, "Bob"), (3, "Charlie")]),
)
.execute()
.await?;
let predicate = "id = 3";
let selected: Vec<RecordBatch> = table
.query()
.only_if(predicate)
.execute()
.await?
.try_collect()
.await?;
println!("Selected: {selected:?}");
table.delete(predicate).await?;
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
println!("Remaining: {rows:?}");
assert_eq!(table.count_rows(None).await?, 2);
assert_eq!(
table
.count_rows(Some("id = 1 AND name = 'Alice'".into()))
.await?,
1
);
assert_eq!(
table
.count_rows(Some("id = 2 AND name = 'Bob'".into()))
.await?,
1
);
assert_eq!(table.count_rows(Some("id = 3".into())).await?, 0);
Ok(())
}
// --8<-- [end:example]
docs/web-tests/rs/ingestion_examples.rs
+97
View File
@@ -0,0 +1,97 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:write_workflow]
use std::sync::Arc;
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase};
fn make_users_reader(rows: &[(i64, &str, i64)]) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, false),
Field::new("login_count", DataType::Int64, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table(
"users",
make_users_reader(&[(1, "Alice", 10), (2, "Bob", 20)]),
)
.execute()
.await?;
table
.add(make_users_reader(&[(3, "Charlie", 5)]))
.execute()
.await?;
table
.update()
.only_if("id = 2")
.column("name", "'Bobby'")
.execute()
.await?;
table
.update()
.only_if("id = 2")
.column("login_count", "login_count + 1")
.execute()
.await?;
let mut merge = table.merge_insert(&["id"]);
merge
.when_matched_update_all(None)
.when_not_matched_insert_all();
merge
.execute(make_users_reader(&[(3, "Charlie", 6), (4, "Dana", 1)]))
.await?;
let selected: Vec<RecordBatch> = table
.query()
.only_if("id = 4")
.execute()
.await?
.try_collect()
.await?;
println!("Rows selected for deletion: {selected:?}");
table.delete("id = 4").await?;
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
println!("Remaining rows: {rows:?}");
assert_eq!(table.count_rows(None).await?, 3);
assert_eq!(
table
.count_rows(Some(
"id = 2 AND name = 'Bobby' AND login_count = 21".into()
))
.await?,
1
);
assert_eq!(
table
.count_rows(Some("id = 3 AND login_count = 6".into()))
.await?,
1
);
Ok(())
}
// --8<-- [end:write_workflow]
docs/web-tests/rs/ingestion_progress.rs
+64
View File
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example]
use std::sync::Arc;
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::ExecutableQuery;
fn make_users_reader(rows: &[(i64, &str)]) -> Box<dyn RecordBatchReader + Send> {
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
],
)
.unwrap();
Box::new(RecordBatchIterator::new(
vec![Ok(batch)].into_iter(),
schema,
))
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table("users", make_users_reader(&[(1, "Alice")]))
.execute()
.await?;
table
.add(make_users_reader(&[(2, "Bob"), (3, "Charlie")]))
.progress(|progress| println!("Rows written: {}", progress.output_rows()))
.execute()
.await?;
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
println!("After insertion: {rows:?}");
assert_eq!(table.count_rows(None).await?, 3);
assert_eq!(
table
.count_rows(Some("id = 1 AND name = 'Alice'".into()))
.await?,
1
);
assert_eq!(
table
.count_rows(Some("id = 2 AND name = 'Bob'".into()))
.await?,
1
);
assert_eq!(table.count_rows(Some("id = 3".into())).await?, 1);
Ok(())
}
// --8<-- [end:example]
docs/web-tests/rs/quickstart.rs
+64 -42
View File
@@ -1,23 +1,20 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::sync::Arc;
use arrow_array::types::Float32Type;
use arrow_array::{
FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray,
};
use arrow_schema::{DataType, Field, FieldRef, Schema};
use lancedb::arrow::IntoPolars;
use lancedb::database::CreateTableMode;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use lancedb::{connect, table::NewColumnTransform};
use polars::prelude::DataFrame;
use serde::{Deserialize, Serialize};
// --8<-- [start:quickstart_define_struct]
use std::sync::Arc;
use lancedb::arrow::arrow_array::types::Float32Type;
use lancedb::arrow::arrow_array::{
FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator,
RecordBatchReader, StructArray,
};
use lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
// Define structs representing the data schema
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone)]
struct Stats {
strength: i8,
magic: i8,
@@ -25,7 +22,7 @@ struct Stats {
wisdom: i8,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone)]
struct Character {
id: String,
name: String,
@@ -43,7 +40,7 @@ fn characters_schema() -> Arc<Schema> {
Field::new("description", DataType::LargeUtf8, false),
Field::new(
"stats",
DataType::Struct(arrow_schema::Fields::from(vec![
DataType::Struct(Fields::from(vec![
Arc::new(Field::new("strength", DataType::Int8, false)),
Arc::new(Field::new("magic", DataType::Int8, false)),
Arc::new(Field::new("leadership", DataType::Int8, false)),
@@ -58,11 +55,11 @@ fn characters_schema() -> Arc<Schema> {
),
]))
}
// --8<-- [end:quickstart_define_struct]
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
fn characters_to_reader(
schema: Arc<Schema>,
rows: &[Character],
) -> Box<dyn RecordBatchReader + Send> {
let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));
let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));
let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));
@@ -114,6 +111,7 @@ fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
schema,
))
}
// --8<-- [end:quickstart_define_struct]
#[tokio::main]
async fn main() {
@@ -166,6 +164,8 @@ async fn main() {
// --8<-- [end:quickstart_data]
// --8<-- [start:quickstart_create_table]
use lancedb::database::CreateTableMode;
let schema = characters_schema();
let table = db
.create_table("characters", characters_to_reader(schema.clone(), &data))
@@ -175,22 +175,37 @@ async fn main() {
.unwrap();
// --8<-- [end:quickstart_create_table]
assert_eq!(table.count_rows(None).await.unwrap(), 3);
db.drop_table("characters", &[]).await.unwrap();
// --8<-- [start:quickstart_create_table_no_overwrite]
let table = db
.create_table("characters", characters_to_reader(schema.clone(), &data))
// --8<-- [start:quickstart_add_data]
let more_data = vec![Character {
id: "4".to_string(),
name: "Morgana".to_string(),
role: "Sorceress".to_string(),
description: "Powerful sorceress of Avalon.".to_string(),
stats: Stats {
strength: 2,
magic: 5,
leadership: 4,
wisdom: 4,
},
vector: [0.3, 0.9, 0.6, 0.8],
}];
table
.add(characters_to_reader(schema.clone(), &more_data))
.execute()
.await
.unwrap();
// --8<-- [end:quickstart_create_table_no_overwrite]
assert_eq!(table.count_rows(None).await.unwrap(), 3);
// --8<-- [end:quickstart_add_data]
assert_eq!(table.count_rows(None).await.unwrap(), 4);
// --8<-- [start:quickstart_vector_search_1]
use lancedb::arrow::IntoPolars;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
// Search for examples similar to a "wise magical advisor"
let query_vector = [0.2, 0.8, 0.4, 0.9];
let result: DataFrame = table
let result = table
.query()
.nearest_to(&query_vector)
.unwrap()
@@ -211,9 +226,10 @@ async fn main() {
// --8<-- [end:quickstart_vector_search_1]
let name_col = result.column("name").unwrap().str().unwrap();
assert_eq!(name_col.get(0).unwrap(), "Merlin");
assert_eq!(name_col.get(1).unwrap(), "Morgana");
// --8<-- [start:quickstart_curate_with_metadata]
let curated: DataFrame = table
let curated = table
.query()
.nearest_to(&query_vector)
.unwrap()
@@ -235,9 +251,10 @@ async fn main() {
// --8<-- [end:quickstart_curate_with_metadata]
let curated_name_col = curated.column("name").unwrap().str().unwrap();
assert_eq!(curated_name_col.get(0).unwrap(), "Merlin");
assert_eq!(curated_name_col.get(1).unwrap(), "Morgana");
// --8<-- [start:quickstart_output_array]
let result: DataFrame = table
let result = table
.query()
.nearest_to(&query_vector)
.unwrap()
@@ -274,7 +291,7 @@ async fn main() {
// --8<-- [end:quickstart_add_feature]
// --8<-- [start:quickstart_query_feature]
let features: DataFrame = table
let features = table
.query()
.select(Select::Columns(vec![
"name".to_string(),
@@ -289,19 +306,23 @@ async fn main() {
.unwrap();
println!("{features:?}");
// --8<-- [end:quickstart_query_feature]
assert!(features.column("power_score").is_ok());
assert_eq!(features.column("power_score").unwrap().len(), 4);
// What the quickstart's download step leaves in an application's directory.
let app = tempfile::tempdir().unwrap();
std::fs::create_dir(app.path().join("data")).unwrap();
std::fs::copy(
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../web/static/assets/images/quickstart/sir-lancelot.jpg"),
app.path().join("data/sir-lancelot.jpg"),
)
.unwrap();
std::env::set_current_dir(app.path()).unwrap();
// --8<-- [start:quickstart_multimodal_bytes]
use std::sync::Arc;
use lancedb::arrow::arrow_array::BinaryArray;
use arrow_array::{
BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,
};
use arrow_schema::{DataType, Field, Schema};
let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../web/static/assets/images/quickstart/sir-lancelot.jpg");
let image_bytes = std::fs::read(image_path).unwrap();
let image_bytes = std::fs::read("data/sir-lancelot.jpg").unwrap();
let image_schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::LargeUtf8, false),
@@ -333,9 +354,10 @@ async fn main() {
],
)
.unwrap();
let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),
);
let image_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
vec![Ok(image_batch)].into_iter(),
image_schema,
));
let multimodal_table = db
.create_table("character_images", image_reader)
.mode(CreateTableMode::Overwrite)
docs/web-tests/rs/search_filter_search.rs
+92
View File
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example_body]
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("text", DataType::Utf8, false),
Field::new("category", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
true,
),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec![
"A puppy playing in the park",
"A dog resting at home",
"A guide to growing tomatoes",
])),
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
.into_iter()
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
2,
),
),
],
)?;
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
let table = db
.create_table(
"documents",
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
)
.execute()
.await?;
let query = table
.vector_search(&[0.0_f32, 1.0])?
.only_if("category = 'animals'")
.select(Select::columns(&["id", "text", "category", "_distance"]))
.limit(2);
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
println!("{rows:?}");
// --8<-- [end:example_body]
let ids: Vec<i64> = rows
.iter()
.flat_map(|b| {
b.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert_eq!(
ids.into_iter().collect::<std::collections::BTreeSet<_>>(),
std::collections::BTreeSet::from([1, 2])
);
assert!(rows.iter().all(|b| {
b.column_by_name("category")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap()
.iter()
.all(|v| v == Some("animals"))
}));
// --8<-- [start:example_tail]
Ok(())
}
// --8<-- [end:example_tail]
docs/web-tests/rs/search_hybrid_search.rs
+99
View File
@@ -0,0 +1,99 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example_body]
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lance_index::scalar::FullTextSearchQuery;
use lancedb::index::{Index, scalar::FtsIndexBuilder};
use lancedb::query::{ExecutableQuery, QueryBase};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("text", DataType::Utf8, false),
Field::new("category", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
true,
),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec![
"A puppy playing in the park",
"A dog resting at home",
"A guide to growing tomatoes",
])),
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
.into_iter()
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
2,
),
),
],
)?;
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
let table = db
.create_table(
"documents",
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
)
.execute()
.await?;
table
.create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
.execute()
.await?;
let query = table
.vector_search(&[1.0_f32, 0.0])?
.full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?)
.rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default()))
.limit(2);
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
println!("{rows:?}");
// --8<-- [end:example_body]
let ids: Vec<i64> = rows
.iter()
.flat_map(|b| {
b.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert_eq!(ids, vec![1, 2]);
let scores: Vec<f32> = rows
.iter()
.flat_map(|b| {
b.column_by_name("_relevance_score")
.unwrap()
.as_any()
.downcast_ref::<arrow_array::Float32Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert!(scores[0] > scores[1]);
// --8<-- [start:example_tail]
Ok(())
}
// --8<-- [end:example_tail]
docs/web-tests/rs/search_query_plan.rs
+81
View File
@@ -0,0 +1,81 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example_body]
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("text", DataType::Utf8, false),
Field::new("category", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
true,
),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec![
"A puppy playing in the park",
"A dog resting at home",
"A guide to growing tomatoes",
])),
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
.into_iter()
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
2,
),
),
],
)?;
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
let table = db
.create_table(
"documents",
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
)
.execute()
.await?;
let query = table
.vector_search(&[1.0_f32, 0.0])?
.select(Select::columns(&["id", "text", "_distance"]))
.limit(2);
println!("Plan: {}", query.explain_plan(false).await?);
println!("Measured execution: {}", query.analyze_plan().await?);
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
println!("{rows:?}");
// --8<-- [end:example_body]
let ids: Vec<i64> = rows
.iter()
.flat_map(|b| {
b.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert_eq!(ids, vec![1, 2]);
// --8<-- [start:example_tail]
Ok(())
}
// --8<-- [end:example_tail]
docs/web-tests/rs/search_text_search.rs
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example_body]
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lance_index::scalar::FullTextSearchQuery;
use lancedb::index::{Index, scalar::FtsIndexBuilder};
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("text", DataType::Utf8, false),
Field::new("category", DataType::Utf8, false),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec![
"A puppy playing in the park",
"A dog resting at home",
"A guide to growing tomatoes",
])),
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
],
)?;
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
let table = db
.create_table(
"documents",
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
)
.execute()
.await?;
table
.create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
.execute()
.await?;
let query = table
.query()
.full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?)
.select(Select::columns(&["id", "text", "_score"]))
.limit(2);
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
println!("{rows:?}");
// --8<-- [end:example_body]
let ids: Vec<i64> = rows
.iter()
.flat_map(|b| {
b.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert_eq!(ids, vec![1]);
let scores: Vec<f32> = rows
.iter()
.flat_map(|b| {
b.column_by_name("_score")
.unwrap()
.as_any()
.downcast_ref::<arrow_array::Float32Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert!(scores[0] > 0.0);
// --8<-- [start:example_tail]
Ok(())
}
// --8<-- [end:example_tail]
docs/web-tests/rs/search_vector_search.rs
+93
View File
@@ -0,0 +1,93 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// --8<-- [start:example_body]
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lancedb::query::{ExecutableQuery, QueryBase, Select};
use std::sync::Arc;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("text", DataType::Utf8, false),
Field::new("category", DataType::Utf8, false),
Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
true,
),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec![
"A puppy playing in the park",
"A dog resting at home",
"A guide to growing tomatoes",
])),
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
Arc::new(
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
.into_iter()
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
2,
),
),
],
)?;
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
let table = db
.create_table(
"documents",
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
)
.execute()
.await?;
let query = table
.vector_search(&[1.0_f32, 0.0])?
.distance_type(lancedb::DistanceType::L2)
.select(Select::columns(&["id", "text", "_distance"]))
.limit(2);
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
println!("{rows:?}");
// --8<-- [end:example_body]
let ids: Vec<i64> = rows
.iter()
.flat_map(|b| {
b.column_by_name("id")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert_eq!(ids, vec![1, 2]);
let scores: Vec<f32> = rows
.iter()
.flat_map(|b| {
b.column_by_name("_distance")
.unwrap()
.as_any()
.downcast_ref::<arrow_array::Float32Array>()
.unwrap()
.values()
.to_vec()
})
.collect();
assert_eq!(scores[0], 0.0);
// --8<-- [start:example_tail]
Ok(())
}
// --8<-- [end:example_tail]
docs/web-tests/ts/connection.test.ts
+82 -36
View File
@@ -1,65 +1,89 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, jest, test } from "@jest/globals";
import * as path from "node:path";
import { withTempDirectory } from "./util.ts";
// --8<-- [start:connect]
import { afterEach, describe, expect, jest, test } from "@jest/globals";
import * as lancedb from "@lancedb/lancedb";
async function connectExample(uri: string) {
const db = await lancedb.connect(uri);
return db;
}
// --8<-- [end:connect]
import { withTempDirectory } from "./util.ts";
test("connect to a local database", async () => {
await withTempDirectory(async (tempDir) => {
const uri = path.join(tempDir, "ex_lancedb");
const db = await connectExample(uri);
expect(db).toBeDefined();
const cwd = process.cwd();
process.chdir(tempDir);
try {
// --8<-- [start:connect]
const db = await lancedb.connect("ex_lancedb");
// --8<-- [end:connect]
await db.createTable("probe", [{ id: 1 }]);
expect((await db.listTables()).tables).toEqual(["probe"]);
} finally {
process.chdir(cwd);
}
const reopened = await lancedb.connect(`${tempDir}/ex_lancedb`);
expect((await reopened.listTables()).tables).toEqual(["probe"]);
});
});
async function connectEnterpriseQuickstart() {
// --8<-- [start:connect_enterprise_quickstart]
const uri = "db://your-database-uri";
const apiKey = "your-api-key";
const region = "us-east-1";
const hostOverride = "https://your-enterprise-endpoint.com";
const setting = (name: string) => {
const value = process.env[name];
if (!value) throw new Error(`Set ${name}`);
return value;
};
const db = await lancedb.connect(uri, {
apiKey,
region,
hostOverride,
const db = await lancedb.connect(setting("LANCEDB_URI"), {
apiKey: setting("LANCEDB_API_KEY"),
region: process.env.LANCEDB_REGION ?? "us-east-1",
hostOverride: setting("LANCEDB_HOST_OVERRIDE"),
});
// --8<-- [end:connect_enterprise_quickstart]
return db;
}
test("enterprise quickstart connect uses placeholder config", async () => {
const mockDb = { __mock: true } as unknown as Awaited<
ReturnType<typeof lancedb.connect>
>;
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
const db = await connectEnterpriseQuickstart();
expect(db).toBe(mockDb);
expect(spy).toHaveBeenCalledWith("db://your-database-uri", {
apiKey: "your-api-key",
region: "us-east-1",
hostOverride: "https://your-enterprise-endpoint.com",
describe("enterprise quickstart connect", () => {
const saved = { ...process.env };
afterEach(() => {
process.env = { ...saved };
});
test("reads the connection from the environment", async () => {
const mockDb = { __mock: true } as unknown as Awaited<
ReturnType<typeof lancedb.connect>
>;
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
process.env.LANCEDB_URI = "db://your-database";
process.env.LANCEDB_API_KEY = "your-api-key";
delete process.env.LANCEDB_REGION;
process.env.LANCEDB_HOST_OVERRIDE = "https://your-endpoint";
const db = await connectEnterpriseQuickstart();
expect(db).toBe(mockDb);
expect(spy).toHaveBeenCalledWith("db://your-database", {
apiKey: "your-api-key",
region: "us-east-1",
hostOverride: "https://your-endpoint",
});
spy.mockRestore();
});
test("requires an endpoint", async () => {
process.env.LANCEDB_URI = "db://your-database";
process.env.LANCEDB_API_KEY = "your-api-key";
delete process.env.LANCEDB_HOST_OVERRIDE;
await expect(connectEnterpriseQuickstart()).rejects.toThrow(
"Set LANCEDB_HOST_OVERRIDE",
);
});
spy.mockRestore();
});
// --8<-- [start:connect_object_storage]
async function connectObjectStorageExample() {
// --8<-- [start:connect_object_storage]
const uri = "s3://your-bucket/path";
// You can also use "gs://your-bucket/path" or "az://your-container/path".
const db = await lancedb.connect(uri);
// --8<-- [end:connect_object_storage]
return db;
}
// --8<-- [end:connect_object_storage]
async function namespaceTableOpsExample() {
// --8<-- [start:namespace_table_ops]
@@ -119,3 +143,25 @@ void [
namespaceTableOpsExample,
namespaceAdminOpsExample,
];
test("enterprise placeholders use an explicit endpoint", async () => {
const mockDb = {} as Awaited<ReturnType<typeof lancedb.connect>>;
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
try {
// --8<-- [start:connect_enterprise_placeholders]
const db = await lancedb.connect("db://your-database", {
apiKey: "your-api-key",
region: "us-east-1",
hostOverride: "https://your-endpoint",
});
// --8<-- [end:connect_enterprise_placeholders]
expect(db).toBe(mockDb);
expect(spy).toHaveBeenCalledWith("db://your-database", {
apiKey: "your-api-key",
region: "us-east-1",
hostOverride: "https://your-endpoint",
});
} finally {
spy.mockRestore();
}
});
docs/web-tests/ts/ingestion_examples.test.ts
+117
View File
@@ -0,0 +1,117 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
test("complete ingestion workflow", async () => {
// --8<-- [start:write_workflow_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("users", [
{ id: 1, name: "Alice", login_count: 10 },
{ id: 2, name: "Bob", login_count: 20 },
]);
await table.add([{ id: 3, name: "Charlie", login_count: 5 }]);
await table.update({ where: "id = 2", values: { name: "Bobby" } });
await table.update({
where: "id = 2",
valuesSql: { login_count: "login_count + 1" },
});
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute([
{ id: 3, name: "Charlie", login_count: 6 },
{ id: 4, name: "Dana", login_count: 1 },
]);
console.log(
"Rows selected for deletion:",
await table.query().where("id = 4").toArray(),
);
await table.delete("id = 4");
const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);
console.log(rows);
// --8<-- [end:write_workflow_body]
expect(rows.map((row) => [row.id, row.name, row.login_count])).toEqual([
[1, "Alice", 10],
[2, "Bobby", 21],
[3, "Charlie", 6],
]);
// --8<-- [start:write_workflow_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:write_workflow_tail]
});
test("complete delete_rows", async () => {
// --8<-- [start:delete_rows_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("users", [
{ id: 1, name: "Alice" },
{ id: 2, name: "Bob" },
{ id: 3, name: "Charlie" },
]);
const predicate = "id = 3";
console.log("Selected:", await table.query().where(predicate).toArray());
await table.delete(predicate);
const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);
console.log("Remaining:", rows);
// --8<-- [end:delete_rows_body]
expect(rows.map((row) => [row.id, row.name])).toEqual([
[1, "Alice"],
[2, "Bob"],
]);
// --8<-- [start:delete_rows_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:delete_rows_tail]
});
test("complete add_progress", async () => {
// --8<-- [start:add_progress_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("users", [{ id: 1, name: "Alice" }]);
await table.add(
[
{ id: 2, name: "Bob" },
{ id: 3, name: "Charlie" },
],
{
progress: (progress) =>
console.log(`Rows written: ${progress.outputRows}`),
},
);
const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);
console.log("After insertion:", rows);
// --8<-- [end:add_progress_body]
expect(rows.map((row) => [row.id, row.name])).toEqual([
[1, "Alice"],
[2, "Bob"],
[3, "Charlie"],
]);
// --8<-- [start:add_progress_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:add_progress_tail]
});
docs/web-tests/ts/quickstart.test.ts
+241 -154
View File
@@ -1,176 +1,263 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
import { describe, expect, test } from "@jest/globals";
import { randomUUID } from "node:crypto";
import * as fs from "node:fs";
import * as path from "node:path";
import * as lancedb from "@lancedb/lancedb";
import { withTempDirectory } from "./util.ts";
test("quickstart example (async)", async () => {
// What the quickstart's download step leaves in an application's directory.
const IMAGE = path.join(
__dirname,
"../../web/static/assets/images/quickstart/sir-lancelot.jpg",
);
const deployment = ["LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY"];
// The connection settings alone never send these tests to a deployment.
const optIn = "LANCEDB_DOCS_TEST_DEPLOYMENT";
const connections =
process.env[optIn] === "1" ? ["local", "enterprise"] : ["local"];
async function tableNames(db: lancedb.Connection) {
const names: string[] = [];
let pageToken: string | undefined;
do {
const page = await db.listTables({ pageToken });
names.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
return names;
}
// A deployment connection on which a test creates only tables of its own:
// each name gets a suffix unique to the test and is claimed only while no
// table has it; the test drops what it claimed, whether or not it passed.
async function withOwnedTables(
db: lancedb.Connection,
fn: (db: lancedb.Connection) => Promise<void>,
) {
const suffix = randomUUID().replaceAll("-", "").slice(0, 12);
const claimed: string[] = [];
const createTable = async (
name: string,
data: Parameters<lancedb.Connection["createTable"]>[1],
options: Partial<lancedb.CreateTableOptions> = {},
) => {
const owned = `${name}_${suffix}`;
let mode = options.mode;
if (!claimed.includes(owned)) {
if ((await tableNames(db)).includes(owned)) {
throw new Error(`${owned} already exists and is not this test's`);
}
claimed.push(owned);
mode = "create";
}
return db.createTable(owned, data, { ...options, mode });
};
try {
await fn(
new Proxy(db, {
get: (target, property, receiver) =>
property === "createTable"
? createTable
: Reflect.get(target, property, receiver),
}),
);
} finally {
const existing = await tableNames(db);
for (const name of claimed) {
if (existing.includes(name)) {
await db.dropTable(name);
}
}
}
}
async function withDatabase(
connection: string,
fn: (db: lancedb.Connection) => Promise<void>,
) {
if (connection === "enterprise") {
const missing = deployment.filter((name) => !process.env[name]);
if (missing.length) {
throw new Error(`${optIn}=1 needs ${missing.join(", ")}`);
}
const db = await lancedb.connect(process.env.LANCEDB_URI!, {
apiKey: process.env.LANCEDB_API_KEY!,
region: process.env.LANCEDB_REGION ?? "us-east-1",
hostOverride: process.env.LANCEDB_HOST_OVERRIDE!,
});
await withOwnedTables(db, fn);
return;
}
await withTempDirectory(async (databaseDir) => {
const db = await lancedb.connect(databaseDir);
await fn(await lancedb.connect(databaseDir));
});
}
// --8<-- [start:quickstart_data]
const data = [
{
id: "1",
name: "King Arthur",
role: "King",
description: "Leader of Camelot and wielder of Excalibur.",
stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
vector: [0.7, 0.1, 0.9, 0.7],
},
{
id: "2",
name: "Merlin",
role: "Wizard",
description: "Advisor and prophet with deep magical knowledge.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
vector: [0.2, 0.9, 0.4, 0.9],
},
{
id: "3",
name: "Sir Lancelot",
role: "Knight",
description: "Legendary knight known for courage and combat skill.",
stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
vector: [0.9, 0.1, 0.5, 0.4],
},
];
// --8<-- [end:quickstart_data]
async function createCharacters(db: lancedb.Connection) {
// --8<-- [start:quickstart_data]
const data = [
{
id: "1",
name: "King Arthur",
role: "King",
description: "Leader of Camelot and wielder of Excalibur.",
stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
vector: [0.7, 0.1, 0.9, 0.7],
},
{
id: "2",
name: "Merlin",
role: "Wizard",
description: "Advisor and prophet with deep magical knowledge.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
vector: [0.2, 0.9, 0.4, 0.9],
},
{
id: "3",
name: "Sir Lancelot",
role: "Knight",
description: "Legendary knight known for courage and combat skill.",
stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
vector: [0.9, 0.1, 0.5, 0.4],
},
];
// --8<-- [end:quickstart_data]
// --8<-- [start:quickstart_create_table]
let table = await db.createTable("characters", data, { mode: "overwrite" });
// --8<-- [end:quickstart_create_table]
expect(await table.countRows()).toBe(3);
await db.dropTable("characters");
// --8<-- [start:quickstart_create_table]
const table = await db.createTable("characters", data, { mode: "overwrite" });
// --8<-- [end:quickstart_create_table]
expect(await table.countRows()).toBe(3);
// --8<-- [start:quickstart_create_table_no_overwrite]
table = await db.createTable("characters", data);
// --8<-- [end:quickstart_create_table_no_overwrite]
expect(await table.countRows()).toBe(3);
// --8<-- [start:quickstart_add_data]
await table.add([
{
id: "4",
name: "Morgana",
role: "Sorceress",
description: "Powerful sorceress of Avalon.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
vector: [0.3, 0.9, 0.6, 0.8],
},
]);
// --8<-- [end:quickstart_add_data]
expect(await table.countRows()).toBe(4);
return table;
}
// --8<-- [start:quickstart_vector_search_1]
// Search for examples similar to a "wise magical advisor"
let queryVector = [0.2, 0.8, 0.4, 0.9];
describe.each(connections)("quickstart (%s)", (connection) => {
test("create, add and search", async () => {
await withDatabase(connection, async (db) => {
const table = await createCharacters(db);
let result = await table
.search(queryVector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(result);
// --8<-- [end:quickstart_vector_search_1]
expect(result[0].name).toBe("Merlin");
// --8<-- [start:quickstart_vector_search_1]
// Search for examples similar to a "wise magical advisor"
const queryVector = [0.2, 0.8, 0.4, 0.9];
// --8<-- [start:quickstart_curate_with_metadata]
const curated = await table
.search(queryVector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(curated);
// --8<-- [end:quickstart_curate_with_metadata]
expect(curated[0].name).toBe("Merlin");
const result = await table
.search(queryVector)
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(result);
// --8<-- [end:quickstart_vector_search_1]
expect(result.map((row) => row.name)).toEqual(["Merlin", "Morgana"]);
// --8<-- [start:quickstart_output_array]
result = await table.search(queryVector).limit(2).toArray();
console.table(result);
// --8<-- [end:quickstart_output_array]
expect(result[0].name).toBe("Merlin");
// --8<-- [start:quickstart_curate_with_metadata]
const curated = await table
.search(queryVector)
.where("stats.magic >= 4")
.select(["name", "role", "description", "_distance"])
.limit(2)
.toArray();
console.table(curated);
// --8<-- [end:quickstart_curate_with_metadata]
expect(curated.map((row) => row.name)).toEqual(["Merlin", "Morgana"]);
});
});
// --8<-- [start:quickstart_add_feature]
await table.addColumns([
{
name: "power_score",
valueSql:
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
},
]);
// --8<-- [end:quickstart_add_feature]
const schemaWithFeature = await table.schema();
expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe(
true,
);
test("a derived feature", async () => {
await withDatabase(connection, async (db) => {
const table = await createCharacters(db);
// --8<-- [start:quickstart_query_feature]
const features = await table
.query()
.select(["name", "role", "power_score"])
.toArray();
console.table(features);
// --8<-- [end:quickstart_query_feature]
expect(features[0]).toHaveProperty("power_score");
// --8<-- [start:quickstart_multimodal_bytes]
const arrow = await import("apache-arrow");
const path = await import("node:path");
const { readFile } = await import("node:fs/promises");
const imagePath = path.join(
__dirname,
"../../web/static/assets/images/quickstart/sir-lancelot.jpg",
);
const imageBytes = await readFile(imagePath);
const imageSchema = new arrow.Schema([
new arrow.Field("id", new arrow.Utf8()),
new arrow.Field("description", new arrow.Utf8()),
new arrow.Field("image", new arrow.Binary()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
4,
new arrow.Field("item", new arrow.Float32(), true),
),
),
]);
const imageData = lancedb.makeArrowTable(
[
// --8<-- [start:quickstart_add_feature]
await table.addColumns([
{
id: "lancelot",
description: "Portrait of Sir Lancelot",
image: imageBytes,
vector: [0.9, 0.1, 0.5, 0.4],
name: "power_score",
valueSql:
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
},
],
{ schema: imageSchema },
);
const multimodalTable = await db.createTable(
"character_images",
imageData,
{ mode: "overwrite" },
);
// --8<-- [end:quickstart_multimodal_bytes]
expect(await multimodalTable.countRows()).toBe(1);
]);
// --8<-- [end:quickstart_add_feature]
// --8<-- [start:quickstart_open_table]
table = await db.openTable("characters");
// --8<-- [end:quickstart_open_table]
// --8<-- [start:quickstart_query_feature]
const features = await table
.query()
.select(["name", "role", "power_score"])
.toArray();
console.table(features);
// --8<-- [end:quickstart_query_feature]
expect(
features.map((row) => [row.name, row.power_score]).sort(),
).toEqual([
["King Arthur", 3.5],
["Merlin", 4],
["Morgana", 3.75],
["Sir Lancelot", 3],
]);
});
});
// --8<-- [start:quickstart_add_data]
const moreData = [
{
id: "4",
name: "Morgana",
role: "Sorceress",
description: "Powerful sorceress of Avalon.",
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
vector: [0.3, 0.9, 0.6, 0.8],
power_score: 3.75,
},
];
test("multimodal data", async () => {
await withDatabase(connection, async (db) => {
await withTempDirectory(async (app) => {
fs.mkdirSync(path.join(app, "data"));
fs.copyFileSync(IMAGE, path.join(app, "data/sir-lancelot.jpg"));
const cwd = process.cwd();
process.chdir(app);
try {
// --8<-- [start:quickstart_multimodal_bytes]
const arrow = await import("apache-arrow");
const { readFile } = await import("node:fs/promises");
// Add data to table
await table.add(moreData);
// --8<-- [end:quickstart_add_data]
expect(await table.countRows()).toBe(4);
// --8<-- [start:quickstart_vector_search_2]
// Search for examples similar to a "powerful sorceress"
queryVector = [0.3, 0.9, 0.6, 0.8];
const results = await table.search(queryVector).limit(2).toArray();
console.table(results);
// --8<-- [end:quickstart_vector_search_2]
expect(results[0].name).toBe("Morgana");
const imageBytes = await readFile("data/sir-lancelot.jpg");
const imageSchema = new arrow.Schema([
new arrow.Field("id", new arrow.Utf8()),
new arrow.Field("description", new arrow.Utf8()),
new arrow.Field("image", new arrow.Binary()),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
4,
new arrow.Field("item", new arrow.Float32(), true),
),
),
]);
const imageData = lancedb.makeArrowTable(
[
{
id: "lancelot",
description: "Portrait of Sir Lancelot",
image: imageBytes,
vector: [0.9, 0.1, 0.5, 0.4],
},
],
{ schema: imageSchema },
);
const multimodalTable = await db.createTable(
"character_images",
imageData,
{ mode: "overwrite" },
);
// --8<-- [end:quickstart_multimodal_bytes]
expect(await multimodalTable.countRows()).toBe(1);
const [row] = await multimodalTable.query().select(["image"]).toArray();
expect(Buffer.from(row.image)).toEqual(fs.readFileSync(IMAGE));
} finally {
process.chdir(cwd);
}
});
});
});
});
docs/web-tests/ts/search_examples.test.ts
+222
View File
@@ -0,0 +1,222 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { expect, test } from "@jest/globals";
test("complete vector_search", async () => {
// --8<-- [start:vector_search_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("documents", [
{
id: 1,
text: "A puppy playing in the park",
category: "animals",
vector: [1.0, 0.0],
},
{
id: 2,
text: "A dog resting at home",
category: "animals",
vector: [0.9, 0.1],
},
{
id: 3,
text: "A guide to growing tomatoes",
category: "gardening",
vector: [0.0, 1.0],
},
]);
const rows = await table
.vectorSearch([1.0, 0.0])
.distanceType("l2")
.select(["id", "text", "_distance"])
.limit(2)
.toArray();
console.log(rows);
// --8<-- [end:vector_search_body]
expect(rows.map((row) => row.id)).toEqual([1, 2]);
expect(rows[0]._distance).toBe(0);
// --8<-- [start:vector_search_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:vector_search_tail]
});
test("complete text_search", async () => {
// --8<-- [start:text_search_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("documents", [
{ id: 1, text: "A puppy playing in the park", category: "animals" },
{ id: 2, text: "A dog resting at home", category: "animals" },
{ id: 3, text: "A guide to growing tomatoes", category: "gardening" },
]);
await table.createIndex("text", { config: lancedb.Index.fts() });
const rows = await table
.query()
.fullTextSearch("puppy", { columns: ["text"] })
.select(["id", "text", "_score"])
.limit(2)
.toArray();
console.log(rows);
// --8<-- [end:text_search_body]
expect(rows.map((row) => row.id)).toEqual([1]);
expect(rows[0]._score).toBeGreaterThan(0);
// --8<-- [start:text_search_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:text_search_tail]
});
test("complete hybrid_search", async () => {
// --8<-- [start:hybrid_search_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("documents", [
{
id: 1,
text: "A puppy playing in the park",
category: "animals",
vector: [1.0, 0.0],
},
{
id: 2,
text: "A dog resting at home",
category: "animals",
vector: [0.9, 0.1],
},
{
id: 3,
text: "A guide to growing tomatoes",
category: "gardening",
vector: [0.0, 1.0],
},
]);
await table.createIndex("text", { config: lancedb.Index.fts() });
const reranker = await lancedb.rerankers.RRFReranker.create();
const rows = await table
.vectorSearch([1.0, 0.0])
.fullTextSearch("puppy", { columns: ["text"] })
.rerank(reranker)
.limit(2)
.toArray();
console.log(rows);
// --8<-- [end:hybrid_search_body]
expect(rows.map((row) => row.id)).toEqual([1, 2]);
expect(rows[0]._relevance_score).toBeGreaterThan(rows[1]._relevance_score);
// --8<-- [start:hybrid_search_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:hybrid_search_tail]
});
test("complete filter_search", async () => {
// --8<-- [start:filter_search_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("documents", [
{
id: 1,
text: "A puppy playing in the park",
category: "animals",
vector: [1.0, 0.0],
},
{
id: 2,
text: "A dog resting at home",
category: "animals",
vector: [0.9, 0.1],
},
{
id: 3,
text: "A guide to growing tomatoes",
category: "gardening",
vector: [0.0, 1.0],
},
]);
const rows = await table
.vectorSearch([0.0, 1.0])
.where("category = 'animals'")
.select(["id", "text", "category", "_distance"])
.limit(2)
.toArray();
console.log(rows);
// --8<-- [end:filter_search_body]
expect(rows.map((row) => row.id).sort()).toEqual([1, 2]);
expect(rows.every((row) => row.category === "animals")).toBe(true);
// --8<-- [start:filter_search_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:filter_search_tail]
});
test("complete query_plan", async () => {
// --8<-- [start:query_plan_body]
const lancedb = await import("@lancedb/lancedb");
const { mkdtemp, rm } = await import("node:fs/promises");
const { tmpdir } = await import("node:os");
const { join } = await import("node:path");
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
try {
const db = await lancedb.connect(uri);
const table = await db.createTable("documents", [
{
id: 1,
text: "A puppy playing in the park",
category: "animals",
vector: [1.0, 0.0],
},
{
id: 2,
text: "A dog resting at home",
category: "animals",
vector: [0.9, 0.1],
},
{
id: 3,
text: "A guide to growing tomatoes",
category: "gardening",
vector: [0.0, 1.0],
},
]);
const query = table
.vectorSearch([1.0, 0.0])
.select(["id", "text", "_distance"])
.limit(2);
console.log("Plan:", await query.explainPlan());
console.log("Measured execution:", await query.analyzePlan());
const rows = await query.toArray();
console.log(rows);
// --8<-- [end:query_plan_body]
expect(rows.map((row) => row.id)).toEqual([1, 2]);
// --8<-- [start:query_plan_tail]
} finally {
await rm(uri, { recursive: true, force: true });
}
// --8<-- [end:query_plan_tail]
});
docs/web/.mintignore
+6
View File
@@ -0,0 +1,6 @@
# Functions are documented before their release. At release, remove these
# entries, add the pages back to the navigation in docs.json, restore the
# /feature-engineering/functions redirect to /management/function, and publish
# the drafts marked "Draft for the Functions release".
management/function.mdx
sql/ddl/function.mdx
docs/web/api-reference/rest/index.mdx
+19 -9
View File
@@ -35,10 +35,13 @@ To authenticate to the Enterprise REST API, you need the endpoint for your deplo
```bash
export LANCEDB_API_KEY="<your-api-key>"
export LANCEDB_URI="https://your-enterprise-endpoint.com"
export LANCEDB_HOST_OVERRIDE="https://your-enterprise-endpoint.com"
export LANCEDB_DATABASE="your-database-name"
```
`LANCEDB_HOST_OVERRIDE` is the endpoint the SDKs take as `host_override`; the
database name is the part of its `db://` URI after `db://`.
3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint.
### Verify authentication {#verify-authentication}
@@ -46,33 +49,40 @@ export LANCEDB_DATABASE="your-database-name"
4. Check that you can reach the deployment and list tables:
```bash
curl -X GET "$LANCEDB_URI/v1/management/table" \
-H "Content-Type: application/json" \
curl -X GET "$LANCEDB_HOST_OVERRIDE/v1/table/" \
-H "x-api-key: $LANCEDB_API_KEY" \
-H "x-lancedb-database: $LANCEDB_DATABASE"
```
The response lists one page of table names, with a token for the next page when
there are more.
If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header.
5. Create a table to confirm write access. Let's call it `words`.
5. Create a table to confirm write access. Let's call it `words`. A table is
created from an Arrow IPC stream, which this writes with PyArrow:
```bash
curl -X POST "$LANCEDB_URI/v1/tables/words" \
python -c "import pyarrow as pa, sys; t = pa.table({'word': ['hello', 'world']}); w = pa.ipc.new_stream(sys.stdout.buffer, t.schema); w.write_table(t); w.close()" > words.arrow
curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/create" \
-H "Content-Type: application/vnd.apache.arrow.stream" \
-H "x-api-key: $LANCEDB_API_KEY" \
-H "x-lancedb-database: $LANCEDB_DATABASE"
-H "x-lancedb-database: $LANCEDB_DATABASE" \
--data-binary @words.arrow
```
6. Check that the table has been created:
```bash
curl -X GET "$LANCEDB_URI/v1/management/table" \
curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \
-H "Content-Type: application/json" \
-H "x-api-key: $LANCEDB_API_KEY" \
-H "x-lancedb-database: $LANCEDB_DATABASE"
-H "x-lancedb-database: $LANCEDB_DATABASE" \
-d '{}'
```
That's it -- you're connected! Now, you can start adding data and querying it.
That's it. You're connected! Now, you can start adding data and querying it.
You can visit the tutorial section to build your own applications with LanceDB.
<Card
docs/web/architecture.mdx
+85 -114
View File
@@ -1,153 +1,124 @@
---
title: "Architecture"
sidebarTitle: "Architecture"
description: "How LanceDB is put together, and what changes between running it embedded and running it distributed."
description: "How LanceDB Enterprise separates query serving, background work, and storage, and how it compares with embedded LanceDB OSS."
icon: cubes
keywords: ["architecture", "embedded", "distributed", "object storage", "compute", "deployment"]
---
LanceDB has one storage layer and two ways to put compute on top of it. Which
one you run decides where queries execute, what runs on your behalf, and what
you have to operate — and almost nothing else. The data is the same
[Lance](/basics/lance) tables either way.
LanceDB OSS is an embedded library for working with Lance tables. LanceDB
Enterprise is a multimodal lakehouse for data curation, feature engineering,
search, and training data access. Its architecture separates query serving,
read execution, caching, and background work. Both use the open
[Lance](/basics/lance) format.
## One storage layer {#storage}
Teams can curate datasets, derive features, search multimodal records, and
read the resulting tables into training frameworks. Enterprise separates the
resources that serve shared data from those that build indexes and run
background work, so teams can size them for these different workloads.
Everything durable lives in object storage: table data, manifests, indexes. No
machine owns a dataset, and no server has to be running for the data to exist.
![LanceDB OSS runs the application and embedded library in one process, using local disk or object storage. Enterprise applications connect to query serving, with separate read execution and caching, indexing and background workers, and durable object storage. Both use the Lance format, but they do not share a storage location automatically.](/static/assets/images/overview/lancedb-architecture.svg)
```
┌──────────────────────────────────────────┐
│ object storage │
│ table data · manifests · indexes │
└──────────────────────────────────────────┘
```
The diagram shows the main compute and storage boundaries, not every request
path. The control plane governs the Enterprise service.
That is the whole persistence story, and it is why the two deployment shapes can
differ so much above it while staying compatible below it. A table written
embedded is opened by a deployment, and the reverse, without conversion.
## A shared format, different storage choices {#storage}
## Embedded {#embedded}
Lance stores table data, versions, and indexes. OSS can read and write tables on
local disk or in object storage. Enterprise keeps durable table data and index
artifacts in object storage, independently of the machines serving requests.
Its caches accelerate reads; they are not the durable source of truth.
```
your process
┌────────────────────────────────┐
│ application │
│ LanceDB library │ ← plans, executes, indexes, compacts
└────────────────────────────────┘
│
┌─────────▼─────────┐
│ object storage │
└───────────────────┘
```
Sharing a format does not mean that an Enterprise deployment automatically
opens an existing OSS database. Plan how to move or connect your data and
configure access for your deployment.
`lancedb.connect("./data")` or `connect("s3://bucket/path")` gives you a
**library**, not a client. Your process opens the tables, plans the queries,
runs the scans, builds the indexes, and writes the new versions. There is no
server in the picture at all.
## OSS: an embedded library inside your application {#embedded}
**What follows from that.** Nothing runs unless you call it — no background
compaction, no server-side [jobs](/basics/jobs), no
[SQL endpoint](/basics/sql). Scale is your process's scale: a query gets the
cores and memory your process has. Access control is whatever your object store
enforces, since there is no other gatekeeper.
With an embedded connection, the LanceDB library plans and executes queries,
builds indexes, and updates tables inside your process. There is no separate
database server to deploy.
**What it is good at.** Development, notebooks, single-node pipelines, embedding
LanceDB inside an application, and anything where "no service to operate" is
worth more than elasticity. Comfortably handles millions of vectors on one
node -- disk-based indexes keep it performant without extra infrastructure.
This fits local retrieval applications, notebooks, development, and
single-node data pipelines. Your application controls the compute resources and
when to run [table maintenance](/maintenance/compaction). You also manage access
to the local files or object storage that hold your tables.
## Distributed {#distributed}
## Enterprise: a multimodal lakehouse for shared workloads {#distributed}
<Badge color="red">Enterprise</Badge>
Applications connect to remote tables through a LanceDB SDK. The service handles
requests across a fleet, with components sized for different kinds of work:
```
your process
┌──────────────┐
│ application │
│ LanceDB SDK │ ← a thin client
└──────┬───────┘
│ db:// (REST / Flight SQL)
┌──────▼──────────────────────────────────────┐
│ control plane config · identity · policy │
├─────────────────────────────────────────────┤
│ data plane query serving · writes │
│ indexers indexing · compaction │
└──────┬──────────────────────────────────────┘
│
┌──────▼────────────┐
│ object storage │
└───────────────────┘
```
- **Query serving** receives requests, resolves tables, and plans operations.
Add serving capacity as more applications and users depend on your data.
- **Read execution and caching** use separately provisioned resources. Executors
cache table data to reduce repeated reads from object storage.
- **Indexing and background workers** build indexes and perform maintenance
outside the query request path. Size this work separately from query serving
as your data grows.
`connect("db://...")` gives you a **client**. Planning and execution move to a
cluster, and the work splits across tiers that scale independently: a control
plane governing configuration, identity and policy; a data plane serving reads
and writes; and indexers building indexes and compacting outside the request
path.
The control plane manages configuration, identity, policy, and cluster
lifecycle. It governs the service rather than executing table queries.
[Authorization](/governance/authorization) lets teams control access to catalog
objects when enabled and configured by the administrator. Deployment settings govern background
[compaction](/maintenance/compaction) and
[cleanup](/maintenance/cleanup), including retention.
**What follows from that.** Compute and storage are separated, so a query fleet
scales for interactive traffic without also scaling indexing capacity, and heavy
background work does not steal resources from user queries. Because a server
exists, so do the things that need one: [SQL](/sql),
[functions](/management/function), server-side jobs, and
[authorization](/governance/authorization) per object rather than per bucket.
These boundaries support different parts of an AI data workflow.
[SQL curation](/curation) requires a deployment with SQL enabled.
[Feature engineering](/feature-engineering) can produce derived columns and
tables; computed columns and materialized views require a deployment that
enables them. [Geneva](/geneva) runs distributed Python feature backfills while
its capabilities continue to transition to LanceDB APIs.
[Training data loaders](/training/data-loading) read tables into model training
frameworks in both offerings. LanceDB provides data access for training; the
training framework runs the model.
**What it is good at.** Many concurrent queries, continuous ingestion and
indexing, multi-team access with real permissions, and workloads too large for
one machine -- customers regularly operate on billions of rows and tens of
terabytes.
Size and monitor each tier for your workload. Read the
[Enterprise architecture](/enterprise/architecture) for the component details
and request flows.
## What changes, precisely {#differences}
## Application changes for Enterprise {#differences}
| | Embedded | Distributed |
| --- | --- | --- |
| What you connect to | a library in your process | a cluster |
| Where queries plan and execute | your process | the data plane |
| Table handle | `LanceTable` | `RemoteTable` |
| Indexing and compaction | when you call it | dedicated indexers |
| [Jobs](/basics/jobs) | run in-process, no server record | real jobs: id, status, events, cancel |
| [SQL](/sql) | unavailable | Flight SQL endpoint |
| [Functions](/management/function) | unavailable | run on workers |
| Access control | object-store permissions | [per-object grants](/governance/authorization) |
| Scale bound | your process | the cluster |
| What you operate | nothing | a deployment |
Core operations remain familiar: create a table, add data, and run a search.
With OSS, the application executes those operations. With Enterprise, the SDK
sends requests to the service, and the service manages execution and configured
background maintenance.
Both expose the same table API — create, open, mutate, query — so application
code moves between them by changing the URI. What does not move is code that
depends on a row in the right-hand column.
To use Enterprise, configure the cluster endpoint, database URI and credentials,
and make your data available in the Enterprise database. Review SDK methods
that behave differently on remote tables, including result retrieval and
background jobs. [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise)
describes these application changes. If you plan to use materialized views, check the
[stable row ID requirements](/basics/limits#schema) before creating source tables.
## Choosing, and changing your mind {#choosing}
## Choose for the workload you need to run {#choosing}
Start embedded. It has no operational cost and the API is the same one you will
keep using.
Choose OSS when you want an embedded library and can manage its resources and
maintenance in your application.
Move when you hit something in the right-hand column — usually concurrent
serving, ingestion that outgrows one machine, or a permissions requirement.
Because the storage layer is identical, moving is a connection-string change
plus a data migration, not a rewrite.
Choose Enterprise when multiple applications or teams need a shared multimodal
lakehouse for curation, feature engineering, search, and training data access.
It also fits workloads whose query and indexing demand grow at different rates,
or that need configured service-level access controls and background operations.
You can begin with Enterprise.
<Note>
The one thing to decide early is **stable row ids**: the SDK can give them to a
table only when it creates one on a local path or an object store, and a
deployment may require them on a
[materialized view](/management/materialized-view) source. See
[Limits](/basics/limits#schema).
</Note>
[Talk to us about your workload](https://www.lancedb.com/contact) to plan capacity,
deployment options, and the path from your existing data to Enterprise.
## Where to go next {#next}
<CardGroup cols={2}>
<Card title="Enterprise architecture" icon="cubes" href="/enterprise/architecture">
Query serving, execution, and background work in detail.
</Card>
<Card title="Connect to Enterprise" icon="plug" href="/basics/connect#enterprise">
Connect an application to your deployment.
</Card>
<Card title="Guarantees" icon="shield-check" href="/basics/guarantees">
What a write promises, and what a reader sees.
</Card>
<Card title="Lance format" icon="layer-group" href="/basics/lance">
The open format underneath both shapes.
</Card>
<Card title="Enterprise architecture" icon="cubes" href="/enterprise/architecture">
The distributed tiers in detail.
</Card>
<Card title="Limits" icon="ruler" href="/basics/limits">
The bounds you can actually hit.
The open format underneath both offerings.
</Card>
</CardGroup>
docs/web/basics/agentic-skills.mdx
+8 -7
View File
@@ -13,7 +13,7 @@ import {
} from '/snippets/build_with_ai_agents.mdx';
A coding agent writing LanceDB code from memory writes whatever was in its
training data -- often an older API, sometimes one that never existed. The
training data: often an older API, sometimes one that never existed. The
LanceDB agent plugin fixes that by supplying the current reference at the point
the agent needs it.
@@ -42,8 +42,8 @@ npx plugins add lancedb/lancedb-agent-plugins
The [`plugins`](https://www.npmjs.com/package/plugins) installer shown under
**Other agents** is a cross-tool option. It detects which agent CLIs are on your
`PATH` — Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well
as Claude Code and Codex — and installs through each one's native plugin system.
`PATH` (Cursor, GitHub Copilot CLI, VS Code, Grok Build, and Kimi Code, as well
as Claude Code and Codex) and installs through each one's native plugin system.
The plugin supplements the agent's training data with current LanceDB
instructions. To pick up later revisions, refresh the marketplace:
@@ -269,10 +269,11 @@ where appropriate.
Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes
shared production infrastructure, and the workload needs distributed capacity,
private deployment, or platform-managed operations. The underlying data format and table
API stay the same, so the pipeline does not need to be redesigned. The agent
instead connects to a remote `db://` table and lets the cluster handle
maintenance and background work.
private deployment, or platform-managed operations. The data format and the core
table API stay the same, so the pipeline's table and search code carries over:
the agent connects to a remote `db://` table, leaves maintenance to the
deployment instead of calling `optimize()`, and submits long-running work as
jobs. See [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise).
## What agents are good at here {#workflows}
docs/web/basics/catalog.mdx
+49 -90
View File
@@ -1,107 +1,66 @@
---
title: "Lance Catalog"
sidebarTitle: "Lance Catalog"
description: "The naming layer: what objects exist, how they nest, and who may reach them."
description: "Organize, find and control access to your data in LanceDB."
icon: sitemap
keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view", "function"]
keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view"]
---
A catalog answers one question: **what exists, and what is it called.** It holds
names, the hierarchy they sit in, and the rules about who may reach them. It
does not store rows, run queries, or build indexes -- that is the
[lakehouse](/basics/lakehouse) on top of it.
## Catalog overview {#catalog}
Keeping the two apart is what lets a Lance table be reached by something that is
not LanceDB. The catalog is a specification -- the
[Lance Namespace](/basics/lance#standard) -- so any engine that implements it
resolves the same names to the same tables.
A catalog keeps track of named data objects, such as tables and views, so
applications can discover and access them. In LanceDB, it organizes those
objects into databases and namespaces.
## The objects {#objects}
## Databases, namespaces and tables {#objects}
| Object | What it is |
A database contains namespaces that group related tables and views. Namespaces
can also contain other namespaces, allowing teams to organize data by project,
environment or purpose.
For example, a search application can keep production and staging data in
separate namespaces within the same database:
```mermaid
flowchart TD
database["ai_data · database"] --> prod["prod · namespace"]
database --> staging["staging · namespace"]
prod --> prod_docs["documents · table"]
staging --> staging_docs["documents · table"]
```
<span id="views" />
| Object | What it represents |
| --- | --- |
| [Database](/management/database) | The root. A connection opens exactly one. |
| [Namespace](/management/namespace) | A named grouping inside a database. Can nest. |
| [Table](/management/table) | Rows and columns, versioned. What most operations act on. |
| [Materialized view](/management/materialized-view) | A table whose contents come from a query, computed and stored. |
| [View](/management/view) | A saved query. Computed when read, not stored. |
| [Function](/management/function) | Registered code, applied to produce column values. |
| [Database](/management/database) | A container for namespaces and their data objects. |
| [Namespace](/management/namespace) | A named group of tables, views or nested namespaces. |
| [Table](/management/table) | Data stored as rows and columns. |
| [View](/management/view) | A saved query, evaluated when read. Available in Enterprise. |
| [Materialized view](/management/materialized-view) | The stored results of a query, updated by a refresh. |
```
database
└── namespace
├── table
├── materialized view (a table, defined by a query)
├── view (a query, not stored)
└── function
```
{/*
Draft for the Functions release. Add this row to the table above:
| [Function](/management/function) | Registered code used to compute column values. Available in Enterprise. |
*/}
A namespace can contain further namespaces, so the hierarchy goes as deep as
your organisation needs. A table is reached by its path through it:
`prod.events.clicks`.
<span id="resolution" />
This page is about what the objects *are*. For the operations on them, see
[Management](/management).
## Organization and access control {#governance}
## Resolution {#resolution}
- **Separate projects and environments.** Keep production and staging tables
in distinct namespaces, even when they use the same table names.
- **Find shared datasets.** Give teams a consistent place to discover and
access the data they work with.
- **Control access in Enterprise.** Use permissions on databases, namespaces
and tables to manage who can access shared data and what they can do with it.
A name is resolved by walking the path, and the walk is the point: reaching a
table means traversing the namespaces above it. That is what makes the hierarchy
more than decoration -- it is the unit of both organisation and access.
<span id="elsewhere" />
A flat set of tables needs none of this. Namespaces start earning their keep when
several teams, environments or domains share one catalog and you want
`prod.search.documents` and `staging.search.documents` to be unambiguous.
## Learn more {#next}
## Governance {#governance}
<Badge color="red">Enterprise</Badge>
Because the catalog knows the hierarchy, it is the layer that can enforce
access. A grant names **one privilege** on **one object** for **one subject**,
and it inherits down the path: reaching a table requires `USAGE` on the
namespace containing it and on the database containing that.
That is the practical reason to put a boundary in the hierarchy -- a namespace
per team gives you something to grant on. See
[Authorization](/governance/authorization).
Embedded LanceDB has no such layer. Access is whatever the object store enforces
on the bucket, which has no concept of a table.
## Views and materialized views are different objects {#views}
The names are close enough to mislead. Both are catalog objects defined by a
query over other tables; the difference is when that query runs.
- A **materialized view** stores its rows, and a refresh brings them up to
date. Reads are reads of stored data -- fast, and as current as the last
refresh.
- A **view** stores only the query. Every read runs it, so a view is never stale
and never faster than the query it wraps.
Reach for a materialized view when the query is expensive and the data changes
less often than it is read. Reach for a view when correctness matters more than
latency, or when the query is cheap.
## What the catalog does not hold {#elsewhere}
**Indexes** belong to a table rather than standing alone. Creating one is a
performance decision, not a structural one -- see [Indexing](/indexing).
**Jobs** are not stored data at all; they are handles to work in progress. See
[Jobs](/basics/jobs).
**Rows** are in the [Lance](/basics/lance) files the catalog points at. The
catalog knows a table exists and where it lives; it does not know what is in it.
## Next {#next}
<CardGroup cols={2}>
<Card title="Lance Lakehouse" icon="layer-group" href="/basics/lakehouse">
What operates on these objects: search, compute, maintenance.
</Card>
<Card title="Management" icon="folder-tree" href="/management">
Creating, listing and dropping each one.
</Card>
</CardGroup>
- [Databases](/management/database): connect to and manage a database.
- [Namespaces](/management/namespace): create groups and address tables within them.
- [Tables](/management/table): create, inspect and manage your data.
- [Views](/management/view) and [materialized views](/management/materialized-view): work with query-defined data.
- [Authorization](/governance/authorization): configure Enterprise access permissions.
docs/web/basics/connect.mdx
+132
View File
@@ -0,0 +1,132 @@
---
title: "Connect to LanceDB"
sidebarTitle: "Connect"
description: "Use the embedded LanceDB OSS library with tables in a local directory or object storage, or connect to LanceDB Enterprise."
icon: plug
keywords: ["connect", "uri", "embedded", "object storage", "s3", "enterprise", "db://"]
---
import {
PyConnect,
PyConnectAsync,
PyConnectEnterprisePlaceholders,
PyConnectEnterpriseQuickstart,
PyConnectObjectStorage,
PyConnectObjectStorageAsync,
RsConnect,
TsConnect,
TsConnectEnterprisePlaceholders,
TsConnectEnterpriseQuickstart,
TsConnectObjectStorage,
} from '/snippets/connection.mdx';
The URI you pass to `connect` decides where the work runs. The same Python,
TypeScript and Rust SDKs open all three kinds;
[OSS and Enterprise](/basics/offerings) describes how the two offerings differ.
| URI | Where queries run |
| --- | --- |
| A local path, such as `./data` | OSS: in your process |
| `s3://`, `gs://` or `az://` | OSS: in your process, reading object storage |
| `db://` | LanceDB Enterprise: queries run on your cluster |
## A local directory {#local}
Embedded in your application, LanceDB reads and writes the tables in a
directory on disk.
<CodeGroup>
<CodeBlock filename="Python (sync)" language="Python" icon="python">
{PyConnect}
</CodeBlock>
<CodeBlock filename="Python (async)" language="Python" icon="python">
{PyConnectAsync}
</CodeBlock>
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
{TsConnect}
</CodeBlock>
<CodeBlock filename="Rust" language="Rust" icon="rust">
{ "use lancedb::connect;\n" }
{ "\n" }
{RsConnect}
</CodeBlock>
</CodeGroup>
## Object storage {#object-storage}
With an object storage URI the library is still embedded: your process runs
every query and reads and writes the bucket directly, with the credentials from
its environment or its storage options. [Configuring storage](/deployment/storage)
covers credentials, endpoints and provider options.
<CodeGroup>
<CodeBlock filename="Python (sync)" language="Python" icon="python">
{PyConnectObjectStorage}
</CodeBlock>
<CodeBlock filename="Python (async)" language="Python" icon="python">
{PyConnectObjectStorageAsync}
</CodeBlock>
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
{TsConnectObjectStorage}
</CodeBlock>
</CodeGroup>
In Rust, pass the same URI to `connect`.
## LanceDB Enterprise {#enterprise}
A `db://` URI connects to LanceDB Enterprise, where your cluster runs queries
and background work. Get the database URI, API key, region and endpoint from
your administrator, then replace the placeholders below. Use your cluster's region
in place of `us-east-1`. In application code, load credentials from environment
variables or a secrets manager.
<CodeGroup>
<CodeBlock filename="Python" language="Python" icon="python">
{PyConnectEnterprisePlaceholders}
</CodeBlock>
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
{TsConnectEnterprisePlaceholders}
</CodeBlock>
</CodeGroup>
In Rust, enable the crate's `remote` feature; the connection builder then takes
the same settings through `api_key`, `region` and `host_override`. To use OAuth
instead of an API key, see
[Authentication](/enterprise/authentication); for every connection option, see
[Database](/management/database#deployment).
<Accordion title="Load production credentials from your environment">
Set these values in the shell that starts your application. Replace the URI, API
key, region and endpoint with your Enterprise connection settings. In production, a secrets
manager can supply these environment variables for you.
```shell
export LANCEDB_URI="db://your-database"
export LANCEDB_API_KEY="your-api-key"
export LANCEDB_REGION="us-east-1"
export LANCEDB_HOST_OVERRIDE="https://your-endpoint"
```
<CodeGroup>
<CodeBlock filename="Python" language="Python" icon="python">
{PyConnectEnterpriseQuickstart}
</CodeBlock>
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
{TsConnectEnterpriseQuickstart}
</CodeBlock>
</CodeGroup>
</Accordion>
docs/web/basics/guarantees.mdx
+59 -102
View File
@@ -1,134 +1,91 @@
---
title: "Guarantees"
sidebarTitle: "Guarantees"
description: "What a write promises when it returns, what a reader sees, and what survives a failure."
description: "Understand table commits, concurrent writes and the boundaries of recovery."
icon: shield-check
keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency", "acid"]
keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency"]
---
What the system promises, and — just as usefully — what it does not. Most
surprises in practice come from assuming one of the guarantees below is
stronger or weaker than it is.
## Atomic table writes {#writes}
## Writes {#writes}
A table transaction commits its changes atomically: readers see the committed
table version, not a partially applied transaction. For example, a committed
append adds its rows together.
**A write is atomic.** Every write commits a new table version. Either the
version exists and holds all of the change, or it does not exist and holds none
of it. There is no state in which a reader sees half an `add()`.
This boundary is one table transaction. A workflow that updates several tables
or runs a background job can contain multiple commits. Do not assume the whole
workflow is atomic.
**A write returns only after the commit.** When the call returns, the version
is durable in object storage. Nothing is buffered in your process waiting to be
flushed.
<span id="schema" />
**Every write advances the version**, including one that affects zero rows. A
delete whose predicate matches nothing still commits. Version numbers count
commits, not changes.
Schema changes are also part of table history. Changing a column's type can
rewrite data; adding a derived column can write the new column independently.
**Concurrent writes are resolved optimistically.** A transaction records the
change and the version it was based on. At commit time, a conflicting
concurrent operation is detected rather than silently overwritten, and the
commit is retried against the newer version. Operations that do not collide —
appending while another process appends — both succeed.
## Consistent reads {#reads}
<Note>
`merge_insert` reports its retry attempts in the result. A high count means
contention on the same keys, not an error.
</Note>
A read of a table uses a consistent snapshot. A consistent snapshot can still
be older than another writer's latest commit. [Consistency](/tables/consistency)
explains how OSS and Enterprise choose the version a reader sees.
## Reads {#reads}
For an experiment that must keep using the same data, select a specific retained
table version rather than relying on a connection's default freshness.
**A read sees a consistent snapshot.** A query reads one table version
throughout; it never sees a mixture of two.
## Concurrent writes {#concurrency}
**Which version, by default, is the one open at the time the table was opened.**
This is the guarantee most often assumed to be stronger than it is:
Lance uses optimistic concurrency control. Writers prepare changes, then check
compatibility with intervening commits. Compatible changes can be committed;
conflicts may be retried or returned as errors.
<Warning>
A long-lived connection **does not observe other processes' writes** until it
reopens the table or its `read_consistency_interval` elapses. The default is
right for a batch job reading a stable snapshot and wrong for a service
expected to see fresh data.
</Warning>
Applications must handle failed writes. Concurrent appends are compatible, but
that does not promise success through storage failures or exhausted retries.
Do not rely on a particular ordering between independent writers.
Set `read_consistency_interval` when several processes write the same table and
readers must observe those writes. The cost is a metadata check on that
interval. See [Consistency](/tables/consistency).
The [Lance transaction specification](https://lance.org/format/table/transaction/)
describes commit behavior and compatibility between operations.
**Your own writes are always visible to you.** The interval governs other
writers, not yourself.
## Storage durability {#durability}
## Durability and history {#durability}
A committed table version is stored in the table’s configured storage. LanceDB
OSS uses your local filesystem or object store; LanceDB Enterprise uses the
cluster’s configured storage.
**Old versions are retained, not overwritten.** A new version does not destroy
the one before it, which is what makes [time travel](/management/table#versioning) work
and what lets a reader finish a scan that started on an older version.
Table version history does not replace backups. Configure storage redundancy
and backups to meet your recovery requirements.
**Deletion is a mark, not an erase.** A deleted row is recorded in a deletion
file. The data is recoverable for as long as the version that contains it
exists.
## Background job completion {#jobs}
<Warning>
Retention is finite. [Compaction](/maintenance/compaction) prunes versions
older than the retention window — **7 days by default** — and a tag or branch
pointing at a pruned version no longer resolves. If you rely on reading old
versions, widen the window before running it, not after.
</Warning>
In LanceDB Enterprise, operations such as index builds and column refreshes
can run as background jobs. Receiving a job handle confirms submission. Wait
for the job to succeed before relying on its result.
**Dropping a table ends its history.** Versioning does not survive
`drop_table`; there is no undrop.
Some operations save results in batches. If a job fails or is cancelled,
batches already committed can remain in the table. Each operation’s guide
explains whether it can leave partial results.
## Jobs {#jobs}
## Writes after a client timeout {#timeouts}
**A refresh recomputes only what changed.** It gives values to rows that lack
them and recomputes rows whose inputs changed since they were computed; it
leaves the rest alone. Two consequences, both relied on elsewhere:
If a client times out while waiting for a write, the write may already have
committed. Retrying an append can therefore insert the same records again.
- **Retrying is safe.** Re-running a refresh fills what is still empty or stale.
It is not guaranteed to cost only the rows that did not complete: a retry can
recompute values that were already written.
- **A filled value catches up at the next refresh.** If a column derives from
`price` and a row's `price` later changes, the derived value keeps its old
value until the next refresh recomputes it.
Check whether the intended records were written before retrying. For an
operation submitted as a background job, check its status using the job handle.
**What a failure keeps depends on where the refresh ran.** `refresh_column` in
your process commits the column once, so a failure keeps none of its values. A
refresh job on a deployment commits finished batches as it goes, so some values
can remain.
## Version history and recovery {#history}
**Cancellation is a request.** `cancel()` asks a job to stop; it does not
guarantee it stops before doing more work, and cancelling a finished job does
nothing.
Retained table versions let you inspect earlier data or restore a table to a
previous version. Cleanup can remove those versions and their underlying files,
so configure retention to cover the history you need.
**A job id is opaque.** It correlates with server logs and the jobs API.
Parsing it, or storing it to resume the job later, is not supported.
Dropping a table is different from deleting records. Table version history
cannot undo `drop_table`.
See [Jobs](/basics/jobs).
<span id="not-guaranteed" />
## Schema {#schema}
## Learn more {#next}
**Schema changes are versioned like data changes.** Adding, renaming, retyping
or dropping a column commits a version, so a schema change can be travelled back
through.
**A computed column's inputs are protected.** While a declaration references a
column, that column cannot be renamed, retyped, or dropped — the declaration is
part of the schema and will not be invalidated out from under itself.
**Most schema changes do not rewrite data.** Adding a derived column writes only
the new column. Type changes are the exception and can be substantial.
## What is not guaranteed {#not-guaranteed}
Worth stating plainly, because each is occasionally assumed:
| | |
| --- | --- |
| Multi-table transactions | A commit covers one table. There is no cross-table atomic write. |
| Read-your-writes across processes | Governed by `read_consistency_interval`, not automatic. |
| Unenforced primary keys | `SET UNENFORCED PRIMARY KEY` records intent; it does not enforce uniqueness. |
| Ordering of concurrent appends | Two appends both succeed; their relative row order is not defined. |
| Index freshness | An index does not cover rows added after it was built until [optimized](/maintenance/compaction). |
| Automatic compaction | Nothing runs in the background. `optimize()` runs when you call it. |
The last two are the most common source of "why is this query slow" — see
[Limits](/basics/limits) for the bounds that pair with them.
- [Consistency](/tables/consistency): control when reads observe updates.
- [Table versioning](/management/table#versioning): inspect and restore history.
- [Cleanup](/maintenance/cleanup): configure retention and understand protected versions.
- [Jobs](/basics/jobs): distinguish submission, completion and cancellation.
- [Column refresh](/feature-engineering/refresh-column): understand partial progress.
- [Architecture](/architecture): understand storage and background maintenance.
docs/web/basics/jobs.mdx
+61 -182
View File
@@ -1,196 +1,75 @@
---
title: "Jobs"
sidebarTitle: "Job"
description: "Work that outlives the call that started it: submitting, waiting, inspecting, cancelling."
description: "Track background work from submission to completion."
icon: hourglass-half
keywords: ["job", "async", "background", "events", "cancel", "refresh"]
keywords: ["job", "background", "progress", "events", "cancel"]
availability:
oss: available
enterprise: available
summary: >-
Job handles track operations in OSS and Enterprise. Enterprise also
supports listing server jobs, reopening them by ID and reading their events.
---
Some operations take longer than a request should. Building an index over a
large table, refreshing a computed column, registering a function -- these run
on the server after the call that started them has returned.
## Job handles {#convention}
Each is submitted the same way and returns the same handle.
A job handle lets you track an operation, such as an index build or a computed
column refresh. Use it to check progress, wait for completion and inspect
failures.
<Note>
A [query](/basics/query) also has an `_async` form, but its handle streams rows
rather than waiting for an outcome. Both answer the same four questions --
which one is this, how is it going, give me the result, stop it -- and differ
only in the third.
</Note>
Methods such as Python's `create_index_async` and TypeScript's
`createIndexAsync` return job handles. In OSS, the work runs in your application
process; in Enterprise, it runs on server workers. A returned handle can
represent an operation that has already finished, so check its status before
waiting or requesting cancellation. See the operation's API reference for when
the method returns.
## The `_async` convention {#convention}
## Submit and wait for a job {#waiting}
An operation that can run in the background has two forms. The plain name blocks
until the work is done. The `_async` / `Async` name submits the work and returns
a **job** immediately.
1. **Submit the operation.** Keep the returned job handle. For an Enterprise
job, also keep its ID if another process needs to look it up later.
2. **Check progress.** Read the current status while the operation runs.
3. **Wait for completion.** Check that the job succeeded before using its
result. If it failed or was cancelled, inspect the failure details and any
results it committed before stopping.
| Blocking | Returns a job | SQL |
In Python, `job.status()` reports the current state and `job.wait()` waits for
completion, raising if the job fails or is cancelled. If `job.wait()` times out, the client stops waiting.
The timeout does not cancel the server job.
<span id="inspecting" />
<span id="events" />
## Inspect and cancel jobs {#managing}
| Task | What to do |
| --- | --- |
| Inspect progress | Check status and refresh the handle to retrieve details. Enterprise job events can help explain ongoing work or failures. |
| Reattach from another process | Open an existing server job with its ID, such as Python's `db.open_job(job_id)`. The server must still retain the job record. |
| Diagnose a failure | Inspect failure details and relevant events. Enterprise SQL also provides `SHOW ERRORS`. |
| Request cancellation | Call the job's cancellation method. Cancellation may take time, and work already committed is not rolled back. |
Reopening a job retrieves the existing operation; it does not restart failed
work. Treat the ID as an opaque identifier and pass it unchanged to the jobs API.
Event responses have [row limits](/basics/limits#jobs). An explicit limit can
request more than the default, but it does not guarantee a complete event
history.
## OSS and Enterprise execution {#in-process}
| | OSS | Enterprise |
| --- | --- | --- |
| `create_index` | `create_index_async` | `CREATE INDEX` |
| `refresh_column` | `refresh_column_async` | `REFRESH COLUMN` |
| `create_function` | `create_function_async` | `CREATE FUNCTION` |
| `drop_table` | `drop_table_async` | `DROP TABLE` |
| Execution | In your application process | On server workers |
| Job identity | No server ID | A server ID for looking up the job |
| Event history | No server event history | Available through the jobs API |
| Reattaching after a client restart | No server job to reopen | Reopen by ID while the record is retained |
<Note>
The blocking form is not a different operation -- it submits the same job and
waits. Reach for the `_async` form when you want to do something else meanwhile,
run several at once, or survive the client going away.
</Note>
## Learn more {#next}
## Submitting and waiting {#waiting}
<CodeGroup>
```python Python icon="python"
from lancedb.index import IvfPq
job = table.create_index_async("vector", config=IvfPq())
print(job.id) # the server's id for this operation
print(job.status()) # "running" | "finished" | "failed" | "cancelled"
job.wait() # blocks until terminal; raises if it failed
```
```typescript TypeScript icon="square-js"
const job = await table.createIndexAsync("vector");
console.log(job.id); // the server's id for this operation
console.log(await job.status()); // "running" | "finished" | "failed" | "cancelled"
await job.wait(); // resolves when terminal; rejects if it failed
```
</CodeGroup>
`wait` is the one call that raises on failure. `status` is a point snapshot and
returns the failed state rather than raising, which is what you want in a
polling loop.
In Python `wait` takes an optional timeout and returns the operation's typed
result -- a `RefreshColumnJobResult`, a `FunctionVersion`, or `None` for an
operation that has no result:
```python
from datetime import timedelta
result = table.refresh_column_async("embedding").wait(timeout=timedelta(minutes=30))
```
`TimeoutError` means the timeout elapsed, not that the job stopped. The job is
still running; the handle just gave up watching.
<Note>
The TypeScript `wait()` resolves to `void`. Read the outcome from `result` after
it resolves.
</Note>
## Inspecting a job {#inspecting}
Submitting an operation returns only an id, so the detail fields are empty until
the handle fetches the record. `refresh()` does that; `status()` does it as a
side effect.
<CodeGroup>
```python Python icon="python"
job.refresh()
job.state # last observed state, without another round trip
job.job_type # what kind of job, as the server names it
job.creation_ms # when it was created, epoch milliseconds
job.spec # what it was submitted with
job.result # its terminal result, once it succeeds
job.failure # why it failed, when it did
```
```typescript TypeScript icon="square-js"
await job.refresh();
job.state; // last observed state, without another round trip
job.jobType; // what kind of job, as the server names it
job.creationMs; // when it was created, epoch milliseconds
job.spec; // what it was submitted with
job.result; // its terminal result, once it succeeds
job.failure; // why it failed, when it did
```
</CodeGroup>
Printing the handle renders every field it knows, which is usually faster than
reading them one at a time.
## Progress, from events {#events}
`result` only tells you anything once the job has finished. Events are written
as it runs, and they outlive the workers that produced them -- so a job that is
still going, or one that never finished, can still account for what it did.
<CodeGroup>
```python Python icon="python"
events = job.events(limit=5000, filter="state = 'claim_complete'")
```
```typescript TypeScript icon="square-js"
const events = await job.events({ limit: 5000, filter: "state = 'claim_complete'" });
```
```sql SQL
SHOW ERRORS ON my_table FOR JOB 'job-id' LIMIT 100
```
</CodeGroup>
A distributed job records a `claim` / `claim_complete` pair per unit of work,
each carrying `rows_processed`. Summing `rows_processed` over the completed
claims is how you get progress on a job that has no percentage.
`filter` is a SQL-like expression over `state`, `updated_by`, `emitted_from`,
`emitted_by`, and `claim_entity`.
<Warning>
The server returns 1000 event rows by default and 10,000 at most, and
**truncates without telling you**. A job that emits an event per fragment will
silently lose its tail unless you pass `limit`.
</Warning>
Events are recorded server-side, so this raises for an operation that ran
in-process.
## Finding and cancelling jobs {#managing}
The connection lists and reopens jobs, so a handle lost to a restarted process
is recoverable from its id.
<CodeGroup>
```python Python icon="python"
db.list_jobs() # every job the server knows about
job = db.open_job(job_id) # reattach to one
db.cancel_job(job_id) # ask it to stop
```
```typescript TypeScript icon="square-js"
await db.listJobs();
const job = await db.openJob(jobId);
await db.cancelJob(jobId);
```
```sql SQL
SHOW JOBS
```
</CodeGroup>
Cancellation is a request, not a guarantee, and cancelling a job that already
finished does nothing. `JobNotFoundError` means the id does not name a job the
server still has.
<Warning>
A job id is opaque. It correlates with server logs and the calls above, but
parsing it or storing it to resume the job later is not supported.
</Warning>
## In-process jobs {#in-process}
A local database has no server to run anything, so its operations run in your
process. They still return a handle, so the same code works against both -- but
one with nothing behind it: `id` is `None`, `status()` reports `"finished"`,
`wait()` returns immediately, `cancel()` does nothing, and `events()` raises.
This is the difference to expect when moving between embedded and deployed --
see [SDKs](/basics/sdk).
- [Build and repair indexes](/indexing/reindexing): submit index work.
- [Column refresh](/feature-engineering/refresh-column): populate derived values.
- [Background jobs](/maintenance/background-jobs): understand Enterprise maintenance.
- [SQL job commands](/sql/ddl/job): list jobs and inspect errors.
- [SDKs](/basics/sdk): find the job API for your client.
docs/web/basics/lakehouse.mdx
+55 -82
View File
@@ -1,102 +1,75 @@
---
title: "Lance Lakehouse"
sidebarTitle: "Lance Lakehouse"
description: "What operates on the catalog's objects: storage, search, computation, and maintenance."
title: "LanceDB Lakehouse"
sidebarTitle: "LanceDB Lakehouse"
description: "Bring multimodal data, search and feature engineering together for AI workloads."
icon: layer-group
keywords: ["lakehouse", "multimodal", "search", "compute", "maintenance", "feature engineering"]
keywords: ["lakehouse", "multimodal", "search", "compute", "training", "feature engineering"]
---
The [catalog](/basics/catalog) says what exists. The **lakehouse** is everything
that acts on it: storing the rows, searching them, computing new ones, and
keeping the whole thing fast as it grows.
## Multimodal storage and processing {#multimodal}
The distinction matters because the two layers have different lifetimes. A
catalog entry is a name and a pointer. A lakehouse operation reads and writes
the [Lance](/basics/lance) files that name points at -- and those operations are
where almost all of the cost, and all of the tuning, lives.
LanceDB brings storage, search and data processing together for AI workloads.
You can store multimodal data, derive features, search your records and prepare
training datasets using tables built on the open [Lance format](/basics/lance).
## Multimodal is the point {#multimodal}
The format provides the storage foundation. The [catalog](/basics/catalog)
organizes your tables. LanceDB provides the tools to work with that data.
A lakehouse table holds images, video, audio, text, embeddings, annotations and
the features you derive from them **in one table**, side by side.
## Example: a video search workflow {#capabilities}
That is the difference from assembling a warehouse plus a vector store plus a
feature store. There is one copy of the data, one set of versions, and no
pipeline keeping three systems agreeing about the same rows. A search can filter
on a label, rank by a vector, and return the blob, without leaving the table.
Consider a team building a video search application:
| | |
| --- | --- |
| **Columns** | vectors, blobs, and columns [computed](/feature-engineering/add-computed-column) from other columns |
| **Versions** | every write commits one, so reads can [travel back](/management/table#versioning) |
| **Indexes** | vector, full-text and scalar, built over those columns |
| **Branches** | isolated lines of change you can [merge or discard](/tables/branching) |
<span id="ingest" />
<span id="compute" />
<span id="search" />
## The four capabilities {#capabilities}
1. **Ingest the clips.** Store video clips with metadata such as timestamps,
source and labels.
2. **Add features.** Generate captions and embeddings with your models, and
store them alongside each clip.
3. **Use the dataset.** Search for relevant clips using embeddings and metadata
filters, or select a dataset to read into a training framework.
Everything the lakehouse does falls into one of four groups, and the rest of
these docs is organized the same way.
```mermaid
flowchart TD
clips["Video clips and metadata"] --> table["Store in a Lance table"]
table --> features["Add captions and embeddings"]
features --> search["Search for relevant clips"]
features --> training["Select data for training"]
```
### Ingest {#ingest}
The clips, metadata and derived features stay associated in the same table,
so retrieval and training workflows can use the same dataset. Your training
framework runs the model; LanceDB provides access to the training data.
Getting rows in, and changing the ones that are there: append, merge, update,
delete. Every write commits a version rather than mutating in place, which is
what makes the other three safe to run against a moving table. See
[Data Ingestion](/ingestion).
## Shared compute and maintenance in Enterprise {#where}
### Search {#search}
LanceDB OSS runs as an embedded library in your application. LanceDB Enterprise
provides shared infrastructure for applications and teams working with larger
multimodal workloads.
Vector, full-text and hybrid retrieval over the same rows, with filters that
push into the scan rather than running after it. The search is a lakehouse
operation, not a separate service holding a second copy of your embeddings. See
[Semantic Search](/search).
<span id="maintenance" />
### Compute {#compute}
- **Scale resources for different workloads.** Provision query serving and
background workers independently as search traffic and indexing demand grow.
- **Share data across teams.** Give applications access to shared tables, with
Enterprise permissions controlling who can read or change them.
- **Manage ongoing maintenance.** Configure background indexing, compaction
and cleanup as your datasets grow.
Deriving new columns and tables from what is already stored:
[computed columns](/feature-engineering/add-computed-column),
[functions](/management/function) that run Python where the data is, and
[materialized views](/management/materialized-view) that persist a query's
result.
See [Architecture](/architecture) for how Enterprise separates storage, query
serving and background work.
The reason this belongs in the lakehouse rather than in a pipeline outside it is
the format. Adding a derived column writes only that column -- the images and
videos are never rewritten -- so deriving a feature is an ordinary operation
rather than a migration. See [Feature Engineering](/feature-engineering) and
[Data Curation](/curation).
## Learn more {#next}
### Maintenance {#maintenance}
- [Data ingestion](/ingestion): load and update your data.
- [Search](/search): retrieve records with vector, full-text and hybrid search.
- [Feature engineering](/feature-engineering): derive columns and datasets.
- [Training data access](/training/data-loading): read datasets into training workflows.
- [Enterprise](/enterprise): explore deployment options for shared workloads.
The work that keeps the first three fast: building and repairing
[indexes](/indexing), and [compaction](/maintenance/compaction) to merge small
files, prune old versions and fold new rows into existing indexes.
<Warning>
None of it runs on its own. There is no background compactor -- `optimize()`
runs when you call it, and an index does not cover rows added after it was
built until you do. See [Guarantees](/basics/guarantees#not-guaranteed).
</Warning>
## Where the work happens {#where}
The same lakehouse runs in two shapes, and which one you use decides where these
capabilities execute rather than whether you have them:
- **Embedded** -- your process does all four itself, directly against object
storage. Nothing runs unless you call it.
- **Distributed** -- a cluster does them, with indexing and compaction on
workers separate from the ones serving queries, plus
[SQL](/sql) and server-side [jobs](/basics/jobs).
See [Architecture](/architecture) for the comparison in full.
## Next {#next}
<CardGroup cols={2}>
<Card title="Lance Catalog" icon="sitemap" href="/basics/catalog">
The objects these operations act on.
</Card>
<Card title="Lance format" icon="layer-group" href="/basics/lance">
The open format underneath, and why column evolution is cheap.
</Card>
</CardGroup>
{/*
Draft for the Functions release. Add this item to the list above, after
Feature engineering:
- [Functions](/management/function): run Python to compute column values.
*/}
docs/web/basics/lance.mdx
+29 -106
View File
@@ -1,122 +1,45 @@
---
title: "Lance format"
sidebarTitle: "Lance Format"
description: "Lance is the open format underneath. LanceDB is what implements a lakehouse on top of it."
description: "The open format for multimodal data behind LanceDB."
icon: "/static/assets/logo/lance-logo-gray.svg"
keywords: ["lance", "format", "open standard", "lakehouse", "namespace", "catalog"]
keywords: ["lance", "format", "multimodal", "versioning", "column evolution"]
---
Two names, two things. [Lance](https://lance.org/) is an open format and
specification. LanceDB is an implementation built on it.
## Lance format overview {#standard}
The distinction matters when you are deciding what you are committing to: the
format is a standard other engines read, and your data stays readable by them
whichever LanceDB you run.
[Lance](https://lance.org/) is an open-source, columnar format designed for
multimodal AI. It defines how data is stored and organized into versioned tables.
## Lance is the open standard {#standard}
<span id="embedded" />
<span id="enterprise" />
Lance is an open-source, columnar lakehouse format for multimodal AI. It
specifies three things:
LanceDB OSS and LanceDB Enterprise both use Lance as their storage foundation.
The format is developed as an independent open-source project, with its own
specification and tools.
| | |
| --- | --- |
| **File format** | how bytes are laid out on disk, optimized for random access without giving up scan throughput |
| **Table format** | how files compose into a versioned table -- schema evolution, deletions, indexes |
| **Namespace specification** | how tables are named, grouped, and resolved -- the catalog layer |
## Multimodal data {#storage}
Being a specification rather than a product is the point. Object storage holds
the bytes, and open engines -- PyTorch, Ray, Spark, Trino, DuckDB, Polars --
read them without going through LanceDB at all.
A Lance table can hold images, video, audio, text, embeddings and structured
metadata together. For example, a video dataset can keep each clip alongside
its caption, labels and embeddings, so applications can search and work with
those fields together.
Lance exists because columnar formats derived from Parquet were not designed
for random access on vectors: [benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/)
show Lance up to 1000x faster than Parquet for that access pattern.
## Data access, versioning and column updates {#capabilities}
<Card
title="Lance format documentation"
icon="/static/assets/logo/lance-logo-gray.svg"
href="https://lance.org/format"
>
The format specification: design, layout, and how it enables the multimodal lakehouse.
</Card>
- **Efficient data access.** Read individual rows for retrieval and training,
or scan columns for analytics.
- **Versioned data.** Read an earlier table version to reproduce an experiment
or inspect previous results, while that version is retained.
- **Independent column updates.** Add embeddings or other derived features
without rewriting the existing image, video or text columns.
## LanceDB embedded implements the storage half {#embedded}
## Learn more {#next}
Connecting to a directory or a bucket gives you a **storage-only lakehouse**.
Your process opens Lance tables directly and does the work itself: it plans and
executes queries, builds indexes, and writes new versions.
<span id="versioning" />
<span id="evolution" />
```python
db = lancedb.connect("./.lancedb")
db = lancedb.connect("s3://bucket/path")
```
There is no server, which is what makes it embedded -- and also what bounds it.
There is nothing to run work on your behalf, so there are no server-side
[jobs](/basics/jobs), no [SQL endpoint](/basics/sql), and no shared catalog
beyond what the storage layer itself provides.
LanceDB OSS is Apache 2.0 licensed. LanceDB Enterprise is the commercial,
deployment-only product that implements the catalog specification below.
## LanceDB Enterprise implements the catalog specification {#enterprise}
<Badge color="red">Enterprise</Badge> A `db://` connection talks to a deployment
that implements the **Lance REST Namespace** -- the REST server defined by the
Lance namespace specification, covering the catalog operations in the
[REST API reference](/api-reference/rest).
```python
db = lancedb.connect("db://my-database", api_key="...")
```
Implementing the specification rather than inventing an interface is what makes
the catalog reachable by anything that speaks it, and what makes
[authorization](/governance/authorization), server-side jobs, and
[SQL](/basics/sql) possible: there is now a server to enforce and to run them.
The tables are the same Lance tables either way. What changes is who resolves
their names and who does the work.
## What the format gives you {#capabilities}
These are properties of Lance, so they hold in both forms:
| Capability | What it enables |
| --- | --- |
| Multimodal storage | Images, video, audio, text, embeddings, annotations and metadata in one table. |
| First-class blob API | Large binary objects in blob columns, with lazy reads and streaming byte access. |
| Fast random access and scans | Sample, shuffle, and fetch individual rows without giving up sequential throughput. |
| Zero-copy schema evolution | Add, drop, rename, or alter columns without rewriting existing data files. |
| Versioned tables | Reproduce experiments and tie artifacts to the exact table version they used. |
| Hybrid search and indexing | Vector search, full-text search, and scalar filters over the same dataset. |
| Open interoperability | Object storage underneath, open engines on top. |
Two of them explain behaviour you will meet elsewhere in these docs.
### Versioning is why old data lingers {#versioning}
Every write creates a version, and old versions are not immediately destroyed --
another reader may be part-way through one. A hundred versions are not a hundred
copies of the data, but they do carry a hundred times the metadata, which
eventually slows queries down.
Deletion works the same way: a deleted row is marked in a deletion file rather
than erased, so the data is recoverable for as long as the version is.
[Compaction](/maintenance/compaction) is what resolves both.
### Column evolution is why feature engineering is cheap {#evolution}
Adding a derived column writes only the new column. Existing data -- the
expensive part, the images and videos -- is untouched. That is what makes
[computed columns](/feature-engineering/add-computed-column) a reasonable thing to
do repeatedly on a large table rather than a full rewrite each time.
<Card
title="Learn more about Lance"
icon="/static/assets/logo/lance-logo-gray.svg"
href="https://lance.org/quickstart"
>
Lance is a separate open source project with its own documentation.
</Card>
- [Lance format specification](https://lance.org/format): how the format works.
- [Table versioning](/management/table#versioning): read and manage table versions.
- [Computed columns](/feature-engineering/add-computed-column): add derived features to your data.
- [Architecture](/architecture): how LanceDB OSS and Enterprise build on Lance.
docs/web/basics/limits.mdx
+51 -86
View File
@@ -1,106 +1,71 @@
---
title: "Limits"
sidebarTitle: "Limits"
description: "The bounds you can actually hit, in one place."
description: "Find fixed constraints, configurable bounds and what happens when you reach them."
icon: ruler
keywords: ["limits", "maximum", "quota", "constraints", "bounds"]
---
Limits worth designing around, collected so you do not have to find them by
hitting them. Each one is enforced -- where a value is refused rather than
clamped, that is stated, because the two fail very differently.
<span id="scope" />
This page covers bounds that are the same for everyone. Deployment-specific
quotas -- storage, throughput, concurrency -- come from your deployment.
This page lists limits for job events, SQL error reports, sampling and
full-text indexing. Each table identifies the affected API and what happens
when a request reaches its limit. SQL and server job limits apply to Enterprise.
## Names {#names}
<span id="names" />
| Object | Maximum | |
[Naming Rules](/basics/naming) is the reference for name lengths, characters and
namespace depth. Client and server versions can differ in which rules they
enforce.
<span id="jobs" />
<span id="sql" />
## Fixed API limits {#fixed-limits}
| Operation | Scope | Limit and behavior |
| --- | --- | --- |
| Database | 64 characters | |
| Namespace | 64 characters | |
| Table | 128 characters | |
| Branch | 64 characters | |
| Job event retrieval | Enterprise | At most 10,000 rows per response. Results can be truncated without a truncation indicator. |
| `SHOW ERRORS ... LIMIT` | Enterprise SQL | Accepts 1 through 100,000. Values outside the range are rejected. |
| `SHOW ERRORS` without `FOR JOB` | Enterprise SQL | Inspects up to the 32 most recent failed jobs. Use `FOR JOB` to inspect a specific job. |
| Percentage sampling | Enterprise SQL | `TABLESAMPLE` accepts percentages from 0 through 100. Values outside the range are rejected. |
| Row-count sampling | Enterprise SQL | `TABLESAMPLE (... ROWS)` requires a whole number. |
A name must not be empty, and must not contain `$` or `:` -- both are reserved
as path delimiters, so a name holding one could not be addressed unambiguously.
Length is counted in characters, not bytes.
See [SQL job commands](/sql/ddl/job#errors) and
[Sampling](/sql/dql/sampling) for syntax and restrictions on supported table types.
## Jobs {#jobs}
<span id="schema" />
| | |
Data-type and index constraints belong to the operation you choose. Check
[Data types](/sql/data-types) and [Vector indexes](/indexing/vector-index)
before choosing a schema or index. For materialized views, check
[stable row ID requirements](/feature-engineering/materialized-views#stable-row-ids)
before creating source tables.
## Configurable limits {#not-quotas}
| Setting | What it controls |
| --- | --- |
| `job.events()` default | 1,000 rows |
| `job.events()` maximum | 10,000 rows |
| `SHOW ERRORS ... LIMIT` | between 1 and 100,000 |
| Failed jobs scanned by one `SHOW ERRORS` | 32 |
| Job event `limit` | Requests more or fewer event rows. The default is 1,000; the 10,000-row maximum still applies. |
| FTS `max_token_length` | The maximum indexed token length. The default is 40; longer tokens are excluded from the index. |
| Enterprise capacity and quotas | Storage, request rate and concurrency are governed by your deployment's resources and configuration. Confirm these with your administrator or LanceDB support. |
<Warning>
`job.events()` **truncates silently**. A job that emits an event per fragment
will lose its tail unless you pass `limit` explicitly. This is the limit on this
page most likely to mislead you, because nothing in the result says it was cut.
</Warning>
## Handling rejected or incomplete results {#behavior}
`SHOW ERRORS` without `FOR JOB` scans the most recent failed jobs and stops at
32 of them. Narrow with `FOR JOB` when you need a specific one rather than a
survey.
- **Rejected requests:** correct the input before retrying, such as a sampling
percentage outside the accepted range.
- **Truncated responses:** do not assume that a returned event list is the full
history. Increasing `limit` cannot exceed the maximum; use filters to narrow
the events you inspect.
- **Excluded data:** an FTS token-length setting can affect which words are
searchable. Choose it for your text and indexing requirements.
See [Jobs](/basics/jobs).
<span id="availability" />
## Schema and types {#schema}
## Learn more {#next}
| | |
| --- | --- |
| `map` with `keys_sorted = true` | rejected |
| `IVF_RQ` vector dimensions | must be divisible by 8 |
| FTS `max_token_length` | 40 by default; longer tokens are dropped |
Through the SDK, a table gets **stable row ids** only at creation, from the
`new_table_enable_stable_row_ids` storage option on a local path or an object
store; no SDK call adds them later, so decide early. A local
[materialized view](/feature-engineering/materialized-views#stable-row-ids) does
not need them. On a deployment, whether a view's source needs them and how a
table gets them depend on the deployment; see
[Against a deployment](/feature-engineering/materialized-views#availability).
See [Data types](/sql/data-types).
## SQL {#sql}
<Badge color="red">Enterprise</Badge>
| | |
| --- | --- |
| `TABLESAMPLE` percentage | 0 to 100, **refused** outside -- not clamped |
| `TABLESAMPLE (n ROWS)` | whole numbers only |
| `TABLESAMPLE BUCKET` | not supported |
| `TABLESAMPLE` on a WAL-backed table | not supported |
| `CREATE INDEX` | one column, server-chosen name |
| `REFRESH MATERIALIZED VIEW ... CASCADE` | depends on the deployment's version; see [REFRESH](/sql/dml/refresh#refresh-materialized-view) |
A sampling percentage over 100 is an error rather than a full scan, because a
query asking for 150 percent has a mistake in it and reading it as 100 would
hide the mistake. See [Sampling](/sql/dql/sampling).
## What is unavailable where {#availability}
Not quotas, but the same class of surprise -- things that exist in one place and
not another.
| | |
| --- | --- |
| SQL on a local database | unavailable; `execute_query` raises |
| SQL methods in the TypeScript SDK | none; use a Flight SQL client |
| Function methods in the TypeScript SDK | none; use Python or SQL |
| Vector search in SQL | no SQL form; use a query builder |
| Geospatial `st_*` functions | behind a build feature LanceDB does not ship |
| `tableExists` / `namespaceExists` in TypeScript | not present |
| Server-side jobs on a local database | run in-process; `events()` raises |
See [SQL](/basics/sql) for why the first two hold.
## Nothing here is a quota {#not-quotas}
None of the above is a billing or capacity limit. Rows per table, table count,
storage, and request rate are bounded by your deployment rather than by the
format -- ask your deployment, not this page.
- [Naming Rules](/basics/naming): choose portable names and paths.
- [Jobs](/basics/jobs): track work and inspect events.
- [Full-text indexes](/indexing/fts-index): configure tokenization.
- [SQL](/basics/sql) and [SDKs](/basics/sdk): check interface support.
- [Enterprise support](/deployment/support): discuss deployment capacity.
docs/web/basics/naming.mdx
+69
View File
@@ -0,0 +1,69 @@
---
title: "Naming rules"
sidebarTitle: "Naming Rules"
description: "Choose portable names for your data objects and namespace paths."
icon: signature
keywords: ["names", "naming", "table name", "namespace", "identifier", "limits"]
---
## Naming rules {#rules}
<span id="objects" />
Databases, namespaces, tables, views and branches all have names. Choose names
that describe their purpose, such as `documents`, `prod` or `training_2026`.
Consistent names make data easier to find and application code easier to read.
Use the following rules for names that work across LanceDB OSS and Enterprise.
| Object | Characters | Length |
| --- | --- | --- |
| Table, materialized view, view | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters |
| Namespace component | ASCII letters and digits, `-`, `_`, `.` | 1 to 64 characters; at most 10 components in a path |
| Database | ASCII letters and digits, `-`, `_`, `.`, with `/` between components | 1 to 64 characters for the whole name |
| Branch | Avoid `$` and `:`; use ASCII letters, digits, `-` and `_` for portability | Keep to 1 to 64 characters |
- Names and individual namespace or database components must not be empty.
- Table, view, namespace and database components cannot be `.` or `..`.
- Avoid `$` and `:` because they separate parts of qualified identifiers.
- A namespace path beginning with `public` cannot contain another component.
{/*
Draft for the Functions release. Add these rows to the end of the table above:
| Secret | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters |
| Function | ASCII letters and digits, `_` | 1 to 128 characters |
Add this item to the list above, after the rule for `.` and `..`:
- Secret names cannot be `.` or `..`.
*/}
## Namespace paths {#paths}
A namespace path is a list of names. `["prod", "search"]` identifies the
`search` namespace within `prod`. A table named `documents` belongs to that
namespace; its table name remains `documents`.
Use the SDK's namespace-path argument to supply the list. A period inside an
SDK name is a character in that name, not a separator that creates another
namespace. SQL uses its own qualified-name syntax; see
[SQL namespace paths](/sql/dql/select#namespaces).
The empty path `[]` identifies the root namespace.
## Examples {#examples}
| Name or path | Guidance |
| --- | --- |
| `documents`, `user-events.v2`, `train_2026` | Portable table names |
| `["prod", "search"]` | A two-component namespace path |
| `my table` | Use `my_table` instead of a space |
| `orders$2026` | Use `orders_2026`; `$` is reserved |
| `résumés` | Use an ASCII name such as `resumes` |
| `["public", "search"]` | Invalid nesting under `public`; use a different parent namespace |
<span id="which-versions-check-them" />
## Learn more {#next}
- [Namespaces](/management/namespace): create namespaces and address tables.
- [Branches](/tables/branching): name and manage separate lines of table history.
- [Limits](/basics/limits): check other constraints before designing your schema.
docs/web/basics/offerings.mdx
+70
View File
@@ -0,0 +1,70 @@
---
title: "LanceDB OSS or Enterprise?"
sidebarTitle: "OSS and Enterprise"
description: "Start with an embedded library, or use the LanceDB Enterprise multimodal lakehouse for shared AI workloads."
icon: compass
keywords: ["oss", "enterprise", "embedded", "deployment", "availability", "comparison"]
---
LanceDB gives you two ways to work with multimodal data, both built on the open
[Lance](/basics/lance) format.
**LanceDB OSS** is an embedded library. Run it inside your application
with tables on local disk or in object storage. You control the application,
its resources, and table maintenance, without a separate database service.
[Try the quickstart](/quickstart) to create a table and run your first search.
**LanceDB Enterprise** is a multimodal lakehouse for data curation, feature
engineering, search, and training data access. It adds query-serving capacity,
data caching, and background operations for teams working with shared
multimodal datasets.
[Connect to Enterprise](/basics/connect#enterprise) when your cluster is ready.
<span id="labels" />
<span id="availability" />
## Why choose Enterprise? {#why-enterprise}
### Serve growing workloads {#serving}
Add query-serving capacity as more applications and users depend on your data.
Enterprise spreads query traffic across a fleet and caches frequently read
table data near compute, reducing repeated reads from object storage. Query
serving and background work use separate resources, so you can size each for
your workload. See [Enterprise architecture](/enterprise/architecture).
### Reduce operational work {#operations}
Let the service handle routine table maintenance instead of scheduling it in
application code. Enterprise handles background [compaction](/maintenance/compaction) and can
[clean up old table versions](/maintenance/cleanup) according to your retention
policy. Configure these maintenance tasks for your Enterprise cluster so each
application does not have to schedule them.
### Fit your deployment requirements {#deployment}
Choose a managed deployment or bring LanceDB into your own cloud account.
Enterprise integrates with OpenTelemetry for metrics and traces, helping your
team monitor the service alongside your applications. Talk to us about your
cloud, networking, and operational requirements to plan the right deployment.
The [Enterprise overview](/enterprise) explains the available options.
### Connect your data workflows {#workflows}
Use the same tables for retrieval, dataset preparation, and training data access.
With its [SQL endpoint](/basics/sql) enabled, Enterprise supports analytical
[curation](/curation), including joins, grouping, and sampling.
[Feature engineering](/feature-engineering) adds derived columns and tables;
computed columns and materialized views require the corresponding features to
be enabled for your Enterprise cluster.
[Geneva](/geneva) runs Python feature backfills as distributed jobs.
[Training data loaders](/training/data-loading), available in both offerings,
read the resulting datasets for model training.
## Plan your Enterprise deployment {#get-started}
[Talk to us about your workload](https://www.lancedb.com/contact) to discuss your
data, query traffic, pipelines, and deployment requirements.
For a closer look at how the service runs, read the
[Enterprise architecture](/enterprise/architecture).
docs/web/basics/query.mdx
+37 -76
View File
@@ -1,94 +1,55 @@
---
title: "Queries"
sidebarTitle: "Query"
description: "Running a query, and reading its results before it finishes."
description: "Choose how to query your data and read the results."
icon: magnifying-glass
keywords: ["query", "async", "streaming", "arrow", "reader", "execute_query"]
keywords: ["query", "search", "SQL", "async", "streaming", "results"]
---
A query returns rows. How you get them depends on two choices that are easy to
conflate: whether **your thread** blocks, and whether you take the results all
at once or stream them as they arrive.
## Querying tables {#queries}
## Blocking or not is a language concern {#async-client}
A query selects data from your tables. You can filter rows, choose columns,
search for similar content or combine those operations. For example, a video
application might find clips similar to a sample and return only those recorded
in the last month.
A concern of your language, not of LanceDB. The work is identical either way.
## SDK searches and SQL queries {#searches}
<CodeGroup>
```python Python icon="python"
db = lancedb.connect("db://my-db", api_key="...") # blocking
tbl = db.open_table("documents")
| Interface | When to use it |
| --- | --- |
| **SDK query builders** | Build vector, full-text and hybrid searches, apply filters and choose the fields to return. Available in OSS and Enterprise. |
| **Enterprise SQL** | Write SQL queries for filtering, aggregation, joins and other data curation tasks. Connect through Python or a Flight SQL client. |
db = await lancedb.connect_async("db://my-db", api_key="...") # coroutines
tbl = await db.open_table("documents")
```
Both interfaces work with LanceDB tables. Choose the one that supports your
operation and fits your application. See [SQL](/basics/sql) for connection
instructions and SQL search support, and [Search](/search) for the query builders.
```typescript TypeScript icon="square-js"
// TypeScript is async throughout; there is no blocking form.
const db = await lancedb.connect("db://my-db", { apiKey: "..." });
const tbl = await db.openTable("documents");
```
</CodeGroup>
## Collect or stream results {#results}
Python offers both. `AsyncConnection` exposes the same surface as coroutines, so
choosing between them is about the program you are writing -- a service wants
the async form, a script does not care.
- **Collect the results** when you want a list, Arrow table or DataFrame that
fits in memory. Use a result limit and select only the columns you need.
- **Read batches** when you want to process a large result incrementally.
Reading batches avoids loading the entire result into application memory.
It does not guarantee that the query can return its first batch immediately.
## Reading before the query finishes {#queries}
### Asynchronous clients {#async-client}
A query is not a job, but it has the same shape of choice: wait for everything,
or take a handle and start reading.
An async client lets your application do other work while a request is waiting.
That is separate from whether you collect results or read them in batches.
Python offers synchronous and asynchronous clients; TypeScript uses async APIs.
Check the SDK reference for the methods available in your client.
<CodeGroup>
```python Python icon="python"
reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 1000")
table = reader.read_all()
### Query handles and background jobs {#jobs}
query = db.execute_query_async("SELECT id, text FROM my_docs")
```
An Enterprise SQL query handle lets you inspect query execution and request
cancellation. Background operations such as index builds return a
[job](/basics/jobs) handle instead. Use the API for the returned handle to
check status and retrieve results.
```sql SQL
SELECT id, text FROM my_docs LIMIT 1000
```
</CodeGroup>
## Learn more {#next}
`execute_query` returns a blocking Arrow reader **once results start arriving**
-- it does not wait for the whole query to finish. Results stream as record
batches, so a large result set does not have to fit in memory before you see its
first row.
`execute_query_async` returns the query handle instead:
```python
query = db.execute_query_async("SELECT id, text FROM my_docs")
query.id # identifies it on the server
query.describe() # what it is doing, without consuming results
reader = query.reader() # stream the rows when you want them
query.cancel() # stop it
```
`describe_query(id)` reports on one from elsewhere, the way `open_job(id)` does
for a job.
<Note>
SQL requires a deployment. See [SQL](/basics/sql#sql-is-remote).
</Note>
Query-builder searches -- vector, full-text, hybrid -- are ordinary calls that
return results. They are fast enough not to need a handle. See
[Query](/search).
## Searches are ordinary calls {#searches}
Query-builder searches -- [vector](/search/vector-search),
[full-text](/search/full-text-search), [hybrid](/search/hybrid-search) -- return
results directly. They are fast enough not to need a handle, and they have no
SQL form. See [Semantic Search](/search).
## When the work outlives the call {#jobs}
Some operations take longer than a query should: building an index, refreshing a
computed column, registering a function. Those return a **job** rather than
rows, and a job is a different handle with a different shape. See
[Jobs](/basics/jobs).
- [Quickstart](/quickstart): run a complete create, insert and search example.
- [Search](/search): choose vector, full-text or hybrid retrieval.
- [Filtering](/search/filtering): narrow results with predicates.
- [SQL](/basics/sql): connect a client and run SQL queries.
- [SDKs](/basics/sdk): choose a language and find its API reference.
docs/web/basics/sdk.mdx
+82 -64
View File
@@ -1,99 +1,117 @@
---
title: "SDKs"
sidebarTitle: "SDKs"
description: "The language clients — Python, TypeScript, Rust — and what stays true across all of them."
description: "Choose a LanceDB SDK and find the API reference for your language."
icon: code-compare
keywords: ["python", "typescript", "rust", "java", "sdk", "rest", "api reference"]
availability:
oss: available
enterprise: available
summary: >-
Python, TypeScript and Rust support both LanceDB OSS and LanceDB Enterprise.
Java provides a client for the Enterprise REST API.
---
A language SDK is the primary way to work with LanceDB: create and manage
tables, write and query data, build indexes, run feature engineering. The same
table API works whether you're connected to an embedded database or an
Enterprise deployment -- application code moves between the two by changing
the connection URI.
Use a LanceDB SDK to create tables, add data, build indexes and run searches
from your application. Python, TypeScript and Rust work with both the embedded
OSS library and LanceDB Enterprise.
## Officially supported SDKs {#supported-sdks}
The [quickstart](/quickstart) includes installation instructions and complete
examples in all three languages. For connection settings, see
[Connect to LanceDB](/basics/connect).
Python, TypeScript and Rust are supported against both embedded and deployed
LanceDB.
## Official SDKs {#supported-sdks}
| Reference | Description |
|:--------------|-------------------|
| <Icon icon="python" /> [Python](https://lancedb.github.io/lancedb/python/python/) | Full-featured Python client with pandas and numpy integration |
| <Icon icon="square-js" /> [TypeScript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs` |
| <Icon icon="rust" /> [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent storage and high performance |
| SDK | Use it with | API reference |
| --- | --- | --- |
| Python | Python applications, notebooks, pandas and PyArrow | [Python reference](https://lancedb.github.io/lancedb/python/python/) |
| TypeScript | Node.js and TypeScript applications | [TypeScript reference](https://lancedb.github.io/lancedb/js/) |
| Rust | Rust applications using the native library | [Rust reference](https://docs.rs/lancedb/latest/lancedb/index.html) |
| Java | Applications using the Enterprise REST API | [Java reference](https://lancedb.github.io/lancedb/java/java/) |
Not every SDK covers the same surface, though:
Guides show the available language examples together. Use the API reference for
method signatures and options; SDKs do not expose every feature through the same
methods. In Rust, enable the `remote` feature to connect to Enterprise.
| | Python | TypeScript |
| --- | :---: | :---: |
| Create and manage catalog objects | yes | yes |
| Insert, merge, update, delete | yes | yes |
| Filter, project, aggregate, join | yes | yes |
| Vector search | yes | yes |
| Full-text search | yes | yes |
| Feature engineering, jobs | yes | partly |
| Runs against a local database | yes | yes |
TypeScript has no methods for [functions](/management/function). Reach the
gap with SQL instead -- see [SQL](/basics/sql).
For SQL joins, aggregations and window functions, use
[SQL](/basics/sql). Python provides a SQL client, and applications in other
languages can use a Flight SQL client. TypeScript's table query API does not
provide a SQL execution method.
## REST API {#rest}
<Badge color="red">Enterprise</Badge> A deployment implements the
[Lance REST Namespace](/basics/lance#enterprise), so the catalog is reachable
over HTTP by anything that speaks it -- see the
[REST API reference](/api-reference/rest).
| Reference | Description |
|:--------------|-------------------|
| <Icon icon="java" /> [Java](https://lancedb.github.io/lancedb/java/java/) | REST API Enterprise SDK in Java |
LanceDB Enterprise also exposes an HTTP API for catalog and table operations.
Use it when integrating a language or tool without a LanceDB SDK. The
[REST API reference](/api-reference/rest) describes its endpoints and request
formats. For SQL queries, connect through [Flight SQL](/basics/sql).
## Community SDKs {#community-sdks}
Contributed by the community rather than supported by LanceDB, so feature parity
and release cadence vary.
These clients are maintained by the community. Check each project's supported
features, installation instructions and releases before choosing one.
| Reference | Description |
|:--------------|-------------------|
| <Icon icon="golang" /> [Go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | Community-contributed Go SDK |
| <Icon icon="gem" /> [Ruby](https://github.com/scientist-labs/lancelot) | Community-contributed Ruby bindings |
| <Icon icon="swift" /> [Swift](https://github.com/RyanLisse/LanceDbSwiftKit) | Community-contributed Swift SDK |
| <Icon icon="R" /> [R](https://github.com/CathalByrneGit/lancedb) | Community-contributed R package |
| <Icon icon="flutter" /> [Flutter](https://github.com/Alexcn/flutter_lancedb) | Community-contributed Flutter bindings |
For any other language, [SQL](/basics/sql) reaches the same tables without a
LanceDB-specific SDK at all.
| Language | Project |
| --- | --- |
| Go | [lancedb-go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) |
| Ruby | [Lancelot](https://github.com/scientist-labs/lancelot) |
| Swift | [LanceDbSwiftKit](https://github.com/RyanLisse/LanceDbSwiftKit) |
| R | [lancedb](https://github.com/CathalByrneGit/lancedb) |
| Flutter | [flutter_lancedb](https://github.com/Alexcn/flutter_lancedb) |
## Python multiprocessing {#multiprocessing}
<Warning>
Do not use `fork` as the start method with Python's `multiprocessing` module.
Lance is multi-threaded internally, and `fork` with alive threads is unsafe --
use `spawn` instead. See
[this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)
for why.
Use `spawn` when starting Python worker processes with `multiprocessing`.
LanceDB uses threads internally, and forking a process with active threads can
cause deadlocks. Create a connection inside each worker instead of sharing an
open connection from the parent process.
</Warning>
## What stays the same {#invariants}
For PyTorch worker setup and connection factories, see
[PyTorch integration](/training/torch).
Whichever you pick:
## Moving to LanceDB Enterprise {#moving-to-enterprise}
- **Types are Arrow types.** A vector column is a `FixedSizeList<Float32>`
everywhere, not a SQL type mapped onto storage. See
[Functions and types](/sql/functions).
- **Reads are Arrow.** Nothing is serialized into language objects on the way out
unless you convert it.
- **The catalog is one catalog.** A table created from TypeScript is visible to
Python and to SQL immediately -- there is no per-language namespace.
The core table workflow stays familiar: connect, create or open a table, write
records, and query them. Enterprise runs those operations on remote compute and
manages background maintenance.
## What to read next {#next}
- **Configure the connection.** Replace the local path with your database URI,
and supply the endpoint, region and credentials provided by your administrator.
[Connect to Enterprise](/basics/connect#enterprise) shows these settings.
- **Retrieve query results.** Python's remote tables do not support whole-table
`to_arrow()` or `to_pandas()` calls. Use a query and select the columns and
number of rows you need, for example `table.search().limit(10).to_arrow()`.
- **Handle table listings and namespaces.** Remote table listings are paginated.
Use the namespace and pagination options described in
[Namespaces](/management/namespace).
- **Use server-side jobs.** Index builds and feature refreshes can run as
[jobs](/basics/jobs). Each operation's guide explains how to submit work and
track completion.
- **Let Enterprise manage maintenance.** Use
[background maintenance](/maintenance/background-jobs) rather than calling
the embedded table's `optimize()` method.
For SQL connection details, see [SQL](/basics/sql). For the operating model and
deployment options, see [LanceDB Enterprise](/enterprise).
## Shared data across languages {#invariants}
LanceDB uses Arrow schemas across its SDKs. A fixed-size vector column stores
vectors of a consistent dimension; the [data types reference](/sql/data-types)
describes the supported element types and other column types.
Applications in different languages can work with the same tables. Connect to
the same database and namespace, use the appropriate credentials, and account
for [read consistency](/tables/consistency) when another process writes data.
## Related guides {#next}
<CardGroup cols={2}>
<Card title="SQL" icon="table-list" href="/basics/sql">
The same tables, reached by any Flight SQL client -- and the one way in for a language with no SDK.
Query Enterprise tables with SQL from Python or a Flight SQL client.
</Card>
<Card title="Agentic Skills" icon="robot" href="/basics/agentic-skills">
Give a coding agent a current reference to this same API surface.
Use the LanceDB plugin to help a coding agent work with your data.
</Card>
</CardGroup>
docs/web/basics/sql.mdx
+115 -81
View File
@@ -1,125 +1,159 @@
---
title: "SQL"
sidebarTitle: "SQL"
description: "The same tables, reached with SQL — the universal access path, and where every operation is formally documented."
description: "Query LanceDB Enterprise with SQL from Python or an Arrow Flight SQL client."
icon: table-list
keywords: ["sql", "flight sql", "datafusion", "reference", "dialect"]
availability:
oss: unavailable
enterprise: available
summary: >-
Enterprise supports SQL through Arrow Flight SQL. Use the SDK table APIs for embedded OSS tables. See [Connection settings](#sql-is-remote) to connect.
---
<Badge color="red">Enterprise</Badge> SQL runs on a deployment. A local
database has no engine to plan a statement, so `execute_query` raises there.
Use SQL in LanceDB Enterprise to filter records, join tables, calculate
aggregates, and prepare datasets. SQL queries work with the same Lance tables
you create and update through the SDKs, without exporting the data to another
system.
SQL is a second way to reach the same tables the [SDKs](/basics/sdk) do --
not a different product, a different interface onto one engine. A table
written from Python is queried from SQL without an export step, and a table
created from SQL is visible to every SDK immediately.
You can run SQL through the Python SDK or an Arrow Flight SQL client. This
guide covers connection settings, a first query, and how SQL fits alongside
the SDK table APIs.
## Any language, via Flight SQL {#flight-sql}
## Connect a SQL client {#flight-sql}
SQL is served over an
Enterprise exposes an
[Arrow Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html)
endpoint, so any Flight SQL client reaches it -- which makes SQL the way in
for a language LanceDB has no SDK for. You don't need a LanceDB-specific
client at all, just a Flight SQL driver for your language.
endpoint. A compatible client sends SQL statements to this endpoint and
receives results as Arrow record batches.
The TypeScript SDK itself has no SQL method, for the same reason a Go or C#
application would reach for SQL first: connect a Flight SQL client instead of
waiting on a native binding. See [SDKs](/basics/sdk) for the officially
supported and community clients, and reach for Flight SQL whenever yours
isn't among them.
- **Python:** use `execute_query()` on an Enterprise connection, as shown below.
- **TypeScript:** use a separate Flight SQL client. The LanceDB TypeScript SDK
does not expose a SQL execution method.
- **Other languages and tools:** use a Flight SQL client or a tool with Flight
SQL integration. A LanceDB-specific SDK is not required to submit SQL.
## Every SDK operation, in SQL {#sdk-parity}
The [SQL reference](/sql#connecting) shows a query from Python. See
[SDKs](/basics/sdk) for the LanceDB language clients.
Broadly, a public SDK method has a corresponding SQL statement -- creating a
table is `CREATE TABLE`, inserting rows is `INSERT INTO`, building an index is
`CREATE INDEX`. The two describe the same operations, so whichever one your
code speaks, the other is available too.
### Connection settings {#sql-is-remote}
| | SDK | SQL |
| --- | :---: | :---: |
| Create and manage catalog objects | yes | partly |
| Insert, merge, update, delete | yes | yes |
| Filter, project, aggregate, join | yes | yes |
| Vector search | yes | version-dependent |
| Full-text search | yes | yes |
| Feature engineering, jobs | yes | yes |
| Runs against a local database | yes | **no** |
Get the following values from your Enterprise administrator:
## Vector search depends on the deployment {#no-vector-sql}
- Database URI, such as `db://your-database`.
- API key and region.
- API endpoint for `host_override`.
- Flight SQL endpoint for `sql_host_override`, including its scheme and port.
The SDK query builder provides vector search across embedded and deployed
databases. SQL support depends on the deployment version and supported query
shape. Some versions recognize `ORDER BY distance(vector, query_vector) LIMIT k`
over a table scan and plan vector search, including indexed and flat search.
Confirm support on your deployment and use SQL
[`EXPLAIN`](/sql/dql/explain) to check the chosen plan; this is not a guarantee
that every deployed release supports that form.
The Flight SQL endpoint must be enabled and reachable from your client. Pass
it explicitly as `sql_host_override` in Python, as in the example below. The
API and SQL endpoints may use different hosts or ports.
Full-text search is available through the [`fts()`](/sql/functions#fts) table
function and the [`contains_tokens`](/sql/scalar-functions#contains-tokens)
predicate.
See [Connect to Enterprise](/basics/connect#enterprise) for other connection options.
The reverse gap also exists: a [view](/management/view) is a saved SQL query,
so SQL is the only way to create one -- there's nothing for an SDK method to
call.
SQL runs in Enterprise. Connections to a local directory or an object-storage
bucket use the embedded OSS library and cannot execute SQL statements. Use
SDK table operations to query those tables.
## SQL needs a deployment {#sql-is-remote}
## Run a query in Python {#sql-from-python}
A connection to a directory or a bucket has no server to plan and execute a
statement, and asking it to raises `NotImplementedError`. This is the largest
behavioral difference between the embedded and deployed forms, and the reason
the SDKs are the portable choice: SDK code moves between the two by changing
the URI, SQL does not.
Install the SDK with `pip install lancedb`. Replace the connection placeholders
with your Enterprise settings, including the region and both endpoint URLs.
This example reads an existing table named `my_docs` with `id` and `text`
columns. Change the query to match your table.
```python
db = lancedb.connect("./.lancedb") # no SQL
db = lancedb.connect("db://my-database") # SQL
```
import lancedb
## Running SQL from Python {#sql-from-python}
db = lancedb.connect(
"db://your-database",
api_key="your-api-key",
region="us-east-1",
host_override="https://your-api-endpoint",
sql_host_override="https://your-sql-endpoint:443",
)
```python
reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 10")
table = reader.read_all()
print(table.to_pylist())
```
`execute_query` returns a blocking Arrow reader once results start arriving; it
does not wait for the whole query. `execute_query_async` returns a handle
instead -- see [Queries](/basics/query).
In application code, load credentials from environment variables or a secrets
manager. The [connection guide](/basics/connect#enterprise) shows that setup.
## Choosing between them {#choosing}
`execute_query()` returns a blocking Arrow reader when results start arriving.
The example uses `read_all()` to collect the small result in memory. For larger
results, consume the reader in batches. To submit a query and check its status
separately, use `execute_query_async()`. See [Queries](/basics/query) for query
handles, streaming, and cancellation.
Reach for **the SDKs** when you are writing data, managing the catalog, running
vector search, or building an application that should work the same embedded and
deployed.
## Choose SQL or SDK table operations {#choosing}
Reach for **SQL** when the question is analytical -- joins across tables,
aggregates, window functions -- when the thing asking is a BI tool, a
notebook, or a person who already thinks in SQL, or when it's the only client
available for your language.
Use **SQL** for joins, aggregations, window functions, and analytical queries
written in SQL. Use **SDK table operations** to build applications around table
management, ingestion, filtering, and search, including applications that run
with the embedded OSS library.
The two compose. A common shape is SDK code to build and maintain tables, and
SQL for the analysis on top.
You can combine them. For example, create and populate a table with an SDK,
then query it with SQL to prepare a training dataset. In Python, both workflows
can use the same Enterprise connection.
## The complete reference {#reference}
### Common SQL operations {#sdk-parity}
This page is the concept; [SQL Reference](/sql) is the specification. It has
one page per statement, syntax first -- and since every SDK operation has a
SQL form, those pages end up defining the precise behavior of the SDK
operations too, not just the SQL dialect.
Several tasks are available through both interfaces, but the methods and SQL
statements do not have identical coverage. SDK support also differs by
language. These SQL reference pages describe the supported statements:
| Task | SQL reference |
| --- | --- |
| Create a table | [CREATE TABLE](/sql/ddl/table) |
| Insert or change rows | [INSERT](/sql/dml/insert), [UPDATE](/sql/dml/update), [DELETE](/sql/dml/delete) |
| Build an index | [CREATE INDEX](/sql/ddl/index) |
| Join and aggregate data | [SELECT](/sql/dql/select) |
| Save a query as a view | [CREATE VIEW](/sql/ddl/view) |
[Computed columns](/feature-engineering/add-computed-column) store SQL
expressions whose values are populated by a refresh. {/* Draft for the Functions release:
They are separate from registered Python Functions. */}
See the computed-column and
[materialized-view](/management/materialized-view) guides for their SDK and SQL
operations and Enterprise setup requirements.
## Vector and full-text search {#no-vector-sql}
SQL vector search support depends on the Enterprise server version and query
form. Versions that support `ORDER BY distance(vector, query_vector) LIMIT k`
over a table scan can plan indexed or flat vector search. Check that your
server version supports this form, then use
[`EXPLAIN`](/sql/dql/explain) to inspect the query plan.
The [SDK vector search API](/search/vector-search) is also available in
Enterprise and OSS. Use it when you prefer the SDK query builder or need to
search embedded tables.
For full-text search in SQL, use the [`fts()`](/sql/functions#fts) table
function or the [`contains_tokens`](/sql/scalar-functions#contains-tokens)
predicate.
## SQL syntax and API references {#reference}
The [SQL reference](/sql) documents statement syntax, supported expressions,
and data types. For SDK methods and their arguments, use the
[Python](https://lancedb.github.io/lancedb/python/python/),
[TypeScript](https://lancedb.github.io/lancedb/js/), or
[Rust](https://docs.rs/lancedb/latest/lancedb/index.html) API reference.
<Card title="SQL Reference" icon="table-list" href="/sql">
DDL by object, then DML, then DQL -- the detailed technical documentation
that defines the behavior of every operation and SDK API.
Statement syntax, expressions, and data types for LanceDB SQL.
</Card>
## What to read next {#next}
## Next steps {#next}
<CardGroup cols={2}>
<Card title="SDKs" icon="code-compare" href="/basics/sdk">
The language clients, and what stays true across all of them.
<Card title="Queries" icon="magnifying-glass" href="/basics/query">
Read results, track long-running queries, and cancel work.
</Card>
<Card title="Agentic Skills" icon="robot" href="/basics/agentic-skills">
Give a coding agent a current reference to this same API surface.
<Card title="SDKs" icon="code-compare" href="/basics/sdk">
Compare language clients and find their API references.
</Card>
</CardGroup>
docs/web/curation/aggregate.mdx
+110 -79
View File
@@ -1,113 +1,144 @@
---
title: "Profiling a dataset"
sidebarTitle: "Aggregate"
description: "Counting, grouping and summarizing — finding out what your data actually contains."
description: "Measure coverage, missing values and class balance before selecting a dataset."
icon: chart-column
keywords: ["aggregate", "group by", "class balance", "count", "profiling", "distinct"]
availability:
oss: unavailable
enterprise: available
summary: >-
Enterprise supports SQL aggregation.
---
<Badge color="red">Enterprise</Badge> Aggregation is SQL, and
[SQL needs a deployment](/basics/sql#sql-is-remote).
Use SQL aggregates to summarize a dataset: count records, find missing values,
and compare groups. These summaries help you understand the data before
selecting records for search, evaluation or training.
Before curating anything, find out what you have. Most of the surprises in a
training set — a class that is 90% of the rows, a column that is half null, a
"unique" id that is not — are one `GROUP BY` away from being visible.
## Example data {#example-data}
## How big, and how complete {#size}
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
document table and a separate label table. Expand the setup below to create them.
<Accordion title="Create the example tables">
`curation_documents` has six records describing four documents.
`curation_labels` has one label for each of three documents. `row_id` identifies
a record; `doc_id` identifies the document it describes.
Run each statement separately in a scratch database or namespace. Create these
tables once for the whole section, using fresh names if they already exist.
```sql
SELECT
count(*) AS rows,
count(embedding) AS embedded,
count(caption) AS captioned
FROM images
CREATE TABLE curation_documents (
row_id BIGINT, doc_id BIGINT, version BIGINT,
source VARCHAR, caption VARCHAR, score DOUBLE
);
INSERT INTO curation_documents VALUES
(1, 10, 1, 'web', NULL, 0.80),
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
(3, 20, 1, 'books', 'A dog in a field', 0.90),
(4, 30, 1, 'web', NULL, 0.70),
(5, 30, 1, 'web', 'A cat by a window', 0.85),
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
INSERT INTO curation_labels VALUES
(10, 'cat'), (20, 'dog'), (30, 'cat');
```
`count(*)` counts rows; `count(col)` counts **non-null values**. The gap between
them is your coverage, which is the fastest way to check how far a
[column refresh](/feature-engineering/refresh-column) has got without waiting
for the job to report.
`score` is a sample quality score, not a search result score. Document 10 has
two versions. Document 30 has two records at the same version. Document 40
has no label. These cases let you check the effect of each operation.
## Class balance {#balance}
</Accordion>
The single most useful curation query:
## Count records and missing values {#size}
Use `count(*)` to count rows and `count(column)` to count non-null values.
Add `DISTINCT` to count unique values:
```sql
SELECT label, count(*) AS n
FROM images
SELECT count(*) AS records,
count(DISTINCT doc_id) AS documents,
count(caption) AS captioned,
count(*) FILTER (WHERE caption IS NULL) AS missing_caption
FROM curation_documents;
```
The example returns **6 records, 4 documents, 4 populated captions and 2 missing
captions**. `count(column)` excludes null values; it does not check whether a
string is empty or whether a value is useful for the task.
## Group records by label {#balance}
Use `GROUP BY` to calculate a summary for each label:
```sql
SELECT label, count(*) AS documents
FROM curation_labels
GROUP BY label
ORDER BY n DESC
ORDER BY documents DESC, label;
```
If the top class is an order of magnitude larger than the rest, sampling
uniformly will reproduce that imbalance. Two ways out: draw a fixed number per
class with [windowing](/curation/windowing#per-group), or oversample the rare
ones and accept the duplication.
There are **2 cats and 1 dog** in the label table. Document 40 is absent, so this
query does not measure labeling coverage across all source documents. Use a
[left join](/curation/join#join-type) to find documents without labels.
## Distinct values {#distinct}
Decide whether the distribution matches your intended use. A uniform random
sample tends to preserve the source distribution; it does not balance classes.
To cap the number selected per label, use
[ranking within a group](/curation/windowing#per-group).
## Count records that match a condition {#conditional}
```sql
SELECT count(DISTINCT source) FROM images
SELECT source,
count(*) AS records,
count(*) FILTER (WHERE score >= 0.9) AS high_score,
count(*) FILTER (WHERE caption IS NULL) AS missing_caption
FROM curation_documents
GROUP BY source
ORDER BY source;
```
On a high-cardinality column this is expensive, because exact distinctness means
holding every value seen. When an estimate will do:
`FILTER` applies a condition to one aggregate. Here, `books` has 2 records,
1 high-score record and no missing captions. `web` has 4 records, 1 high-score
record and 2 missing captions. Counts are per record, so repeated document IDs
still contribute more than once.
<span id="distinct"></span>
## Group numeric values into ranges {#buckets}
Use `count(DISTINCT column)` for the number of distinct non-null values.
For numeric fields, group by a range or summarize their distribution. For
example, this query counts records in score buckets of width 0.1:
```sql
SELECT approx_distinct(source) FROM images
SELECT floor(score * 10) / 10 AS score_bucket, count(*) AS records
FROM curation_documents
GROUP BY score_bucket
ORDER BY score_bucket;
```
Cheap enough to run on a table where the exact version is not worth the wait.
The other approximate functions — `approx_median`, `approx_percentile_cont` —
are useful for the same reason when profiling a numeric column.
For time-based data, group by the time interval relevant to the task. The
[aggregation reference](/sql/dql/aggregation) and
[scalar function reference](/sql/scalar-functions) cover additional summaries,
including approximate counts and percentiles. Use an approximate aggregate when an estimate is sufficient; use an exact
count when the result must be precise.
## Conditional counts {#conditional}
## Profile a sample {#sampling}
`FILTER` narrows one aggregate without narrowing the query, which is how you get
several slices in one pass rather than one query each:
A [sample](/curation/sampling) can help you explore a large dataset before a
full scan. Report the sample size and method alongside the result. A sample can miss rare classes, and fragment sampling can reflect the order
in which records were written.
```sql
SELECT
label,
count(*) AS total,
count(*) FILTER (WHERE score > 0.9) AS confident,
count(*) FILTER (WHERE caption IS NULL) AS uncaptioned
FROM images
GROUP BY label
```
Before accepting a curated dataset, check its required fields and key
constraints directly rather than inferring them from a small sample.
## Grouping by a bucket {#buckets}
## Continue curating {#reference}
Raw values are rarely the grouping you want. `date_bin` buckets timestamps, and
ordinary arithmetic buckets numbers:
```sql
SELECT date_bin(INTERVAL '1 day', created_at) AS day, count(*)
FROM images GROUP BY day ORDER BY day
```
A histogram of a score column is the same trick:
```sql
SELECT floor(score * 10) / 10 AS bucket, count(*)
FROM images GROUP BY bucket ORDER BY bucket
```
## Profiling a sample {#sampling}
On a very large table, profile a [sample](/curation/sampling) and scale:
```sql
SELECT label, count(*) * 100 AS estimated
FROM images TABLESAMPLE (1)
GROUP BY label
```
Treat the result as an estimate. A 1% sample will miss classes rarer than about
1 in 10,000 entirely, which is exactly the tail you are usually looking for — so
use this to check the shape of the head, not the existence of the tail.
## Reference {#reference}
The complete clause and function list is in
[Aggregation](/sql/dql/aggregation).
Use these measurements to choose a [sample](/curation/sampling),
[attach missing metadata](/curation/join), or
[select one record per document](/curation/windowing).
See [Aggregation](/sql/dql/aggregation) for complete SQL syntax.
docs/web/curation/index.mdx
+104 -33
View File
@@ -1,54 +1,125 @@
---
title: "Data curation"
sidebarTitle: "Overview"
description: "Shaping a dataset before you train on it: profiling, sampling, joining, and ranking."
description: "Prepare datasets for search, evaluation and training by profiling, selecting and combining records."
icon: filter
keywords: ["curation", "dataset", "training set", "sampling", "deduplication", "class balance"]
---
Curation is the work between having data and having a *dataset*: finding out
what you actually have, drawing a subset you can afford to train on, attaching
labels that live elsewhere, and picking one row per thing when you have several.
Data curation turns a collection of records into a dataset suited to a task.
You might select labeled examples for training, prepare a fixed evaluation
set, or remove duplicate documents from a search corpus.
These pages are task-oriented. Each shows the operation in the context you would
reach for it, and links to the [SQL reference](/sql) for the complete syntax.
Start with the result you need. Then inspect the source data, choose which
records belong, and save the selected data or the rules used to produce it.
<Note>
Most curation is analytical -- grouping, joining, ranking -- which the query
builders do not do. It is therefore SQL, and SQL needs a deployment. See
[SQL](/basics/sql#sql-is-remote).
</Note>
## Choose a curation task {#questions}
## The four questions {#questions}
| | |
| Task | What it helps you decide |
| --- | --- |
| [What do I have?](/curation/aggregate) | counts, class balance, null coverage, distinct values |
| [What can I afford?](/curation/sampling) | a random subset, reproducible when you need it fixed |
| [What else do I know?](/curation/join) | labels, metadata and scores that live in another table |
| [Which one do I keep?](/curation/windowing) | one row per key, ranking within a group, splits |
| [Profile a dataset](/curation/aggregate) | Whether coverage, missing values and class balance suit the task. |
| [Sample records](/curation/sampling) | Which subset to inspect or use for an experiment. |
| [Join datasets](/curation/join) | How to attach labels, metadata or scores stored elsewhere. |
| [Rank and deduplicate](/curation/windowing) | Which records to keep when several describe the same item. |
## A worked shape {#shape}
The analytical examples in this section use **Enterprise SQL**. Connect with
the [SQL guide](/basics/sql). For sampling through the Python SDK in OSS or
Enterprise, see [Sample without SQL](/curation/sampling#training).
The four compose into the same pipeline most curation ends up being:
## Prepare a labeled document dataset {#shape}
Suppose you need one labeled record per document for a training experiment.
Inspect the source records, choose the latest version of each document, then
join its label. Finally, check the class balance of the selected records.
### Example data {#example-data}
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
document table and a separate label table. Expand the setup below to create them.
<Accordion title="Create the example tables">
`curation_documents` has six records describing four documents.
`curation_labels` has one label for each of three documents. `row_id` identifies
a record; `doc_id` identifies the document it describes.
Run each statement separately in a scratch database or namespace. Create these
tables once for the whole section, using fresh names if they already exist.
```sql
CREATE TABLE curation_documents (
row_id BIGINT, doc_id BIGINT, version BIGINT,
source VARCHAR, caption VARCHAR, score DOUBLE
);
INSERT INTO curation_documents VALUES
(1, 10, 1, 'web', NULL, 0.80),
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
(3, 20, 1, 'books', 'A dog in a field', 0.90),
(4, 30, 1, 'web', NULL, 0.70),
(5, 30, 1, 'web', 'A cat by a window', 0.85),
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
INSERT INTO curation_labels VALUES
(10, 'cat'), (20, 'dog'), (30, 'cat');
```
`score` is a sample quality score, not a search result score. Document 10 has
two versions. Document 30 has two records at the same version. Document 40
has no label. These cases let you check the effect of each operation.
</Accordion>
The following query chooses the latest record for each document, then keeps
only documents with a label:
```sql
WITH ranked AS (
SELECT d.*, c.label,
row_number() OVER (PARTITION BY d.doc_id ORDER BY d.version DESC) AS rn
FROM documents AS d
JOIN labels AS c ON c.doc_id = d.doc_id
SELECT *, row_number() OVER (
PARTITION BY doc_id ORDER BY version DESC, row_id DESC
) AS rn
FROM curation_documents
)
SELECT * FROM ranked WHERE rn = 1
SELECT d.row_id, d.doc_id, d.caption, l.label
FROM ranked AS d
JOIN curation_labels AS l ON l.doc_id = d.doc_id
WHERE d.rn = 1
ORDER BY d.doc_id;
```
Join to bring in what you know, window to pick one row per document, filter to
keep it. Add [`TABLESAMPLE`](/curation/sampling) when the result is still larger
than you want, and [aggregate](/curation/aggregate) to check the balance of what
you ended up with.
The result contains documents **10, 20 and 30**, with labels **cat, dog and
cat**. Document 40 is excluded because this particular task requires a label.
The `row_id` tie-breaker makes the choice for document 30 deterministic;
choose a tie-breaker that fits your own data.
## Then what {#next}
The example assumes one label per document. Duplicate keys in the label table
would multiply the result rows. The [joining guide](/curation/join#labels)
shows how to check that assumption.
A curated result you want to keep is a
[materialized view](/management/materialized-view) -- stored, refreshed, and
readable by a [data loader](/training/data-loading) like any other table.
## Check the selected dataset {#check-result}
Compare the result with the task's requirements: are the expected documents
present, are required fields populated, and is the class distribution suitable?
This example has two cats and one dog. That distribution may be appropriate
for exploration, but it does not establish a representative evaluation set.
[Profiling](/curation/aggregate) provides the counts to inspect.
[Sampling](/curation/sampling) and [ranking](/curation/windowing) help you
adjust the selection without changing the original records.
## Save and reuse the result {#next}
Choose the output according to how you will use it:
- **A fixed dataset:** save the query result in a new table with
[`CREATE TABLE AS SELECT`](/sql/ddl/table#create). Keep that output unchanged
when comparing experiments.
- **A reusable selection:** save the query as a [view](/management/view).
A view stores the query definition, so it is not a frozen evaluation set.
- **Stored results that you refresh:** use a
[materialized view](/management/materialized-view) when its supported query
and refresh behavior fit the workflow.
Record the source versions and selection rules alongside the output. Continue
with [data loading](/training/data-loading) to use the curated data in a
training pipeline.
docs/web/curation/join.mdx
+97 -55
View File
@@ -1,91 +1,133 @@
---
title: "Joining datasets"
sidebarTitle: "Join"
description: "Bringing labels, metadata and scores together with the rows they describe."
description: "Combine records with labels or metadata, and check for missing or duplicate matches."
icon: link
keywords: ["join", "labels", "metadata", "enrichment", "namespace"]
availability:
oss: unavailable
enterprise: available
summary: >-
Enterprise supports SQL joins.
---
<Badge color="red">Enterprise</Badge> Joins are SQL, and
[SQL needs a deployment](/basics/sql#sql-is-remote).
A SQL join combines records from two tables using a shared key. Use joins to
attach labels, metadata or model scores to your documents. Choose an inner join
to keep matching records, or a left join to retain records without a match.
Curation usually means combining things that were produced separately — images
in one table, human labels in another, model scores in a third. A join is how
they become one dataset.
## Example data {#example-data}
## Attaching labels {#labels}
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
document table and a separate label table. Expand the setup below to create them.
<Accordion title="Create the example tables">
`curation_documents` has six records describing four documents.
`curation_labels` has one label for each of three documents. `row_id` identifies
a record; `doc_id` identifies the document it describes.
Run each statement separately in a scratch database or namespace. Create these
tables once for the whole section, using fresh names if they already exist.
```sql
SELECT i.id, i.embedding, l.label
FROM images AS i
JOIN labels AS l ON l.image_id = i.id
CREATE TABLE curation_documents (
row_id BIGINT, doc_id BIGINT, version BIGINT,
source VARCHAR, caption VARCHAR, score DOUBLE
);
INSERT INTO curation_documents VALUES
(1, 10, 1, 'web', NULL, 0.80),
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
(3, 20, 1, 'books', 'A dog in a field', 0.90),
(4, 30, 1, 'web', NULL, 0.70),
(5, 30, 1, 'web', 'A cat by a window', 0.85),
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
INSERT INTO curation_labels VALUES
(10, 'cat'), (20, 'dog'), (30, 'cat');
```
An inner join keeps only rows that matched, which for curation is often exactly
what you want: unlabelled rows are not training data.
`score` is a sample quality score, not a search result score. Document 10 has
two versions. Document 30 has two records at the same version. Document 40
has no label. These cases let you check the effect of each operation.
When you need to *see* what did not match, use a left join and filter:
</Accordion>
## Basic join {#join-type}
An inner join keeps matching records:
```sql
SELECT i.id
FROM images AS i
LEFT JOIN labels AS l ON l.image_id = i.id
WHERE l.image_id IS NULL
SELECT d.row_id, d.doc_id, l.label
FROM curation_documents AS d
JOIN curation_labels AS l ON l.doc_id = d.doc_id
ORDER BY d.row_id;
```
That is your unlabelled backlog, which is the thing worth counting before you
plan an annotation round.
This returns **5 rows**: both records for document 10, one for document 20,
and both records for document 30. To produce one row per document,
[deduplicate the source records](/curation/windowing#dedup).
## Across namespaces {#namespaces}
### Find records without a match
Tables in different [namespaces](/management/namespace) join by their dotted
paths — the catalog is one catalog, so there is nothing to federate:
A left join keeps every record from the left table and fills missing right-side
values with `NULL`. Add `WHERE l.doc_id IS NULL` to return only unlabeled documents:
```sql
SELECT i.id, u.team
FROM prod.media.images AS i
JOIN prod.reference.uploaders AS u ON u.id = i.uploader_id
SELECT DISTINCT d.doc_id
FROM curation_documents AS d
LEFT JOIN curation_labels AS l ON l.doc_id = d.doc_id
WHERE l.doc_id IS NULL
ORDER BY d.doc_id;
```
## Joining a search result {#search}
The result is **document 40**. You might send it for annotation, retain it for
an unlabeled-data workflow, or exclude it from a task that requires labels.
Full-text search is a table function, so its result is a relation and joins like
any other. This is how you ask which categories a query matches, rather than
just which rows:
## Handle duplicate keys {#labels}
This example joins `curation_documents.doc_id` to `curation_labels.doc_id`.
First check whether the label table contains more than one row per document:
```sql
SELECT c.name, count(*) AS hits
FROM fts('documents', 'puppy') AS m
JOIN categories AS c ON c.id = m.category_id
GROUP BY c.name
ORDER BY hits DESC
SELECT doc_id, count(*) AS labels
FROM curation_labels
GROUP BY doc_id
HAVING count(*) > 1;
```
<Note>
Vector search has no SQL form, so a similarity result cannot be joined this way.
Run the search in an SDK, then join over its output — or write the result to a
table first.
</Note>
The example returns no rows. If your data returns duplicates, decide whether
they represent valid multiple labels or records that need consolidation.
A join does not enforce one-to-one matching.
## Making it fast {#performance}
## Join tables across namespaces {#namespaces}
Two things dominate join cost on a lakehouse table.
Use a table's qualified path to address it in another
[namespace](/management/namespace). For example, a table named `images` in
`prod.media` is addressed as `prod.media.images`. The same matching-key and
join-type rules apply; your identity needs access to the source objects.
**Index the join key.** A [scalar index](/indexing/scalar-index) on the key lets
the planner look rows up instead of scanning for them. Check with
[`EXPLAIN`](/sql/dql/explain): an indexed key shows a `ScalarIndexQuery`, an
unindexed one shows a `LanceScan` feeding the join.
## Combine search results with metadata {#search}
**Do not carry wide columns through.** Select the columns you need before
joining, not after. Embeddings and blobs are the expensive part of a multimodal
table, and a join can materialize them many more times than the result has rows.
A [SQL full-text search](/sql/dql/fts) returns rows that can be joined to other
tables. Join using an identifier retained in the search result, then check
whether the metadata has one or several matches per result.
```sql
SELECT i.id, l.label
FROM (SELECT id, uploader_id FROM images) AS i
JOIN labels AS l ON l.image_id = i.id
```
For an SDK search, retain the matching IDs and combine them with metadata in
your application or save the selected records to a table for further SQL work.
See [SQL search interfaces](/basics/sql#no-vector-sql) for supported query forms.
## Reference {#reference}
## Join performance {#performance}
The join types and their semantics are in [Joins](/sql/dql/join).
Check the row count, unmatched keys and duplicate matches before using a
joined dataset. Select only the columns needed for the next step, especially
when source records include large media or embedding columns.
Use [Explain a query plan](/search/explain) to see how the query reads and
combines data. Check the plan to see whether the query uses indexes and how it performs the
join. An index on a key does not guarantee that the join will use it.
## Save the result or continue curating {#reference}
Keep the query for reuse or [save the selected data](/curation#next).
See [Joins](/sql/dql/join) for join types and exact SQL behavior.
docs/web/curation/sampling.mdx
+132 -58
View File
@@ -1,98 +1,172 @@
---
title: "Sampling a dataset"
sidebarTitle: "Sampling"
description: "Drawing a subset you can afford to work with, and fixing it when it has to be reproducible."
title: "SQL sampling"
sidebarTitle: "SQL Sampling"
description: "Choose a sample for exploration or evaluation and preserve the selected records when reproducibility matters."
icon: dice
keywords: ["sampling", "tablesample", "subset", "eval set", "reproducible", "shuffle"]
availability:
oss: unavailable
enterprise: available
summary: >-
Enterprise supports SQL `TABLESAMPLE`. For sampling in OSS and Enterprise through the Python SDK, see [Sample without SQL](#training).
---
Most curation starts with a subset. A question you can answer in ten seconds on
1% of the data is worth asking twenty times; the same question on the whole
table is worth asking once.
import { PySampleDataset } from '/snippets/curation_sampling.mdx';
## A quick look {#quick}
Sampling selects records from a larger table. Use it to explore unfamiliar
data, run a smaller experiment, or prepare an evaluation dataset. Enterprise
SQL provides `TABLESAMPLE` for selecting a percentage or number of rows.
<Badge color="red">Enterprise</Badge>
This page covers Enterprise SQL sampling and a complete
[Python SDK example](#training) that runs locally.
## Example data {#example-data}
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
document table and a separate label table. Expand the setup below to create them.
<Accordion title="Create the example tables">
`curation_documents` has six records describing four documents.
`curation_labels` has one label for each of three documents. `row_id` identifies
a record; `doc_id` identifies the document it describes.
Run each statement separately in a scratch database or namespace. Create these
tables once for the whole section, using fresh names if they already exist.
```sql
SELECT * FROM images TABLESAMPLE (1) -- 1 percent
SELECT * FROM images TABLESAMPLE (500 ROWS) -- 500 rows
CREATE TABLE curation_documents (
row_id BIGINT, doc_id BIGINT, version BIGINT,
source VARCHAR, caption VARCHAR, score DOUBLE
);
INSERT INTO curation_documents VALUES
(1, 10, 1, 'web', NULL, 0.80),
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
(3, 20, 1, 'books', 'A dog in a field', 0.90),
(4, 30, 1, 'web', NULL, 0.70),
(5, 30, 1, 'web', 'A cat by a window', 0.85),
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
INSERT INTO curation_labels VALUES
(10, 'cat'), (20, 'dog'), (30, 'cat');
```
A bare number is a percentage; add `ROWS` for a count. Use it while you are
exploring — the cost of the query drops with the sample, and so does the cost
of being wrong about what you wanted to ask.
`score` is a sample quality score, not a search result score. Document 10 has
two versions. Document 30 has two records at the same version. Document 40
has no label. These cases let you check the effect of each operation.
## Rows or fragments {#methods}
</Accordion>
## Basic sampling {#quick}
A percentage selects a fraction of the source; `ROWS` requests a row count:
```sql
SELECT * FROM images TABLESAMPLE BERNOULLI (10) -- rows
SELECT * FROM images TABLESAMPLE SYSTEM (10) -- fragments
SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (50);
SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (3 ROWS);
```
`BERNOULLI` considers each row independently and reads across the whole table.
`SYSTEM` picks whole Lance fragments and takes the rows in them — far less I/O,
because it skips files rather than rows.
A percentage sample does not guarantee an exact number of rows. Use a count
when the desired sample size is the main constraint, and inspect the result
rather than assuming it contains every category you need.
The trade is uniformity. A fragment sample is only as representative as your
write order. If rows arrived grouped by source, by date, or by label — which is
usual — then a fragment sample inherits that grouping.
`TABLESAMPLE` requires a table that is not WAL-backed. See the
[sampling reference](/sql/dql/sampling#refused) for this restriction; do not
change a production table's write configuration just to try an example.
<Note>
Reach for `SYSTEM` when you want a cheap look, and `BERNOULLI` when the number
you compute has to be defensible. Omitting the method gives `BERNOULLI`.
</Note>
## Sampling methods {#methods}
## A fixed evaluation set {#fixed}
Choose a method according to how records should be selected:
Without a seed, every execution resamples. That is fine for exploring and wrong
for evaluation — a metric computed on different rows each run is not comparable
with itself.
| Method | Selection | When to use it |
| --- | --- | --- |
| `BERNOULLI` (default) | Select individual rows | When records should be sampled independently of their storage fragments |
| `SYSTEM` | Select whole Lance fragments | When reducing reads matters and grouping by fragment is acceptable |
```sql
SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42)
SELECT row_id, doc_id
FROM curation_documents TABLESAMPLE BERNOULLI (50);
SELECT row_id, doc_id
FROM curation_documents TABLESAMPLE SYSTEM (50);
```
The same seed against the same data gives the same rows, which is what makes a
sample usable as a held-out set. If the underlying data changes, so does the
draw — when you need the set frozen against that too, materialize it:
If write order groups records by label, date or source, a fragment sample can
reflect that grouping. On a tiny table stored in one fragment, it may include
all records or none. Row sampling also needs an adequate sample size to
represent rare groups; check that the resulting sample covers the groups you need.
## Repeatable samples {#fixed}
A seed requests a repeatable draw:
```sql
CREATE TABLE eval_set AS
SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42)
SELECT row_id, doc_id
FROM curation_documents TABLESAMPLE (50) REPEATABLE (42);
```
A [materialized view](/management/materialized-view) is the other option when
you want it to track the source on a refresh rather than stay frozen.
## Sampling both sides of a join {#joins}
Keep the source data and sampling configuration fixed when comparing draws.
For a lasting evaluation set, save the selected records rather than relying
on a seed to preserve membership as the source changes:
```sql
SELECT * FROM a TABLESAMPLE (10) JOIN b TABLESAMPLE (10) ON a.id = b.id
CREATE TABLE curation_eval AS
SELECT * FROM curation_documents TABLESAMPLE (50) REPEATABLE (42);
```
Each side samples independently, so a row survives only if both of its sides
were drawn. Sampling 10% of each side does **not** give 10% of the join — it
gives roughly 1%. Sample once, after the join, unless you specifically want the
smaller result.
Create this output once with a fresh name and keep it unchanged. Record the
source version and selection policy. Sampling records independently can place
revisions of the same document into different splits; decide the
[grouping policy](/curation/windowing#splits) before using the result for
training or evaluation.
## Shuffling for training {#training}
## Sampling with joins {#joins}
For a training loop rather than a query, the SDKs read a table in a shuffled
order without SQL at all:
Sampling both input tables independently changes which pairs can match.
For a one-to-one match, independent 10% row samples retain a given pair with
about 1% probability, not 10%.
```python
from lancedb.permutation import Permutation
To sample documents while retaining their available labels, sample only the
document table:
permutation = Permutation.identity(table)
dataloader = torch.utils.data.DataLoader(permutation)
```sql
SELECT curation_documents.row_id, curation_documents.doc_id, l.label
FROM curation_documents TABLESAMPLE (50)
JOIN curation_labels AS l ON l.doc_id = curation_documents.doc_id;
```
This is the right tool for epoch-level shuffling, where you want every row
eventually and only the order varies. `TABLESAMPLE` is for when you want fewer
rows. See [Training](/training/data-loading).
This samples source records, not distinct documents. If you need to sample the
result of a join or deduplication, first save that result as a table suitable
for sampling. `TABLESAMPLE` cannot be applied directly to a subquery result.
## Reference {#reference}
## Sample without SQL {#training}
The full clause, its options and what it refuses are in
[Sampling](/sql/dql/sampling).
The Python SDK provides a permutation builder for selecting and ordering rows
in OSS and Enterprise. A split selects a subset; a shuffle changes the order
in which records are read. `Permutation.identity(table)` retains stored order.
<Accordion title="Complete Python example: read a random subset">
Install the Python SDK with `pip install lancedb`, save the program as
`sample_dataset.py`, and run `python sample_dataset.py`. It creates a temporary
local database with ten rows and reads a random split containing three rows.
The temporary database is removed when the program finishes.
<CodeBlock filename="sample_dataset.py" language="python" icon="python">
{PySampleDataset}
</CodeBlock>
The program prints three distinct IDs. It uses a fixed seed, but does not
promise the same IDs across SDK versions or changes to the source dataset.
</Accordion>
For shuffled training reads, worker configuration and data loaders, see
[Loading data for model training](/training/data-loading).
## Related guides {#reference}
Count the sampled rows and [inspect their distribution](/curation/aggregate)
before using them. Preserve the selected data when the exact membership matters.
See [SQL sampling](/sql/dql/sampling) for syntax and restrictions.
docs/web/curation/windowing.mdx
+127 -89
View File
@@ -1,124 +1,162 @@
---
title: "Ranking and deduplication"
sidebarTitle: "Windowing"
description: "Picking one row per thing, ranking within a group, and splitting a dataset."
description: "Choose records with explicit ranking rules and deterministic tie handling."
icon: chart-simple
keywords: ["window", "deduplication", "row_number", "rank", "train test split", "per class"]
availability:
oss: unavailable
enterprise: available
summary: >-
Enterprise supports SQL window functions.
---
<Badge color="red">Enterprise</Badge> Window functions are SQL, and
[SQL needs a deployment](/basics/sql#sql-is-remote).
Window functions calculate values across related rows while keeping each row
in the result. Use them to rank documents, keep one record per item, or compare
a record with the one before it.
A window function computes across a set of related rows and returns a value
**per row** — nothing is collapsed. That is what makes it the tool for "which of
these do I keep", where an aggregate would throw away the rows you are choosing
between.
## Example data {#example-data}
## One row per thing {#dedup}
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
document table and a separate label table. Expand the setup below to create them.
The most common curation query there is. You have several rows per document —
revisions, re-uploads, repeated annotations — and want one.
<Accordion title="Create the example tables">
`curation_documents` has six records describing four documents.
`curation_labels` has one label for each of three documents. `row_id` identifies
a record; `doc_id` identifies the document it describes.
Run each statement separately in a scratch database or namespace. Create these
tables once for the whole section, using fresh names if they already exist.
```sql
CREATE TABLE curation_documents (
row_id BIGINT, doc_id BIGINT, version BIGINT,
source VARCHAR, caption VARCHAR, score DOUBLE
);
INSERT INTO curation_documents VALUES
(1, 10, 1, 'web', NULL, 0.80),
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
(3, 20, 1, 'books', 'A dog in a field', 0.90),
(4, 30, 1, 'web', NULL, 0.70),
(5, 30, 1, 'web', 'A cat by a window', 0.85),
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
INSERT INTO curation_labels VALUES
(10, 'cat'), (20, 'dog'), (30, 'cat');
```
`score` is a sample quality score, not a search result score. Document 10 has
two versions. Document 30 has two records at the same version. Document 40
has no label. These cases let you check the effect of each operation.
</Accordion>
## Keep one record per document {#dedup}
`row_number()` numbers rows within each group. `PARTITION BY` defines the group,
and `ORDER BY` determines which record comes first. This query selects the
latest version of each document, using `row_id` to break ties:
```sql
WITH ranked AS (
SELECT *,
row_number() OVER (PARTITION BY doc_id ORDER BY version DESC) AS rn
FROM revisions
SELECT *, row_number() OVER (
PARTITION BY doc_id ORDER BY version DESC, row_id DESC
) AS rn
FROM curation_documents
)
SELECT * FROM ranked WHERE rn = 1
SELECT row_id, doc_id, version, caption
FROM ranked
WHERE rn = 1
ORDER BY doc_id;
```
`PARTITION BY` says what "per thing" means; `ORDER BY` says which one wins. The
outer query is not optional — a window function cannot appear in `WHERE`,
because windows are computed after filtering, so the predicate would have
nothing to read.
This returns row IDs **2, 3, 5 and 6**. For document 30, rows 4 and 5 have the
same version; the example deliberately chooses the larger `row_id`.
Without a complete ordering, the selected row among ties is not determined by
the query.
<Note>
Swap `row_number()` for `rank()` when ties should both survive. `row_number()`
always picks exactly one; `rank()` gives tied rows the same rank, so `rank = 1`
can return several.
</Note>
Use `rank()` instead of `row_number()` when tied records should all survive,
and order only by the fields that define a tie. Adding a unique tie-breaker
would make every rank distinct. The outer query filters the calculated rank
because window functions are evaluated after the inner query's `WHERE` clause.
## A fixed number per class {#per-group}
## Select the highest-scoring records per class {#per-group}
Class imbalance found by [profiling](/curation/aggregate#balance) is fixed here:
rank within each class, then take the top N.
Deduplicate first, then rank the labeled documents within each class:
```sql
WITH ranked AS (
SELECT *, row_number() OVER (PARTITION BY label ORDER BY score DESC) AS rn
FROM images
WITH latest AS (
SELECT *, row_number() OVER (
PARTITION BY doc_id ORDER BY version DESC, row_id DESC
) AS version_rank
FROM curation_documents
), ranked AS (
SELECT d.doc_id, l.label, d.score,
row_number() OVER (
PARTITION BY l.label ORDER BY d.score DESC, d.doc_id
) AS class_rank
FROM latest AS d
JOIN curation_labels AS l ON l.doc_id = d.doc_id
WHERE d.version_rank = 1
)
SELECT * FROM ranked WHERE rn <= 1000
SELECT doc_id, label, score
FROM ranked
WHERE class_rank <= 1
ORDER BY label;
```
That gives at most 1,000 rows per label, taking the highest-scoring ones. Order
by something arbitrary but stable if you want an unbiased draw instead, or
combine with [sampling](/curation/sampling) to randomize within each class.
The result is **document 10 for cat and document 20 for dog**. Change the cap
from `1` to the maximum number you want per class. A class with fewer records
cannot supply the full cap.
## Splitting a dataset {#splits}
This selects by score, not at random. For a random subset, use
[sampling](/curation/sampling) instead of ordering by score or ID.
`ntile` divides each partition into equal buckets, which is a train/test split
that stays balanced per class:
## Training and evaluation splits {#splits}
Ranking can divide ordered records into groups, but ordering by ID does not
make those groups random or representative. `ntile()` produces approximately
equal-sized buckets within a partition; small classes may not populate every
bucket.
Choose the split policy for the task. Keep related records, such as revisions
of one document or frames from one video, in the same split to avoid leakage.
A time-based evaluation may instead require a cutoff date. Save the final
assignments or resulting datasets so later updates do not silently change the
evaluation population.
## Compare neighboring records {#offset}
`lag()` and `lead()` read neighboring rows in the specified order. For example,
inspect how scores change across a document's records:
```sql
WITH split AS (
SELECT *, ntile(10) OVER (PARTITION BY label ORDER BY id) AS bucket
FROM images
)
SELECT * FROM split WHERE bucket <= 8 -- train
SELECT doc_id, row_id, score,
lag(score) OVER (
PARTITION BY doc_id ORDER BY version, row_id
) AS previous_score
FROM curation_documents
ORDER BY doc_id, version, row_id;
```
Buckets 9 and 10 are your validation and test sets. Because the partition is by
label, each split keeps the class proportions of the whole.
The first record in each document has no previous score. In this dataset,
row 2 follows a score of 0.80 and row 5 follows a score of 0.70.
<Warning>
Order by something stable. Ordering by a value that changes — a score you
recompute, a timestamp that gets updated — means the split changes with it, and
a row can move from train to test between runs.
</Warning>
<span id="running"></span>
## Window query performance {#cost}
## Comparing a row to its neighbours {#offset}
Window operations can require sorting. Filter out records you do not need
before ranking, and inspect the
[query plan](/search/explain) when cost matters.
`lag` and `lead` read the previous and next row in the window, which is how you
find gaps and changes without a self-join:
For cumulative counts, sums and other running summaries, use an aggregate
with an explicit window order and frame. The
[window reference](/sql/dql/window#window) covers those expressions.
```sql
SELECT
id,
created_at,
created_at - lag(created_at) OVER (ORDER BY created_at) AS gap
FROM events
```
## Continue with the curated data {#reference}
Useful for spotting ingestion gaps, duplicate timestamps, and runs of identical
values that suggest a broken upstream job.
## Running totals {#running}
Any [aggregate](/curation/aggregate) becomes a window function with `OVER`,
which is how cumulative coverage is expressed:
```sql
SELECT
label,
count(*) AS n,
sum(count(*)) OVER (ORDER BY count(*) DESC) AS cumulative
FROM images
GROUP BY label
```
Reading down the `cumulative` column tells you how many classes cover most of
your data — usually far fewer than you expect.
## Cost {#cost}
Window functions sort within each partition, so on a large table they are among
the more expensive things you can ask for. Filter first, partition on something
selective, and check the plan with
[Explain Analyze](/search/explain).
## Reference {#reference}
The full window syntax, framing, and function list are in
[Window functions](/sql/dql/window).
[Profile the selected records](/curation/aggregate) and
[save the result](/curation#next) for downstream use.
See [Window functions](/sql/dql/window) for the full syntax and function list.
docs/web/deployment/monitoring.mdx
+11 -9
View File
@@ -11,10 +11,10 @@ LanceDB emits internal metrics (currently object store request counts, bytes tra
The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper over LanceDB's `metrics` recorder; your application supplies and configures the OpenTelemetry SDK.
<Note>
This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side — see [Enterprise monitoring](#enterprise) below.
This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side. See [Enterprise monitoring](#enterprise) below.
</Note>
## What you get {#what-you-get}
## Available metrics {#what-you-get}
Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer:
@@ -22,8 +22,10 @@ Once instrumented, LanceDB registers one observable instrument per metric on you
|--------|------|-------------|
| `lance_object_store_requests_total` | Counter | Total object store requests, labelled by `operation` and `base` (store scheme). |
| `lance_object_store_request_duration_seconds` | Histogram | Request latency in seconds. |
| `lance_object_store_bytes_transferred_total` | Counter | Bytes read from or written to the store. |
| `lance_object_store_retryable_responses_total` | Counter | Requests that returned a retryable error (throttles, transient failures). |
| `lance_object_store_request_bytes_total` | Counter | Bytes read from or written to the store. |
| `lance_object_store_errors_total` | Counter | Object store requests that failed. |
| `lance_object_store_retryable_responses_total` | Counter | Retryable HTTP responses (5xx, 429 and 408) from a store reached over HTTP. |
| `lance_object_store_throttle_total` | Counter | Throttle responses (429 and 503) from a store reached over HTTP. |
| `lance_object_store_in_flight_requests` | Gauge | Currently outstanding object store requests. |
The recorder is process-global and pull-based: your configured `MetricReader` collects on its own schedule, so there is no hot-path overhead beyond the atomic aggregation that LanceDB does anyway.
@@ -37,7 +39,7 @@ The recorder is process-global and pull-based: your configured `MetricReader` co
Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects.
```bash
pip install "lancedb[otel]" opentelemetry-sdk
pip install "lancedb[otel]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc
```
Call `instrument_lancedb_metrics()` once at startup, before opening any tables. It returns `True` when the recorder is installed and instruments are registered.
@@ -70,7 +72,7 @@ If you omit `meter_provider`, LanceDB uses the global provider returned by `open
## TypeScript {#typescript}
The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export.
Install the OpenTelemetry SDK and exporter in your application:
```bash
npm install @opentelemetry/sdk-metrics @opentelemetry/exporter-metrics-otlp-grpc
@@ -93,7 +95,7 @@ const db = await connect("s3://my-bucket/lancedb");
`instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call.
## What to watch {#what-to-watch}
## Build dashboards and alerts {#what-to-watch}
A few starting points for dashboards and alerts:
@@ -106,7 +108,7 @@ A few starting points for dashboards and alerts:
<Badge color="red">Enterprise</Badge> A deployment emits its own metrics from
the server side, using Prometheus for collection and OpenTelemetry (OTel) to
export them with data enrichment -- no client-side instrumentation needed.
export them with data enrichment. No client-side instrumentation is needed.
| Component | Metrics |
| --- | --- |
@@ -116,7 +118,7 @@ export them with data enrichment -- no client-side instrumentation needed.
[Contact support](/deployment/support) for the comprehensive metric list and
help integrating them with your monitoring dashboard.
## Where to go next {#where-to-go-next}
## Related guides {#where-to-go-next}
<Columns cols={2}>
<Card title="Performance tips" icon="gauge-high" href="/search#performance">
docs/web/deployment/storage.mdx
+19 -6
View File
@@ -1,11 +1,24 @@
---
title: "Storage"
sidebarTitle: "Storage"
description: "Choosing and configuring where LanceDB stores its data -- how you deploy an embedded database."
title: "Embedded storage configuration"
sidebarTitle: "Embedded Storage"
description: "Choose and configure local disk or object storage for tables accessed by the embedded LanceDB OSS library."
icon: database
keywords: ["storage", "s3", "gcs", "azure", "object store", "configuration", "deploy"]
availability:
oss: available
enterprise: unavailable
summary: >-
The embedded library chooses and configures its storage backend from the
connection URI and `storage_options`. In LanceDB Enterprise, storage is part of
the deployment's own configuration instead.
---
This page is about the embedded library, where your connection chooses and
configures the storage backend. A LanceDB Enterprise deployment sets up its own
storage when it is deployed, so a `db://` connection passes none of these
options except an Azure storage account name. See
[Deployment](/enterprise/deployment) for how a deployment is set up.
LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs).
Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload.
@@ -36,7 +49,7 @@ LanceDB separates storage and compute and writes immutable fragments, making it
<Note>
**Concurrent writers on S3**
S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box — no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier.
S3 and S3 Express now support atomic writes natively, so LanceDB handles concurrent writers against the same table out-of-the-box: no external commit coordinator is required. Bucket-level [server-side encryption with KMS](/storage/configuration#server-side-encryption-with-kms) and [S3 Express One Zone](/storage/configuration#s3-express) are also supported on this tier.
</Note>
### 2\. File storage (EFS, GCS Filestore, Azure File) {#2-file-storage-efs-gcs-filestore-azure-file}
@@ -176,9 +189,9 @@ Table-level `storage_options` inherit every key from the connection and override
</CodeGroup>
<Tip>
**Inspect the effective options**
**Inspect a table's own options**
On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release.
On `AsyncTable`, `await table.initial_storage_options()` returns the options passed when the table was opened, and `await table.latest_storage_options()` returns them after any provider-driven refresh. Neither includes what the table inherits from its connection's `storage_options`, which still apply. The deprecated `table.storage_options()` method will be removed in a future release.
</Tip>
#### General object store options {#general-object-store-options}
docs/web/deployment/support.mdx
+3 -4
View File
@@ -9,11 +9,10 @@ keywords: ["support", "help", "discord", "community"]
## LanceDB OSS {#oss}
For a quick question, the
[Discord community](https://discord.gg/AUEWnJ7Txb) is fastest -- ask under the
relevant language channel for Python, TypeScript, or Rust, and you reach both
the community and the engineering team.
[Discord community](https://discord.gg/AUEWnJ7Txb) has channels for
Python, TypeScript and Rust where you can ask the community for help.
For anything worth keeping searchable for the next person, use GitHub instead:
Use GitHub for discussions and issues that others can search and follow:
| | |
| --- | --- |
docs/web/docs.json
+6 -12
View File
@@ -63,10 +63,6 @@
"source": "/feature-engineering/computed-columns",
"destination": "/feature-engineering/add-computed-column"
},
{
"source": "/feature-engineering/functions",
"destination": "/management/function"
},
{
"source": "/tables/create",
"destination": "/ingestion"
@@ -163,10 +159,6 @@
"source": "/storage/monitoring",
"destination": "/deployment/monitoring"
},
{
"source": "/enterprise/deployment",
"destination": "/enterprise/deployment/index"
},
{
"source": "/search/sql",
"destination": "/sql"
@@ -253,6 +245,8 @@
"group": "Getting Started",
"pages": [
"index",
"basics/offerings",
"basics/connect",
"quickstart",
"architecture",
"basics/sdk",
@@ -270,7 +264,8 @@
"basics/jobs",
"tables/consistency",
"basics/guarantees",
"basics/limits"
"basics/limits",
"basics/naming"
]
},
{
@@ -332,9 +327,9 @@
"management/namespace",
"management/table",
"tables/branching",
"tables/realtime-engine",
"management/materialized-view",
"management/view",
"management/function"
"management/view"
]
},
{
@@ -384,7 +379,6 @@
"sql/ddl/view",
"sql/ddl/materialized-view",
"sql/ddl/index",
"sql/ddl/function",
"sql/ddl/job"
],
"icon": "sitemap"
docs/web/embedding/index.mdx
+37 -3
View File
@@ -1,8 +1,15 @@
---
title: "Managing Embeddings"
sidebarTitle: "Managing Embeddings"
description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support."
description: "Use the embedding API in LanceDB: registry, functions, schemas, and multi-language SDK support."
icon: "bars"
availability:
oss: available
enterprise: available
summary: >-
Embedding functions run in your process, whether the table is local or on a
deployment, which stores their configuration but never runs them. Python
and TypeScript also embed query text; in Rust you embed queries yourself.
---
import {
@@ -30,8 +37,9 @@ the data, so vectors that are close to each other are considered similar.
LanceDB provides an embedding function registry in OSS as well as its Enterprise versions
([see below](#embeddings-in-lancedb-enterprise))
that automatically generates vector embeddings during data ingestion. Automatic query-time embedding
generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts
that automatically generates vector embeddings during data ingestion. Query text is embedded
automatically too, in the client, for local and Enterprise tables alike, with SDK-specific query
ergonomics. The API abstracts
embedding generation, allowing you to focus on your application logic.
## Embedding Registry {#embedding-registry}
@@ -41,6 +49,9 @@ Once configured, the embedding function will automatically generate embeddings w
into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly,
while Rust examples typically compute query embeddings explicitly before vector search.
In the Rust examples, replace `your-openai-api-key` with your OpenAI API key. In
application code, load credentials from environment variables or a secrets manager.
<CodeGroup>
<CodeBlock filename="Python" language="python" icon="python">
{PyOpenaiEmbeddings}
@@ -123,6 +134,29 @@ For non-sensitive settings such as inference device selection, you can also use
Find the full list of arguments for each provider in the [integrations](/integrations/embedding) section.
### Trusting remote code and custom endpoints {#trusting-remote-code-and-custom-endpoints}
In Python, the embedding definition is stored in the table, so anyone who can write to the table can change it.
To keep a table from running code or sending data where you didn't intend, two kinds of settings are only honored
when this process has approved them:
- `trust_remote_code=True`, which runs code from the model's Hugging Face repository.
- Custom endpoints such as OpenAI `base_url`, Ollama `host`, or watsonx `url`, which receive your data and API keys.
Embedding functions you create in the current process are approved automatically. When you open a table in a new
process, approve these settings explicitly before computing embeddings:
```python
from lancedb.embeddings import get_registry
registry = get_registry()
# Pinning a commit hash is recommended, since the code at a branch can change.
registry.allow_remote_code("nomic-ai/nomic-embed-text-v1.5", revision="<commit-hash>")
registry.allow_endpoint("http://my-ollama-host:11434")
```
Until then, computing embeddings raises `UntrustedEmbeddingConfigError`. Other operations on the table still work.
## Multiple embedding columns {#multiple-embedding-columns}
A single table can include more than one embedding definition when you want to store multiple semantic views

Some files were not shown because too many files have changed in this diff Show More