mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-16 16:52:34 +00:00
Merge main and retain sliced Arrow validity fix
This commit is contained in:
Gatefixer
@@ -0,0 +1,80 @@
|
||||
name: Setup protoc
|
||||
description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves
|
||||
|
||||
inputs:
|
||||
install:
|
||||
description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there."
|
||||
required: false
|
||||
default: "true"
|
||||
|
||||
outputs:
|
||||
version:
|
||||
description: "The pinned protoc version, e.g. 36.2"
|
||||
value: ${{ steps.pin.outputs.version }}
|
||||
linux-x86_64-sha256:
|
||||
description: "SHA-256 of the protoc linux-x86_64 zip"
|
||||
value: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
|
||||
linux-aarch_64-sha256:
|
||||
description: "SHA-256 of the protoc linux-aarch_64 zip"
|
||||
value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# protobuf does not publish checksum files for protoc. These are the digests
|
||||
# GitHub records for the release assets; when bumping, copy them from
|
||||
# `gh api repos/protocolbuffers/protobuf/releases/tags/v<version>`.
|
||||
- name: Set protoc pin
|
||||
id: pin
|
||||
shell: bash
|
||||
run: |
|
||||
{
|
||||
echo "version=36.2"
|
||||
echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278"
|
||||
echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a"
|
||||
echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b"
|
||||
echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
- name: Install protoc
|
||||
if: ${{ inputs.install == 'true' }}
|
||||
shell: bash
|
||||
env:
|
||||
PROTOC_VERSION: ${{ steps.pin.outputs.version }}
|
||||
RUNNER_OS_NAME: ${{ runner.os }}
|
||||
RUNNER_ARCH_NAME: ${{ runner.arch }}
|
||||
LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }}
|
||||
LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }}
|
||||
OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }}
|
||||
WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in
|
||||
Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;;
|
||||
Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;;
|
||||
macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;;
|
||||
Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;;
|
||||
*) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;;
|
||||
esac
|
||||
zip="$RUNNER_TEMP/protoc.zip"
|
||||
dest="$RUNNER_TEMP/protoc"
|
||||
curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip"
|
||||
# Hash stdin rather than a path: Windows paths contain backslashes,
|
||||
# which make sha256sum escape its output.
|
||||
if [ "$RUNNER_OS_NAME" = "macOS" ]; then
|
||||
actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1)
|
||||
else
|
||||
actual=$(sha256sum < "$zip" | cut -d' ' -f1)
|
||||
fi
|
||||
if [ "$actual" != "$sha256" ]; then
|
||||
echo "protoc checksum mismatch: expected $sha256, got $actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$RUNNER_OS_NAME" = "Windows" ]; then
|
||||
# Git Bash on Windows runners does not reliably ship unzip.
|
||||
7z x -y -o"$dest" "$zip" > /dev/null
|
||||
else
|
||||
unzip -q -o "$zip" -d "$dest"
|
||||
fi
|
||||
rm "$zip"
|
||||
echo "$dest/bin" >> "$GITHUB_PATH"
|
||||
"$dest/bin/protoc" --version
|
||||
@@ -29,6 +29,11 @@ inputs:
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Get protoc version
|
||||
id: protoc
|
||||
uses: ./.github/actions/setup-protoc
|
||||
with:
|
||||
install: "false"
|
||||
- name: CONFIRM ARM BUILD
|
||||
shell: bash
|
||||
env:
|
||||
@@ -54,13 +59,14 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
|
||||
target: x86_64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip
|
||||
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
@@ -71,14 +77,15 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}"
|
||||
target: aarch64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
yum install -y clang
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip
|
||||
echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c -
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
|
||||
@@ -12,10 +12,7 @@ inputs:
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Install macos dependency
|
||||
shell: bash
|
||||
run: |
|
||||
brew install protobuf
|
||||
- uses: ./.github/actions/setup-protoc
|
||||
- name: Build wheel
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
|
||||
@@ -12,15 +12,7 @@ inputs:
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Install Protoc v21.12
|
||||
working-directory: C:\
|
||||
run: |
|
||||
New-Item -Path 'C:\protoc' -ItemType Directory
|
||||
Set-Location C:\protoc
|
||||
Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
|
||||
7z x protoc.zip
|
||||
Add-Content $env:GITHUB_PATH "C:\protoc\bin"
|
||||
shell: powershell
|
||||
- uses: ./.github/actions/setup-protoc
|
||||
- name: Build wheel
|
||||
uses: PyO3/maturin-action@v1
|
||||
with:
|
||||
|
||||
@@ -11,6 +11,7 @@ on:
|
||||
- .github/workflows/build_linux_wheel/action.yml
|
||||
- .github/workflows/build_mac_wheel/action.yml
|
||||
- .github/workflows/build_windows_wheel/action.yml
|
||||
- .github/actions/setup-protoc/action.yml
|
||||
- Cargo.toml # Change in dependency frequently breaks builds
|
||||
- Cargo.lock
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ on:
|
||||
- .github/workflows/build_linux_wheel/**
|
||||
- .github/workflows/build_mac_wheel/**
|
||||
- .github/workflows/build_windows_wheel/**
|
||||
- .github/actions/setup-protoc/**
|
||||
- .github/workflows/run_tests/**
|
||||
|
||||
permissions:
|
||||
|
||||
+39
-1
@@ -94,7 +94,7 @@ what the capability is, and what the embedded form does instead. A separate
|
||||
private repository supplies the fuller version of each, which replaces the page
|
||||
at the same path when the published site is assembled.
|
||||
|
||||
Two things to know before editing:
|
||||
Four things to know before editing:
|
||||
|
||||
**Code examples are not written into the pages.** They live in real tests under
|
||||
`docs/web-tests/{py,ts,rs}`, are extracted into `docs/web/snippets/`, and are
|
||||
@@ -116,6 +116,44 @@ Enterprise pages attach their additions to the right section, so they are
|
||||
assigned once and never regenerated. Leave an existing anchor alone even when
|
||||
you reword the heading above it; only new headings need a new one.
|
||||
|
||||
**A page can declare where it applies.** Its frontmatter says whether LanceDB
|
||||
OSS and LanceDB Enterprise support the feature: use only `available` or
|
||||
`unavailable`, and one sentence that explains its scope. Availability describes
|
||||
product support, not whether a particular deployment has enabled or configured
|
||||
the feature. A supported feature stays `available` when it requires setup, a
|
||||
supported SDK or server version, or has differences between offerings. State
|
||||
those requirements in the summary or the relevant section, including unsupported
|
||||
subfeatures. Do not use `varies`, infer support from an unknown status, or mark an
|
||||
absent capability available. Narrow a page to the capability it actually covers
|
||||
when necessary. For example:
|
||||
|
||||
```yaml
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Requests to a deployment carry an API key or an OAuth token. Embedded
|
||||
LanceDB has no server to sign in to.
|
||||
```
|
||||
|
||||
The assembled site renders this as a label above the page's content and as a
|
||||
sidebar tag when only one offering has the feature. Do not also open the page
|
||||
with a badge of its own, and keep a qualification that only one section needs
|
||||
in that section's text. A `mint dev` preview of `docs/web` alone renders
|
||||
neither the labels nor the tags.
|
||||
|
||||
**A feature can be documented before its release.** List a draft page in
|
||||
`docs/web/.mintignore`, keep it out of the navigation in `docs.json`, and do not
|
||||
link to it from published pages. Wrap a draft section in an MDX comment,
|
||||
`{/* ... */}`, whose first line names the release, such as
|
||||
`Draft for the Feature X release.` Keep draft table rows and list items in a
|
||||
comment below the table or list, since a comment between rows or items breaks
|
||||
it. Drafts are hidden from readers but stay in the source, and availability does
|
||||
not change what is published. At release, remove the comments and the
|
||||
`.mintignore` entries, and restore the navigation and links. See
|
||||
[Writing docs before release](https://github.com/lancedb/docs#writing-docs-before-release)
|
||||
in the documentation repository's README for previews and release checks.
|
||||
|
||||
## Release process
|
||||
|
||||
For information on the release process, see: [release_process.md](release_process.md)
|
||||
|
||||
Generated
+49
-47
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5133,7 +5133,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "14.0.0-beta.15"
|
||||
version = "14.0.0-beta.18"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5155,7 +5155,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5169,7 +5169,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5178,8 +5178,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -5189,8 +5189,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5227,8 +5227,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5259,8 +5259,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5277,8 +5277,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5287,8 +5287,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5321,8 +5321,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5354,8 +5354,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-traits",
|
||||
@@ -5369,8 +5369,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5441,8 +5441,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5465,8 +5465,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5493,6 +5493,7 @@ dependencies = [
|
||||
"path_abs",
|
||||
"pin-project",
|
||||
"prost",
|
||||
"quick-xml 0.41.0",
|
||||
"rand 0.9.5",
|
||||
"reqsign-core",
|
||||
"reqsign-file-read-tokio",
|
||||
@@ -5508,8 +5509,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-jsonb"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"ethnum",
|
||||
@@ -5526,8 +5527,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5541,8 +5542,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5556,8 +5557,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5611,8 +5612,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5626,8 +5627,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5667,8 +5668,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5681,8 +5682,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "14.0.0-beta.15"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4"
|
||||
version = "14.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -5778,6 +5779,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"sha2 0.10.9",
|
||||
"snafu 0.8.9",
|
||||
"strum_macros 0.28.0",
|
||||
"tempfile",
|
||||
"test-log",
|
||||
"tokenizers",
|
||||
@@ -6818,8 +6820,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "object_store"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94"
|
||||
source = "git+https://github.com/apache/arrow-rs-object-store.git?rev=c10758c207c0103355f940f0c736fb8ecb157b04#c10758c207c0103355f940f0c736fb8ecb157b04"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
@@ -9445,6 +9446,7 @@ version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
|
||||
+20
-16
@@ -13,21 +13,21 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.94.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-jsonb = { version = "=14.0.0-beta.15", tag = "v14.0.0-beta.15", git = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { version = "=14.0.0-beta.15", path = "python/vendor/lance-arrow" }
|
||||
lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-jsonb = { version = "=14.0.0-beta.18", tag = "v14.0.0-beta.18", git = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { version = "=14.0.0-beta.18", path = "python/vendor/lance-arrow" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
@@ -73,7 +73,7 @@ oauth2 = { version = "5.0", default-features = false }
|
||||
regex = "1.10"
|
||||
semver = "1.0.25"
|
||||
serde = "1"
|
||||
serde_json = "1"
|
||||
serde_json = { version = "1", features = ["preserve_order"] }
|
||||
tempfile = "3.5.0"
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
|
||||
tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] }
|
||||
@@ -84,6 +84,10 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
# Carry the offset-safe validity fix until it is available in a Lance release.
|
||||
lance-arrow = { path = "python/vendor/lance-arrow" }
|
||||
|
||||
[patch.crates-io]
|
||||
# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision.
|
||||
object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" }
|
||||
|
||||
[profile.ci]
|
||||
debug = "line-tables-only"
|
||||
inherits = "dev"
|
||||
|
||||
@@ -78,6 +78,27 @@ ignore = [
|
||||
{ id = "RUSTSEC-2026-0098", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
||||
{ id = "RUSTSEC-2026-0099", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
||||
|
||||
# rustls 0.23.45: four advisories fixed in 0.23.46 (Debug impls leaking
|
||||
# TLS secrets, ECH rejection buffering, Brotli certificate decompression
|
||||
# allocation, ServerHello key-exchange downgrade). 0.23.46 was published
|
||||
# 2026-10-14 and the dependency age policy requires releases to be 48
|
||||
# hours old, so the upgrade cannot land yet. Remove these entries and run
|
||||
# `cargo update -p rustls` once the window has passed. Mirrors lance #9985.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0346
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0347
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0348
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0349
|
||||
{ id = "RUSTSEC-2026-0346", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
|
||||
{ id = "RUSTSEC-2026-0347", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
|
||||
{ id = "RUSTSEC-2026-0348", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
|
||||
{ id = "RUSTSEC-2026-0349", reason = "fixed in rustls 0.23.46; blocked on the 48-hour release age policy" },
|
||||
|
||||
# unsafe-libyaml: unmaintained. Transitive via lance-tokenizer → lindera
|
||||
# → serde_yaml_ng. No maintained upgrade exists; clearing it requires
|
||||
# lindera to move off serde_yaml_ng.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2024-0449
|
||||
{ id = "RUSTSEC-2024-0449", reason = "transitive via lindera (lance-tokenizer); no maintained upgrade available" },
|
||||
|
||||
# rustls-webpki 0.101.7: reachable panic in CRL parsing. Same legacy
|
||||
# rustls 0.21 chain from aws-smithy-http-client as above. The 0.103 line
|
||||
# we actively use is upgraded to 0.103.13 which contains the fix.
|
||||
@@ -197,4 +218,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
# releases are cut to crates.io. Allow that specific host.
|
||||
allow-git = [
|
||||
"https://github.com/lance-format/lance",
|
||||
# Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml).
|
||||
"https://github.com/apache/arrow-rs-object-store",
|
||||
]
|
||||
|
||||
@@ -143,7 +143,8 @@ containing the new version number of the table after altering the columns.
|
||||
abstract blobColumns(): Promise<string[]>
|
||||
```
|
||||
|
||||
Blob v2 columns, including nested dotted paths.
|
||||
Blob v2 columns addressable by row ID, including struct-nested dotted paths.
|
||||
Blobs inside lists are omitted because a row ID does not identify an element.
|
||||
|
||||
#### Returns
|
||||
|
||||
@@ -527,6 +528,8 @@ table's current checkout.
|
||||
|
||||
Preserves input order, duplicates, and nulls. Use this for large payloads.
|
||||
See [Table.fetchBlobs](Table.md#fetchblobs) for row-ID validity across versions.
|
||||
One missing or deleted row ID rejects the entire batch, including on remote
|
||||
tables; no partial handles are returned.
|
||||
|
||||
#### Parameters
|
||||
|
||||
@@ -556,6 +559,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
|
||||
Reads the table's current checkout. IDs from another version can fail after
|
||||
compaction unless stable row ids are enabled. Results keep input order and
|
||||
duplicates. Null blobs are `null`. Empty blobs are empty buffers.
|
||||
One missing or deleted row ID rejects the entire batch, including on remote
|
||||
tables; no partial results are returned.
|
||||
Blobs inside lists cannot be fetched by row ID.
|
||||
|
||||
Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
|
||||
The client splits requests automatically and reads an individual larger
|
||||
blob through the Range route. This method still materializes all bytes in
|
||||
|
||||
@@ -14,6 +14,8 @@ Declares a `lance.blob.v2` column.
|
||||
|
||||
Query results are descriptors, not payload bytes. Use [Table.fetchBlobs](../classes/Table.md#fetchblobs)
|
||||
or [Table.fetchBlobFiles](../classes/Table.md#fetchblobfiles) to read bytes.
|
||||
Blobs inside lists can be written and queried as descriptors, but cannot
|
||||
currently be fetched because a row ID does not identify a list element.
|
||||
|
||||
## Parameters
|
||||
|
||||
|
||||
@@ -8,6 +8,18 @@
|
||||
|
||||
## Properties
|
||||
|
||||
### blobRequestConcurrency?
|
||||
|
||||
```ts
|
||||
optional blobRequestConcurrency: number;
|
||||
```
|
||||
|
||||
Maximum number of concurrent blob HTTP requests across all tables and
|
||||
blob handles on this connection. Defaults to 8. Can also be set with
|
||||
`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
|
||||
|
||||
***
|
||||
|
||||
### extraHeaders?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -48,6 +48,169 @@ are ordinary connections. Dropping a database requires it to be empty.
|
||||
::: lancedb.catalog.DatabaseNames
|
||||
|
||||
|
||||
## Authorization
|
||||
|
||||
Some catalogs support the authorization API. This is an API that can be used to
|
||||
bind access control lists (ACLs) to individual catalog objects.
|
||||
|
||||
Each ACL contains a subject, an object, and privilege.
|
||||
|
||||
### Subjects
|
||||
|
||||
There are three kinds of subjects: principals, groups, and roles.
|
||||
|
||||
A principal uniquely identifies each user accessing the system. Each LanceDB
|
||||
Enterprise API key corresponds to a unique principal, and each OpenID user has
|
||||
a unique principal.
|
||||
|
||||
All principals have an immutable ID which never changes, and is safe to include
|
||||
in log files. For example, the API key
|
||||
`sk_AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPQ====` maps to the
|
||||
principal ID `pk_aaaqeayeaudaocajbifqydiob4`. APIs such as `list_acls` will
|
||||
return the principal ID, never the API key.
|
||||
|
||||
OpenID principals also typically have a human-readable name associated with
|
||||
them. For example, a principal might be named `bob@example.com` and might have
|
||||
a principal ID of `8a004400-9443-4fab-9997-b9372a7ca8fd`. Whenever
|
||||
possible, APIs will return the principal name for convenience. However,
|
||||
principal names are mutable and can be changed by the identity provider.
|
||||
|
||||
Here is an example of getting information about an API key:
|
||||
```python
|
||||
from lancedb.authz import Subject
|
||||
|
||||
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
|
||||
principal = catalog.authz.get_principal(Subject.principal_api_key(api_key))
|
||||
print(f"Found principal {principal.principal_id}")
|
||||
```
|
||||
|
||||
OpenID principals can be members of groups. Like principals, groups also have
|
||||
IDs and human-readable names. It is not possible to modify group membership via
|
||||
this API; it is managed by the OpenID identity provider.
|
||||
|
||||
Here is an example of getting information about a group based on its name:
|
||||
```python
|
||||
from lancedb.authz import Subject
|
||||
|
||||
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
|
||||
group = catalog.authz.get_group(Subject.group_name("Engineering"))
|
||||
print(f"Found group {group.group_id} with name {group.group_name}")
|
||||
```
|
||||
|
||||
Roles can contain either principals or groups. Unlike groups, they are managed
|
||||
by the catalog, not the identity provider. Roles can also contain API key
|
||||
principals.
|
||||
|
||||
Here is an example of managing the `reader` role:
|
||||
```python
|
||||
from lancedb.authz import Subject
|
||||
|
||||
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
|
||||
catalog.authz.add_role_binding(subject=Subject.principal_name("Alice"), role="reader")
|
||||
```
|
||||
It may take a few milliseconds for changes to role bindings to become visible
|
||||
to all clients.
|
||||
|
||||
More subject types may be added in the future.
|
||||
|
||||
### Objects
|
||||
|
||||
An object is any resource in the system that we can attach authorization rules to.
|
||||
The Python builders and Rust client share resource validation and wire encoding.
|
||||
Methods also accept literal strings such as `table:tenant/db:public$events`;
|
||||
malformed names for known resource types raise `ValueError` locally before a
|
||||
request is sent. Unknown resource types are preserved for compatibility with
|
||||
newer servers.
|
||||
|
||||
Object types include:
|
||||
|
||||
The System object, a singleton which represents the entire system. You can
|
||||
refer to it as `Object.system()`.
|
||||
|
||||
Database objects which represents specific databases. For example,
|
||||
`Object.database("analytics")` represents the "analytics" database.
|
||||
|
||||
Table objects which represent specific tables. For example,
|
||||
`Object.table(database="analytics", name="events")` represents the "events"
|
||||
table inside the "analytics" database. Tables are in the default namespace (which
|
||||
is named "public") unless a specific namespace is given.
|
||||
|
||||
Similarly, View objects represent specific views. For example,
|
||||
`Object.view(database="analytics", name="recent_events")` represents the
|
||||
"recent_events" view inside the "analytics" database. Views are in the default
|
||||
namespace (which is named "public") unless a specific namespace is given.
|
||||
|
||||
Secret objects identify named credentials, for example,
|
||||
`Object.secret(database="analytics", name="api-key")`. Function objects identify
|
||||
registered functions, for example,
|
||||
`Object.function(database="analytics", name="caption")`. Both default to the
|
||||
`public` namespace. Pass `namespace_path=["platform", "ml"]` to select a nested
|
||||
namespace. These objects identify resources for ACL operations; a secret object
|
||||
contains the credential's name, not its value.
|
||||
|
||||
More object types may be added in the future.
|
||||
|
||||
### Access control lists
|
||||
|
||||
An access control list attaches a specific set of privileges to a specific
|
||||
(subject, object) pair. So, for example, we might have an access control list
|
||||
that gives `bob@example.com` the ability to read the foo table, that looks like
|
||||
this:
|
||||
|
||||
```python
|
||||
from lancedb.authz import Object, Subject, Privilege
|
||||
|
||||
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
|
||||
catalog.authz.add_acl(
|
||||
object=Object.table(database="analytics", name="foo"),
|
||||
subject=Subject.principal_name("bob@example.com"),
|
||||
privilege=Privilege.SELECT,
|
||||
)
|
||||
```
|
||||
|
||||
Reading a table generally requires USAGE on its database and namespace, plus
|
||||
SELECT on the table. Each object has at most one owner.
|
||||
|
||||
OWNERSHIP is a special privilege which grants all the other privileges. By
|
||||
default, the principal who created an object owns it. Only one subject can own
|
||||
an object; granting OWNERSHIP to a new subject removes it from the previous
|
||||
owner. OWNERSHIP cannot be deleted.
|
||||
|
||||
To delegate secret or function creation, grant `CREATE_SECRET` or
|
||||
`CREATE_FUNCTION` on the containing namespace. This preserves namespace ownership:
|
||||
|
||||
```python
|
||||
for privilege in [Privilege.CREATE_SECRET, Privilege.CREATE_FUNCTION]:
|
||||
catalog.authz.add_acl(
|
||||
object=Object.namespace(database="analytics", namespace_path=["public"]),
|
||||
subject=Subject.role("developer"),
|
||||
privilege=privilege,
|
||||
)
|
||||
```
|
||||
|
||||
Both privileges also accept their string names in synchronous and asynchronous
|
||||
ACL methods.
|
||||
|
||||
By default, the `list_acls` function lists all the access control entries in
|
||||
the system. By adding a function parameter, the output is filtered to just ACLs
|
||||
matching that parameter. For example, this code lists all the ACLs for the
|
||||
"reader" role:
|
||||
|
||||
```python
|
||||
from lancedb.authz import Subject
|
||||
|
||||
catalog = lancedb.connect_catalog("https://catalog.example.com", api_key=api_key)
|
||||
for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
|
||||
print(entry)
|
||||
```
|
||||
|
||||
Or, when using the async API:
|
||||
```python
|
||||
catalog = await lancedb.connect_catalog_async(endpoint, api_key=api_key)
|
||||
async for entry in catalog.authz.list_acls(subject=Subject.role("reader")):
|
||||
print(entry)
|
||||
```
|
||||
|
||||
## Remote SQL
|
||||
|
||||
Submit SQL against a remote LanceDB database through the connection.
|
||||
@@ -342,6 +505,12 @@ pylance, LanceDB uses a matching `lance.blob.v2` extension type so blob columns
|
||||
still work. Queries return descriptors. Call
|
||||
[`fetch_blob_files`][lancedb.table.Table.fetch_blob_files] for lazy reads or
|
||||
[`fetch_blobs`][lancedb.table.Table.fetch_blobs] for eager bytes.
|
||||
One missing or deleted row ID rejects the entire blob-read batch; the APIs do
|
||||
not return partial results.
|
||||
|
||||
Blobs inside lists can be written and queried as descriptors. They are omitted
|
||||
from `blob_columns()` because row ids do not identify individual list elements,
|
||||
so the fetch APIs cannot read their bytes yet.
|
||||
|
||||
::: lancedb.blob
|
||||
|
||||
@@ -359,6 +528,8 @@ still work. Queries return descriptors. Call
|
||||
|
||||
## Exceptions
|
||||
|
||||
::: lancedb.remote.errors.HttpError
|
||||
|
||||
::: lancedb.exceptions.MissingValueError
|
||||
|
||||
::: lancedb.exceptions.MissingColumnError
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Reader examples with explicit placeholders.
|
||||
|
||||
These functions are documentation source. test_provider_examples.py runs the
|
||||
reranker examples offline, with stand-ins for the model and the provider, to
|
||||
check that each one writes only to its own new directory. Corresponding opt-in
|
||||
integration tests remain in test_integrations.py and retain real credential
|
||||
loading. Replace placeholders before running these examples.
|
||||
"""
|
||||
|
||||
|
||||
def embedding_ibm_usage():
|
||||
# --8<-- [start:embedding_ibm_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import EmbeddingFunctionRegistry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
registry = EmbeddingFunctionRegistry.get_instance()
|
||||
registry.set_var("watsonx-api-key", "your-api-key")
|
||||
watsonx_embed = registry.get("watsonx").create(
|
||||
name="ibm/slate-125m-english-rtrvr",
|
||||
api_key="$var:watsonx-api-key",
|
||||
project_id="your-project-id",
|
||||
)
|
||||
|
||||
class TextModel(LanceModel):
|
||||
text: str = watsonx_embed.SourceField()
|
||||
vector: Vector(watsonx_embed.ndims()) = watsonx_embed.VectorField()
|
||||
|
||||
data = [
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-demo"))
|
||||
tbl = db.create_table("watsonx_test", schema=TextModel, mode="overwrite")
|
||||
tbl.add(data)
|
||||
|
||||
rs = tbl.search("hello").limit(1).to_pandas()
|
||||
print(rs.head())
|
||||
# --8<-- [end:embedding_ibm_usage]
|
||||
|
||||
|
||||
def embedding_jina_text():
|
||||
# --8<-- [start:embedding_jina_text]
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import EmbeddingFunctionRegistry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
os.environ["JINA_API_KEY"] = "your-api-key"
|
||||
|
||||
jina_embed = (
|
||||
EmbeddingFunctionRegistry.get_instance()
|
||||
.get("jina")
|
||||
.create(name="jina-embeddings-v2-base-en")
|
||||
)
|
||||
|
||||
class TextModel(LanceModel):
|
||||
text: str = jina_embed.SourceField()
|
||||
vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
|
||||
|
||||
data = [{"text": "hello world"}, {"text": "goodbye world"}]
|
||||
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-text"))
|
||||
tbl = db.create_table("test", schema=TextModel, mode="overwrite")
|
||||
|
||||
tbl.add(data)
|
||||
# --8<-- [end:embedding_jina_text]
|
||||
|
||||
|
||||
def embedding_jina_multimodal():
|
||||
# --8<-- [start:embedding_jina_multimodal]
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
import pandas as pd
|
||||
import requests
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
os.environ["JINA_API_KEY"] = "your-api-key"
|
||||
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-images"))
|
||||
func = get_registry().get("jina").create()
|
||||
|
||||
class Images(LanceModel):
|
||||
label: str
|
||||
image_uri: str = func.SourceField()
|
||||
image_bytes: bytes = func.SourceField()
|
||||
vector: Vector(func.ndims()) = func.VectorField()
|
||||
vec_from_bytes: Vector(func.ndims()) = func.VectorField()
|
||||
|
||||
table = db.create_table("images", schema=Images)
|
||||
labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
|
||||
uris = [
|
||||
"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
|
||||
"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
|
||||
"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
|
||||
"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
|
||||
"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
|
||||
"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
|
||||
]
|
||||
image_bytes = [requests.get(uri).content for uri in uris]
|
||||
table.add(
|
||||
pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
|
||||
)
|
||||
# --8<-- [end:embedding_jina_multimodal]
|
||||
|
||||
|
||||
def reranking_cohere_usage():
|
||||
# --8<-- [start:reranking_cohere_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import CohereReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cohere-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
vector: Vector(embedder.ndims()) = embedder.VectorField()
|
||||
|
||||
data = [
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = CohereReranker(api_key="your-api-key")
|
||||
|
||||
# Run vector search with a reranker
|
||||
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
print(vector_results)
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
fts_results = (
|
||||
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(fts_results)
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
hybrid_results = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(hybrid_results)
|
||||
# --8<-- [end:reranking_cohere_usage]
|
||||
|
||||
|
||||
def reranking_jina_usage():
|
||||
# --8<-- [start:reranking_jina_usage]
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import JinaReranker
|
||||
|
||||
os.environ["JINA_API_KEY"] = "your-api-key"
|
||||
embedder = get_registry().get("jina").create()
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "jina-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
vector: Vector(embedder.ndims()) = embedder.VectorField()
|
||||
|
||||
data = [
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = JinaReranker(api_key="your-api-key")
|
||||
|
||||
# Run vector search with a reranker
|
||||
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
print(vector_results)
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
fts_results = (
|
||||
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(fts_results)
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
hybrid_results = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(hybrid_results)
|
||||
# --8<-- [end:reranking_jina_usage]
|
||||
|
||||
|
||||
def reranking_watsonx_usage():
|
||||
# --8<-- [start:reranking_watsonx_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import WatsonxReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "watsonx-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
vector: Vector(embedder.ndims()) = embedder.VectorField()
|
||||
|
||||
data = [
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
|
||||
# Supply your API key and project ID (or use space_id instead of project_id).
|
||||
reranker = WatsonxReranker(
|
||||
api_key="your-api-key",
|
||||
project_id="your-project-id",
|
||||
)
|
||||
|
||||
# Run vector search with a reranker
|
||||
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
print(vector_results)
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
fts_results = (
|
||||
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(fts_results)
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
hybrid_results = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(hybrid_results)
|
||||
# --8<-- [end:reranking_watsonx_usage]
|
||||
|
||||
|
||||
def reranking_openai_usage():
|
||||
# --8<-- [start:reranking_openai_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import OpenaiReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "openai-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
vector: Vector(embedder.ndims()) = embedder.VectorField()
|
||||
|
||||
data = [
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = OpenaiReranker(api_key="your-api-key")
|
||||
|
||||
# Run vector search with a reranker
|
||||
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
print(vector_results)
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
fts_results = (
|
||||
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(fts_results)
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
hybrid_results = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(hybrid_results)
|
||||
# --8<-- [end:reranking_openai_usage]
|
||||
|
||||
|
||||
def reranking_voyageai_usage():
|
||||
# --8<-- [start:reranking_voyageai_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import VoyageAIReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "voyageai-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
vector: Vector(embedder.ndims()) = embedder.VectorField()
|
||||
|
||||
data = [
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = VoyageAIReranker(api_key="your-api-key", model_name="rerank-2")
|
||||
|
||||
# Run vector search with a reranker
|
||||
vector_results = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
print(vector_results)
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
fts_results = (
|
||||
tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(fts_results)
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
hybrid_results = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
print(hybrid_results)
|
||||
# --8<-- [end:reranking_voyageai_usage]
|
||||
@@ -4,6 +4,8 @@
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_connection():
|
||||
# --8<-- [start:connect]
|
||||
@@ -29,19 +31,16 @@ async def connect_async_example():
|
||||
|
||||
|
||||
def connect_enterprise_quickstart_config():
|
||||
# --8<-- [start:connect_enterprise_quickstart]
|
||||
import os
|
||||
|
||||
import lancedb
|
||||
|
||||
# --8<-- [start:connect_enterprise_quickstart]
|
||||
uri = "db://your-database-uri"
|
||||
api_key = "your-api-key"
|
||||
region = "us-east-1"
|
||||
host_override = "https://your-enterprise-endpoint.com"
|
||||
|
||||
db = lancedb.connect(
|
||||
uri=uri,
|
||||
api_key=api_key,
|
||||
region=region,
|
||||
host_override=host_override,
|
||||
uri=os.environ["LANCEDB_URI"], # db://your-database
|
||||
api_key=os.environ["LANCEDB_API_KEY"],
|
||||
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
|
||||
host_override=os.environ["LANCEDB_HOST_OVERRIDE"], # https://your-endpoint
|
||||
)
|
||||
# --8<-- [end:connect_enterprise_quickstart]
|
||||
return db
|
||||
@@ -57,17 +56,29 @@ def test_connect_enterprise_quickstart(monkeypatch):
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(lancedb, "connect", fake_connect)
|
||||
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
|
||||
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
|
||||
monkeypatch.delenv("LANCEDB_REGION", raising=False)
|
||||
monkeypatch.setenv("LANCEDB_HOST_OVERRIDE", "https://your-endpoint")
|
||||
|
||||
db = connect_enterprise_quickstart_config()
|
||||
assert db is not None
|
||||
assert connect_enterprise_quickstart_config() is not None
|
||||
assert captured == {
|
||||
"uri": "db://your-database-uri",
|
||||
"uri": "db://your-database",
|
||||
"api_key": "your-api-key",
|
||||
"region": "us-east-1",
|
||||
"host_override": "https://your-enterprise-endpoint.com",
|
||||
"host_override": "https://your-endpoint",
|
||||
}
|
||||
|
||||
|
||||
def test_connect_enterprise_quickstart_requires_an_endpoint(monkeypatch):
|
||||
monkeypatch.setenv("LANCEDB_URI", "db://your-database")
|
||||
monkeypatch.setenv("LANCEDB_API_KEY", "your-api-key")
|
||||
monkeypatch.delenv("LANCEDB_HOST_OVERRIDE", raising=False)
|
||||
|
||||
with pytest.raises(KeyError, match="LANCEDB_HOST_OVERRIDE"):
|
||||
connect_enterprise_quickstart_config()
|
||||
|
||||
|
||||
def connect_object_storage_config():
|
||||
# --8<-- [start:connect_object_storage]
|
||||
import lancedb
|
||||
@@ -105,14 +116,26 @@ def namespace_table_ops_example():
|
||||
mode="create", # use "overwrite" only if you want to replace existing table
|
||||
)
|
||||
|
||||
# Verify
|
||||
print(db.list_namespaces()) # ['prod']
|
||||
print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search']
|
||||
print(db.list_tables(namespace_path=["prod", "search"])) # ['user']
|
||||
print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user']
|
||||
# Verify: listings return a response with the names and a page token
|
||||
print(db.list_namespaces().namespaces) # ['prod']
|
||||
print(db.list_namespaces(namespace_path=["prod"]).namespaces)
|
||||
# ['recommendations', 'search']
|
||||
print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user']
|
||||
print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user']
|
||||
# --8<-- [end:namespace_table_ops]
|
||||
|
||||
|
||||
def test_namespace_table_ops(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
namespace_table_ops_example()
|
||||
assert capsys.readouterr().out.splitlines() == [
|
||||
"['prod']",
|
||||
"['recommendations', 'search']",
|
||||
"['user']",
|
||||
"['user']",
|
||||
]
|
||||
|
||||
|
||||
def namespace_admin_ops_example():
|
||||
# --8<-- [start:namespace_admin_ops]
|
||||
import lancedb
|
||||
@@ -124,8 +147,8 @@ def namespace_admin_ops_example():
|
||||
db.create_namespace(["prod", "search"])
|
||||
|
||||
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
|
||||
print(f"Child namespaces under {namespace}: {child_namespaces}")
|
||||
# Child namespaces under ['prod', 'search']: ['search']
|
||||
print(f"Child namespaces under ['prod']: {child_namespaces}")
|
||||
# Child namespaces under ['prod']: ['search']
|
||||
|
||||
metadata = db.describe_namespace(["prod", "search"])
|
||||
print(f"Metadata for namespace {namespace}: {metadata}")
|
||||
@@ -134,7 +157,17 @@ def namespace_admin_ops_example():
|
||||
db.drop_namespace(["prod", "search"], mode="skip")
|
||||
db.drop_namespace(["prod"], mode="skip")
|
||||
# --8<-- [end:namespace_admin_ops]
|
||||
return child_namespaces, metadata
|
||||
return db, child_namespaces, metadata
|
||||
|
||||
|
||||
def test_namespace_admin_ops(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
db, child_namespaces, metadata = namespace_admin_ops_example()
|
||||
assert child_namespaces == ["search"]
|
||||
assert capsys.readouterr().out.splitlines()[0] == (
|
||||
"Child namespaces under ['prod']: ['search']"
|
||||
)
|
||||
assert db.list_namespaces().namespaces == []
|
||||
|
||||
async def connect_object_storage_config_async():
|
||||
# --8<-- [start:connect_object_storage_async]
|
||||
@@ -146,3 +179,32 @@ async def connect_object_storage_config_async():
|
||||
# --8<-- [end:connect_object_storage_async]
|
||||
|
||||
return async_db
|
||||
|
||||
|
||||
def test_connect_enterprise_placeholders(monkeypatch):
|
||||
import lancedb
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_connect(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(lancedb, "connect", fake_connect)
|
||||
# --8<-- [start:connect_enterprise_placeholders]
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect(
|
||||
uri="db://your-database",
|
||||
api_key="your-api-key",
|
||||
region="us-east-1",
|
||||
host_override="https://your-endpoint",
|
||||
)
|
||||
# --8<-- [end:connect_enterprise_placeholders]
|
||||
assert db is not None
|
||||
assert captured == {
|
||||
"uri": "db://your-database",
|
||||
"api_key": "your-api-key",
|
||||
"region": "us-east-1",
|
||||
"host_override": "https://your-endpoint",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
|
||||
def test_sample_dataset():
|
||||
# --8<-- [start:sample_dataset]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
from lancedb.permutation import Permutation, permutation_builder
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-sampling-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table("records", [{"id": i} for i in range(10)])
|
||||
splits = (
|
||||
permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()
|
||||
)
|
||||
sample = Permutation.from_tables(table, splits, split=0).with_format("arrow")
|
||||
selected_ids = []
|
||||
for batch in sample.iter(batch_size=3):
|
||||
selected_ids.extend(batch.column("id").to_pylist())
|
||||
print(f"Selected {len(selected_ids)} rows: {selected_ids}")
|
||||
# --8<-- [end:sample_dataset]
|
||||
|
||||
assert len(selected_ids) == 3
|
||||
assert len(set(selected_ids)) == 3
|
||||
assert all(0 <= value < 10 for value in selected_ids)
|
||||
@@ -4,16 +4,18 @@
|
||||
"""Examples for docs/web/management/function.mdx.
|
||||
|
||||
Registering a Function and filling its column need a LanceDB Enterprise
|
||||
deployment, so `test_function_enterprise` runs only when the connection
|
||||
variables the Enterprise examples already use are set: LANCEDB_URI,
|
||||
LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the
|
||||
deployment needs one. The walkthrough calls its table `function_demo` and its
|
||||
Function `double`; the test creates both under names unique to its run, never
|
||||
replaces an existing object, and drops only what it created.
|
||||
deployment, so `test_function_enterprise` runs only when
|
||||
LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the
|
||||
Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and
|
||||
LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The
|
||||
walkthrough calls its table `function_demo` and its Function `double`; the test
|
||||
creates both under names unique to its run, never replaces an existing object,
|
||||
and drops only what it created.
|
||||
|
||||
The other tests run locally and check the client side of the same code: what
|
||||
`@udf` sends for registration, how a registered version binds a column, and
|
||||
where a local connection stops. They do not run a Function remotely.
|
||||
`@udf` sends for registration, how a registered version binds a column, where a
|
||||
local connection stops, and when the Enterprise test connects. They do not run
|
||||
a Function remotely.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -70,12 +72,17 @@ def registered_version(definition) -> FunctionVersion:
|
||||
return FunctionVersion.from_json(json.dumps(reply))
|
||||
|
||||
|
||||
def require_env(var_name: str) -> str:
|
||||
"""Skip the test unless the required environment variable is provided."""
|
||||
value = os.environ.get(var_name)
|
||||
if not value:
|
||||
pytest.skip(f"Set {var_name} to run this Enterprise example")
|
||||
return value
|
||||
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
|
||||
# The connection settings alone never send this test to a deployment.
|
||||
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
|
||||
|
||||
|
||||
def require_deployment():
|
||||
if os.environ.get(OPT_IN) != "1":
|
||||
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
|
||||
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
|
||||
if missing:
|
||||
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
|
||||
|
||||
|
||||
def test_function_runs_locally_as_python():
|
||||
@@ -160,9 +167,7 @@ def test_function_version_binds_a_column(tmp_path):
|
||||
|
||||
|
||||
def test_function_enterprise(monkeypatch):
|
||||
require_env("LANCEDB_URI")
|
||||
require_env("LANCEDB_API_KEY")
|
||||
require_env("LANCEDB_HOST_OVERRIDE")
|
||||
require_deployment()
|
||||
|
||||
# --8<-- [start:function_connect]
|
||||
import os
|
||||
@@ -240,3 +245,91 @@ def test_function_enterprise(monkeypatch):
|
||||
db.drop_table(table_name)
|
||||
if version is not None:
|
||||
db.drop_function(version.name, version=version.version)
|
||||
|
||||
|
||||
class ReachedConnection(Exception):
|
||||
pass
|
||||
|
||||
|
||||
SETTINGS = {
|
||||
"LANCEDB_URI": "db://docs-test",
|
||||
"LANCEDB_HOST_OVERRIDE": "https://deployment.invalid",
|
||||
"LANCEDB_API_KEY": "docs-test-key",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def connect_attempts(monkeypatch):
|
||||
"""`lancedb.connect`, replaced so that it records each call and stops."""
|
||||
attempts = []
|
||||
|
||||
def connect(**kwargs):
|
||||
attempts.append(kwargs)
|
||||
raise ReachedConnection
|
||||
|
||||
monkeypatch.setattr(lancedb, "connect", connect)
|
||||
for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
return attempts
|
||||
|
||||
|
||||
def enterprise_outcome(monkeypatch):
|
||||
try:
|
||||
test_function_enterprise(monkeypatch)
|
||||
except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome:
|
||||
return outcome
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"opt_in",
|
||||
[None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"],
|
||||
)
|
||||
def test_function_enterprise_needs_the_exact_opt_in(
|
||||
monkeypatch, connect_attempts, opt_in
|
||||
):
|
||||
for name, value in SETTINGS.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
if opt_in is not None:
|
||||
monkeypatch.setenv(OPT_IN, opt_in)
|
||||
outcome = enterprise_outcome(monkeypatch)
|
||||
assert isinstance(outcome, pytest.skip.Exception), outcome
|
||||
assert str(outcome).startswith(f"set {OPT_IN}=1 ")
|
||||
assert connect_attempts == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blank", ["unset", "empty"])
|
||||
@pytest.mark.parametrize(
|
||||
"missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)]
|
||||
)
|
||||
def test_function_enterprise_fails_when_asked_without_settings(
|
||||
monkeypatch, connect_attempts, missing, blank
|
||||
):
|
||||
monkeypatch.setenv(OPT_IN, "1")
|
||||
for name, value in SETTINGS.items():
|
||||
if name not in missing:
|
||||
monkeypatch.setenv(name, value)
|
||||
elif blank == "empty":
|
||||
monkeypatch.setenv(name, "")
|
||||
outcome = enterprise_outcome(monkeypatch)
|
||||
assert isinstance(outcome, pytest.fail.Exception), outcome
|
||||
assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}"
|
||||
assert connect_attempts == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("region", [None, "eu-west-1"])
|
||||
def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region):
|
||||
monkeypatch.setenv(OPT_IN, "1")
|
||||
for name, value in SETTINGS.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
if region:
|
||||
monkeypatch.setenv("LANCEDB_REGION", region)
|
||||
outcome = enterprise_outcome(monkeypatch)
|
||||
assert isinstance(outcome, ReachedConnection), outcome
|
||||
assert connect_attempts == [
|
||||
{
|
||||
"uri": "db://docs-test",
|
||||
"api_key": "docs-test-key",
|
||||
"region": region or "us-east-1",
|
||||
"host_override": "https://deployment.invalid",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,739 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
|
||||
def test_create_table_from_dicts():
|
||||
# --8<-- [start:create_table_from_dicts]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
data = [
|
||||
{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},
|
||||
{"vector": [0.2, 1.8], "lat": 40.1, "long": -74.1},
|
||||
]
|
||||
db.create_table("test_table", data, mode="overwrite")
|
||||
tbl = db["test_table"]
|
||||
tbl.head()
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_from_dicts]
|
||||
assert tbl.count_rows() == 2
|
||||
|
||||
|
||||
def test_create_table_conflict_handling():
|
||||
# --8<-- [start:create_table_conflict_handling]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
|
||||
db.create_table("conflict_table", data)
|
||||
# Idempotent open: reuse the existing table if it exists.
|
||||
# The provided data is ignored; the schema is validated against the
|
||||
# existing table and a mismatch raises an error.
|
||||
tbl = db.create_table("conflict_table", data, exist_ok=True)
|
||||
|
||||
# Overwrite: replace the contents with the provided data.
|
||||
tbl = db.create_table("conflict_table", data, mode="overwrite")
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_conflict_handling]
|
||||
|
||||
|
||||
def test_create_table_custom_schema():
|
||||
# --8<-- [start:create_table_custom_schema]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
custom_schema = pa.schema(
|
||||
[
|
||||
pa.field("vector", pa.list_(pa.float32(), 4)),
|
||||
pa.field("lat", pa.float32()),
|
||||
pa.field("long", pa.float32()),
|
||||
]
|
||||
)
|
||||
|
||||
data = [
|
||||
{"vector": [1.1, 1.2, 1.3, 1.4], "lat": 45.5, "long": -122.7},
|
||||
{"vector": [0.2, 1.8, 0.4, 3.6], "lat": 40.1, "long": -74.1},
|
||||
]
|
||||
tbl = db.create_table(
|
||||
"my_table_custom_schema", data, schema=custom_schema, mode="overwrite"
|
||||
)
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_custom_schema]
|
||||
assert tbl.count_rows() == 2
|
||||
|
||||
|
||||
def test_create_table_from_pandas():
|
||||
# --8<-- [start:create_table_from_pandas]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pandas as pd
|
||||
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
"vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
|
||||
"lat": [45.5, 40.1],
|
||||
"long": [-122.7, -74.1],
|
||||
}
|
||||
)
|
||||
db.create_table("my_table_pandas", data, mode="overwrite")
|
||||
db["my_table_pandas"].head()
|
||||
# --8<-- [end:create_table_from_pandas]
|
||||
|
||||
|
||||
def test_create_table_from_polars():
|
||||
# --8<-- [start:create_table_from_polars]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import polars as pl
|
||||
|
||||
data = pl.DataFrame(
|
||||
{
|
||||
"vector": [[3.1, 4.1], [5.9, 26.5]],
|
||||
"item": ["foo", "bar"],
|
||||
"price": [10.0, 20.0],
|
||||
}
|
||||
)
|
||||
tbl = db.create_table("my_table_pl", data, mode="overwrite")
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_from_polars]
|
||||
assert tbl.count_rows() == 2
|
||||
|
||||
|
||||
def test_create_table_from_arrow():
|
||||
# --8<-- [start:create_table_from_arrow]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
|
||||
dim = 16
|
||||
total = 2
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("vector", pa.list_(pa.float16(), dim)),
|
||||
pa.field("text", pa.string()),
|
||||
]
|
||||
)
|
||||
data = pa.Table.from_arrays(
|
||||
[
|
||||
pa.array(
|
||||
[np.random.randn(dim).astype(np.float16) for _ in range(total)],
|
||||
pa.list_(pa.float16(), dim),
|
||||
),
|
||||
pa.array(["foo", "bar"]),
|
||||
],
|
||||
["vector", "text"],
|
||||
)
|
||||
tbl = db.create_table("f16_tbl", data, schema=schema, mode="overwrite")
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_from_arrow]
|
||||
assert tbl.count_rows() == 2
|
||||
|
||||
|
||||
def test_create_table_from_pydantic():
|
||||
# --8<-- [start:create_table_from_pydantic]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
class Content(LanceModel):
|
||||
movie_id: int
|
||||
vector: Vector(128)
|
||||
genres: str
|
||||
title: str
|
||||
imdb_id: int
|
||||
|
||||
@property
|
||||
def imdb_url(self) -> str:
|
||||
return f"https://www.imdb.com/title/tt{self.imdb_id}"
|
||||
|
||||
tbl = db.create_table("movielens_small", schema=Content, mode="overwrite")
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_from_pydantic]
|
||||
assert tbl.count_rows() == 0
|
||||
|
||||
|
||||
def test_create_table_nested_schema():
|
||||
# --8<-- [start:create_table_nested_schema]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from pydantic import BaseModel
|
||||
|
||||
class Document(BaseModel):
|
||||
content: str
|
||||
source: str
|
||||
|
||||
class NestedSchema(LanceModel):
|
||||
id: str
|
||||
vector: Vector(1536)
|
||||
document: Document
|
||||
|
||||
tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_table_nested_schema]
|
||||
assert tbl.count_rows() == 0
|
||||
|
||||
|
||||
def test_create_empty_table():
|
||||
# --8<-- [start:create_empty_table]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("vector", pa.list_(pa.float32(), 2)),
|
||||
pa.field("item", pa.string()),
|
||||
pa.field("price", pa.float32()),
|
||||
]
|
||||
)
|
||||
tbl = db.create_table("test_empty_table", schema=schema, mode="overwrite")
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_empty_table]
|
||||
assert tbl.count_rows() == 0
|
||||
|
||||
|
||||
def test_create_empty_table_pydantic():
|
||||
# --8<-- [start:create_empty_table_pydantic]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
class Item(LanceModel):
|
||||
vector: Vector(2)
|
||||
item: str
|
||||
price: float
|
||||
|
||||
tbl = db.create_table(
|
||||
"test_empty_table_new", schema=Item.to_arrow_schema(), mode="overwrite"
|
||||
)
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:create_empty_table_pydantic]
|
||||
assert tbl.count_rows() == 0
|
||||
|
||||
|
||||
def test_open_existing_table():
|
||||
# --8<-- [start:open_existing_table]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
# Create a table first
|
||||
data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7}]
|
||||
db.create_table("test_table", data, mode="overwrite")
|
||||
|
||||
# List table names
|
||||
print(db.list_tables().tables)
|
||||
|
||||
# Open existing table
|
||||
tbl = db.open_table("test_table")
|
||||
|
||||
print("Rows:", tbl.count_rows())
|
||||
# --8<-- [end:open_existing_table]
|
||||
assert tbl.count_rows() == 1
|
||||
|
||||
|
||||
def test_drop_table():
|
||||
# --8<-- [start:drop_table]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
# Create a table first
|
||||
data = [{"vector": [1.1, 1.2], "lat": 45.5}]
|
||||
db.create_table("my_table", data, mode="overwrite")
|
||||
|
||||
# Drop the table
|
||||
db.drop_table("my_table")
|
||||
# --8<-- [end:drop_table]
|
||||
|
||||
|
||||
def test_add_from_dataset():
|
||||
# --8<-- [start:add_from_dataset]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
data_path = str(Path(uri) / "input.parquet")
|
||||
pq.write_table(pa.table({"id": [1, 2], "name": ["Alice", "Bob"]}), data_path)
|
||||
import pyarrow.dataset as ds
|
||||
|
||||
dataset = ds.dataset(data_path, format="parquet")
|
||||
table = db.create_table("my_table", schema=dataset.schema, mode="overwrite")
|
||||
table.add(dataset)
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:add_from_dataset]
|
||||
assert table.count_rows() == 2
|
||||
|
||||
|
||||
def test_create_table_from_iterator():
|
||||
# --8<-- [start:create_table_from_iterator]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("vector", pa.list_(pa.float32(), 4)),
|
||||
pa.field("item", pa.utf8()),
|
||||
pa.field("price", pa.float32()),
|
||||
]
|
||||
)
|
||||
|
||||
def make_batches():
|
||||
for i in range(5):
|
||||
yield pa.RecordBatch.from_arrays(
|
||||
[
|
||||
pa.array(
|
||||
[[3.1, 4.1, 5.1, 6.1], [5.9, 26.5, 4.7, 32.8]],
|
||||
pa.list_(pa.float32(), 4),
|
||||
),
|
||||
pa.array(["foo", "bar"]),
|
||||
pa.array([10.0, 20.0]),
|
||||
],
|
||||
["vector", "item", "price"],
|
||||
)
|
||||
|
||||
table = db.create_table("batched_table", schema=schema)
|
||||
table.add(make_batches())
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:create_table_from_iterator]
|
||||
assert table.count_rows() == 10
|
||||
|
||||
|
||||
def test_update_operation():
|
||||
# --8<-- [start:update_operation]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"name": ["Alice", "Bob"],
|
||||
"login_count": [10, 20],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
table.update(where="id = 2", values={"name": "Bobby"})
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:update_operation]
|
||||
assert table.count_rows() == 2
|
||||
assert table.search().where("id = 2").to_list()[0]["name"] == "Bobby"
|
||||
|
||||
|
||||
def test_update_using_sql():
|
||||
# --8<-- [start:update_using_sql]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"name": ["Alice", "Bob"],
|
||||
"login_count": [10, 20],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:update_using_sql]
|
||||
assert table.count_rows() == 2
|
||||
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
|
||||
|
||||
|
||||
def test_merge_matched_update_only():
|
||||
# --8<-- [start:merge_matched_update_only]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"name": ["Alice", "Bob"],
|
||||
"login_count": [10, 20],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
|
||||
incoming_users = pa.table(
|
||||
{
|
||||
"id": [2, 3],
|
||||
"name": ["Bobby", "Charlie"],
|
||||
"login_count": [21, 5],
|
||||
}
|
||||
)
|
||||
|
||||
(table.merge_insert("id").when_matched_update_all().execute(incoming_users))
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:merge_matched_update_only]
|
||||
assert table.count_rows() == 2
|
||||
|
||||
|
||||
def test_insert_if_not_exists():
|
||||
# --8<-- [start:insert_if_not_exists]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"name": ["Alice", "Bob"],
|
||||
"login_count": [10, 20],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
|
||||
incoming_users = pa.table(
|
||||
{
|
||||
"id": [2, 3],
|
||||
"name": ["Bobby", "Charlie"],
|
||||
"login_count": [21, 5],
|
||||
}
|
||||
)
|
||||
|
||||
(table.merge_insert("id").when_not_matched_insert_all().execute(incoming_users))
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:insert_if_not_exists]
|
||||
assert table.count_rows() == 3
|
||||
|
||||
|
||||
def test_merge_update_insert():
|
||||
# --8<-- [start:merge_update_insert]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"name": ["Alice", "Bob"],
|
||||
"login_count": [10, 20],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
|
||||
incoming_users = pa.table(
|
||||
{
|
||||
"id": [2, 3],
|
||||
"name": ["Bobby", "Charlie"],
|
||||
"login_count": [21, 5],
|
||||
}
|
||||
)
|
||||
|
||||
(
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(incoming_users)
|
||||
)
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:merge_update_insert]
|
||||
assert table.count_rows() == 3
|
||||
assert table.search().where("id = 2").to_list()[0]["login_count"] == 21
|
||||
|
||||
|
||||
def test_merge_delete_missing_by_source():
|
||||
# --8<-- [start:merge_delete_missing_by_source]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2, 3],
|
||||
"name": ["Alice", "Bob", "Charlie"],
|
||||
"login_count": [10, 20, 5],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
|
||||
incoming_users = pa.table(
|
||||
{
|
||||
"id": [2, 3],
|
||||
"name": ["Bobby", "Charlie"],
|
||||
"login_count": [21, 5],
|
||||
}
|
||||
)
|
||||
|
||||
(
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.when_not_matched_by_source_delete()
|
||||
.execute(incoming_users)
|
||||
)
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:merge_delete_missing_by_source]
|
||||
assert table.count_rows() == 2
|
||||
assert table.count_rows("id = 1") == 0
|
||||
|
||||
|
||||
def test_merge_partial_columns():
|
||||
# --8<-- [start:merge_partial_columns]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
import pyarrow as pa
|
||||
|
||||
table = db.create_table(
|
||||
"users_example",
|
||||
data=pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"name": ["Alice", "Bob"],
|
||||
"login_count": [10, 20],
|
||||
}
|
||||
),
|
||||
mode="overwrite",
|
||||
)
|
||||
|
||||
incoming_users = pa.table(
|
||||
{
|
||||
"id": [2, 3],
|
||||
"name": ["Bobby", "Charlie"],
|
||||
}
|
||||
)
|
||||
|
||||
(
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(incoming_users)
|
||||
)
|
||||
|
||||
print("Rows:", table.count_rows())
|
||||
# --8<-- [end:merge_partial_columns]
|
||||
assert table.count_rows() == 3
|
||||
assert table.search().where("id = 3").to_list()[0]["login_count"] is None
|
||||
|
||||
|
||||
def test_write_workflow():
|
||||
# --8<-- [start:write_workflow]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"users",
|
||||
[
|
||||
{"id": 1, "name": "Alice", "login_count": 10},
|
||||
{"id": 2, "name": "Bob", "login_count": 20},
|
||||
],
|
||||
)
|
||||
table.add([{"id": 3, "name": "Charlie", "login_count": 5}])
|
||||
table.update(where="id = 2", values={"name": "Bobby"})
|
||||
table.update(where="id = 2", values_sql={"login_count": "login_count + 1"})
|
||||
(
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(
|
||||
[
|
||||
{"id": 3, "name": "Charlie", "login_count": 6},
|
||||
{"id": 4, "name": "Dana", "login_count": 1},
|
||||
]
|
||||
)
|
||||
)
|
||||
print("Rows selected for deletion:", table.search().where("id = 4").to_list())
|
||||
table.delete("id = 4")
|
||||
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
|
||||
print(rows)
|
||||
# --8<-- [end:write_workflow]
|
||||
|
||||
assert rows == [
|
||||
{"id": 1, "name": "Alice", "login_count": 10},
|
||||
{"id": 2, "name": "Bobby", "login_count": 21},
|
||||
{"id": 3, "name": "Charlie", "login_count": 6},
|
||||
]
|
||||
|
||||
|
||||
def test_append_rows():
|
||||
# --8<-- [start:append_rows]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
|
||||
table.add([{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}])
|
||||
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
|
||||
print(rows)
|
||||
# --8<-- [end:append_rows]
|
||||
|
||||
assert [row["id"] for row in rows] == [1, 2, 3]
|
||||
|
||||
|
||||
def test_delete_rows():
|
||||
# --8<-- [start:delete_rows]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"users",
|
||||
[
|
||||
{"id": 1, "name": "Alice"},
|
||||
{"id": 2, "name": "Bob"},
|
||||
{"id": 3, "name": "Charlie"},
|
||||
],
|
||||
)
|
||||
predicate = "id = 3"
|
||||
print("Selected:", table.search().where(predicate).to_list())
|
||||
table.delete(predicate)
|
||||
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
|
||||
print("Remaining:", rows)
|
||||
# --8<-- [end:delete_rows]
|
||||
|
||||
assert rows == [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
|
||||
|
||||
|
||||
def test_add_progress():
|
||||
# --8<-- [start:add_progress]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
def report_progress(progress):
|
||||
print(f"Rows written: {progress['output_rows']}")
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-ingestion-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table("users", [{"id": 1, "name": "Alice"}])
|
||||
table.add(
|
||||
[{"id": 2, "name": "Bob"}, {"id": 3, "name": "Charlie"}],
|
||||
progress=report_progress,
|
||||
)
|
||||
rows = sorted(table.search().to_list(), key=lambda row: row["id"])
|
||||
print("After insertion:", rows)
|
||||
# --8<-- [end:add_progress]
|
||||
|
||||
assert rows == [
|
||||
{"id": 1, "name": "Alice"},
|
||||
{"id": 2, "name": "Bob"},
|
||||
{"id": 3, "name": "Charlie"},
|
||||
]
|
||||
@@ -740,13 +740,16 @@ def test_reranking_answerdotai_usage() -> None:
|
||||
require_flag("RUN_RERANKER_SNIPPETS")
|
||||
|
||||
# --8<-- [start:reranking_answerdotai_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import AnswerdotaiRerankers
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect("~/.lancedb")
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "answerdotai-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
@@ -756,18 +759,20 @@ def test_reranking_answerdotai_usage() -> None:
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema, mode="overwrite")
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = AnswerdotaiRerankers()
|
||||
|
||||
# Run vector search with a reranker
|
||||
result = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
result = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
@@ -819,13 +824,16 @@ def test_reranking_colbert_usage() -> None:
|
||||
require_flag("RUN_RERANKER_SNIPPETS")
|
||||
|
||||
# --8<-- [start:reranking_colbert_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import ColbertReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect("~/.lancedb")
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "colbert-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
@@ -835,18 +843,20 @@ def test_reranking_colbert_usage() -> None:
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema, mode="overwrite")
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = ColbertReranker()
|
||||
|
||||
# Run vector search with a reranker
|
||||
result = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
result = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
@@ -857,13 +867,16 @@ def test_reranking_cross_encoder_usage() -> None:
|
||||
require_flag("RUN_RERANKER_SNIPPETS")
|
||||
|
||||
# --8<-- [start:reranking_cross_encoder_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import CrossEncoderReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect("~/.lancedb")
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "cross-encoder-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
@@ -873,18 +886,20 @@ def test_reranking_cross_encoder_usage() -> None:
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema, mode="overwrite")
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = CrossEncoderReranker()
|
||||
|
||||
# Run vector search with a reranker
|
||||
result = tbl.search("hello").rerank(reranker=reranker).to_list()
|
||||
|
||||
# Build the full-text index before either FTS or hybrid search.
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
|
||||
# Run FTS search with a reranker
|
||||
result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
|
||||
|
||||
# Run hybrid search with a reranker
|
||||
tbl.create_fts_index("text", replace=True)
|
||||
result = (
|
||||
tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
|
||||
)
|
||||
@@ -936,13 +951,16 @@ def test_reranking_linear_combination_usage() -> None:
|
||||
require_flag("RUN_RERANKER_SNIPPETS")
|
||||
|
||||
# --8<-- [start:reranking_linear_combination_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import LinearCombinationReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect("~/.lancedb")
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "linear-combination-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
@@ -952,7 +970,7 @@ def test_reranking_linear_combination_usage() -> None:
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema, mode="overwrite")
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = LinearCombinationReranker()
|
||||
|
||||
@@ -1007,13 +1025,16 @@ def test_reranking_rrf_usage() -> None:
|
||||
require_flag("RUN_RERANKER_SNIPPETS")
|
||||
|
||||
# --8<-- [start:reranking_rrf_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import RRFReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect("~/.lancedb")
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "rrf-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
@@ -1023,7 +1044,7 @@ def test_reranking_rrf_usage() -> None:
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema, mode="overwrite")
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = RRFReranker()
|
||||
|
||||
@@ -1039,13 +1060,16 @@ def test_reranking_mrr_usage() -> None:
|
||||
require_flag("RUN_RERANKER_SNIPPETS")
|
||||
|
||||
# --8<-- [start:reranking_mrr_usage]
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from lancedb.rerankers import MRRReranker
|
||||
|
||||
embedder = get_registry().get("sentence-transformers").create()
|
||||
db = lancedb.connect("~/.lancedb")
|
||||
db = lancedb.connect(str(Path(tempfile.mkdtemp()) / "mrr-rerank"))
|
||||
|
||||
class Schema(LanceModel):
|
||||
text: str = embedder.SourceField()
|
||||
@@ -1055,7 +1079,7 @@ def test_reranking_mrr_usage() -> None:
|
||||
{"text": "hello world"},
|
||||
{"text": "goodbye world"},
|
||||
]
|
||||
tbl = db.create_table("test", schema=Schema, mode="overwrite")
|
||||
tbl = db.create_table("test", schema=Schema)
|
||||
tbl.add(data)
|
||||
reranker = MRRReranker(weight_vector=0.7, weight_fts=0.3)
|
||||
|
||||
@@ -1390,9 +1414,12 @@ def test_frameworks_pydantic_examples() -> None:
|
||||
{"text": "goodbye world", "vector": [0.0, 1.0]},
|
||||
]
|
||||
)
|
||||
results = table.search("hello world").limit(1).to_pydantic(LanceDocs)
|
||||
results = table.search([1.0, 0.0]).limit(1).to_pydantic(LanceDocs)
|
||||
print(results[0].text)
|
||||
# hello world
|
||||
# --8<-- [end:frameworks_pydantic_base_example]
|
||||
assert isinstance(results[0], LanceDocs)
|
||||
assert results[0].text == "hello world"
|
||||
|
||||
|
||||
# Platform integrations
|
||||
@@ -1686,7 +1713,7 @@ def test_platforms_pandas_examples() -> None:
|
||||
# --8<-- [end:platforms_pandas_vector_search]
|
||||
|
||||
# --8<-- [start:platforms_pandas_async_example]
|
||||
async def run_pandas_async_example() -> None:
|
||||
async def run_pandas_async_example() -> pd.DataFrame:
|
||||
async_db = await lancedb.connect_async(
|
||||
str(Path(tempfile.mkdtemp()) / "pandas-async")
|
||||
)
|
||||
@@ -1699,16 +1726,16 @@ def test_platforms_pandas_examples() -> None:
|
||||
async_table = await async_db.create_table(
|
||||
"creatures_async", data=async_df, mode="overwrite"
|
||||
)
|
||||
async_results = await (
|
||||
async_table.search([0.6, 0.4, 0.8])
|
||||
.select(["text", "_distance"])
|
||||
.limit(1)
|
||||
.to_pandas()
|
||||
)
|
||||
# An async search returns its query builder from an await
|
||||
query = await async_table.search([0.6, 0.4, 0.8])
|
||||
async_results = await query.select(["text", "_distance"]).limit(1).to_pandas()
|
||||
print(async_results)
|
||||
return async_results
|
||||
|
||||
asyncio.run(run_pandas_async_example())
|
||||
async_results = asyncio.run(run_pandas_async_example())
|
||||
# --8<-- [end:platforms_pandas_async_example]
|
||||
assert async_results["text"].tolist() == ["sage"]
|
||||
assert async_results["_distance"].tolist() == [0.0]
|
||||
|
||||
|
||||
def test_platforms_polars_examples() -> None:
|
||||
|
||||
@@ -12,7 +12,9 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
|
||||
|
||||
db = lancedb.connect("./.lancedb")
|
||||
table = db.create_table(
|
||||
"people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
|
||||
"people",
|
||||
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
|
||||
storage_options={"new_table_enable_stable_row_ids": "true"},
|
||||
)
|
||||
|
||||
view = db.create_materialized_view(
|
||||
@@ -29,12 +31,11 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch):
|
||||
# --8<-- [start:materialized_view_refresh]
|
||||
table.add([{"name": "bea", "age": 41}])
|
||||
|
||||
result = view.refresh()
|
||||
print(result.rows_written)
|
||||
view.refresh()
|
||||
print(view.table.count_rows())
|
||||
# 2
|
||||
# --8<-- [end:materialized_view_refresh]
|
||||
assert result.mode == "rebuild"
|
||||
assert result.rows_written == 2
|
||||
assert view.table.count_rows() == 2
|
||||
assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [
|
||||
"ADA",
|
||||
"BEA",
|
||||
@@ -45,7 +46,11 @@ def test_materialized_view_with_no_data(tmp_path):
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect(tmp_path)
|
||||
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
|
||||
db.create_table(
|
||||
"people",
|
||||
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
|
||||
storage_options={"new_table_enable_stable_row_ids": "true"},
|
||||
)
|
||||
view = db.create_materialized_view(
|
||||
"adults", "people", where="age >= 18", with_no_data=True
|
||||
)
|
||||
@@ -58,11 +63,16 @@ def test_materialized_view_refresh_to_source_version(tmp_path):
|
||||
import lancedb
|
||||
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("people", [{"name": "ada", "age": 36}])
|
||||
table = db.create_table(
|
||||
"people",
|
||||
[{"name": "ada", "age": 36}],
|
||||
storage_options={"new_table_enable_stable_row_ids": "true"},
|
||||
)
|
||||
view = db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
first = table.version
|
||||
table.add([{"name": "bea", "age": 41}])
|
||||
assert view.refresh().rows_written == 2
|
||||
view.refresh()
|
||||
assert view.table.count_rows() == 2
|
||||
|
||||
result = view.refresh(source_version=first)
|
||||
assert (result.source_version, result.rows_written) == (first, 1)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""The provider reranker examples write only to a directory they create.
|
||||
|
||||
Each example runs offline. A stand-in embedding function answers for the model
|
||||
the example asks the registry for, a stand-in reranker replaces the provider
|
||||
client, and outbound connections are refused, so no model is downloaded and no
|
||||
provider is called. Sentinel tables in the owned home directory, where these
|
||||
examples used to write, and in the temporary directory must be left unchanged.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import socket
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import ClassVar
|
||||
|
||||
import lancedb
|
||||
import lancedb.embeddings
|
||||
import lancedb.rerankers
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from lancedb.embeddings import TextEmbeddingFunction, register
|
||||
from lancedb.rerankers import Reranker
|
||||
|
||||
EXAMPLES = Path(__file__).with_name("provider_examples.py")
|
||||
RERANKERS = {
|
||||
"reranking_cohere_usage": "CohereReranker",
|
||||
"reranking_jina_usage": "JinaReranker",
|
||||
"reranking_watsonx_usage": "WatsonxReranker",
|
||||
"reranking_openai_usage": "OpenaiReranker",
|
||||
"reranking_voyageai_usage": "VoyageAIReranker",
|
||||
}
|
||||
|
||||
|
||||
@register("docs-offline-text")
|
||||
class OfflineEmbedding(TextEmbeddingFunction):
|
||||
"""Deterministic four-dimensional vectors, computed locally."""
|
||||
|
||||
def ndims(self):
|
||||
return 4
|
||||
|
||||
def generate_embeddings(self, texts):
|
||||
return [[float(len(text)), 1.0, 0.0, 0.0] for text in texts]
|
||||
|
||||
|
||||
class OfflineReranker(Reranker):
|
||||
"""Scores every row 1.0 and records which searches it reranked."""
|
||||
|
||||
instances: ClassVar[list["OfflineReranker"]] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.kwargs = kwargs
|
||||
self.searches = []
|
||||
OfflineReranker.instances.append(self)
|
||||
|
||||
def _score(self, results, search):
|
||||
self.searches.append(search)
|
||||
drop = [c for c in ("_distance", "_score") if c in results.column_names]
|
||||
results = results.drop_columns(drop)
|
||||
scores = pa.array([1.0] * results.num_rows, pa.float32())
|
||||
return results.append_column("_relevance_score", scores)
|
||||
|
||||
def rerank_vector(self, query, vector_results):
|
||||
return self._score(vector_results, "vector")
|
||||
|
||||
def rerank_fts(self, query, fts_results):
|
||||
return self._score(fts_results, "fts")
|
||||
|
||||
def rerank_hybrid(self, query, vector_results, fts_results):
|
||||
merged = self.merge_results(vector_results, fts_results)
|
||||
return self._score(merged, "hybrid")
|
||||
|
||||
|
||||
def snapshot(connect, uri):
|
||||
db = connect(str(uri))
|
||||
tables = {}
|
||||
for name in db.list_tables().tables:
|
||||
table = db.open_table(name)
|
||||
tables[name] = (table.version, table.schema, table.to_arrow().to_pylist())
|
||||
return tables
|
||||
|
||||
|
||||
def load_examples():
|
||||
spec = importlib.util.spec_from_file_location("provider_examples", EXAMPLES)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def owned(tmp_path, monkeypatch):
|
||||
home, temp_root = tmp_path / "home", tmp_path / "tmp"
|
||||
home.mkdir()
|
||||
temp_root.mkdir()
|
||||
monkeypatch.setenv("HOME", str(home))
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(temp_root))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# Anything the examples resolve from "~" must land in the owned home.
|
||||
assert Path("~/.lancedb").expanduser().resolve().is_relative_to(home.resolve())
|
||||
|
||||
def refuse(*args, **kwargs):
|
||||
raise OSError("network access is disabled in this test")
|
||||
|
||||
monkeypatch.setattr(socket.socket, "connect", refuse)
|
||||
monkeypatch.setattr(socket, "create_connection", refuse)
|
||||
|
||||
real_connect = lancedb.connect
|
||||
sentinels = [home / ".lancedb", temp_root / "existing"]
|
||||
for uri in sentinels:
|
||||
row = {"text": "keep me", "vector": [0.5, 0.5, 0.5, 0.5]}
|
||||
real_connect(str(uri)).create_table("test", [row])
|
||||
before = [snapshot(real_connect, uri) for uri in sentinels]
|
||||
|
||||
registry = lancedb.embeddings.get_registry()
|
||||
monkeypatch.setattr(registry, "get", lambda name: OfflineEmbedding)
|
||||
for name in RERANKERS.values():
|
||||
monkeypatch.setattr(lancedb.rerankers, name, OfflineReranker)
|
||||
monkeypatch.setenv("JINA_API_KEY", "replaced-by-the-example")
|
||||
OfflineReranker.instances.clear()
|
||||
|
||||
connected = []
|
||||
|
||||
def recording_connect(uri, *args, **kwargs):
|
||||
connected.append(Path(uri).expanduser().resolve())
|
||||
return real_connect(uri, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(lancedb, "connect", recording_connect)
|
||||
return SimpleNamespace(
|
||||
home=home,
|
||||
temp_root=temp_root,
|
||||
sentinels=sentinels,
|
||||
before=before,
|
||||
connect=real_connect,
|
||||
connected=connected,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("example", sorted(RERANKERS))
|
||||
def test_reranker_example_writes_only_to_its_own_directory(owned, example):
|
||||
existing = set(owned.temp_root.iterdir())
|
||||
|
||||
getattr(load_examples(), example)()
|
||||
|
||||
after = [snapshot(owned.connect, uri) for uri in owned.sentinels]
|
||||
assert after == owned.before
|
||||
created = set(owned.temp_root.iterdir()) - existing
|
||||
assert len(created) == 1
|
||||
(own_directory,) = created
|
||||
assert len(owned.connected) == 1
|
||||
assert owned.connected[0].is_relative_to(own_directory.resolve())
|
||||
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
|
||||
|
||||
(reranker,) = OfflineReranker.instances
|
||||
assert reranker.searches == ["vector", "fts", "hybrid"]
|
||||
assert reranker.kwargs["api_key"] == "your-api-key"
|
||||
assert reranker.kwargs.get("project_id", "your-project-id") == "your-project-id"
|
||||
@@ -1,14 +1,125 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
# What the quickstart's download step leaves in an application's directory.
|
||||
IMAGE = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "web/static/assets/images/quickstart/sir-lancelot.jpg"
|
||||
)
|
||||
DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY")
|
||||
# The connection settings alone never send these tests to a deployment.
|
||||
OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT"
|
||||
|
||||
|
||||
def test_quickstart(db_path_factory):
|
||||
uri = db_path_factory("quickstart_db")
|
||||
db = lancedb.connect(uri)
|
||||
def deployment_settings():
|
||||
if os.environ.get(OPT_IN) != "1":
|
||||
pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment")
|
||||
missing = [name for name in DEPLOYMENT if not os.environ.get(name)]
|
||||
if missing:
|
||||
pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}")
|
||||
return dict(
|
||||
uri=os.environ["LANCEDB_URI"],
|
||||
api_key=os.environ["LANCEDB_API_KEY"],
|
||||
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
|
||||
host_override=os.environ["LANCEDB_HOST_OVERRIDE"],
|
||||
)
|
||||
|
||||
|
||||
class OwnedTables:
|
||||
"""A deployment connection on which a test creates only tables of its own.
|
||||
|
||||
Each name gets a suffix unique to the test and is claimed only while no
|
||||
table has it; the test drops what it claimed, whether or not it passed.
|
||||
"""
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
self.suffix = uuid.uuid4().hex[:12]
|
||||
self.claimed = []
|
||||
|
||||
def claim(self, name, existing, mode):
|
||||
owned = f"{name}_{self.suffix}"
|
||||
if owned in self.claimed:
|
||||
return owned, mode
|
||||
if owned in existing:
|
||||
raise RuntimeError(f"{owned} already exists and is not this test's")
|
||||
self.claimed.append(owned)
|
||||
return owned, "create"
|
||||
|
||||
def table_names(self):
|
||||
names, token = [], None
|
||||
while True:
|
||||
page = self.db.list_tables(page_token=token)
|
||||
names += page.tables
|
||||
if not (token := page.page_token):
|
||||
return names
|
||||
|
||||
def create_table(self, name, *args, mode="create", **kwargs):
|
||||
owned, mode = self.claim(name, self.table_names(), mode)
|
||||
return self.db.create_table(owned, *args, mode=mode, **kwargs)
|
||||
|
||||
def drop_claimed(self):
|
||||
existing = self.table_names()
|
||||
for name in self.claimed:
|
||||
if name in existing:
|
||||
self.db.drop_table(name)
|
||||
|
||||
|
||||
class AsyncOwnedTables(OwnedTables):
|
||||
async def table_names(self):
|
||||
names, token = [], None
|
||||
while True:
|
||||
page = await self.db.list_tables(page_token=token)
|
||||
names += page.tables
|
||||
if not (token := page.page_token):
|
||||
return names
|
||||
|
||||
async def create_table(self, name, *args, mode="create", **kwargs):
|
||||
owned, mode = self.claim(name, await self.table_names(), mode)
|
||||
return await self.db.create_table(owned, *args, mode=mode, **kwargs)
|
||||
|
||||
async def drop_claimed(self):
|
||||
existing = await self.table_names()
|
||||
for name in self.claimed:
|
||||
if name in existing:
|
||||
await self.db.drop_table(name)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deployment():
|
||||
owned = OwnedTables(lancedb.connect(**deployment_settings()))
|
||||
yield owned
|
||||
owned.drop_claimed()
|
||||
|
||||
|
||||
@pytest.fixture(params=["local", "enterprise"])
|
||||
def db(request, db_path_factory):
|
||||
if request.param == "local":
|
||||
yield lancedb.connect(db_path_factory("quickstart_db"))
|
||||
else:
|
||||
yield request.getfixturevalue("deployment")
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(params=["local", "enterprise"])
|
||||
async def async_db(request, db_path_factory):
|
||||
if request.param == "local":
|
||||
yield await lancedb.connect_async(db_path_factory("quickstart_async_db"))
|
||||
return
|
||||
owned = AsyncOwnedTables(await lancedb.connect_async(**deployment_settings()))
|
||||
yield owned
|
||||
await owned.drop_claimed()
|
||||
|
||||
|
||||
def create_characters(db):
|
||||
# --8<-- [start:quickstart_data]
|
||||
data = [
|
||||
{
|
||||
@@ -41,14 +152,29 @@ def test_quickstart(db_path_factory):
|
||||
# --8<-- [start:quickstart_create_table]
|
||||
table = db.create_table("characters", data=data, mode="overwrite")
|
||||
# --8<-- [end:quickstart_create_table]
|
||||
assert len(table) == 3
|
||||
# Drop the table to test create without overwrite
|
||||
db.drop_table("characters")
|
||||
assert table.count_rows() == 3
|
||||
|
||||
# --8<-- [start:quickstart_add_data]
|
||||
table.add(
|
||||
[
|
||||
{
|
||||
"id": "4",
|
||||
"name": "Morgana",
|
||||
"role": "Sorceress",
|
||||
"description": "Powerful sorceress of Avalon.",
|
||||
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
|
||||
"vector": [0.3, 0.9, 0.6, 0.8],
|
||||
}
|
||||
]
|
||||
)
|
||||
# --8<-- [end:quickstart_add_data]
|
||||
assert table.count_rows() == 4
|
||||
return table
|
||||
|
||||
|
||||
def test_quickstart(db):
|
||||
table = create_characters(db)
|
||||
|
||||
# --8<-- [start:quickstart_create_table_no_overwrite]
|
||||
table = db.create_table("characters", data=data)
|
||||
# --8<-- [end:quickstart_create_table_no_overwrite]
|
||||
assert len(table) == 3
|
||||
# --8<-- [start:quickstart_vector_search_1]
|
||||
# Search for examples similar to a "wise magical advisor"
|
||||
query_vector = [0.2, 0.8, 0.4, 0.9]
|
||||
@@ -62,7 +188,14 @@ def test_quickstart(db_path_factory):
|
||||
)
|
||||
print(result)
|
||||
# --8<-- [end:quickstart_vector_search_1]
|
||||
assert result.head(1)["name"][0] == "Merlin"
|
||||
assert result["name"].to_list() == ["Merlin", "Morgana"]
|
||||
|
||||
# --8<-- [start:quickstart_output_pandas]
|
||||
# Ensure you run `pip install pandas` beforehand
|
||||
result = table.search(query_vector).limit(2).to_pandas()
|
||||
print(result)
|
||||
# --8<-- [end:quickstart_output_pandas]
|
||||
assert result["name"].to_list() == ["Merlin", "Morgana"]
|
||||
|
||||
# --8<-- [start:quickstart_curate_with_metadata]
|
||||
curated = (
|
||||
@@ -74,131 +207,15 @@ def test_quickstart(db_path_factory):
|
||||
)
|
||||
print(curated)
|
||||
# --8<-- [end:quickstart_curate_with_metadata]
|
||||
assert curated.head(1)["name"][0] == "Merlin"
|
||||
assert curated["name"].to_list() == ["Merlin", "Morgana"]
|
||||
|
||||
# --8<-- [start:quickstart_output_pandas]
|
||||
# Ensure you run `pip install pandas` beforehand
|
||||
result = table.search(query_vector).limit(2).to_pandas()
|
||||
print(result)
|
||||
# --8<-- [end:quickstart_output_pandas]
|
||||
assert result.iloc[0]["name"] == "Merlin"
|
||||
|
||||
# --8<-- [start:quickstart_add_feature]
|
||||
table.add_columns(
|
||||
{
|
||||
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
|
||||
}
|
||||
)
|
||||
# --8<-- [end:quickstart_add_feature]
|
||||
assert "power_score" in table.schema.names
|
||||
|
||||
# --8<-- [start:quickstart_query_feature]
|
||||
features = table.search().select(["name", "role", "power_score"]).to_polars()
|
||||
print(features)
|
||||
# --8<-- [end:quickstart_query_feature]
|
||||
assert "power_score" in features.columns
|
||||
|
||||
# --8<-- [start:quickstart_multimodal_bytes]
|
||||
from pathlib import Path
|
||||
|
||||
image_path = Path("docs/web/static/assets/images/quickstart/sir-lancelot.jpg")
|
||||
image_bytes = image_path.read_bytes()
|
||||
|
||||
multimodal_table = db.create_table(
|
||||
"character_images",
|
||||
data=[
|
||||
{
|
||||
"id": "lancelot",
|
||||
"description": "Portrait of Sir Lancelot",
|
||||
"image": image_bytes,
|
||||
"vector": [0.9, 0.1, 0.5, 0.4],
|
||||
}
|
||||
],
|
||||
mode="overwrite",
|
||||
)
|
||||
# --8<-- [end:quickstart_multimodal_bytes]
|
||||
assert len(multimodal_table) == 1
|
||||
|
||||
# --8<-- [start:quickstart_open_table]
|
||||
table = db.open_table("characters")
|
||||
# --8<-- [end:quickstart_open_table]
|
||||
|
||||
# --8<-- [start:quickstart_add_data]
|
||||
more_data = [
|
||||
{
|
||||
"id": "4",
|
||||
"name": "Morgana",
|
||||
"role": "Sorceress",
|
||||
"description": "Powerful sorceress of Avalon.",
|
||||
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 4},
|
||||
"vector": [0.3, 0.9, 0.6, 0.8],
|
||||
"power_score": 3.75,
|
||||
},
|
||||
]
|
||||
|
||||
# Add data to table
|
||||
table.add(more_data)
|
||||
# --8<-- [end:quickstart_add_data]
|
||||
assert len(table) == 4
|
||||
|
||||
# --8<-- [start:quickstart_vector_search_2]
|
||||
# Search for examples similar to a "powerful sorceress"
|
||||
query_vector = [0.3, 0.9, 0.6, 0.8]
|
||||
|
||||
results = table.search(query_vector).limit(2).to_polars()
|
||||
print(results)
|
||||
# --8<-- [end:quickstart_vector_search_2]
|
||||
assert results.head(1)["name"][0] == "Morgana"
|
||||
|
||||
# --8<-- [start:quickstart_computed_feature]
|
||||
table.add_columns(computed={"combat_score": "(stats.magic + stats.strength) / 2.0"})
|
||||
table.refresh_column("combat_score")
|
||||
# --8<-- [end:quickstart_computed_feature]
|
||||
scores = (
|
||||
table.search().select(["combat_score"]).to_arrow()["combat_score"].to_pylist()
|
||||
)
|
||||
assert scores == [2.5, 3.5, 3.0, 3.5]
|
||||
|
||||
# --8<-- [start:quickstart_training]
|
||||
import torch
|
||||
from lancedb.permutation import Permutation
|
||||
|
||||
dataset = Permutation.identity(table).select_columns(
|
||||
["power_score", "combat_score"]
|
||||
)
|
||||
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
|
||||
# --8<-- [end:quickstart_training]
|
||||
batches = list(dataloader)
|
||||
assert len(batches) == 1
|
||||
assert set(batches[0]) == {"power_score", "combat_score"}
|
||||
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
|
||||
assert sorted(batches[0]["combat_score"].tolist()) == sorted(scores)
|
||||
|
||||
# --8<-- [start:quickstart_training_projection]
|
||||
dataset = Permutation.identity(table).select_columns(["power_score"])
|
||||
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024)
|
||||
# --8<-- [end:quickstart_training_projection]
|
||||
batch = next(iter(dataloader))
|
||||
assert set(batch) == {"power_score"}
|
||||
assert batch["power_score"].tolist() == [3.5, 4.0, 3.0, 3.75]
|
||||
curated = table.search(query_vector).where("stats.magic < 4").limit(2).to_list()
|
||||
assert [row["name"] for row in curated] == ["King Arthur", "Sir Lancelot"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quickstart_async_api(db_path_factory):
|
||||
db_uri = db_path_factory("quickstart_async_db")
|
||||
import lancedb
|
||||
async_db = await lancedb.connect_async(db_uri)
|
||||
|
||||
# --8<-- [start:quickstart_data_async]
|
||||
async def test_quickstart_async_api(async_db):
|
||||
data = [
|
||||
{
|
||||
"id": "1",
|
||||
"name": "King Arthur",
|
||||
"role": "King",
|
||||
"description": "Leader of Camelot and wielder of Excalibur.",
|
||||
"stats": {"strength": 4, "magic": 1, "leadership": 5, "wisdom": 4},
|
||||
"vector": [0.7, 0.1, 0.9, 0.7],
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"name": "Merlin",
|
||||
@@ -206,17 +223,8 @@ async def test_quickstart_async_api(db_path_factory):
|
||||
"description": "Advisor and prophet with deep magical knowledge.",
|
||||
"stats": {"strength": 2, "magic": 5, "leadership": 4, "wisdom": 5},
|
||||
"vector": [0.2, 0.9, 0.4, 0.9],
|
||||
},
|
||||
{
|
||||
"id": "3",
|
||||
"name": "Sir Lancelot",
|
||||
"role": "Knight",
|
||||
"description": "Legendary knight known for courage and combat skill.",
|
||||
"stats": {"strength": 5, "magic": 1, "leadership": 3, "wisdom": 3},
|
||||
"vector": [0.9, 0.1, 0.5, 0.4],
|
||||
},
|
||||
}
|
||||
]
|
||||
# --8<-- [end:quickstart_data_async]
|
||||
|
||||
# --8<-- [start:quickstart_create_table_async]
|
||||
async_table = await async_db.create_table(
|
||||
@@ -236,5 +244,110 @@ async def test_quickstart_async_api(db_path_factory):
|
||||
).select(["name", "role", "description", "_distance"]).limit(2).to_polars()
|
||||
print(async_result)
|
||||
# --8<-- [end:quickstart_vector_search_1_async]
|
||||
assert async_result["name"].to_list() == ["Merlin"]
|
||||
|
||||
assert async_result.head(1)["name"][0] == "Merlin"
|
||||
|
||||
def test_quickstart_derived_feature(db):
|
||||
table = create_characters(db)
|
||||
|
||||
# --8<-- [start:quickstart_add_feature]
|
||||
table.add_columns(
|
||||
{
|
||||
"power_score": "cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)"
|
||||
}
|
||||
)
|
||||
# --8<-- [end:quickstart_add_feature]
|
||||
|
||||
# --8<-- [start:quickstart_query_feature]
|
||||
features = table.search().select(["name", "role", "power_score"]).to_polars()
|
||||
print(features)
|
||||
# --8<-- [end:quickstart_query_feature]
|
||||
assert sorted(zip(features["name"], features["power_score"])) == [
|
||||
("King Arthur", 3.5),
|
||||
("Merlin", 4.0),
|
||||
("Morgana", 3.75),
|
||||
("Sir Lancelot", 3.0),
|
||||
]
|
||||
|
||||
|
||||
def description_lengths(table):
|
||||
rows = table.search().select(["description", "description_length"]).to_list()
|
||||
return {row["description"]: row["description_length"] for row in rows}
|
||||
|
||||
|
||||
def test_quickstart_computed_column(db_path_factory):
|
||||
table = create_characters(lancedb.connect(db_path_factory("computed")))
|
||||
|
||||
# --8<-- [start:quickstart_computed_feature]
|
||||
table.add_columns(computed={"description_length": "length(description)"})
|
||||
table.refresh_column("description_length")
|
||||
# --8<-- [end:quickstart_computed_feature]
|
||||
lengths = description_lengths(table)
|
||||
assert lengths == {text: len(text) for text in lengths}
|
||||
|
||||
|
||||
def test_quickstart_computed_column_on_a_deployment(deployment):
|
||||
table = create_characters(deployment)
|
||||
|
||||
# --8<-- [start:quickstart_computed_feature_enterprise]
|
||||
table.add_columns(computed={"description_length": "length(description)"})
|
||||
job = table.refresh_column_async("description_length")
|
||||
job.wait()
|
||||
# --8<-- [end:quickstart_computed_feature_enterprise]
|
||||
lengths = description_lengths(table)
|
||||
assert len(lengths) == 4
|
||||
assert lengths == {text: len(text) for text in lengths}
|
||||
with pytest.raises(Exception):
|
||||
table.refresh_column("description_length")
|
||||
|
||||
|
||||
def test_quickstart_multimodal(db, tmp_path, monkeypatch):
|
||||
app = tmp_path / "app"
|
||||
(app / "data").mkdir(parents=True)
|
||||
shutil.copy(IMAGE, app / "data/sir-lancelot.jpg")
|
||||
monkeypatch.chdir(app)
|
||||
|
||||
# --8<-- [start:quickstart_multimodal_bytes]
|
||||
from pathlib import Path
|
||||
|
||||
image_bytes = Path("data/sir-lancelot.jpg").read_bytes()
|
||||
|
||||
multimodal_table = db.create_table(
|
||||
"character_images",
|
||||
data=[
|
||||
{
|
||||
"id": "lancelot",
|
||||
"description": "Portrait of Sir Lancelot",
|
||||
"image": image_bytes,
|
||||
"vector": [0.9, 0.1, 0.5, 0.4],
|
||||
}
|
||||
],
|
||||
mode="overwrite",
|
||||
)
|
||||
# --8<-- [end:quickstart_multimodal_bytes]
|
||||
assert multimodal_table.count_rows() == 1
|
||||
stored = multimodal_table.search().select(["image"]).to_list()[0]["image"]
|
||||
assert stored == IMAGE.read_bytes()
|
||||
|
||||
|
||||
def test_quickstart_training(db):
|
||||
table = create_characters(db)
|
||||
|
||||
# --8<-- [start:quickstart_training]
|
||||
import torch
|
||||
from lancedb.permutation import Permutation
|
||||
|
||||
table.add_columns(
|
||||
{
|
||||
"magic": "cast(stats.magic as float)",
|
||||
"strength": "cast(stats.strength as float)",
|
||||
}
|
||||
)
|
||||
dataset = Permutation.identity(table).select_columns(["magic", "strength"])
|
||||
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1024, shuffle=True)
|
||||
# --8<-- [end:quickstart_training]
|
||||
batches = list(dataloader)
|
||||
assert len(batches) == 1
|
||||
assert set(batches[0]) == {"magic", "strength"}
|
||||
assert all(isinstance(value, torch.Tensor) for value in batches[0].values())
|
||||
assert sorted(batches[0]["magic"].tolist()) == [1.0, 1.0, 5.0, 5.0]
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
|
||||
def test_read_consistency():
|
||||
# --8<-- [start:read_consistency]
|
||||
from datetime import timedelta
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-consistency-") as uri:
|
||||
writer = lancedb.connect(uri)
|
||||
table = writer.create_table("documents", [{"id": 1, "text": "first"}])
|
||||
|
||||
default_reader = lancedb.connect(uri).open_table("documents")
|
||||
latest_reader = lancedb.connect(
|
||||
uri, read_consistency_interval=timedelta(0)
|
||||
).open_table("documents")
|
||||
|
||||
table.add([{"id": 2, "text": "second"}])
|
||||
|
||||
default_count = default_reader.count_rows()
|
||||
latest_count = latest_reader.count_rows()
|
||||
print(f"Default reader: {default_count} rows") # 1
|
||||
print(f"Zero-interval reader: {latest_count} rows") # 2
|
||||
|
||||
default_reader.checkout_latest()
|
||||
refreshed_count = default_reader.count_rows()
|
||||
print(f"After explicit refresh: {refreshed_count} rows") # 2
|
||||
# --8<-- [end:read_consistency]
|
||||
|
||||
assert default_count == 1
|
||||
assert latest_count == 2
|
||||
assert refreshed_count == 2
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""The published reranker examples in test_integrations.py write only to a directory they create.
|
||||
|
||||
Each example runs as readers see it: its marked region is extracted the way the snippet
|
||||
generator extracts it, saved as a script and run offline. A stand-in embedding function answers for the
|
||||
sentence-transformers model, stand-in rerankers replace the model-backed AnswerDotAI, ColBERT
|
||||
and cross-encoder rerankers, and outbound connections are refused. The RRF, linear-combination
|
||||
and MRR rerankers need no model and run unchanged. Sentinel tables in the owned home directory,
|
||||
where these examples used to write, and in the temporary directory must be left unchanged.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import lancedb.rerankers
|
||||
import pytest
|
||||
import test_provider_examples as provider
|
||||
|
||||
owned = provider.owned
|
||||
SOURCE = Path(__file__).with_name("test_integrations.py")
|
||||
GENERATOR = Path(__file__).parents[1] / "mdx_snippets_gen.py"
|
||||
MODEL_RERANKERS = {
|
||||
"reranking_answerdotai_usage": "AnswerdotaiRerankers",
|
||||
"reranking_colbert_usage": "ColbertReranker",
|
||||
"reranking_cross_encoder_usage": "CrossEncoderReranker",
|
||||
}
|
||||
LOCAL_RERANKERS = [
|
||||
"reranking_linear_combination_usage",
|
||||
"reranking_mrr_usage",
|
||||
"reranking_rrf_usage",
|
||||
]
|
||||
|
||||
|
||||
def load_generator():
|
||||
if "mdx_snippets_gen" not in sys.modules:
|
||||
spec = importlib.util.spec_from_file_location("mdx_snippets_gen", GENERATOR)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module # its dataclasses resolve annotations by module
|
||||
spec.loader.exec_module(module)
|
||||
return sys.modules["mdx_snippets_gen"]
|
||||
|
||||
|
||||
def reader_code(name):
|
||||
"""The example exactly as the snippet generator publishes it."""
|
||||
generator = load_generator()
|
||||
start, end = generator.markers_for_lang("py")
|
||||
lines = SOURCE.read_text().splitlines(keepends=True)
|
||||
snippet = generator.parse_snippets(lines, start, end)[name]
|
||||
return generator.snippet_text(generator.dedent(snippet))
|
||||
|
||||
|
||||
def run_example(owned, tmp_path, name):
|
||||
script = tmp_path / "scripts" / f"{name}.py"
|
||||
script.parent.mkdir(exist_ok=True)
|
||||
script.write_text(reader_code(name))
|
||||
existing = set(owned.temp_root.iterdir())
|
||||
namespace = runpy.run_path(str(script))
|
||||
|
||||
after = [provider.snapshot(owned.connect, uri) for uri in owned.sentinels]
|
||||
assert after == owned.before
|
||||
created = set(owned.temp_root.iterdir()) - existing
|
||||
assert len(created) == 1
|
||||
(own_directory,) = created
|
||||
assert len(owned.connected) == 1
|
||||
assert owned.connected[0].is_relative_to(own_directory.resolve())
|
||||
assert owned.connect(str(owned.connected[0])).list_tables().tables == ["test"]
|
||||
return namespace
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(MODEL_RERANKERS))
|
||||
def test_model_reranker_example_writes_only_to_its_own_directory(
|
||||
owned, monkeypatch, tmp_path, name
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
lancedb.rerankers, MODEL_RERANKERS[name], provider.OfflineReranker
|
||||
)
|
||||
|
||||
namespace = run_example(owned, tmp_path, name)
|
||||
|
||||
(reranker,) = provider.OfflineReranker.instances
|
||||
assert reranker.searches == ["vector", "fts", "hybrid"]
|
||||
assert namespace["result"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", LOCAL_RERANKERS)
|
||||
def test_local_reranker_example_writes_only_to_its_own_directory(owned, tmp_path, name):
|
||||
namespace = run_example(owned, tmp_path, name)
|
||||
|
||||
assert namespace["result"]
|
||||
assert all("_relevance_score" in row for row in namespace["result"])
|
||||
if name == "reranking_mrr_usage":
|
||||
assert namespace["combined"].num_rows > 0
|
||||
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
|
||||
def test_vector_search():
|
||||
# --8<-- [start:vector_search]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-search-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"documents",
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "A puppy playing in the park",
|
||||
"category": "animals",
|
||||
"vector": [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "A dog resting at home",
|
||||
"category": "animals",
|
||||
"vector": [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"text": "A guide to growing tomatoes",
|
||||
"category": "gardening",
|
||||
"vector": [0.0, 1.0],
|
||||
},
|
||||
],
|
||||
)
|
||||
rows = (
|
||||
table.search([1.0, 0.0])
|
||||
.distance_type("l2")
|
||||
.select(["id", "text", "_distance"])
|
||||
.limit(2)
|
||||
.to_list()
|
||||
)
|
||||
print(rows)
|
||||
# --8<-- [end:vector_search]
|
||||
|
||||
assert [row["id"] for row in rows] == [1, 2]
|
||||
assert rows[0]["_distance"] == 0.0
|
||||
|
||||
|
||||
def test_text_search():
|
||||
# --8<-- [start:text_search]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
from lancedb.index import FTS
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-search-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"documents",
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "A puppy playing in the park",
|
||||
"category": "animals",
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "A dog resting at home",
|
||||
"category": "animals",
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"text": "A guide to growing tomatoes",
|
||||
"category": "gardening",
|
||||
},
|
||||
],
|
||||
)
|
||||
table.create_index("text", config=FTS())
|
||||
rows = (
|
||||
table.search("puppy", query_type="fts", fts_columns="text")
|
||||
.select(["id", "text", "_score"])
|
||||
.limit(2)
|
||||
.to_list()
|
||||
)
|
||||
print(rows)
|
||||
# --8<-- [end:text_search]
|
||||
|
||||
assert [row["id"] for row in rows] == [1]
|
||||
assert rows[0]["_score"] > 0
|
||||
|
||||
|
||||
def test_hybrid_search():
|
||||
# --8<-- [start:hybrid_search]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
from lancedb.index import FTS
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-search-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"documents",
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "A puppy playing in the park",
|
||||
"category": "animals",
|
||||
"vector": [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "A dog resting at home",
|
||||
"category": "animals",
|
||||
"vector": [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"text": "A guide to growing tomatoes",
|
||||
"category": "gardening",
|
||||
"vector": [0.0, 1.0],
|
||||
},
|
||||
],
|
||||
)
|
||||
from lancedb.rerankers import RRFReranker
|
||||
|
||||
table.create_index("text", config=FTS())
|
||||
rows = (
|
||||
table.search(
|
||||
query_type="hybrid", vector_column_name="vector", fts_columns="text"
|
||||
)
|
||||
.vector([1.0, 0.0])
|
||||
.text("puppy")
|
||||
.rerank(RRFReranker())
|
||||
.limit(2)
|
||||
.to_list()
|
||||
)
|
||||
print(rows)
|
||||
# --8<-- [end:hybrid_search]
|
||||
|
||||
assert [row["id"] for row in rows] == [1, 2]
|
||||
assert rows[0]["_relevance_score"] > rows[1]["_relevance_score"]
|
||||
|
||||
|
||||
def test_filter_search():
|
||||
# --8<-- [start:filter_search]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-search-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"documents",
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "A puppy playing in the park",
|
||||
"category": "animals",
|
||||
"vector": [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "A dog resting at home",
|
||||
"category": "animals",
|
||||
"vector": [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"text": "A guide to growing tomatoes",
|
||||
"category": "gardening",
|
||||
"vector": [0.0, 1.0],
|
||||
},
|
||||
],
|
||||
)
|
||||
rows = (
|
||||
table.search([0.0, 1.0])
|
||||
.where("category = 'animals'", prefilter=True)
|
||||
.select(["id", "text", "category", "_distance"])
|
||||
.limit(2)
|
||||
.to_list()
|
||||
)
|
||||
print(rows)
|
||||
# --8<-- [end:filter_search]
|
||||
|
||||
assert {row["id"] for row in rows} == {1, 2}
|
||||
assert all(row["category"] == "animals" for row in rows)
|
||||
|
||||
|
||||
def test_query_plan():
|
||||
# --8<-- [start:query_plan]
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
import lancedb
|
||||
|
||||
with TemporaryDirectory(prefix="lancedb-search-") as uri:
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table(
|
||||
"documents",
|
||||
[
|
||||
{
|
||||
"id": 1,
|
||||
"text": "A puppy playing in the park",
|
||||
"category": "animals",
|
||||
"vector": [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"text": "A dog resting at home",
|
||||
"category": "animals",
|
||||
"vector": [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"text": "A guide to growing tomatoes",
|
||||
"category": "gardening",
|
||||
"vector": [0.0, 1.0],
|
||||
},
|
||||
],
|
||||
)
|
||||
query = table.search([1.0, 0.0]).select(["id", "text", "_distance"]).limit(2)
|
||||
print("Plan:", query.explain_plan())
|
||||
print("Measured execution:", query.analyze_plan())
|
||||
rows = query.to_list()
|
||||
print(rows)
|
||||
# --8<-- [end:query_plan]
|
||||
|
||||
assert [row["id"] for row in rows] == [1, 2]
|
||||
@@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db):
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def _setup_schema_add_table(tmp_db, data=None):
|
||||
def _setup_schema_add_table(db):
|
||||
# --8<-- [start:schema_add_setup]
|
||||
table_name = "schema_evolution_add_example"
|
||||
if data is None:
|
||||
data = [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Laptop",
|
||||
"price": 1200.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Smartphone",
|
||||
"price": 800.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Headphones",
|
||||
"price": 150.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
]
|
||||
table = tmp_db.create_table(table_name, data, mode="overwrite")
|
||||
import numpy as np
|
||||
|
||||
data = [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Laptop",
|
||||
"price": 1200.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Smartphone",
|
||||
"price": 800.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"name": "Headphones",
|
||||
"price": 150.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
},
|
||||
]
|
||||
table = db.create_table("schema_evolution_add_example", data, mode="overwrite")
|
||||
# --8<-- [end:schema_add_setup]
|
||||
return table
|
||||
|
||||
@@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db):
|
||||
|
||||
|
||||
def test_add_columns_nullable(tmp_db):
|
||||
table = _setup_schema_add_table(
|
||||
tmp_db,
|
||||
data=[
|
||||
{
|
||||
"id": 1,
|
||||
"name": "Laptop",
|
||||
"price": 1200.00,
|
||||
"vector": np.random.random(128).tolist(),
|
||||
}
|
||||
],
|
||||
)
|
||||
table = _setup_schema_add_table(tmp_db)
|
||||
|
||||
# --8<-- [start:add_columns_nullable]
|
||||
# Add a nullable timestamp column
|
||||
|
||||
Generated
+208
-138
@@ -452,6 +452,61 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.7.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"sync_wrapper",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.76"
|
||||
@@ -845,6 +900,16 @@ version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.10.1"
|
||||
@@ -1769,26 +1834,6 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deepsize"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
|
||||
dependencies = [
|
||||
"deepsize_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deepsize_derive"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.5"
|
||||
@@ -2416,6 +2461,12 @@ version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.3.0"
|
||||
@@ -2436,6 +2487,7 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
@@ -2480,9 +2532,11 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"windows-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2938,15 +2992,15 @@ dependencies = [
|
||||
"half",
|
||||
"humantime",
|
||||
"itertools 0.13.0",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-datafusion",
|
||||
"lance-encoding",
|
||||
"lance-file",
|
||||
"lance-index",
|
||||
"lance-io",
|
||||
"lance-linalg",
|
||||
"lance-namespace 8.0.0",
|
||||
"lance-namespace",
|
||||
"lance-select",
|
||||
"lance-table",
|
||||
"lance-tokenizer",
|
||||
@@ -2974,28 +3028,6 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "7.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "253f4a0a70580c985b91e65e9ca6cad644825a4078de28d8efbacf3ffbd7ecdc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-data",
|
||||
"arrow-ipc",
|
||||
"arrow-ord",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"bytes",
|
||||
"futures",
|
||||
"getrandom 0.2.16",
|
||||
"half",
|
||||
"jsonb",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "8.0.0"
|
||||
@@ -3056,41 +3088,6 @@ dependencies = [
|
||||
"seq-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "7.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"deepsize",
|
||||
"futures",
|
||||
"itertools 0.13.0",
|
||||
"lance-arrow 7.0.0",
|
||||
"libc",
|
||||
"log",
|
||||
"moka",
|
||||
"num_cpus",
|
||||
"object_store",
|
||||
"pin-project",
|
||||
"prost",
|
||||
"rand 0.9.5",
|
||||
"roaring",
|
||||
"serde_json",
|
||||
"snafu 0.9.1",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "8.0.0"
|
||||
@@ -3108,7 +3105,7 @@ dependencies = [
|
||||
"datafusion-sql",
|
||||
"futures",
|
||||
"itertools 0.13.0",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-derive",
|
||||
"libc",
|
||||
"libm",
|
||||
@@ -3152,8 +3149,8 @@ dependencies = [
|
||||
"datafusion-physical-expr",
|
||||
"futures",
|
||||
"jsonb",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-datagen",
|
||||
"log",
|
||||
"pin-project",
|
||||
@@ -3214,9 +3211,9 @@ dependencies = [
|
||||
"hex",
|
||||
"hyperloglogplus",
|
||||
"itertools 0.13.0",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-bitpacking",
|
||||
"lance-core 8.0.0",
|
||||
"lance-core",
|
||||
"log",
|
||||
"lz4",
|
||||
"num-traits",
|
||||
@@ -3248,8 +3245,8 @@ dependencies = [
|
||||
"bytes",
|
||||
"datafusion-common",
|
||||
"futures",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-encoding",
|
||||
"lance-io",
|
||||
"log",
|
||||
@@ -3294,9 +3291,9 @@ dependencies = [
|
||||
"itertools 0.13.0",
|
||||
"jieba-rs",
|
||||
"jsonb",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-arrow-stats",
|
||||
"lance-core 8.0.0",
|
||||
"lance-core",
|
||||
"lance-datafusion",
|
||||
"lance-datagen",
|
||||
"lance-encoding",
|
||||
@@ -3351,9 +3348,9 @@ dependencies = [
|
||||
"futures",
|
||||
"http",
|
||||
"io-uring",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-namespace 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-namespace",
|
||||
"log",
|
||||
"moka",
|
||||
"object_store",
|
||||
@@ -3379,26 +3376,12 @@ dependencies = [
|
||||
"arrow-schema",
|
||||
"cc",
|
||||
"half",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"num-traits",
|
||||
"rand 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "7.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "014e8332ca0615506342e0d3af608639864b68396973be14239f09c9f21f1fc2"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"lance-core 7.0.0",
|
||||
"lance-namespace-reqwest-client 0.7.7",
|
||||
"snafu 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "8.0.0"
|
||||
@@ -3408,8 +3391,8 @@ dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"lance-core 8.0.0",
|
||||
"lance-namespace-reqwest-client 0.8.6",
|
||||
"lance-core",
|
||||
"lance-namespace-reqwest-client",
|
||||
"snafu 0.9.1",
|
||||
]
|
||||
|
||||
@@ -3423,42 +3406,33 @@ dependencies = [
|
||||
"arrow-ipc",
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"bytes",
|
||||
"datafusion-common",
|
||||
"datafusion-physical-plan",
|
||||
"futures",
|
||||
"lance",
|
||||
"lance-core 8.0.0",
|
||||
"lance-core",
|
||||
"lance-index",
|
||||
"lance-io",
|
||||
"lance-linalg",
|
||||
"lance-namespace 8.0.0",
|
||||
"lance-namespace",
|
||||
"lance-table",
|
||||
"log",
|
||||
"object_store",
|
||||
"rand 0.9.5",
|
||||
"reqwest",
|
||||
"roaring",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-http 0.5.2",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-reqwest-client"
|
||||
version = "0.7.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6369eee4682fb11edf538388b43c61ce288b8302fe89bb40944d7daa7faaae99"
|
||||
dependencies = [
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_repr",
|
||||
"serde_with",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-reqwest-client"
|
||||
version = "0.8.6"
|
||||
@@ -3485,7 +3459,7 @@ dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"itertools 0.13.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-core",
|
||||
"roaring",
|
||||
"tracing",
|
||||
]
|
||||
@@ -3506,8 +3480,8 @@ dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-file",
|
||||
"lance-io",
|
||||
"lance-select",
|
||||
@@ -3538,7 +3512,7 @@ dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
"criterion",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-arrow",
|
||||
"num-traits",
|
||||
"pprof",
|
||||
"rand 0.9.5",
|
||||
@@ -3589,9 +3563,10 @@ dependencies = [
|
||||
"datafusion-sql",
|
||||
"futures",
|
||||
"half",
|
||||
"http",
|
||||
"lance",
|
||||
"lance-arrow 8.0.0",
|
||||
"lance-core 8.0.0",
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-datafusion",
|
||||
"lance-datagen",
|
||||
"lance-encoding",
|
||||
@@ -3599,7 +3574,7 @@ dependencies = [
|
||||
"lance-index",
|
||||
"lance-io",
|
||||
"lance-linalg",
|
||||
"lance-namespace 8.0.0",
|
||||
"lance-namespace",
|
||||
"lance-namespace-impls",
|
||||
"lance-table",
|
||||
"lance-testing",
|
||||
@@ -3613,6 +3588,7 @@ dependencies = [
|
||||
"polars-arrow",
|
||||
"rand 0.9.5",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -3621,6 +3597,7 @@ dependencies = [
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -3867,6 +3844,12 @@ dependencies = [
|
||||
"regex-automata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.3.10"
|
||||
@@ -5391,7 +5374,7 @@ dependencies = [
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tower-http 0.6.8",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
@@ -5472,7 +5455,8 @@ dependencies = [
|
||||
"arrow-schema",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"lance-namespace 7.0.0",
|
||||
"lance-index",
|
||||
"lance-namespace",
|
||||
"lancedb",
|
||||
"polars",
|
||||
"polars-arrow",
|
||||
@@ -5659,7 +5643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"core-foundation",
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"security-framework-sys",
|
||||
@@ -5730,6 +5714,17 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_path_to_error"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_repr"
|
||||
version = "0.1.20"
|
||||
@@ -6203,6 +6198,27 @@ dependencies = [
|
||||
"windows 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"core-foundation 0.9.4",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration-sys"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tagptr"
|
||||
version = "0.2.0"
|
||||
@@ -6444,6 +6460,24 @@ dependencies = [
|
||||
"tokio",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower-http"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"bytes",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6487,6 +6521,7 @@ version = "0.1.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647"
|
||||
dependencies = [
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
@@ -6641,6 +6676,12 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urlencoding"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-ranges"
|
||||
version = "1.0.5"
|
||||
@@ -6927,8 +6968,8 @@ dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link 0.1.3",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
"windows-result 0.3.4",
|
||||
"windows-strings 0.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6986,6 +7027,17 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
"windows-strings 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
@@ -6995,6 +7047,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.4.2"
|
||||
@@ -7004,6 +7065,15 @@ dependencies = [
|
||||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
|
||||
dependencies = [
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
|
||||
@@ -9,8 +9,9 @@ arrow-json = "58.3.0"
|
||||
arrow-schema = "58.3.0"
|
||||
futures = "0.3.31"
|
||||
futures-util = "0.3.31"
|
||||
lance-namespace = "7.0.0"
|
||||
lancedb = { version = "0.31.0", features = ["polars"] }
|
||||
lance-namespace = "8.0.0"
|
||||
lance-index = "=8.0.0"
|
||||
lancedb = { version = "0.31.0", features = ["polars", "remote"] }
|
||||
polars = ">=0.37,<0.40.0"
|
||||
polars-arrow = ">=0.37,<0.40.0"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -41,3 +42,35 @@ path = "multimodal.rs"
|
||||
[[example]]
|
||||
name = "search"
|
||||
path = "search.rs"
|
||||
|
||||
[[example]]
|
||||
name = "ingestion_examples"
|
||||
path = "ingestion_examples.rs"
|
||||
|
||||
[[example]]
|
||||
name = "ingestion_delete"
|
||||
path = "ingestion_delete.rs"
|
||||
|
||||
[[example]]
|
||||
name = "ingestion_progress"
|
||||
path = "ingestion_progress.rs"
|
||||
|
||||
[[example]]
|
||||
name = "search_vector_search"
|
||||
path = "search_vector_search.rs"
|
||||
|
||||
[[example]]
|
||||
name = "search_text_search"
|
||||
path = "search_text_search.rs"
|
||||
|
||||
[[example]]
|
||||
name = "search_hybrid_search"
|
||||
path = "search_hybrid_search.rs"
|
||||
|
||||
[[example]]
|
||||
name = "search_filter_search"
|
||||
path = "search_filter_search.rs"
|
||||
|
||||
[[example]]
|
||||
name = "search_query_plan"
|
||||
path = "search_query_plan.rs"
|
||||
|
||||
@@ -3,48 +3,69 @@
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lancedb::connect;
|
||||
use lancedb::{Connection, connect};
|
||||
|
||||
// --8<-- [start:connect]
|
||||
async fn connect_example(uri: &str) {
|
||||
let db = connect(uri).execute().await.unwrap();
|
||||
let _ = db;
|
||||
async fn connect_local() -> lancedb::Result<Connection> {
|
||||
// --8<-- [start:connect]
|
||||
let db = connect("ex_lancedb").execute().await?;
|
||||
// --8<-- [end:connect]
|
||||
Ok(db)
|
||||
}
|
||||
// --8<-- [end:connect]
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let uri = temp_dir.path().join("ex_lancedb");
|
||||
connect_example(uri.to_str().unwrap()).await;
|
||||
|
||||
// Keep the enterprise snippet in this file, but don't run it in CI.
|
||||
let _ = connect_enterprise_quickstart_config();
|
||||
let _ = connect_object_storage_config();
|
||||
}
|
||||
|
||||
fn connect_enterprise_quickstart_config() -> (String, String, String, String) {
|
||||
// --8<-- [start:connect_enterprise_quickstart]
|
||||
let uri = "db://your-database-uri";
|
||||
let api_key = "your-api-key";
|
||||
let region = "us-east-1";
|
||||
let host_override = "https://your-enterprise-endpoint.com";
|
||||
// --8<-- [end:connect_enterprise_quickstart]
|
||||
(
|
||||
uri.to_string(),
|
||||
api_key.to_string(),
|
||||
region.to_string(),
|
||||
host_override.to_string(),
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let temp_dir = tempfile::tempdir()?;
|
||||
std::env::set_current_dir(temp_dir.path())?;
|
||||
let db = connect_local().await?;
|
||||
db.create_empty_table(
|
||||
"probe",
|
||||
std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
|
||||
"id",
|
||||
arrow_schema::DataType::Int64,
|
||||
false,
|
||||
)])),
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
let reopened = connect(temp_dir.path().join("ex_lancedb").to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
assert_eq!(reopened.table_names().execute().await?, vec!["probe"]);
|
||||
|
||||
// Connects to a deployment only when asked, and only lists its tables.
|
||||
if std::env::var("LANCEDB_DOCS_TEST_DEPLOYMENT").as_deref() == Ok("1") {
|
||||
let db = connect_enterprise_quickstart().await?;
|
||||
println!("{:?}", db.table_names().execute().await?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn connect_object_storage_config() -> &'static str {
|
||||
async fn connect_enterprise_quickstart() -> Result<Connection, Box<dyn std::error::Error>> {
|
||||
// --8<-- [start:connect_enterprise_quickstart]
|
||||
let uri = std::env::var("LANCEDB_URI")?; // db://your-database
|
||||
let api_key = std::env::var("LANCEDB_API_KEY")?;
|
||||
let region = std::env::var("LANCEDB_REGION").unwrap_or_else(|_| "us-east-1".to_string());
|
||||
let host_override = std::env::var("LANCEDB_HOST_OVERRIDE")?; // https://your-endpoint
|
||||
|
||||
let db = connect(&uri)
|
||||
.api_key(&api_key)
|
||||
.region(®ion)
|
||||
.host_override(&host_override)
|
||||
.execute()
|
||||
.await?;
|
||||
// --8<-- [end:connect_enterprise_quickstart]
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
// Compiled as documentation, never called by the example runner.
|
||||
#[allow(dead_code)]
|
||||
async fn connect_object_storage() -> lancedb::Result<Connection> {
|
||||
// --8<-- [start:connect_object_storage]
|
||||
let uri = "s3://your-bucket/path";
|
||||
// You can also use "gs://your-bucket/path" or "az://your-container/path".
|
||||
let db = connect(uri).execute().await?;
|
||||
// --8<-- [end:connect_object_storage]
|
||||
|
||||
uri
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
async fn namespace_table_ops_example(uri: &str) -> lancedb::Result<()> {
|
||||
@@ -132,3 +153,17 @@ async fn namespace_admin_ops_example() -> lancedb::Result<()> {
|
||||
fn repo_root() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
|
||||
}
|
||||
|
||||
// Compiled as documentation, never called by the example runner.
|
||||
#[allow(dead_code)]
|
||||
async fn connect_enterprise_placeholders() -> Result<Connection, Box<dyn std::error::Error>> {
|
||||
// --8<-- [start:connect_enterprise_placeholders]
|
||||
let db = connect("db://your-database")
|
||||
.api_key("your-api-key")
|
||||
.region("us-east-1")
|
||||
.host_override("https://your-endpoint")
|
||||
.execute()
|
||||
.await?;
|
||||
// --8<-- [end:connect_enterprise_placeholders]
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ use lancedb::{
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let db = connect("./mydb").execute().await?;
|
||||
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
|
||||
let api_key = "your-openai-api-key".to_string();
|
||||
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
|
||||
api_key,
|
||||
"text-embedding-3-large",
|
||||
@@ -54,7 +54,7 @@ use std::sync::Arc;
|
||||
|
||||
use lancedb::embeddings::openai::OpenAIEmbeddingFunction;
|
||||
|
||||
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
|
||||
let api_key = "your-openai-api-key".to_string();
|
||||
let embedding = Arc::new(
|
||||
OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small")
|
||||
.expect("failed to create OpenAI embedding function"),
|
||||
@@ -77,7 +77,7 @@ use lancedb::{
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let db = connect("./mydb").execute().await?;
|
||||
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
|
||||
let api_key = "your-openai-api-key".to_string();
|
||||
let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
|
||||
api_key,
|
||||
"text-embedding-3-large",
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example]
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase};
|
||||
|
||||
fn make_users_reader(rows: &[(i64, &str)]) -> Box<dyn RecordBatchReader + Send> {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("name", DataType::Utf8, false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
|
||||
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
Box::new(RecordBatchIterator::new(
|
||||
vec![Ok(batch)].into_iter(),
|
||||
schema,
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let table = db
|
||||
.create_table(
|
||||
"users",
|
||||
make_users_reader(&[(1, "Alice"), (2, "Bob"), (3, "Charlie")]),
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
let predicate = "id = 3";
|
||||
let selected: Vec<RecordBatch> = table
|
||||
.query()
|
||||
.only_if(predicate)
|
||||
.execute()
|
||||
.await?
|
||||
.try_collect()
|
||||
.await?;
|
||||
println!("Selected: {selected:?}");
|
||||
table.delete(predicate).await?;
|
||||
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
|
||||
println!("Remaining: {rows:?}");
|
||||
assert_eq!(table.count_rows(None).await?, 2);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some("id = 1 AND name = 'Alice'".into()))
|
||||
.await?,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some("id = 2 AND name = 'Bob'".into()))
|
||||
.await?,
|
||||
1
|
||||
);
|
||||
assert_eq!(table.count_rows(Some("id = 3".into())).await?, 0);
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example]
|
||||
@@ -0,0 +1,97 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:write_workflow]
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase};
|
||||
|
||||
fn make_users_reader(rows: &[(i64, &str, i64)]) -> Box<dyn RecordBatchReader + Send> {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("name", DataType::Utf8, false),
|
||||
Field::new("login_count", DataType::Int64, false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
|
||||
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
|
||||
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.2))),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
Box::new(RecordBatchIterator::new(
|
||||
vec![Ok(batch)].into_iter(),
|
||||
schema,
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let table = db
|
||||
.create_table(
|
||||
"users",
|
||||
make_users_reader(&[(1, "Alice", 10), (2, "Bob", 20)]),
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
table
|
||||
.add(make_users_reader(&[(3, "Charlie", 5)]))
|
||||
.execute()
|
||||
.await?;
|
||||
table
|
||||
.update()
|
||||
.only_if("id = 2")
|
||||
.column("name", "'Bobby'")
|
||||
.execute()
|
||||
.await?;
|
||||
table
|
||||
.update()
|
||||
.only_if("id = 2")
|
||||
.column("login_count", "login_count + 1")
|
||||
.execute()
|
||||
.await?;
|
||||
let mut merge = table.merge_insert(&["id"]);
|
||||
merge
|
||||
.when_matched_update_all(None)
|
||||
.when_not_matched_insert_all();
|
||||
merge
|
||||
.execute(make_users_reader(&[(3, "Charlie", 6), (4, "Dana", 1)]))
|
||||
.await?;
|
||||
let selected: Vec<RecordBatch> = table
|
||||
.query()
|
||||
.only_if("id = 4")
|
||||
.execute()
|
||||
.await?
|
||||
.try_collect()
|
||||
.await?;
|
||||
println!("Rows selected for deletion: {selected:?}");
|
||||
table.delete("id = 4").await?;
|
||||
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
|
||||
println!("Remaining rows: {rows:?}");
|
||||
assert_eq!(table.count_rows(None).await?, 3);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some(
|
||||
"id = 2 AND name = 'Bobby' AND login_count = 21".into()
|
||||
))
|
||||
.await?,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some("id = 3 AND login_count = 6".into()))
|
||||
.await?,
|
||||
1
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:write_workflow]
|
||||
@@ -0,0 +1,64 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example]
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::query::ExecutableQuery;
|
||||
|
||||
fn make_users_reader(rows: &[(i64, &str)]) -> Box<dyn RecordBatchReader + Send> {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("name", DataType::Utf8, false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.0))),
|
||||
Arc::new(StringArray::from_iter_values(rows.iter().map(|r| r.1))),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
Box::new(RecordBatchIterator::new(
|
||||
vec![Ok(batch)].into_iter(),
|
||||
schema,
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let table = db
|
||||
.create_table("users", make_users_reader(&[(1, "Alice")]))
|
||||
.execute()
|
||||
.await?;
|
||||
table
|
||||
.add(make_users_reader(&[(2, "Bob"), (3, "Charlie")]))
|
||||
.progress(|progress| println!("Rows written: {}", progress.output_rows()))
|
||||
.execute()
|
||||
.await?;
|
||||
let rows: Vec<RecordBatch> = table.query().execute().await?.try_collect().await?;
|
||||
println!("After insertion: {rows:?}");
|
||||
assert_eq!(table.count_rows(None).await?, 3);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some("id = 1 AND name = 'Alice'".into()))
|
||||
.await?,
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
table
|
||||
.count_rows(Some("id = 2 AND name = 'Bob'".into()))
|
||||
.await?,
|
||||
1
|
||||
);
|
||||
assert_eq!(table.count_rows(Some("id = 3".into())).await?, 1);
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example]
|
||||
@@ -1,23 +1,20 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{
|
||||
FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator, StructArray,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, FieldRef, Schema};
|
||||
use lancedb::arrow::IntoPolars;
|
||||
use lancedb::database::CreateTableMode;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use lancedb::{connect, table::NewColumnTransform};
|
||||
use polars::prelude::DataFrame;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
// --8<-- [start:quickstart_define_struct]
|
||||
use std::sync::Arc;
|
||||
|
||||
use lancedb::arrow::arrow_array::types::Float32Type;
|
||||
use lancedb::arrow::arrow_array::{
|
||||
FixedSizeListArray, Int8Array, LargeStringArray, RecordBatch, RecordBatchIterator,
|
||||
RecordBatchReader, StructArray,
|
||||
};
|
||||
use lancedb::arrow::arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
|
||||
|
||||
// Define structs representing the data schema
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone)]
|
||||
struct Stats {
|
||||
strength: i8,
|
||||
magic: i8,
|
||||
@@ -25,7 +22,7 @@ struct Stats {
|
||||
wisdom: i8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone)]
|
||||
struct Character {
|
||||
id: String,
|
||||
name: String,
|
||||
@@ -43,7 +40,7 @@ fn characters_schema() -> Arc<Schema> {
|
||||
Field::new("description", DataType::LargeUtf8, false),
|
||||
Field::new(
|
||||
"stats",
|
||||
DataType::Struct(arrow_schema::Fields::from(vec![
|
||||
DataType::Struct(Fields::from(vec![
|
||||
Arc::new(Field::new("strength", DataType::Int8, false)),
|
||||
Arc::new(Field::new("magic", DataType::Int8, false)),
|
||||
Arc::new(Field::new("leadership", DataType::Int8, false)),
|
||||
@@ -58,11 +55,11 @@ fn characters_schema() -> Arc<Schema> {
|
||||
),
|
||||
]))
|
||||
}
|
||||
// --8<-- [end:quickstart_define_struct]
|
||||
|
||||
type BatchIter = Box<dyn arrow_array::RecordBatchReader + Send>;
|
||||
|
||||
fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
|
||||
fn characters_to_reader(
|
||||
schema: Arc<Schema>,
|
||||
rows: &[Character],
|
||||
) -> Box<dyn RecordBatchReader + Send> {
|
||||
let ids = LargeStringArray::from_iter_values(rows.iter().map(|row| row.id.as_str()));
|
||||
let names = LargeStringArray::from_iter_values(rows.iter().map(|row| row.name.as_str()));
|
||||
let roles = LargeStringArray::from_iter_values(rows.iter().map(|row| row.role.as_str()));
|
||||
@@ -114,6 +111,7 @@ fn characters_to_reader(schema: Arc<Schema>, rows: &[Character]) -> BatchIter {
|
||||
schema,
|
||||
))
|
||||
}
|
||||
// --8<-- [end:quickstart_define_struct]
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
@@ -166,6 +164,8 @@ async fn main() {
|
||||
// --8<-- [end:quickstart_data]
|
||||
|
||||
// --8<-- [start:quickstart_create_table]
|
||||
use lancedb::database::CreateTableMode;
|
||||
|
||||
let schema = characters_schema();
|
||||
let table = db
|
||||
.create_table("characters", characters_to_reader(schema.clone(), &data))
|
||||
@@ -175,22 +175,37 @@ async fn main() {
|
||||
.unwrap();
|
||||
// --8<-- [end:quickstart_create_table]
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 3);
|
||||
db.drop_table("characters", &[]).await.unwrap();
|
||||
|
||||
// --8<-- [start:quickstart_create_table_no_overwrite]
|
||||
let table = db
|
||||
.create_table("characters", characters_to_reader(schema.clone(), &data))
|
||||
// --8<-- [start:quickstart_add_data]
|
||||
let more_data = vec![Character {
|
||||
id: "4".to_string(),
|
||||
name: "Morgana".to_string(),
|
||||
role: "Sorceress".to_string(),
|
||||
description: "Powerful sorceress of Avalon.".to_string(),
|
||||
stats: Stats {
|
||||
strength: 2,
|
||||
magic: 5,
|
||||
leadership: 4,
|
||||
wisdom: 4,
|
||||
},
|
||||
vector: [0.3, 0.9, 0.6, 0.8],
|
||||
}];
|
||||
table
|
||||
.add(characters_to_reader(schema.clone(), &more_data))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
// --8<-- [end:quickstart_create_table_no_overwrite]
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 3);
|
||||
// --8<-- [end:quickstart_add_data]
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 4);
|
||||
|
||||
// --8<-- [start:quickstart_vector_search_1]
|
||||
use lancedb::arrow::IntoPolars;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
|
||||
// Search for examples similar to a "wise magical advisor"
|
||||
let query_vector = [0.2, 0.8, 0.4, 0.9];
|
||||
|
||||
let result: DataFrame = table
|
||||
let result = table
|
||||
.query()
|
||||
.nearest_to(&query_vector)
|
||||
.unwrap()
|
||||
@@ -211,9 +226,10 @@ async fn main() {
|
||||
// --8<-- [end:quickstart_vector_search_1]
|
||||
let name_col = result.column("name").unwrap().str().unwrap();
|
||||
assert_eq!(name_col.get(0).unwrap(), "Merlin");
|
||||
assert_eq!(name_col.get(1).unwrap(), "Morgana");
|
||||
|
||||
// --8<-- [start:quickstart_curate_with_metadata]
|
||||
let curated: DataFrame = table
|
||||
let curated = table
|
||||
.query()
|
||||
.nearest_to(&query_vector)
|
||||
.unwrap()
|
||||
@@ -235,9 +251,10 @@ async fn main() {
|
||||
// --8<-- [end:quickstart_curate_with_metadata]
|
||||
let curated_name_col = curated.column("name").unwrap().str().unwrap();
|
||||
assert_eq!(curated_name_col.get(0).unwrap(), "Merlin");
|
||||
assert_eq!(curated_name_col.get(1).unwrap(), "Morgana");
|
||||
|
||||
// --8<-- [start:quickstart_output_array]
|
||||
let result: DataFrame = table
|
||||
let result = table
|
||||
.query()
|
||||
.nearest_to(&query_vector)
|
||||
.unwrap()
|
||||
@@ -274,7 +291,7 @@ async fn main() {
|
||||
// --8<-- [end:quickstart_add_feature]
|
||||
|
||||
// --8<-- [start:quickstart_query_feature]
|
||||
let features: DataFrame = table
|
||||
let features = table
|
||||
.query()
|
||||
.select(Select::Columns(vec![
|
||||
"name".to_string(),
|
||||
@@ -289,19 +306,23 @@ async fn main() {
|
||||
.unwrap();
|
||||
println!("{features:?}");
|
||||
// --8<-- [end:quickstart_query_feature]
|
||||
assert!(features.column("power_score").is_ok());
|
||||
assert_eq!(features.column("power_score").unwrap().len(), 4);
|
||||
|
||||
// What the quickstart's download step leaves in an application's directory.
|
||||
let app = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(app.path().join("data")).unwrap();
|
||||
std::fs::copy(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../web/static/assets/images/quickstart/sir-lancelot.jpg"),
|
||||
app.path().join("data/sir-lancelot.jpg"),
|
||||
)
|
||||
.unwrap();
|
||||
std::env::set_current_dir(app.path()).unwrap();
|
||||
|
||||
// --8<-- [start:quickstart_multimodal_bytes]
|
||||
use std::sync::Arc;
|
||||
use lancedb::arrow::arrow_array::BinaryArray;
|
||||
|
||||
use arrow_array::{
|
||||
BinaryArray, FixedSizeListArray, LargeStringArray, RecordBatch, RecordBatchIterator,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
|
||||
let image_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../web/static/assets/images/quickstart/sir-lancelot.jpg");
|
||||
let image_bytes = std::fs::read(image_path).unwrap();
|
||||
let image_bytes = std::fs::read("data/sir-lancelot.jpg").unwrap();
|
||||
|
||||
let image_schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::LargeUtf8, false),
|
||||
@@ -333,9 +354,10 @@ async fn main() {
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let image_reader: Box<dyn arrow_array::RecordBatchReader + Send> = Box::new(
|
||||
RecordBatchIterator::new(vec![Ok(image_batch)].into_iter(), image_schema),
|
||||
);
|
||||
let image_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
|
||||
vec![Ok(image_batch)].into_iter(),
|
||||
image_schema,
|
||||
));
|
||||
let multimodal_table = db
|
||||
.create_table("character_images", image_reader)
|
||||
.mode(CreateTableMode::Overwrite)
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example_body]
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("text", DataType::Utf8, false),
|
||||
Field::new("category", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
|
||||
true,
|
||||
),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec![
|
||||
"A puppy playing in the park",
|
||||
"A dog resting at home",
|
||||
"A guide to growing tomatoes",
|
||||
])),
|
||||
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
|
||||
Arc::new(
|
||||
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
|
||||
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
|
||||
.into_iter()
|
||||
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
|
||||
2,
|
||||
),
|
||||
),
|
||||
],
|
||||
)?;
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
|
||||
let table = db
|
||||
.create_table(
|
||||
"documents",
|
||||
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
let query = table
|
||||
.vector_search(&[0.0_f32, 1.0])?
|
||||
.only_if("category = 'animals'")
|
||||
.select(Select::columns(&["id", "text", "category", "_distance"]))
|
||||
.limit(2);
|
||||
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
|
||||
println!("{rows:?}");
|
||||
// --8<-- [end:example_body]
|
||||
let ids: Vec<i64> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int64Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
ids.into_iter().collect::<std::collections::BTreeSet<_>>(),
|
||||
std::collections::BTreeSet::from([1, 2])
|
||||
);
|
||||
assert!(rows.iter().all(|b| {
|
||||
b.column_by_name("category")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|v| v == Some("animals"))
|
||||
}));
|
||||
// --8<-- [start:example_tail]
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example_tail]
|
||||
@@ -0,0 +1,99 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example_body]
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lance_index::scalar::FullTextSearchQuery;
|
||||
use lancedb::index::{Index, scalar::FtsIndexBuilder};
|
||||
use lancedb::query::{ExecutableQuery, QueryBase};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("text", DataType::Utf8, false),
|
||||
Field::new("category", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
|
||||
true,
|
||||
),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec![
|
||||
"A puppy playing in the park",
|
||||
"A dog resting at home",
|
||||
"A guide to growing tomatoes",
|
||||
])),
|
||||
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
|
||||
Arc::new(
|
||||
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
|
||||
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
|
||||
.into_iter()
|
||||
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
|
||||
2,
|
||||
),
|
||||
),
|
||||
],
|
||||
)?;
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
|
||||
let table = db
|
||||
.create_table(
|
||||
"documents",
|
||||
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
table
|
||||
.create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
|
||||
.execute()
|
||||
.await?;
|
||||
let query = table
|
||||
.vector_search(&[1.0_f32, 0.0])?
|
||||
.full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?)
|
||||
.rerank(Arc::new(lancedb::rerankers::rrf::RRFReranker::default()))
|
||||
.limit(2);
|
||||
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
|
||||
println!("{rows:?}");
|
||||
// --8<-- [end:example_body]
|
||||
let ids: Vec<i64> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int64Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(ids, vec![1, 2]);
|
||||
let scores: Vec<f32> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("_relevance_score")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::Float32Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert!(scores[0] > scores[1]);
|
||||
// --8<-- [start:example_tail]
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example_tail]
|
||||
@@ -0,0 +1,81 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example_body]
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("text", DataType::Utf8, false),
|
||||
Field::new("category", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
|
||||
true,
|
||||
),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec![
|
||||
"A puppy playing in the park",
|
||||
"A dog resting at home",
|
||||
"A guide to growing tomatoes",
|
||||
])),
|
||||
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
|
||||
Arc::new(
|
||||
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
|
||||
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
|
||||
.into_iter()
|
||||
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
|
||||
2,
|
||||
),
|
||||
),
|
||||
],
|
||||
)?;
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
|
||||
let table = db
|
||||
.create_table(
|
||||
"documents",
|
||||
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
let query = table
|
||||
.vector_search(&[1.0_f32, 0.0])?
|
||||
.select(Select::columns(&["id", "text", "_distance"]))
|
||||
.limit(2);
|
||||
println!("Plan: {}", query.explain_plan(false).await?);
|
||||
println!("Measured execution: {}", query.analyze_plan().await?);
|
||||
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
|
||||
println!("{rows:?}");
|
||||
// --8<-- [end:example_body]
|
||||
let ids: Vec<i64> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int64Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(ids, vec![1, 2]);
|
||||
// --8<-- [start:example_tail]
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example_tail]
|
||||
@@ -0,0 +1,85 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example_body]
|
||||
use arrow_array::{Int64Array, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lance_index::scalar::FullTextSearchQuery;
|
||||
use lancedb::index::{Index, scalar::FtsIndexBuilder};
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("text", DataType::Utf8, false),
|
||||
Field::new("category", DataType::Utf8, false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec![
|
||||
"A puppy playing in the park",
|
||||
"A dog resting at home",
|
||||
"A guide to growing tomatoes",
|
||||
])),
|
||||
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
|
||||
],
|
||||
)?;
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
|
||||
let table = db
|
||||
.create_table(
|
||||
"documents",
|
||||
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
table
|
||||
.create_index(&["text"], Index::FTS(FtsIndexBuilder::default()))
|
||||
.execute()
|
||||
.await?;
|
||||
let query = table
|
||||
.query()
|
||||
.full_text_search(FullTextSearchQuery::new("puppy".into()).with_column("text".into())?)
|
||||
.select(Select::columns(&["id", "text", "_score"]))
|
||||
.limit(2);
|
||||
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
|
||||
println!("{rows:?}");
|
||||
// --8<-- [end:example_body]
|
||||
let ids: Vec<i64> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int64Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(ids, vec![1]);
|
||||
let scores: Vec<f32> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("_score")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::Float32Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert!(scores[0] > 0.0);
|
||||
// --8<-- [start:example_tail]
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example_tail]
|
||||
@@ -0,0 +1,93 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
// --8<-- [start:example_body]
|
||||
use arrow_array::types::Float32Type;
|
||||
use arrow_array::{FixedSizeListArray, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lancedb::query::{ExecutableQuery, QueryBase, Select};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("text", DataType::Utf8, false),
|
||||
Field::new("category", DataType::Utf8, false),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
|
||||
true,
|
||||
),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(Int64Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec![
|
||||
"A puppy playing in the park",
|
||||
"A dog resting at home",
|
||||
"A guide to growing tomatoes",
|
||||
])),
|
||||
Arc::new(StringArray::from(vec!["animals", "animals", "gardening"])),
|
||||
Arc::new(
|
||||
FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
|
||||
[[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]]
|
||||
.into_iter()
|
||||
.map(|v| Some(v.into_iter().map(Some).collect::<Vec<_>>())),
|
||||
2,
|
||||
),
|
||||
),
|
||||
],
|
||||
)?;
|
||||
let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema);
|
||||
let table = db
|
||||
.create_table(
|
||||
"documents",
|
||||
Box::new(reader) as Box<dyn arrow_array::RecordBatchReader + Send>,
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
let query = table
|
||||
.vector_search(&[1.0_f32, 0.0])?
|
||||
.distance_type(lancedb::DistanceType::L2)
|
||||
.select(Select::columns(&["id", "text", "_distance"]))
|
||||
.limit(2);
|
||||
let rows: Vec<RecordBatch> = query.execute().await?.try_collect().await?;
|
||||
println!("{rows:?}");
|
||||
// --8<-- [end:example_body]
|
||||
let ids: Vec<i64> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("id")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<Int64Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(ids, vec![1, 2]);
|
||||
let scores: Vec<f32> = rows
|
||||
.iter()
|
||||
.flat_map(|b| {
|
||||
b.column_by_name("_distance")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref::<arrow_array::Float32Array>()
|
||||
.unwrap()
|
||||
.values()
|
||||
.to_vec()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(scores[0], 0.0);
|
||||
// --8<-- [start:example_tail]
|
||||
Ok(())
|
||||
}
|
||||
// --8<-- [end:example_tail]
|
||||
@@ -1,65 +1,89 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
import { expect, jest, test } from "@jest/globals";
|
||||
import * as path from "node:path";
|
||||
import { withTempDirectory } from "./util.ts";
|
||||
// --8<-- [start:connect]
|
||||
import { afterEach, describe, expect, jest, test } from "@jest/globals";
|
||||
import * as lancedb from "@lancedb/lancedb";
|
||||
|
||||
async function connectExample(uri: string) {
|
||||
const db = await lancedb.connect(uri);
|
||||
return db;
|
||||
}
|
||||
// --8<-- [end:connect]
|
||||
import { withTempDirectory } from "./util.ts";
|
||||
|
||||
test("connect to a local database", async () => {
|
||||
await withTempDirectory(async (tempDir) => {
|
||||
const uri = path.join(tempDir, "ex_lancedb");
|
||||
const db = await connectExample(uri);
|
||||
expect(db).toBeDefined();
|
||||
const cwd = process.cwd();
|
||||
process.chdir(tempDir);
|
||||
try {
|
||||
// --8<-- [start:connect]
|
||||
const db = await lancedb.connect("ex_lancedb");
|
||||
// --8<-- [end:connect]
|
||||
await db.createTable("probe", [{ id: 1 }]);
|
||||
expect((await db.listTables()).tables).toEqual(["probe"]);
|
||||
} finally {
|
||||
process.chdir(cwd);
|
||||
}
|
||||
const reopened = await lancedb.connect(`${tempDir}/ex_lancedb`);
|
||||
expect((await reopened.listTables()).tables).toEqual(["probe"]);
|
||||
});
|
||||
});
|
||||
|
||||
async function connectEnterpriseQuickstart() {
|
||||
// --8<-- [start:connect_enterprise_quickstart]
|
||||
const uri = "db://your-database-uri";
|
||||
const apiKey = "your-api-key";
|
||||
const region = "us-east-1";
|
||||
const hostOverride = "https://your-enterprise-endpoint.com";
|
||||
const setting = (name: string) => {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(`Set ${name}`);
|
||||
return value;
|
||||
};
|
||||
|
||||
const db = await lancedb.connect(uri, {
|
||||
apiKey,
|
||||
region,
|
||||
hostOverride,
|
||||
const db = await lancedb.connect(setting("LANCEDB_URI"), {
|
||||
apiKey: setting("LANCEDB_API_KEY"),
|
||||
region: process.env.LANCEDB_REGION ?? "us-east-1",
|
||||
hostOverride: setting("LANCEDB_HOST_OVERRIDE"),
|
||||
});
|
||||
// --8<-- [end:connect_enterprise_quickstart]
|
||||
return db;
|
||||
}
|
||||
|
||||
test("enterprise quickstart connect uses placeholder config", async () => {
|
||||
const mockDb = { __mock: true } as unknown as Awaited<
|
||||
ReturnType<typeof lancedb.connect>
|
||||
>;
|
||||
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
|
||||
|
||||
const db = await connectEnterpriseQuickstart();
|
||||
expect(db).toBe(mockDb);
|
||||
expect(spy).toHaveBeenCalledWith("db://your-database-uri", {
|
||||
apiKey: "your-api-key",
|
||||
region: "us-east-1",
|
||||
hostOverride: "https://your-enterprise-endpoint.com",
|
||||
describe("enterprise quickstart connect", () => {
|
||||
const saved = { ...process.env };
|
||||
afterEach(() => {
|
||||
process.env = { ...saved };
|
||||
});
|
||||
|
||||
test("reads the connection from the environment", async () => {
|
||||
const mockDb = { __mock: true } as unknown as Awaited<
|
||||
ReturnType<typeof lancedb.connect>
|
||||
>;
|
||||
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
|
||||
process.env.LANCEDB_URI = "db://your-database";
|
||||
process.env.LANCEDB_API_KEY = "your-api-key";
|
||||
delete process.env.LANCEDB_REGION;
|
||||
process.env.LANCEDB_HOST_OVERRIDE = "https://your-endpoint";
|
||||
|
||||
const db = await connectEnterpriseQuickstart();
|
||||
expect(db).toBe(mockDb);
|
||||
expect(spy).toHaveBeenCalledWith("db://your-database", {
|
||||
apiKey: "your-api-key",
|
||||
region: "us-east-1",
|
||||
hostOverride: "https://your-endpoint",
|
||||
});
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
test("requires an endpoint", async () => {
|
||||
process.env.LANCEDB_URI = "db://your-database";
|
||||
process.env.LANCEDB_API_KEY = "your-api-key";
|
||||
delete process.env.LANCEDB_HOST_OVERRIDE;
|
||||
|
||||
await expect(connectEnterpriseQuickstart()).rejects.toThrow(
|
||||
"Set LANCEDB_HOST_OVERRIDE",
|
||||
);
|
||||
});
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
// --8<-- [start:connect_object_storage]
|
||||
async function connectObjectStorageExample() {
|
||||
// --8<-- [start:connect_object_storage]
|
||||
const uri = "s3://your-bucket/path";
|
||||
// You can also use "gs://your-bucket/path" or "az://your-container/path".
|
||||
const db = await lancedb.connect(uri);
|
||||
// --8<-- [end:connect_object_storage]
|
||||
return db;
|
||||
}
|
||||
// --8<-- [end:connect_object_storage]
|
||||
|
||||
async function namespaceTableOpsExample() {
|
||||
// --8<-- [start:namespace_table_ops]
|
||||
@@ -119,3 +143,25 @@ void [
|
||||
namespaceTableOpsExample,
|
||||
namespaceAdminOpsExample,
|
||||
];
|
||||
|
||||
test("enterprise placeholders use an explicit endpoint", async () => {
|
||||
const mockDb = {} as Awaited<ReturnType<typeof lancedb.connect>>;
|
||||
const spy = jest.spyOn(lancedb, "connect").mockResolvedValue(mockDb);
|
||||
try {
|
||||
// --8<-- [start:connect_enterprise_placeholders]
|
||||
const db = await lancedb.connect("db://your-database", {
|
||||
apiKey: "your-api-key",
|
||||
region: "us-east-1",
|
||||
hostOverride: "https://your-endpoint",
|
||||
});
|
||||
// --8<-- [end:connect_enterprise_placeholders]
|
||||
expect(db).toBe(mockDb);
|
||||
expect(spy).toHaveBeenCalledWith("db://your-database", {
|
||||
apiKey: "your-api-key",
|
||||
region: "us-east-1",
|
||||
hostOverride: "https://your-endpoint",
|
||||
});
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
import { expect, test } from "@jest/globals";
|
||||
|
||||
test("complete ingestion workflow", async () => {
|
||||
// --8<-- [start:write_workflow_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("users", [
|
||||
{ id: 1, name: "Alice", login_count: 10 },
|
||||
{ id: 2, name: "Bob", login_count: 20 },
|
||||
]);
|
||||
await table.add([{ id: 3, name: "Charlie", login_count: 5 }]);
|
||||
await table.update({ where: "id = 2", values: { name: "Bobby" } });
|
||||
await table.update({
|
||||
where: "id = 2",
|
||||
valuesSql: { login_count: "login_count + 1" },
|
||||
});
|
||||
await table
|
||||
.mergeInsert("id")
|
||||
.whenMatchedUpdateAll()
|
||||
.whenNotMatchedInsertAll()
|
||||
.execute([
|
||||
{ id: 3, name: "Charlie", login_count: 6 },
|
||||
{ id: 4, name: "Dana", login_count: 1 },
|
||||
]);
|
||||
console.log(
|
||||
"Rows selected for deletion:",
|
||||
await table.query().where("id = 4").toArray(),
|
||||
);
|
||||
await table.delete("id = 4");
|
||||
const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);
|
||||
console.log(rows);
|
||||
// --8<-- [end:write_workflow_body]
|
||||
expect(rows.map((row) => [row.id, row.name, row.login_count])).toEqual([
|
||||
[1, "Alice", 10],
|
||||
[2, "Bobby", 21],
|
||||
[3, "Charlie", 6],
|
||||
]);
|
||||
// --8<-- [start:write_workflow_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:write_workflow_tail]
|
||||
});
|
||||
|
||||
test("complete delete_rows", async () => {
|
||||
// --8<-- [start:delete_rows_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("users", [
|
||||
{ id: 1, name: "Alice" },
|
||||
{ id: 2, name: "Bob" },
|
||||
{ id: 3, name: "Charlie" },
|
||||
]);
|
||||
const predicate = "id = 3";
|
||||
console.log("Selected:", await table.query().where(predicate).toArray());
|
||||
await table.delete(predicate);
|
||||
const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);
|
||||
console.log("Remaining:", rows);
|
||||
// --8<-- [end:delete_rows_body]
|
||||
expect(rows.map((row) => [row.id, row.name])).toEqual([
|
||||
[1, "Alice"],
|
||||
[2, "Bob"],
|
||||
]);
|
||||
// --8<-- [start:delete_rows_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:delete_rows_tail]
|
||||
});
|
||||
|
||||
test("complete add_progress", async () => {
|
||||
// --8<-- [start:add_progress_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-ingestion-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("users", [{ id: 1, name: "Alice" }]);
|
||||
await table.add(
|
||||
[
|
||||
{ id: 2, name: "Bob" },
|
||||
{ id: 3, name: "Charlie" },
|
||||
],
|
||||
{
|
||||
progress: (progress) =>
|
||||
console.log(`Rows written: ${progress.outputRows}`),
|
||||
},
|
||||
);
|
||||
const rows = (await table.query().toArray()).sort((a, b) => a.id - b.id);
|
||||
console.log("After insertion:", rows);
|
||||
// --8<-- [end:add_progress_body]
|
||||
expect(rows.map((row) => [row.id, row.name])).toEqual([
|
||||
[1, "Alice"],
|
||||
[2, "Bob"],
|
||||
[3, "Charlie"],
|
||||
]);
|
||||
// --8<-- [start:add_progress_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:add_progress_tail]
|
||||
});
|
||||
@@ -1,176 +1,263 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
import { expect, test } from "@jest/globals";
|
||||
import { describe, expect, test } from "@jest/globals";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import * as lancedb from "@lancedb/lancedb";
|
||||
import { withTempDirectory } from "./util.ts";
|
||||
|
||||
test("quickstart example (async)", async () => {
|
||||
// What the quickstart's download step leaves in an application's directory.
|
||||
const IMAGE = path.join(
|
||||
__dirname,
|
||||
"../../web/static/assets/images/quickstart/sir-lancelot.jpg",
|
||||
);
|
||||
const deployment = ["LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY"];
|
||||
// The connection settings alone never send these tests to a deployment.
|
||||
const optIn = "LANCEDB_DOCS_TEST_DEPLOYMENT";
|
||||
const connections =
|
||||
process.env[optIn] === "1" ? ["local", "enterprise"] : ["local"];
|
||||
|
||||
async function tableNames(db: lancedb.Connection) {
|
||||
const names: string[] = [];
|
||||
let pageToken: string | undefined;
|
||||
do {
|
||||
const page = await db.listTables({ pageToken });
|
||||
names.push(...page.tables);
|
||||
pageToken = page.pageToken;
|
||||
} while (pageToken);
|
||||
return names;
|
||||
}
|
||||
|
||||
// A deployment connection on which a test creates only tables of its own:
|
||||
// each name gets a suffix unique to the test and is claimed only while no
|
||||
// table has it; the test drops what it claimed, whether or not it passed.
|
||||
async function withOwnedTables(
|
||||
db: lancedb.Connection,
|
||||
fn: (db: lancedb.Connection) => Promise<void>,
|
||||
) {
|
||||
const suffix = randomUUID().replaceAll("-", "").slice(0, 12);
|
||||
const claimed: string[] = [];
|
||||
const createTable = async (
|
||||
name: string,
|
||||
data: Parameters<lancedb.Connection["createTable"]>[1],
|
||||
options: Partial<lancedb.CreateTableOptions> = {},
|
||||
) => {
|
||||
const owned = `${name}_${suffix}`;
|
||||
let mode = options.mode;
|
||||
if (!claimed.includes(owned)) {
|
||||
if ((await tableNames(db)).includes(owned)) {
|
||||
throw new Error(`${owned} already exists and is not this test's`);
|
||||
}
|
||||
claimed.push(owned);
|
||||
mode = "create";
|
||||
}
|
||||
return db.createTable(owned, data, { ...options, mode });
|
||||
};
|
||||
try {
|
||||
await fn(
|
||||
new Proxy(db, {
|
||||
get: (target, property, receiver) =>
|
||||
property === "createTable"
|
||||
? createTable
|
||||
: Reflect.get(target, property, receiver),
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
const existing = await tableNames(db);
|
||||
for (const name of claimed) {
|
||||
if (existing.includes(name)) {
|
||||
await db.dropTable(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function withDatabase(
|
||||
connection: string,
|
||||
fn: (db: lancedb.Connection) => Promise<void>,
|
||||
) {
|
||||
if (connection === "enterprise") {
|
||||
const missing = deployment.filter((name) => !process.env[name]);
|
||||
if (missing.length) {
|
||||
throw new Error(`${optIn}=1 needs ${missing.join(", ")}`);
|
||||
}
|
||||
const db = await lancedb.connect(process.env.LANCEDB_URI!, {
|
||||
apiKey: process.env.LANCEDB_API_KEY!,
|
||||
region: process.env.LANCEDB_REGION ?? "us-east-1",
|
||||
hostOverride: process.env.LANCEDB_HOST_OVERRIDE!,
|
||||
});
|
||||
await withOwnedTables(db, fn);
|
||||
return;
|
||||
}
|
||||
await withTempDirectory(async (databaseDir) => {
|
||||
const db = await lancedb.connect(databaseDir);
|
||||
await fn(await lancedb.connect(databaseDir));
|
||||
});
|
||||
}
|
||||
|
||||
// --8<-- [start:quickstart_data]
|
||||
const data = [
|
||||
{
|
||||
id: "1",
|
||||
name: "King Arthur",
|
||||
role: "King",
|
||||
description: "Leader of Camelot and wielder of Excalibur.",
|
||||
stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
|
||||
vector: [0.7, 0.1, 0.9, 0.7],
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
name: "Merlin",
|
||||
role: "Wizard",
|
||||
description: "Advisor and prophet with deep magical knowledge.",
|
||||
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
|
||||
vector: [0.2, 0.9, 0.4, 0.9],
|
||||
},
|
||||
{
|
||||
id: "3",
|
||||
name: "Sir Lancelot",
|
||||
role: "Knight",
|
||||
description: "Legendary knight known for courage and combat skill.",
|
||||
stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
|
||||
vector: [0.9, 0.1, 0.5, 0.4],
|
||||
},
|
||||
];
|
||||
// --8<-- [end:quickstart_data]
|
||||
async function createCharacters(db: lancedb.Connection) {
|
||||
// --8<-- [start:quickstart_data]
|
||||
const data = [
|
||||
{
|
||||
id: "1",
|
||||
name: "King Arthur",
|
||||
role: "King",
|
||||
description: "Leader of Camelot and wielder of Excalibur.",
|
||||
stats: { strength: 4, magic: 1, leadership: 5, wisdom: 4 },
|
||||
vector: [0.7, 0.1, 0.9, 0.7],
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
name: "Merlin",
|
||||
role: "Wizard",
|
||||
description: "Advisor and prophet with deep magical knowledge.",
|
||||
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 5 },
|
||||
vector: [0.2, 0.9, 0.4, 0.9],
|
||||
},
|
||||
{
|
||||
id: "3",
|
||||
name: "Sir Lancelot",
|
||||
role: "Knight",
|
||||
description: "Legendary knight known for courage and combat skill.",
|
||||
stats: { strength: 5, magic: 1, leadership: 3, wisdom: 3 },
|
||||
vector: [0.9, 0.1, 0.5, 0.4],
|
||||
},
|
||||
];
|
||||
// --8<-- [end:quickstart_data]
|
||||
|
||||
// --8<-- [start:quickstart_create_table]
|
||||
let table = await db.createTable("characters", data, { mode: "overwrite" });
|
||||
// --8<-- [end:quickstart_create_table]
|
||||
expect(await table.countRows()).toBe(3);
|
||||
await db.dropTable("characters");
|
||||
// --8<-- [start:quickstart_create_table]
|
||||
const table = await db.createTable("characters", data, { mode: "overwrite" });
|
||||
// --8<-- [end:quickstart_create_table]
|
||||
expect(await table.countRows()).toBe(3);
|
||||
|
||||
// --8<-- [start:quickstart_create_table_no_overwrite]
|
||||
table = await db.createTable("characters", data);
|
||||
// --8<-- [end:quickstart_create_table_no_overwrite]
|
||||
expect(await table.countRows()).toBe(3);
|
||||
// --8<-- [start:quickstart_add_data]
|
||||
await table.add([
|
||||
{
|
||||
id: "4",
|
||||
name: "Morgana",
|
||||
role: "Sorceress",
|
||||
description: "Powerful sorceress of Avalon.",
|
||||
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
|
||||
vector: [0.3, 0.9, 0.6, 0.8],
|
||||
},
|
||||
]);
|
||||
// --8<-- [end:quickstart_add_data]
|
||||
expect(await table.countRows()).toBe(4);
|
||||
return table;
|
||||
}
|
||||
|
||||
// --8<-- [start:quickstart_vector_search_1]
|
||||
// Search for examples similar to a "wise magical advisor"
|
||||
let queryVector = [0.2, 0.8, 0.4, 0.9];
|
||||
describe.each(connections)("quickstart (%s)", (connection) => {
|
||||
test("create, add and search", async () => {
|
||||
await withDatabase(connection, async (db) => {
|
||||
const table = await createCharacters(db);
|
||||
|
||||
let result = await table
|
||||
.search(queryVector)
|
||||
.select(["name", "role", "description", "_distance"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.table(result);
|
||||
// --8<-- [end:quickstart_vector_search_1]
|
||||
expect(result[0].name).toBe("Merlin");
|
||||
// --8<-- [start:quickstart_vector_search_1]
|
||||
// Search for examples similar to a "wise magical advisor"
|
||||
const queryVector = [0.2, 0.8, 0.4, 0.9];
|
||||
|
||||
// --8<-- [start:quickstart_curate_with_metadata]
|
||||
const curated = await table
|
||||
.search(queryVector)
|
||||
.where("stats.magic >= 4")
|
||||
.select(["name", "role", "description", "_distance"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.table(curated);
|
||||
// --8<-- [end:quickstart_curate_with_metadata]
|
||||
expect(curated[0].name).toBe("Merlin");
|
||||
const result = await table
|
||||
.search(queryVector)
|
||||
.select(["name", "role", "description", "_distance"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.table(result);
|
||||
// --8<-- [end:quickstart_vector_search_1]
|
||||
expect(result.map((row) => row.name)).toEqual(["Merlin", "Morgana"]);
|
||||
|
||||
// --8<-- [start:quickstart_output_array]
|
||||
result = await table.search(queryVector).limit(2).toArray();
|
||||
console.table(result);
|
||||
// --8<-- [end:quickstart_output_array]
|
||||
expect(result[0].name).toBe("Merlin");
|
||||
// --8<-- [start:quickstart_curate_with_metadata]
|
||||
const curated = await table
|
||||
.search(queryVector)
|
||||
.where("stats.magic >= 4")
|
||||
.select(["name", "role", "description", "_distance"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.table(curated);
|
||||
// --8<-- [end:quickstart_curate_with_metadata]
|
||||
expect(curated.map((row) => row.name)).toEqual(["Merlin", "Morgana"]);
|
||||
});
|
||||
});
|
||||
|
||||
// --8<-- [start:quickstart_add_feature]
|
||||
await table.addColumns([
|
||||
{
|
||||
name: "power_score",
|
||||
valueSql:
|
||||
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
|
||||
},
|
||||
]);
|
||||
// --8<-- [end:quickstart_add_feature]
|
||||
const schemaWithFeature = await table.schema();
|
||||
expect(schemaWithFeature.fields.some((f) => f.name === "power_score")).toBe(
|
||||
true,
|
||||
);
|
||||
test("a derived feature", async () => {
|
||||
await withDatabase(connection, async (db) => {
|
||||
const table = await createCharacters(db);
|
||||
|
||||
// --8<-- [start:quickstart_query_feature]
|
||||
const features = await table
|
||||
.query()
|
||||
.select(["name", "role", "power_score"])
|
||||
.toArray();
|
||||
console.table(features);
|
||||
// --8<-- [end:quickstart_query_feature]
|
||||
expect(features[0]).toHaveProperty("power_score");
|
||||
|
||||
// --8<-- [start:quickstart_multimodal_bytes]
|
||||
const arrow = await import("apache-arrow");
|
||||
const path = await import("node:path");
|
||||
const { readFile } = await import("node:fs/promises");
|
||||
|
||||
const imagePath = path.join(
|
||||
__dirname,
|
||||
"../../web/static/assets/images/quickstart/sir-lancelot.jpg",
|
||||
);
|
||||
const imageBytes = await readFile(imagePath);
|
||||
const imageSchema = new arrow.Schema([
|
||||
new arrow.Field("id", new arrow.Utf8()),
|
||||
new arrow.Field("description", new arrow.Utf8()),
|
||||
new arrow.Field("image", new arrow.Binary()),
|
||||
new arrow.Field(
|
||||
"vector",
|
||||
new arrow.FixedSizeList(
|
||||
4,
|
||||
new arrow.Field("item", new arrow.Float32(), true),
|
||||
),
|
||||
),
|
||||
]);
|
||||
const imageData = lancedb.makeArrowTable(
|
||||
[
|
||||
// --8<-- [start:quickstart_add_feature]
|
||||
await table.addColumns([
|
||||
{
|
||||
id: "lancelot",
|
||||
description: "Portrait of Sir Lancelot",
|
||||
image: imageBytes,
|
||||
vector: [0.9, 0.1, 0.5, 0.4],
|
||||
name: "power_score",
|
||||
valueSql:
|
||||
"cast(((stats.strength + stats.magic + stats.leadership + stats.wisdom) / 4.0) as float)",
|
||||
},
|
||||
],
|
||||
{ schema: imageSchema },
|
||||
);
|
||||
const multimodalTable = await db.createTable(
|
||||
"character_images",
|
||||
imageData,
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
// --8<-- [end:quickstart_multimodal_bytes]
|
||||
expect(await multimodalTable.countRows()).toBe(1);
|
||||
]);
|
||||
// --8<-- [end:quickstart_add_feature]
|
||||
|
||||
// --8<-- [start:quickstart_open_table]
|
||||
table = await db.openTable("characters");
|
||||
// --8<-- [end:quickstart_open_table]
|
||||
// --8<-- [start:quickstart_query_feature]
|
||||
const features = await table
|
||||
.query()
|
||||
.select(["name", "role", "power_score"])
|
||||
.toArray();
|
||||
console.table(features);
|
||||
// --8<-- [end:quickstart_query_feature]
|
||||
expect(
|
||||
features.map((row) => [row.name, row.power_score]).sort(),
|
||||
).toEqual([
|
||||
["King Arthur", 3.5],
|
||||
["Merlin", 4],
|
||||
["Morgana", 3.75],
|
||||
["Sir Lancelot", 3],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// --8<-- [start:quickstart_add_data]
|
||||
const moreData = [
|
||||
{
|
||||
id: "4",
|
||||
name: "Morgana",
|
||||
role: "Sorceress",
|
||||
description: "Powerful sorceress of Avalon.",
|
||||
stats: { strength: 2, magic: 5, leadership: 4, wisdom: 4 },
|
||||
vector: [0.3, 0.9, 0.6, 0.8],
|
||||
power_score: 3.75,
|
||||
},
|
||||
];
|
||||
test("multimodal data", async () => {
|
||||
await withDatabase(connection, async (db) => {
|
||||
await withTempDirectory(async (app) => {
|
||||
fs.mkdirSync(path.join(app, "data"));
|
||||
fs.copyFileSync(IMAGE, path.join(app, "data/sir-lancelot.jpg"));
|
||||
const cwd = process.cwd();
|
||||
process.chdir(app);
|
||||
try {
|
||||
// --8<-- [start:quickstart_multimodal_bytes]
|
||||
const arrow = await import("apache-arrow");
|
||||
const { readFile } = await import("node:fs/promises");
|
||||
|
||||
// Add data to table
|
||||
await table.add(moreData);
|
||||
// --8<-- [end:quickstart_add_data]
|
||||
expect(await table.countRows()).toBe(4);
|
||||
|
||||
// --8<-- [start:quickstart_vector_search_2]
|
||||
// Search for examples similar to a "powerful sorceress"
|
||||
queryVector = [0.3, 0.9, 0.6, 0.8];
|
||||
|
||||
const results = await table.search(queryVector).limit(2).toArray();
|
||||
console.table(results);
|
||||
// --8<-- [end:quickstart_vector_search_2]
|
||||
expect(results[0].name).toBe("Morgana");
|
||||
const imageBytes = await readFile("data/sir-lancelot.jpg");
|
||||
const imageSchema = new arrow.Schema([
|
||||
new arrow.Field("id", new arrow.Utf8()),
|
||||
new arrow.Field("description", new arrow.Utf8()),
|
||||
new arrow.Field("image", new arrow.Binary()),
|
||||
new arrow.Field(
|
||||
"vector",
|
||||
new arrow.FixedSizeList(
|
||||
4,
|
||||
new arrow.Field("item", new arrow.Float32(), true),
|
||||
),
|
||||
),
|
||||
]);
|
||||
const imageData = lancedb.makeArrowTable(
|
||||
[
|
||||
{
|
||||
id: "lancelot",
|
||||
description: "Portrait of Sir Lancelot",
|
||||
image: imageBytes,
|
||||
vector: [0.9, 0.1, 0.5, 0.4],
|
||||
},
|
||||
],
|
||||
{ schema: imageSchema },
|
||||
);
|
||||
const multimodalTable = await db.createTable(
|
||||
"character_images",
|
||||
imageData,
|
||||
{ mode: "overwrite" },
|
||||
);
|
||||
// --8<-- [end:quickstart_multimodal_bytes]
|
||||
expect(await multimodalTable.countRows()).toBe(1);
|
||||
const [row] = await multimodalTable.query().select(["image"]).toArray();
|
||||
expect(Buffer.from(row.image)).toEqual(fs.readFileSync(IMAGE));
|
||||
} finally {
|
||||
process.chdir(cwd);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { expect, test } from "@jest/globals";
|
||||
|
||||
test("complete vector_search", async () => {
|
||||
// --8<-- [start:vector_search_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("documents", [
|
||||
{
|
||||
id: 1,
|
||||
text: "A puppy playing in the park",
|
||||
category: "animals",
|
||||
vector: [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
text: "A dog resting at home",
|
||||
category: "animals",
|
||||
vector: [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
text: "A guide to growing tomatoes",
|
||||
category: "gardening",
|
||||
vector: [0.0, 1.0],
|
||||
},
|
||||
]);
|
||||
const rows = await table
|
||||
.vectorSearch([1.0, 0.0])
|
||||
.distanceType("l2")
|
||||
.select(["id", "text", "_distance"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.log(rows);
|
||||
// --8<-- [end:vector_search_body]
|
||||
expect(rows.map((row) => row.id)).toEqual([1, 2]);
|
||||
expect(rows[0]._distance).toBe(0);
|
||||
// --8<-- [start:vector_search_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:vector_search_tail]
|
||||
});
|
||||
|
||||
test("complete text_search", async () => {
|
||||
// --8<-- [start:text_search_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("documents", [
|
||||
{ id: 1, text: "A puppy playing in the park", category: "animals" },
|
||||
{ id: 2, text: "A dog resting at home", category: "animals" },
|
||||
{ id: 3, text: "A guide to growing tomatoes", category: "gardening" },
|
||||
]);
|
||||
await table.createIndex("text", { config: lancedb.Index.fts() });
|
||||
const rows = await table
|
||||
.query()
|
||||
.fullTextSearch("puppy", { columns: ["text"] })
|
||||
.select(["id", "text", "_score"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.log(rows);
|
||||
// --8<-- [end:text_search_body]
|
||||
expect(rows.map((row) => row.id)).toEqual([1]);
|
||||
expect(rows[0]._score).toBeGreaterThan(0);
|
||||
// --8<-- [start:text_search_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:text_search_tail]
|
||||
});
|
||||
|
||||
test("complete hybrid_search", async () => {
|
||||
// --8<-- [start:hybrid_search_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("documents", [
|
||||
{
|
||||
id: 1,
|
||||
text: "A puppy playing in the park",
|
||||
category: "animals",
|
||||
vector: [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
text: "A dog resting at home",
|
||||
category: "animals",
|
||||
vector: [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
text: "A guide to growing tomatoes",
|
||||
category: "gardening",
|
||||
vector: [0.0, 1.0],
|
||||
},
|
||||
]);
|
||||
await table.createIndex("text", { config: lancedb.Index.fts() });
|
||||
const reranker = await lancedb.rerankers.RRFReranker.create();
|
||||
const rows = await table
|
||||
.vectorSearch([1.0, 0.0])
|
||||
.fullTextSearch("puppy", { columns: ["text"] })
|
||||
.rerank(reranker)
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.log(rows);
|
||||
// --8<-- [end:hybrid_search_body]
|
||||
expect(rows.map((row) => row.id)).toEqual([1, 2]);
|
||||
expect(rows[0]._relevance_score).toBeGreaterThan(rows[1]._relevance_score);
|
||||
// --8<-- [start:hybrid_search_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:hybrid_search_tail]
|
||||
});
|
||||
|
||||
test("complete filter_search", async () => {
|
||||
// --8<-- [start:filter_search_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("documents", [
|
||||
{
|
||||
id: 1,
|
||||
text: "A puppy playing in the park",
|
||||
category: "animals",
|
||||
vector: [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
text: "A dog resting at home",
|
||||
category: "animals",
|
||||
vector: [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
text: "A guide to growing tomatoes",
|
||||
category: "gardening",
|
||||
vector: [0.0, 1.0],
|
||||
},
|
||||
]);
|
||||
const rows = await table
|
||||
.vectorSearch([0.0, 1.0])
|
||||
.where("category = 'animals'")
|
||||
.select(["id", "text", "category", "_distance"])
|
||||
.limit(2)
|
||||
.toArray();
|
||||
console.log(rows);
|
||||
// --8<-- [end:filter_search_body]
|
||||
expect(rows.map((row) => row.id).sort()).toEqual([1, 2]);
|
||||
expect(rows.every((row) => row.category === "animals")).toBe(true);
|
||||
// --8<-- [start:filter_search_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:filter_search_tail]
|
||||
});
|
||||
|
||||
test("complete query_plan", async () => {
|
||||
// --8<-- [start:query_plan_body]
|
||||
const lancedb = await import("@lancedb/lancedb");
|
||||
const { mkdtemp, rm } = await import("node:fs/promises");
|
||||
const { tmpdir } = await import("node:os");
|
||||
const { join } = await import("node:path");
|
||||
const uri = await mkdtemp(join(tmpdir(), "lancedb-search-"));
|
||||
try {
|
||||
const db = await lancedb.connect(uri);
|
||||
const table = await db.createTable("documents", [
|
||||
{
|
||||
id: 1,
|
||||
text: "A puppy playing in the park",
|
||||
category: "animals",
|
||||
vector: [1.0, 0.0],
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
text: "A dog resting at home",
|
||||
category: "animals",
|
||||
vector: [0.9, 0.1],
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
text: "A guide to growing tomatoes",
|
||||
category: "gardening",
|
||||
vector: [0.0, 1.0],
|
||||
},
|
||||
]);
|
||||
const query = table
|
||||
.vectorSearch([1.0, 0.0])
|
||||
.select(["id", "text", "_distance"])
|
||||
.limit(2);
|
||||
console.log("Plan:", await query.explainPlan());
|
||||
console.log("Measured execution:", await query.analyzePlan());
|
||||
const rows = await query.toArray();
|
||||
console.log(rows);
|
||||
// --8<-- [end:query_plan_body]
|
||||
expect(rows.map((row) => row.id)).toEqual([1, 2]);
|
||||
// --8<-- [start:query_plan_tail]
|
||||
} finally {
|
||||
await rm(uri, { recursive: true, force: true });
|
||||
}
|
||||
// --8<-- [end:query_plan_tail]
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
# Functions are documented before their release. At release, remove these
|
||||
# entries, add the pages back to the navigation in docs.json, restore the
|
||||
# /feature-engineering/functions redirect to /management/function, and publish
|
||||
# the drafts marked "Draft for the Functions release".
|
||||
management/function.mdx
|
||||
sql/ddl/function.mdx
|
||||
@@ -35,10 +35,13 @@ To authenticate to the Enterprise REST API, you need the endpoint for your deplo
|
||||
|
||||
```bash
|
||||
export LANCEDB_API_KEY="<your-api-key>"
|
||||
export LANCEDB_URI="https://your-enterprise-endpoint.com"
|
||||
export LANCEDB_HOST_OVERRIDE="https://your-enterprise-endpoint.com"
|
||||
export LANCEDB_DATABASE="your-database-name"
|
||||
```
|
||||
|
||||
`LANCEDB_HOST_OVERRIDE` is the endpoint the SDKs take as `host_override`; the
|
||||
database name is the part of its `db://` URI after `db://`.
|
||||
|
||||
3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint.
|
||||
|
||||
### Verify authentication {#verify-authentication}
|
||||
@@ -46,30 +49,37 @@ export LANCEDB_DATABASE="your-database-name"
|
||||
4. Check that you can reach the deployment and list tables:
|
||||
|
||||
```bash
|
||||
curl -X GET "$LANCEDB_URI/v1/management/table" \
|
||||
-H "Content-Type: application/json" \
|
||||
curl -X GET "$LANCEDB_HOST_OVERRIDE/v1/table/" \
|
||||
-H "x-api-key: $LANCEDB_API_KEY" \
|
||||
-H "x-lancedb-database: $LANCEDB_DATABASE"
|
||||
```
|
||||
|
||||
The response lists one page of table names, with a token for the next page when
|
||||
there are more.
|
||||
|
||||
If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header.
|
||||
|
||||
5. Create a table to confirm write access. Let's call it `words`.
|
||||
5. Create a table to confirm write access. Let's call it `words`. A table is
|
||||
created from an Arrow IPC stream, which this writes with PyArrow:
|
||||
|
||||
```bash
|
||||
curl -X POST "$LANCEDB_URI/v1/tables/words" \
|
||||
python -c "import pyarrow as pa, sys; t = pa.table({'word': ['hello', 'world']}); w = pa.ipc.new_stream(sys.stdout.buffer, t.schema); w.write_table(t); w.close()" > words.arrow
|
||||
|
||||
curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/create" \
|
||||
-H "Content-Type: application/vnd.apache.arrow.stream" \
|
||||
-H "x-api-key: $LANCEDB_API_KEY" \
|
||||
-H "x-lancedb-database: $LANCEDB_DATABASE"
|
||||
-H "x-lancedb-database: $LANCEDB_DATABASE" \
|
||||
--data-binary @words.arrow
|
||||
```
|
||||
|
||||
6. Check that the table has been created:
|
||||
|
||||
```bash
|
||||
curl -X GET "$LANCEDB_URI/v1/management/table" \
|
||||
curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-api-key: $LANCEDB_API_KEY" \
|
||||
-H "x-lancedb-database: $LANCEDB_DATABASE"
|
||||
-H "x-lancedb-database: $LANCEDB_DATABASE" \
|
||||
-d '{}'
|
||||
```
|
||||
|
||||
That's it -- you're connected! Now, you can start adding data and querying it.
|
||||
|
||||
+85
-114
@@ -1,153 +1,124 @@
|
||||
---
|
||||
title: "Architecture"
|
||||
sidebarTitle: "Architecture"
|
||||
description: "How LanceDB is put together, and what changes between running it embedded and running it distributed."
|
||||
description: "How LanceDB Enterprise separates query serving, background work, and storage, and how it compares with embedded LanceDB OSS."
|
||||
icon: cubes
|
||||
keywords: ["architecture", "embedded", "distributed", "object storage", "compute", "deployment"]
|
||||
---
|
||||
|
||||
LanceDB has one storage layer and two ways to put compute on top of it. Which
|
||||
one you run decides where queries execute, what runs on your behalf, and what
|
||||
you have to operate — and almost nothing else. The data is the same
|
||||
[Lance](/basics/lance) tables either way.
|
||||
LanceDB OSS is an embedded library for working with Lance tables. LanceDB
|
||||
Enterprise is a multimodal lakehouse for data curation, feature engineering,
|
||||
search, and training data access. Its architecture separates query serving,
|
||||
read execution, caching, and background work. Both use the open
|
||||
[Lance](/basics/lance) format.
|
||||
|
||||
## One storage layer {#storage}
|
||||
Teams can curate datasets, derive features, search multimodal records, and
|
||||
read the resulting tables into training frameworks. Enterprise separates the
|
||||
resources that serve shared data from those that build indexes and run
|
||||
background work, so teams can size them for these different workloads.
|
||||
|
||||
Everything durable lives in object storage: table data, manifests, indexes. No
|
||||
machine owns a dataset, and no server has to be running for the data to exist.
|
||||
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────┐
|
||||
│ object storage │
|
||||
│ table data · manifests · indexes │
|
||||
└──────────────────────────────────────────┘
|
||||
```
|
||||
The diagram shows the main compute and storage boundaries, not every request
|
||||
path. The control plane governs the Enterprise service.
|
||||
|
||||
That is the whole persistence story, and it is why the two deployment shapes can
|
||||
differ so much above it while staying compatible below it. A table written
|
||||
embedded is opened by a deployment, and the reverse, without conversion.
|
||||
## A shared format, different storage choices {#storage}
|
||||
|
||||
## Embedded {#embedded}
|
||||
Lance stores table data, versions, and indexes. OSS can read and write tables on
|
||||
local disk or in object storage. Enterprise keeps durable table data and index
|
||||
artifacts in object storage, independently of the machines serving requests.
|
||||
Its caches accelerate reads; they are not the durable source of truth.
|
||||
|
||||
```
|
||||
your process
|
||||
┌────────────────────────────────┐
|
||||
│ application │
|
||||
│ LanceDB library │ ← plans, executes, indexes, compacts
|
||||
└────────────────────────────────┘
|
||||
│
|
||||
┌─────────▼─────────┐
|
||||
│ object storage │
|
||||
└───────────────────┘
|
||||
```
|
||||
Sharing a format does not mean that an Enterprise deployment automatically
|
||||
opens an existing OSS database. Plan how to move or connect your data and
|
||||
configure access for your deployment.
|
||||
|
||||
`lancedb.connect("./data")` or `connect("s3://bucket/path")` gives you a
|
||||
**library**, not a client. Your process opens the tables, plans the queries,
|
||||
runs the scans, builds the indexes, and writes the new versions. There is no
|
||||
server in the picture at all.
|
||||
## OSS: an embedded library inside your application {#embedded}
|
||||
|
||||
**What follows from that.** Nothing runs unless you call it — no background
|
||||
compaction, no server-side [jobs](/basics/jobs), no
|
||||
[SQL endpoint](/basics/sql). Scale is your process's scale: a query gets the
|
||||
cores and memory your process has. Access control is whatever your object store
|
||||
enforces, since there is no other gatekeeper.
|
||||
With an embedded connection, the LanceDB library plans and executes queries,
|
||||
builds indexes, and updates tables inside your process. There is no separate
|
||||
database server to deploy.
|
||||
|
||||
**What it is good at.** Development, notebooks, single-node pipelines, embedding
|
||||
LanceDB inside an application, and anything where "no service to operate" is
|
||||
worth more than elasticity. Comfortably handles millions of vectors on one
|
||||
node -- disk-based indexes keep it performant without extra infrastructure.
|
||||
This fits local retrieval applications, notebooks, development, and
|
||||
single-node data pipelines. Your application controls the compute resources and
|
||||
when to run [table maintenance](/maintenance/compaction). You also manage access
|
||||
to the local files or object storage that hold your tables.
|
||||
|
||||
## Distributed {#distributed}
|
||||
## Enterprise: a multimodal lakehouse for shared workloads {#distributed}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
Applications connect to remote tables through a LanceDB SDK. The service handles
|
||||
requests across a fleet, with components sized for different kinds of work:
|
||||
|
||||
```
|
||||
your process
|
||||
┌──────────────┐
|
||||
│ application │
|
||||
│ LanceDB SDK │ ← a thin client
|
||||
└──────┬───────┘
|
||||
│ db:// (REST / Flight SQL)
|
||||
┌──────▼──────────────────────────────────────┐
|
||||
│ control plane config · identity · policy │
|
||||
├─────────────────────────────────────────────┤
|
||||
│ data plane query serving · writes │
|
||||
│ indexers indexing · compaction │
|
||||
└──────┬──────────────────────────────────────┘
|
||||
│
|
||||
┌──────▼────────────┐
|
||||
│ object storage │
|
||||
└───────────────────┘
|
||||
```
|
||||
- **Query serving** receives requests, resolves tables, and plans operations.
|
||||
Add serving capacity as more applications and users depend on your data.
|
||||
- **Read execution and caching** use separately provisioned resources. Executors
|
||||
cache table data to reduce repeated reads from object storage.
|
||||
- **Indexing and background workers** build indexes and perform maintenance
|
||||
outside the query request path. Size this work separately from query serving
|
||||
as your data grows.
|
||||
|
||||
`connect("db://...")` gives you a **client**. Planning and execution move to a
|
||||
cluster, and the work splits across tiers that scale independently: a control
|
||||
plane governing configuration, identity and policy; a data plane serving reads
|
||||
and writes; and indexers building indexes and compacting outside the request
|
||||
path.
|
||||
The control plane manages configuration, identity, policy, and cluster
|
||||
lifecycle. It governs the service rather than executing table queries.
|
||||
[Authorization](/governance/authorization) lets teams control access to catalog
|
||||
objects when enabled and configured by the administrator. Deployment settings govern background
|
||||
[compaction](/maintenance/compaction) and
|
||||
[cleanup](/maintenance/cleanup), including retention.
|
||||
|
||||
**What follows from that.** Compute and storage are separated, so a query fleet
|
||||
scales for interactive traffic without also scaling indexing capacity, and heavy
|
||||
background work does not steal resources from user queries. Because a server
|
||||
exists, so do the things that need one: [SQL](/sql),
|
||||
[functions](/management/function), server-side jobs, and
|
||||
[authorization](/governance/authorization) per object rather than per bucket.
|
||||
These boundaries support different parts of an AI data workflow.
|
||||
[SQL curation](/curation) requires a deployment with SQL enabled.
|
||||
[Feature engineering](/feature-engineering) can produce derived columns and
|
||||
tables; computed columns and materialized views require a deployment that
|
||||
enables them. [Geneva](/geneva) runs distributed Python feature backfills while
|
||||
its capabilities continue to transition to LanceDB APIs.
|
||||
[Training data loaders](/training/data-loading) read tables into model training
|
||||
frameworks in both offerings. LanceDB provides data access for training; the
|
||||
training framework runs the model.
|
||||
|
||||
**What it is good at.** Many concurrent queries, continuous ingestion and
|
||||
indexing, multi-team access with real permissions, and workloads too large for
|
||||
one machine -- customers regularly operate on billions of rows and tens of
|
||||
terabytes.
|
||||
Size and monitor each tier for your workload. Read the
|
||||
[Enterprise architecture](/enterprise/architecture) for the component details
|
||||
and request flows.
|
||||
|
||||
## What changes, precisely {#differences}
|
||||
## Application changes for Enterprise {#differences}
|
||||
|
||||
| | Embedded | Distributed |
|
||||
| --- | --- | --- |
|
||||
| What you connect to | a library in your process | a cluster |
|
||||
| Where queries plan and execute | your process | the data plane |
|
||||
| Table handle | `LanceTable` | `RemoteTable` |
|
||||
| Indexing and compaction | when you call it | dedicated indexers |
|
||||
| [Jobs](/basics/jobs) | run in-process, no server record | real jobs: id, status, events, cancel |
|
||||
| [SQL](/sql) | unavailable | Flight SQL endpoint |
|
||||
| [Functions](/management/function) | unavailable | run on workers |
|
||||
| Access control | object-store permissions | [per-object grants](/governance/authorization) |
|
||||
| Scale bound | your process | the cluster |
|
||||
| What you operate | nothing | a deployment |
|
||||
Core operations remain familiar: create a table, add data, and run a search.
|
||||
With OSS, the application executes those operations. With Enterprise, the SDK
|
||||
sends requests to the service, and the service manages execution and configured
|
||||
background maintenance.
|
||||
|
||||
Both expose the same table API — create, open, mutate, query — so application
|
||||
code moves between them by changing the URI. What does not move is code that
|
||||
depends on a row in the right-hand column.
|
||||
To use Enterprise, configure the cluster endpoint, database URI and credentials,
|
||||
and make your data available in the Enterprise database. Review SDK methods
|
||||
that behave differently on remote tables, including result retrieval and
|
||||
background jobs. [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise)
|
||||
describes these application changes. If you plan to use materialized views, check the
|
||||
[stable row ID requirements](/basics/limits#schema) before creating source tables.
|
||||
|
||||
## Choosing, and changing your mind {#choosing}
|
||||
## Choose for the workload you need to run {#choosing}
|
||||
|
||||
Start embedded. It has no operational cost and the API is the same one you will
|
||||
keep using.
|
||||
Choose OSS when you want an embedded library and can manage its resources and
|
||||
maintenance in your application.
|
||||
|
||||
Move when you hit something in the right-hand column — usually concurrent
|
||||
serving, ingestion that outgrows one machine, or a permissions requirement.
|
||||
Because the storage layer is identical, moving is a connection-string change
|
||||
plus a data migration, not a rewrite.
|
||||
Choose Enterprise when multiple applications or teams need a shared multimodal
|
||||
lakehouse for curation, feature engineering, search, and training data access.
|
||||
It also fits workloads whose query and indexing demand grow at different rates,
|
||||
or that need configured service-level access controls and background operations.
|
||||
You can begin with Enterprise.
|
||||
|
||||
<Note>
|
||||
The one thing to decide early is **stable row ids**: the SDK can give them to a
|
||||
table only when it creates one on a local path or an object store, and a
|
||||
deployment may require them on a
|
||||
[materialized view](/management/materialized-view) source. See
|
||||
[Limits](/basics/limits#schema).
|
||||
</Note>
|
||||
[Talk to us about your workload](https://www.lancedb.com/contact) to plan capacity,
|
||||
deployment options, and the path from your existing data to Enterprise.
|
||||
|
||||
## Where to go next {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Enterprise architecture" icon="cubes" href="/enterprise/architecture">
|
||||
Query serving, execution, and background work in detail.
|
||||
</Card>
|
||||
<Card title="Connect to Enterprise" icon="plug" href="/basics/connect#enterprise">
|
||||
Connect an application to your deployment.
|
||||
</Card>
|
||||
<Card title="Guarantees" icon="shield-check" href="/basics/guarantees">
|
||||
What a write promises, and what a reader sees.
|
||||
</Card>
|
||||
<Card title="Lance format" icon="layer-group" href="/basics/lance">
|
||||
The open format underneath both shapes.
|
||||
</Card>
|
||||
<Card title="Enterprise architecture" icon="cubes" href="/enterprise/architecture">
|
||||
The distributed tiers in detail.
|
||||
</Card>
|
||||
<Card title="Limits" icon="ruler" href="/basics/limits">
|
||||
The bounds you can actually hit.
|
||||
The open format underneath both offerings.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
@@ -269,10 +269,11 @@ where appropriate.
|
||||
|
||||
Choose [LanceDB Enterprise](/enterprise) when the resulting table becomes
|
||||
shared production infrastructure, and the workload needs distributed capacity,
|
||||
private deployment, or platform-managed operations. The underlying data format and table
|
||||
API stay the same, so the pipeline does not need to be redesigned. The agent
|
||||
instead connects to a remote `db://` table and lets the cluster handle
|
||||
maintenance and background work.
|
||||
private deployment, or platform-managed operations. The data format and the core
|
||||
table API stay the same, so the pipeline's table and search code carries over:
|
||||
the agent connects to a remote `db://` table, leaves maintenance to the
|
||||
deployment instead of calling `optimize()`, and submits long-running work as
|
||||
jobs. See [Moving to LanceDB Enterprise](/basics/sdk#moving-to-enterprise).
|
||||
|
||||
## What agents are good at here {#workflows}
|
||||
|
||||
|
||||
+49
-90
@@ -1,107 +1,66 @@
|
||||
---
|
||||
title: "Lance Catalog"
|
||||
sidebarTitle: "Lance Catalog"
|
||||
description: "The naming layer: what objects exist, how they nest, and who may reach them."
|
||||
description: "Organize, find and control access to your data in LanceDB."
|
||||
icon: sitemap
|
||||
keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view", "function"]
|
||||
keywords: ["catalog", "namespace", "hierarchy", "governance", "database", "table", "view"]
|
||||
---
|
||||
|
||||
A catalog answers one question: **what exists, and what is it called.** It holds
|
||||
names, the hierarchy they sit in, and the rules about who may reach them. It
|
||||
does not store rows, run queries, or build indexes -- that is the
|
||||
[lakehouse](/basics/lakehouse) on top of it.
|
||||
## Catalog overview {#catalog}
|
||||
|
||||
Keeping the two apart is what lets a Lance table be reached by something that is
|
||||
not LanceDB. The catalog is a specification -- the
|
||||
[Lance Namespace](/basics/lance#standard) -- so any engine that implements it
|
||||
resolves the same names to the same tables.
|
||||
A catalog keeps track of named data objects, such as tables and views, so
|
||||
applications can discover and access them. In LanceDB, it organizes those
|
||||
objects into databases and namespaces.
|
||||
|
||||
## The objects {#objects}
|
||||
## Databases, namespaces and tables {#objects}
|
||||
|
||||
| Object | What it is |
|
||||
A database contains namespaces that group related tables and views. Namespaces
|
||||
can also contain other namespaces, allowing teams to organize data by project,
|
||||
environment or purpose.
|
||||
|
||||
For example, a search application can keep production and staging data in
|
||||
separate namespaces within the same database:
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
database["ai_data · database"] --> prod["prod · namespace"]
|
||||
database --> staging["staging · namespace"]
|
||||
prod --> prod_docs["documents · table"]
|
||||
staging --> staging_docs["documents · table"]
|
||||
```
|
||||
|
||||
<span id="views" />
|
||||
|
||||
| Object | What it represents |
|
||||
| --- | --- |
|
||||
| [Database](/management/database) | The root. A connection opens exactly one. |
|
||||
| [Namespace](/management/namespace) | A named grouping inside a database. Can nest. |
|
||||
| [Table](/management/table) | Rows and columns, versioned. What most operations act on. |
|
||||
| [Materialized view](/management/materialized-view) | A table whose contents come from a query, computed and stored. |
|
||||
| [View](/management/view) | A saved query. Computed when read, not stored. |
|
||||
| [Function](/management/function) | Registered code, applied to produce column values. |
|
||||
| [Database](/management/database) | A container for namespaces and their data objects. |
|
||||
| [Namespace](/management/namespace) | A named group of tables, views or nested namespaces. |
|
||||
| [Table](/management/table) | Data stored as rows and columns. |
|
||||
| [View](/management/view) | A saved query, evaluated when read. Available in Enterprise. |
|
||||
| [Materialized view](/management/materialized-view) | The stored results of a query, updated by a refresh. |
|
||||
|
||||
```
|
||||
database
|
||||
└── namespace
|
||||
├── table
|
||||
├── materialized view (a table, defined by a query)
|
||||
├── view (a query, not stored)
|
||||
└── function
|
||||
```
|
||||
{/*
|
||||
Draft for the Functions release. Add this row to the table above:
|
||||
| [Function](/management/function) | Registered code used to compute column values. Available in Enterprise. |
|
||||
*/}
|
||||
|
||||
A namespace can contain further namespaces, so the hierarchy goes as deep as
|
||||
your organisation needs. A table is reached by its path through it:
|
||||
`prod.events.clicks`.
|
||||
<span id="resolution" />
|
||||
|
||||
This page is about what the objects *are*. For the operations on them, see
|
||||
[Management](/management).
|
||||
## Organization and access control {#governance}
|
||||
|
||||
## Resolution {#resolution}
|
||||
- **Separate projects and environments.** Keep production and staging tables
|
||||
in distinct namespaces, even when they use the same table names.
|
||||
- **Find shared datasets.** Give teams a consistent place to discover and
|
||||
access the data they work with.
|
||||
- **Control access in Enterprise.** Use permissions on databases, namespaces
|
||||
and tables to manage who can access shared data and what they can do with it.
|
||||
|
||||
A name is resolved by walking the path, and the walk is the point: reaching a
|
||||
table means traversing the namespaces above it. That is what makes the hierarchy
|
||||
more than decoration -- it is the unit of both organisation and access.
|
||||
<span id="elsewhere" />
|
||||
|
||||
A flat set of tables needs none of this. Namespaces start earning their keep when
|
||||
several teams, environments or domains share one catalog and you want
|
||||
`prod.search.documents` and `staging.search.documents` to be unambiguous.
|
||||
## Learn more {#next}
|
||||
|
||||
## Governance {#governance}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
Because the catalog knows the hierarchy, it is the layer that can enforce
|
||||
access. A grant names **one privilege** on **one object** for **one subject**,
|
||||
and it inherits down the path: reaching a table requires `USAGE` on the
|
||||
namespace containing it and on the database containing that.
|
||||
|
||||
That is the practical reason to put a boundary in the hierarchy -- a namespace
|
||||
per team gives you something to grant on. See
|
||||
[Authorization](/governance/authorization).
|
||||
|
||||
Embedded LanceDB has no such layer. Access is whatever the object store enforces
|
||||
on the bucket, which has no concept of a table.
|
||||
|
||||
## Views and materialized views are different objects {#views}
|
||||
|
||||
The names are close enough to mislead. Both are catalog objects defined by a
|
||||
query over other tables; the difference is when that query runs.
|
||||
|
||||
- A **materialized view** stores its rows, and a refresh brings them up to
|
||||
date. Reads are reads of stored data -- fast, and as current as the last
|
||||
refresh.
|
||||
- A **view** stores only the query. Every read runs it, so a view is never stale
|
||||
and never faster than the query it wraps.
|
||||
|
||||
Reach for a materialized view when the query is expensive and the data changes
|
||||
less often than it is read. Reach for a view when correctness matters more than
|
||||
latency, or when the query is cheap.
|
||||
|
||||
## What the catalog does not hold {#elsewhere}
|
||||
|
||||
**Indexes** belong to a table rather than standing alone. Creating one is a
|
||||
performance decision, not a structural one -- see [Indexing](/indexing).
|
||||
|
||||
**Jobs** are not stored data at all; they are handles to work in progress. See
|
||||
[Jobs](/basics/jobs).
|
||||
|
||||
**Rows** are in the [Lance](/basics/lance) files the catalog points at. The
|
||||
catalog knows a table exists and where it lives; it does not know what is in it.
|
||||
|
||||
## Next {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Lance Lakehouse" icon="layer-group" href="/basics/lakehouse">
|
||||
What operates on these objects: search, compute, maintenance.
|
||||
</Card>
|
||||
<Card title="Management" icon="folder-tree" href="/management">
|
||||
Creating, listing and dropping each one.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
- [Databases](/management/database): connect to and manage a database.
|
||||
- [Namespaces](/management/namespace): create groups and address tables within them.
|
||||
- [Tables](/management/table): create, inspect and manage your data.
|
||||
- [Views](/management/view) and [materialized views](/management/materialized-view): work with query-defined data.
|
||||
- [Authorization](/governance/authorization): configure Enterprise access permissions.
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
---
|
||||
title: "Connect to LanceDB"
|
||||
sidebarTitle: "Connect"
|
||||
description: "Use the embedded LanceDB OSS library with tables in a local directory or object storage, or connect to LanceDB Enterprise."
|
||||
icon: plug
|
||||
keywords: ["connect", "uri", "embedded", "object storage", "s3", "enterprise", "db://"]
|
||||
---
|
||||
|
||||
import {
|
||||
PyConnect,
|
||||
PyConnectAsync,
|
||||
PyConnectEnterprisePlaceholders,
|
||||
PyConnectEnterpriseQuickstart,
|
||||
PyConnectObjectStorage,
|
||||
PyConnectObjectStorageAsync,
|
||||
RsConnect,
|
||||
TsConnect,
|
||||
TsConnectEnterprisePlaceholders,
|
||||
TsConnectEnterpriseQuickstart,
|
||||
TsConnectObjectStorage,
|
||||
} from '/snippets/connection.mdx';
|
||||
|
||||
The URI you pass to `connect` decides where the work runs. The same Python,
|
||||
TypeScript and Rust SDKs open all three kinds;
|
||||
[OSS and Enterprise](/basics/offerings) describes how the two offerings differ.
|
||||
|
||||
| URI | Where queries run |
|
||||
| --- | --- |
|
||||
| A local path, such as `./data` | OSS: in your process |
|
||||
| `s3://`, `gs://` or `az://` | OSS: in your process, reading object storage |
|
||||
| `db://` | LanceDB Enterprise: queries run on your cluster |
|
||||
|
||||
## A local directory {#local}
|
||||
|
||||
Embedded in your application, LanceDB reads and writes the tables in a
|
||||
directory on disk.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python (sync)" language="Python" icon="python">
|
||||
{PyConnect}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Python (async)" language="Python" icon="python">
|
||||
{PyConnectAsync}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
|
||||
{TsConnect}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{ "use lancedb::connect;\n" }
|
||||
{ "\n" }
|
||||
{RsConnect}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
## Object storage {#object-storage}
|
||||
|
||||
With an object storage URI the library is still embedded: your process runs
|
||||
every query and reads and writes the bucket directly, with the credentials from
|
||||
its environment or its storage options. [Configuring storage](/deployment/storage)
|
||||
covers credentials, endpoints and provider options.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python (sync)" language="Python" icon="python">
|
||||
{PyConnectObjectStorage}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Python (async)" language="Python" icon="python">
|
||||
{PyConnectObjectStorageAsync}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
|
||||
{TsConnectObjectStorage}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
In Rust, pass the same URI to `connect`.
|
||||
|
||||
## LanceDB Enterprise {#enterprise}
|
||||
|
||||
A `db://` URI connects to LanceDB Enterprise, where your cluster runs queries
|
||||
and background work. Get the database URI, API key, region and endpoint from
|
||||
your administrator, then replace the placeholders below. Use your cluster's region
|
||||
in place of `us-east-1`. In application code, load credentials from environment
|
||||
variables or a secrets manager.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyConnectEnterprisePlaceholders}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
|
||||
{TsConnectEnterprisePlaceholders}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
In Rust, enable the crate's `remote` feature; the connection builder then takes
|
||||
the same settings through `api_key`, `region` and `host_override`. To use OAuth
|
||||
instead of an API key, see
|
||||
[Authentication](/enterprise/authentication); for every connection option, see
|
||||
[Database](/management/database#deployment).
|
||||
|
||||
|
||||
<Accordion title="Load production credentials from your environment">
|
||||
|
||||
Set these values in the shell that starts your application. Replace the URI, API
|
||||
key, region and endpoint with your Enterprise connection settings. In production, a secrets
|
||||
manager can supply these environment variables for you.
|
||||
|
||||
```shell
|
||||
export LANCEDB_URI="db://your-database"
|
||||
export LANCEDB_API_KEY="your-api-key"
|
||||
export LANCEDB_REGION="us-east-1"
|
||||
export LANCEDB_HOST_OVERRIDE="https://your-endpoint"
|
||||
```
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyConnectEnterpriseQuickstart}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{ "import * as lancedb from \"@lancedb/lancedb\";\n\n" }
|
||||
{TsConnectEnterpriseQuickstart}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
</Accordion>
|
||||
+59
-102
@@ -1,134 +1,91 @@
|
||||
---
|
||||
title: "Guarantees"
|
||||
sidebarTitle: "Guarantees"
|
||||
description: "What a write promises when it returns, what a reader sees, and what survives a failure."
|
||||
description: "Understand table commits, concurrent writes and the boundaries of recovery."
|
||||
icon: shield-check
|
||||
keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency", "acid"]
|
||||
keywords: ["guarantees", "consistency", "atomicity", "durability", "concurrency"]
|
||||
---
|
||||
|
||||
What the system promises, and — just as usefully — what it does not. Most
|
||||
surprises in practice come from assuming one of the guarantees below is
|
||||
stronger or weaker than it is.
|
||||
## Atomic table writes {#writes}
|
||||
|
||||
## Writes {#writes}
|
||||
A table transaction commits its changes atomically: readers see the committed
|
||||
table version, not a partially applied transaction. For example, a committed
|
||||
append adds its rows together.
|
||||
|
||||
**A write is atomic.** Every write commits a new table version. Either the
|
||||
version exists and holds all of the change, or it does not exist and holds none
|
||||
of it. There is no state in which a reader sees half an `add()`.
|
||||
This boundary is one table transaction. A workflow that updates several tables
|
||||
or runs a background job can contain multiple commits. Do not assume the whole
|
||||
workflow is atomic.
|
||||
|
||||
**A write returns only after the commit.** When the call returns, the version
|
||||
is durable in object storage. Nothing is buffered in your process waiting to be
|
||||
flushed.
|
||||
<span id="schema" />
|
||||
|
||||
**Every write advances the version**, including one that affects zero rows. A
|
||||
delete whose predicate matches nothing still commits. Version numbers count
|
||||
commits, not changes.
|
||||
Schema changes are also part of table history. Changing a column's type can
|
||||
rewrite data; adding a derived column can write the new column independently.
|
||||
|
||||
**Concurrent writes are resolved optimistically.** A transaction records the
|
||||
change and the version it was based on. At commit time, a conflicting
|
||||
concurrent operation is detected rather than silently overwritten, and the
|
||||
commit is retried against the newer version. Operations that do not collide —
|
||||
appending while another process appends — both succeed.
|
||||
## Consistent reads {#reads}
|
||||
|
||||
<Note>
|
||||
`merge_insert` reports its retry attempts in the result. A high count means
|
||||
contention on the same keys, not an error.
|
||||
</Note>
|
||||
A read of a table uses a consistent snapshot. A consistent snapshot can still
|
||||
be older than another writer's latest commit. [Consistency](/tables/consistency)
|
||||
explains how OSS and Enterprise choose the version a reader sees.
|
||||
|
||||
## Reads {#reads}
|
||||
For an experiment that must keep using the same data, select a specific retained
|
||||
table version rather than relying on a connection's default freshness.
|
||||
|
||||
**A read sees a consistent snapshot.** A query reads one table version
|
||||
throughout; it never sees a mixture of two.
|
||||
## Concurrent writes {#concurrency}
|
||||
|
||||
**Which version, by default, is the one open at the time the table was opened.**
|
||||
This is the guarantee most often assumed to be stronger than it is:
|
||||
Lance uses optimistic concurrency control. Writers prepare changes, then check
|
||||
compatibility with intervening commits. Compatible changes can be committed;
|
||||
conflicts may be retried or returned as errors.
|
||||
|
||||
<Warning>
|
||||
A long-lived connection **does not observe other processes' writes** until it
|
||||
reopens the table or its `read_consistency_interval` elapses. The default is
|
||||
right for a batch job reading a stable snapshot and wrong for a service
|
||||
expected to see fresh data.
|
||||
</Warning>
|
||||
Applications must handle failed writes. Concurrent appends are compatible, but
|
||||
that does not promise success through storage failures or exhausted retries.
|
||||
Do not rely on a particular ordering between independent writers.
|
||||
|
||||
Set `read_consistency_interval` when several processes write the same table and
|
||||
readers must observe those writes. The cost is a metadata check on that
|
||||
interval. See [Consistency](/tables/consistency).
|
||||
The [Lance transaction specification](https://lance.org/format/table/transaction/)
|
||||
describes commit behavior and compatibility between operations.
|
||||
|
||||
**Your own writes are always visible to you.** The interval governs other
|
||||
writers, not yourself.
|
||||
## Storage durability {#durability}
|
||||
|
||||
## Durability and history {#durability}
|
||||
A committed table version is stored in the table’s configured storage. LanceDB
|
||||
OSS uses your local filesystem or object store; LanceDB Enterprise uses the
|
||||
cluster’s configured storage.
|
||||
|
||||
**Old versions are retained, not overwritten.** A new version does not destroy
|
||||
the one before it, which is what makes [time travel](/management/table#versioning) work
|
||||
and what lets a reader finish a scan that started on an older version.
|
||||
Table version history does not replace backups. Configure storage redundancy
|
||||
and backups to meet your recovery requirements.
|
||||
|
||||
**Deletion is a mark, not an erase.** A deleted row is recorded in a deletion
|
||||
file. The data is recoverable for as long as the version that contains it
|
||||
exists.
|
||||
## Background job completion {#jobs}
|
||||
|
||||
<Warning>
|
||||
Retention is finite. [Compaction](/maintenance/compaction) prunes versions
|
||||
older than the retention window — **7 days by default** — and a tag or branch
|
||||
pointing at a pruned version no longer resolves. If you rely on reading old
|
||||
versions, widen the window before running it, not after.
|
||||
</Warning>
|
||||
In LanceDB Enterprise, operations such as index builds and column refreshes
|
||||
can run as background jobs. Receiving a job handle confirms submission. Wait
|
||||
for the job to succeed before relying on its result.
|
||||
|
||||
**Dropping a table ends its history.** Versioning does not survive
|
||||
`drop_table`; there is no undrop.
|
||||
Some operations save results in batches. If a job fails or is cancelled,
|
||||
batches already committed can remain in the table. Each operation’s guide
|
||||
explains whether it can leave partial results.
|
||||
|
||||
## Jobs {#jobs}
|
||||
## Writes after a client timeout {#timeouts}
|
||||
|
||||
**A refresh recomputes only what changed.** It gives values to rows that lack
|
||||
them and recomputes rows whose inputs changed since they were computed; it
|
||||
leaves the rest alone. Two consequences, both relied on elsewhere:
|
||||
If a client times out while waiting for a write, the write may already have
|
||||
committed. Retrying an append can therefore insert the same records again.
|
||||
|
||||
- **Retrying is safe.** Re-running a refresh fills what is still empty or stale.
|
||||
It is not guaranteed to cost only the rows that did not complete: a retry can
|
||||
recompute values that were already written.
|
||||
- **A filled value catches up at the next refresh.** If a column derives from
|
||||
`price` and a row's `price` later changes, the derived value keeps its old
|
||||
value until the next refresh recomputes it.
|
||||
Check whether the intended records were written before retrying. For an
|
||||
operation submitted as a background job, check its status using the job handle.
|
||||
|
||||
**What a failure keeps depends on where the refresh ran.** `refresh_column` in
|
||||
your process commits the column once, so a failure keeps none of its values. A
|
||||
refresh job on a deployment commits finished batches as it goes, so some values
|
||||
can remain.
|
||||
## Version history and recovery {#history}
|
||||
|
||||
**Cancellation is a request.** `cancel()` asks a job to stop; it does not
|
||||
guarantee it stops before doing more work, and cancelling a finished job does
|
||||
nothing.
|
||||
Retained table versions let you inspect earlier data or restore a table to a
|
||||
previous version. Cleanup can remove those versions and their underlying files,
|
||||
so configure retention to cover the history you need.
|
||||
|
||||
**A job id is opaque.** It correlates with server logs and the jobs API.
|
||||
Parsing it, or storing it to resume the job later, is not supported.
|
||||
Dropping a table is different from deleting records. Table version history
|
||||
cannot undo `drop_table`.
|
||||
|
||||
See [Jobs](/basics/jobs).
|
||||
<span id="not-guaranteed" />
|
||||
|
||||
## Schema {#schema}
|
||||
## Learn more {#next}
|
||||
|
||||
**Schema changes are versioned like data changes.** Adding, renaming, retyping
|
||||
or dropping a column commits a version, so a schema change can be travelled back
|
||||
through.
|
||||
|
||||
**A computed column's inputs are protected.** While a declaration references a
|
||||
column, that column cannot be renamed, retyped, or dropped — the declaration is
|
||||
part of the schema and will not be invalidated out from under itself.
|
||||
|
||||
**Most schema changes do not rewrite data.** Adding a derived column writes only
|
||||
the new column. Type changes are the exception and can be substantial.
|
||||
|
||||
## What is not guaranteed {#not-guaranteed}
|
||||
|
||||
Worth stating plainly, because each is occasionally assumed:
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| Multi-table transactions | A commit covers one table. There is no cross-table atomic write. |
|
||||
| Read-your-writes across processes | Governed by `read_consistency_interval`, not automatic. |
|
||||
| Unenforced primary keys | `SET UNENFORCED PRIMARY KEY` records intent; it does not enforce uniqueness. |
|
||||
| Ordering of concurrent appends | Two appends both succeed; their relative row order is not defined. |
|
||||
| Index freshness | An index does not cover rows added after it was built until [optimized](/maintenance/compaction). |
|
||||
| Automatic compaction | Nothing runs in the background. `optimize()` runs when you call it. |
|
||||
|
||||
The last two are the most common source of "why is this query slow" — see
|
||||
[Limits](/basics/limits) for the bounds that pair with them.
|
||||
- [Consistency](/tables/consistency): control when reads observe updates.
|
||||
- [Table versioning](/management/table#versioning): inspect and restore history.
|
||||
- [Cleanup](/maintenance/cleanup): configure retention and understand protected versions.
|
||||
- [Jobs](/basics/jobs): distinguish submission, completion and cancellation.
|
||||
- [Column refresh](/feature-engineering/refresh-column): understand partial progress.
|
||||
- [Architecture](/architecture): understand storage and background maintenance.
|
||||
|
||||
+61
-182
@@ -1,196 +1,75 @@
|
||||
---
|
||||
title: "Jobs"
|
||||
sidebarTitle: "Job"
|
||||
description: "Work that outlives the call that started it: submitting, waiting, inspecting, cancelling."
|
||||
description: "Track background work from submission to completion."
|
||||
icon: hourglass-half
|
||||
keywords: ["job", "async", "background", "events", "cancel", "refresh"]
|
||||
keywords: ["job", "background", "progress", "events", "cancel"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Job handles track operations in OSS and Enterprise. Enterprise also
|
||||
supports listing server jobs, reopening them by ID and reading their events.
|
||||
---
|
||||
|
||||
Some operations take longer than a request should. Building an index over a
|
||||
large table, refreshing a computed column, registering a function -- these run
|
||||
on the server after the call that started them has returned.
|
||||
## Job handles {#convention}
|
||||
|
||||
Each is submitted the same way and returns the same handle.
|
||||
A job handle lets you track an operation, such as an index build or a computed
|
||||
column refresh. Use it to check progress, wait for completion and inspect
|
||||
failures.
|
||||
|
||||
<Note>
|
||||
A [query](/basics/query) also has an `_async` form, but its handle streams rows
|
||||
rather than waiting for an outcome. Both answer the same four questions --
|
||||
which one is this, how is it going, give me the result, stop it -- and differ
|
||||
only in the third.
|
||||
</Note>
|
||||
Methods such as Python's `create_index_async` and TypeScript's
|
||||
`createIndexAsync` return job handles. In OSS, the work runs in your application
|
||||
process; in Enterprise, it runs on server workers. A returned handle can
|
||||
represent an operation that has already finished, so check its status before
|
||||
waiting or requesting cancellation. See the operation's API reference for when
|
||||
the method returns.
|
||||
|
||||
## The `_async` convention {#convention}
|
||||
## Submit and wait for a job {#waiting}
|
||||
|
||||
An operation that can run in the background has two forms. The plain name blocks
|
||||
until the work is done. The `_async` / `Async` name submits the work and returns
|
||||
a **job** immediately.
|
||||
1. **Submit the operation.** Keep the returned job handle. For an Enterprise
|
||||
job, also keep its ID if another process needs to look it up later.
|
||||
2. **Check progress.** Read the current status while the operation runs.
|
||||
3. **Wait for completion.** Check that the job succeeded before using its
|
||||
result. If it failed or was cancelled, inspect the failure details and any
|
||||
results it committed before stopping.
|
||||
|
||||
| Blocking | Returns a job | SQL |
|
||||
In Python, `job.status()` reports the current state and `job.wait()` waits for
|
||||
completion, raising if the job fails or is cancelled. If `job.wait()` times out, the client stops waiting.
|
||||
The timeout does not cancel the server job.
|
||||
|
||||
<span id="inspecting" />
|
||||
<span id="events" />
|
||||
|
||||
## Inspect and cancel jobs {#managing}
|
||||
|
||||
| Task | What to do |
|
||||
| --- | --- |
|
||||
| Inspect progress | Check status and refresh the handle to retrieve details. Enterprise job events can help explain ongoing work or failures. |
|
||||
| Reattach from another process | Open an existing server job with its ID, such as Python's `db.open_job(job_id)`. The server must still retain the job record. |
|
||||
| Diagnose a failure | Inspect failure details and relevant events. Enterprise SQL also provides `SHOW ERRORS`. |
|
||||
| Request cancellation | Call the job's cancellation method. Cancellation may take time, and work already committed is not rolled back. |
|
||||
|
||||
Reopening a job retrieves the existing operation; it does not restart failed
|
||||
work. Treat the ID as an opaque identifier and pass it unchanged to the jobs API.
|
||||
|
||||
Event responses have [row limits](/basics/limits#jobs). An explicit limit can
|
||||
request more than the default, but it does not guarantee a complete event
|
||||
history.
|
||||
|
||||
## OSS and Enterprise execution {#in-process}
|
||||
|
||||
| | OSS | Enterprise |
|
||||
| --- | --- | --- |
|
||||
| `create_index` | `create_index_async` | `CREATE INDEX` |
|
||||
| `refresh_column` | `refresh_column_async` | `REFRESH COLUMN` |
|
||||
| `create_function` | `create_function_async` | `CREATE FUNCTION` |
|
||||
| `drop_table` | `drop_table_async` | `DROP TABLE` |
|
||||
| Execution | In your application process | On server workers |
|
||||
| Job identity | No server ID | A server ID for looking up the job |
|
||||
| Event history | No server event history | Available through the jobs API |
|
||||
| Reattaching after a client restart | No server job to reopen | Reopen by ID while the record is retained |
|
||||
|
||||
<Note>
|
||||
The blocking form is not a different operation -- it submits the same job and
|
||||
waits. Reach for the `_async` form when you want to do something else meanwhile,
|
||||
run several at once, or survive the client going away.
|
||||
</Note>
|
||||
## Learn more {#next}
|
||||
|
||||
## Submitting and waiting {#waiting}
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
from lancedb.index import IvfPq
|
||||
|
||||
job = table.create_index_async("vector", config=IvfPq())
|
||||
|
||||
print(job.id) # the server's id for this operation
|
||||
print(job.status()) # "running" | "finished" | "failed" | "cancelled"
|
||||
|
||||
job.wait() # blocks until terminal; raises if it failed
|
||||
```
|
||||
|
||||
```typescript TypeScript icon="square-js"
|
||||
const job = await table.createIndexAsync("vector");
|
||||
|
||||
console.log(job.id); // the server's id for this operation
|
||||
console.log(await job.status()); // "running" | "finished" | "failed" | "cancelled"
|
||||
|
||||
await job.wait(); // resolves when terminal; rejects if it failed
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
`wait` is the one call that raises on failure. `status` is a point snapshot and
|
||||
returns the failed state rather than raising, which is what you want in a
|
||||
polling loop.
|
||||
|
||||
In Python `wait` takes an optional timeout and returns the operation's typed
|
||||
result -- a `RefreshColumnJobResult`, a `FunctionVersion`, or `None` for an
|
||||
operation that has no result:
|
||||
|
||||
```python
|
||||
from datetime import timedelta
|
||||
|
||||
result = table.refresh_column_async("embedding").wait(timeout=timedelta(minutes=30))
|
||||
```
|
||||
|
||||
`TimeoutError` means the timeout elapsed, not that the job stopped. The job is
|
||||
still running; the handle just gave up watching.
|
||||
|
||||
<Note>
|
||||
The TypeScript `wait()` resolves to `void`. Read the outcome from `result` after
|
||||
it resolves.
|
||||
</Note>
|
||||
|
||||
## Inspecting a job {#inspecting}
|
||||
|
||||
Submitting an operation returns only an id, so the detail fields are empty until
|
||||
the handle fetches the record. `refresh()` does that; `status()` does it as a
|
||||
side effect.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
job.refresh()
|
||||
|
||||
job.state # last observed state, without another round trip
|
||||
job.job_type # what kind of job, as the server names it
|
||||
job.creation_ms # when it was created, epoch milliseconds
|
||||
job.spec # what it was submitted with
|
||||
job.result # its terminal result, once it succeeds
|
||||
job.failure # why it failed, when it did
|
||||
```
|
||||
|
||||
```typescript TypeScript icon="square-js"
|
||||
await job.refresh();
|
||||
|
||||
job.state; // last observed state, without another round trip
|
||||
job.jobType; // what kind of job, as the server names it
|
||||
job.creationMs; // when it was created, epoch milliseconds
|
||||
job.spec; // what it was submitted with
|
||||
job.result; // its terminal result, once it succeeds
|
||||
job.failure; // why it failed, when it did
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
Printing the handle renders every field it knows, which is usually faster than
|
||||
reading them one at a time.
|
||||
|
||||
## Progress, from events {#events}
|
||||
|
||||
`result` only tells you anything once the job has finished. Events are written
|
||||
as it runs, and they outlive the workers that produced them -- so a job that is
|
||||
still going, or one that never finished, can still account for what it did.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
events = job.events(limit=5000, filter="state = 'claim_complete'")
|
||||
```
|
||||
|
||||
```typescript TypeScript icon="square-js"
|
||||
const events = await job.events({ limit: 5000, filter: "state = 'claim_complete'" });
|
||||
```
|
||||
|
||||
```sql SQL
|
||||
SHOW ERRORS ON my_table FOR JOB 'job-id' LIMIT 100
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
A distributed job records a `claim` / `claim_complete` pair per unit of work,
|
||||
each carrying `rows_processed`. Summing `rows_processed` over the completed
|
||||
claims is how you get progress on a job that has no percentage.
|
||||
|
||||
`filter` is a SQL-like expression over `state`, `updated_by`, `emitted_from`,
|
||||
`emitted_by`, and `claim_entity`.
|
||||
|
||||
<Warning>
|
||||
The server returns 1000 event rows by default and 10,000 at most, and
|
||||
**truncates without telling you**. A job that emits an event per fragment will
|
||||
silently lose its tail unless you pass `limit`.
|
||||
</Warning>
|
||||
|
||||
Events are recorded server-side, so this raises for an operation that ran
|
||||
in-process.
|
||||
|
||||
## Finding and cancelling jobs {#managing}
|
||||
|
||||
The connection lists and reopens jobs, so a handle lost to a restarted process
|
||||
is recoverable from its id.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
db.list_jobs() # every job the server knows about
|
||||
job = db.open_job(job_id) # reattach to one
|
||||
db.cancel_job(job_id) # ask it to stop
|
||||
```
|
||||
|
||||
```typescript TypeScript icon="square-js"
|
||||
await db.listJobs();
|
||||
const job = await db.openJob(jobId);
|
||||
await db.cancelJob(jobId);
|
||||
```
|
||||
|
||||
```sql SQL
|
||||
SHOW JOBS
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
Cancellation is a request, not a guarantee, and cancelling a job that already
|
||||
finished does nothing. `JobNotFoundError` means the id does not name a job the
|
||||
server still has.
|
||||
|
||||
<Warning>
|
||||
A job id is opaque. It correlates with server logs and the calls above, but
|
||||
parsing it or storing it to resume the job later is not supported.
|
||||
</Warning>
|
||||
|
||||
## In-process jobs {#in-process}
|
||||
|
||||
A local database has no server to run anything, so its operations run in your
|
||||
process. They still return a handle, so the same code works against both -- but
|
||||
one with nothing behind it: `id` is `None`, `status()` reports `"finished"`,
|
||||
`wait()` returns immediately, `cancel()` does nothing, and `events()` raises.
|
||||
|
||||
This is the difference to expect when moving between embedded and deployed --
|
||||
see [SDKs](/basics/sdk).
|
||||
- [Build and repair indexes](/indexing/reindexing): submit index work.
|
||||
- [Column refresh](/feature-engineering/refresh-column): populate derived values.
|
||||
- [Background jobs](/maintenance/background-jobs): understand Enterprise maintenance.
|
||||
- [SQL job commands](/sql/ddl/job): list jobs and inspect errors.
|
||||
- [SDKs](/basics/sdk): find the job API for your client.
|
||||
|
||||
@@ -1,102 +1,75 @@
|
||||
---
|
||||
title: "Lance Lakehouse"
|
||||
sidebarTitle: "Lance Lakehouse"
|
||||
description: "What operates on the catalog's objects: storage, search, computation, and maintenance."
|
||||
title: "LanceDB Lakehouse"
|
||||
sidebarTitle: "LanceDB Lakehouse"
|
||||
description: "Bring multimodal data, search and feature engineering together for AI workloads."
|
||||
icon: layer-group
|
||||
keywords: ["lakehouse", "multimodal", "search", "compute", "maintenance", "feature engineering"]
|
||||
keywords: ["lakehouse", "multimodal", "search", "compute", "training", "feature engineering"]
|
||||
---
|
||||
|
||||
The [catalog](/basics/catalog) says what exists. The **lakehouse** is everything
|
||||
that acts on it: storing the rows, searching them, computing new ones, and
|
||||
keeping the whole thing fast as it grows.
|
||||
## Multimodal storage and processing {#multimodal}
|
||||
|
||||
The distinction matters because the two layers have different lifetimes. A
|
||||
catalog entry is a name and a pointer. A lakehouse operation reads and writes
|
||||
the [Lance](/basics/lance) files that name points at -- and those operations are
|
||||
where almost all of the cost, and all of the tuning, lives.
|
||||
LanceDB brings storage, search and data processing together for AI workloads.
|
||||
You can store multimodal data, derive features, search your records and prepare
|
||||
training datasets using tables built on the open [Lance format](/basics/lance).
|
||||
|
||||
## Multimodal is the point {#multimodal}
|
||||
The format provides the storage foundation. The [catalog](/basics/catalog)
|
||||
organizes your tables. LanceDB provides the tools to work with that data.
|
||||
|
||||
A lakehouse table holds images, video, audio, text, embeddings, annotations and
|
||||
the features you derive from them **in one table**, side by side.
|
||||
## Example: a video search workflow {#capabilities}
|
||||
|
||||
That is the difference from assembling a warehouse plus a vector store plus a
|
||||
feature store. There is one copy of the data, one set of versions, and no
|
||||
pipeline keeping three systems agreeing about the same rows. A search can filter
|
||||
on a label, rank by a vector, and return the blob, without leaving the table.
|
||||
Consider a team building a video search application:
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| **Columns** | vectors, blobs, and columns [computed](/feature-engineering/add-computed-column) from other columns |
|
||||
| **Versions** | every write commits one, so reads can [travel back](/management/table#versioning) |
|
||||
| **Indexes** | vector, full-text and scalar, built over those columns |
|
||||
| **Branches** | isolated lines of change you can [merge or discard](/tables/branching) |
|
||||
<span id="ingest" />
|
||||
<span id="compute" />
|
||||
<span id="search" />
|
||||
|
||||
## The four capabilities {#capabilities}
|
||||
1. **Ingest the clips.** Store video clips with metadata such as timestamps,
|
||||
source and labels.
|
||||
2. **Add features.** Generate captions and embeddings with your models, and
|
||||
store them alongside each clip.
|
||||
3. **Use the dataset.** Search for relevant clips using embeddings and metadata
|
||||
filters, or select a dataset to read into a training framework.
|
||||
|
||||
Everything the lakehouse does falls into one of four groups, and the rest of
|
||||
these docs is organized the same way.
|
||||
```mermaid
|
||||
flowchart TD
|
||||
clips["Video clips and metadata"] --> table["Store in a Lance table"]
|
||||
table --> features["Add captions and embeddings"]
|
||||
features --> search["Search for relevant clips"]
|
||||
features --> training["Select data for training"]
|
||||
```
|
||||
|
||||
### Ingest {#ingest}
|
||||
The clips, metadata and derived features stay associated in the same table,
|
||||
so retrieval and training workflows can use the same dataset. Your training
|
||||
framework runs the model; LanceDB provides access to the training data.
|
||||
|
||||
Getting rows in, and changing the ones that are there: append, merge, update,
|
||||
delete. Every write commits a version rather than mutating in place, which is
|
||||
what makes the other three safe to run against a moving table. See
|
||||
[Data Ingestion](/ingestion).
|
||||
## Shared compute and maintenance in Enterprise {#where}
|
||||
|
||||
### Search {#search}
|
||||
LanceDB OSS runs as an embedded library in your application. LanceDB Enterprise
|
||||
provides shared infrastructure for applications and teams working with larger
|
||||
multimodal workloads.
|
||||
|
||||
Vector, full-text and hybrid retrieval over the same rows, with filters that
|
||||
push into the scan rather than running after it. The search is a lakehouse
|
||||
operation, not a separate service holding a second copy of your embeddings. See
|
||||
[Semantic Search](/search).
|
||||
<span id="maintenance" />
|
||||
|
||||
### Compute {#compute}
|
||||
- **Scale resources for different workloads.** Provision query serving and
|
||||
background workers independently as search traffic and indexing demand grow.
|
||||
- **Share data across teams.** Give applications access to shared tables, with
|
||||
Enterprise permissions controlling who can read or change them.
|
||||
- **Manage ongoing maintenance.** Configure background indexing, compaction
|
||||
and cleanup as your datasets grow.
|
||||
|
||||
Deriving new columns and tables from what is already stored:
|
||||
[computed columns](/feature-engineering/add-computed-column),
|
||||
[functions](/management/function) that run Python where the data is, and
|
||||
[materialized views](/management/materialized-view) that persist a query's
|
||||
result.
|
||||
See [Architecture](/architecture) for how Enterprise separates storage, query
|
||||
serving and background work.
|
||||
|
||||
The reason this belongs in the lakehouse rather than in a pipeline outside it is
|
||||
the format. Adding a derived column writes only that column -- the images and
|
||||
videos are never rewritten -- so deriving a feature is an ordinary operation
|
||||
rather than a migration. See [Feature Engineering](/feature-engineering) and
|
||||
[Data Curation](/curation).
|
||||
## Learn more {#next}
|
||||
|
||||
### Maintenance {#maintenance}
|
||||
- [Data ingestion](/ingestion): load and update your data.
|
||||
- [Search](/search): retrieve records with vector, full-text and hybrid search.
|
||||
- [Feature engineering](/feature-engineering): derive columns and datasets.
|
||||
- [Training data access](/training/data-loading): read datasets into training workflows.
|
||||
- [Enterprise](/enterprise): explore deployment options for shared workloads.
|
||||
|
||||
The work that keeps the first three fast: building and repairing
|
||||
[indexes](/indexing), and [compaction](/maintenance/compaction) to merge small
|
||||
files, prune old versions and fold new rows into existing indexes.
|
||||
|
||||
<Warning>
|
||||
None of it runs on its own. There is no background compactor -- `optimize()`
|
||||
runs when you call it, and an index does not cover rows added after it was
|
||||
built until you do. See [Guarantees](/basics/guarantees#not-guaranteed).
|
||||
</Warning>
|
||||
|
||||
## Where the work happens {#where}
|
||||
|
||||
The same lakehouse runs in two shapes, and which one you use decides where these
|
||||
capabilities execute rather than whether you have them:
|
||||
|
||||
- **Embedded** -- your process does all four itself, directly against object
|
||||
storage. Nothing runs unless you call it.
|
||||
- **Distributed** -- a cluster does them, with indexing and compaction on
|
||||
workers separate from the ones serving queries, plus
|
||||
[SQL](/sql) and server-side [jobs](/basics/jobs).
|
||||
|
||||
See [Architecture](/architecture) for the comparison in full.
|
||||
|
||||
## Next {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Lance Catalog" icon="sitemap" href="/basics/catalog">
|
||||
The objects these operations act on.
|
||||
</Card>
|
||||
<Card title="Lance format" icon="layer-group" href="/basics/lance">
|
||||
The open format underneath, and why column evolution is cheap.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
{/*
|
||||
Draft for the Functions release. Add this item to the list above, after
|
||||
Feature engineering:
|
||||
- [Functions](/management/function): run Python to compute column values.
|
||||
*/}
|
||||
|
||||
+29
-106
@@ -1,122 +1,45 @@
|
||||
---
|
||||
title: "Lance format"
|
||||
sidebarTitle: "Lance Format"
|
||||
description: "Lance is the open format underneath. LanceDB is what implements a lakehouse on top of it."
|
||||
description: "The open format for multimodal data behind LanceDB."
|
||||
icon: "/static/assets/logo/lance-logo-gray.svg"
|
||||
keywords: ["lance", "format", "open standard", "lakehouse", "namespace", "catalog"]
|
||||
keywords: ["lance", "format", "multimodal", "versioning", "column evolution"]
|
||||
---
|
||||
|
||||
Two names, two things. [Lance](https://lance.org/) is an open format and
|
||||
specification. LanceDB is an implementation built on it.
|
||||
## Lance format overview {#standard}
|
||||
|
||||
The distinction matters when you are deciding what you are committing to: the
|
||||
format is a standard other engines read, and your data stays readable by them
|
||||
whichever LanceDB you run.
|
||||
[Lance](https://lance.org/) is an open-source, columnar format designed for
|
||||
multimodal AI. It defines how data is stored and organized into versioned tables.
|
||||
|
||||
## Lance is the open standard {#standard}
|
||||
<span id="embedded" />
|
||||
<span id="enterprise" />
|
||||
|
||||
Lance is an open-source, columnar lakehouse format for multimodal AI. It
|
||||
specifies three things:
|
||||
LanceDB OSS and LanceDB Enterprise both use Lance as their storage foundation.
|
||||
The format is developed as an independent open-source project, with its own
|
||||
specification and tools.
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| **File format** | how bytes are laid out on disk, optimized for random access without giving up scan throughput |
|
||||
| **Table format** | how files compose into a versioned table -- schema evolution, deletions, indexes |
|
||||
| **Namespace specification** | how tables are named, grouped, and resolved -- the catalog layer |
|
||||
## Multimodal data {#storage}
|
||||
|
||||
Being a specification rather than a product is the point. Object storage holds
|
||||
the bytes, and open engines -- PyTorch, Ray, Spark, Trino, DuckDB, Polars --
|
||||
read them without going through LanceDB at all.
|
||||
A Lance table can hold images, video, audio, text, embeddings and structured
|
||||
metadata together. For example, a video dataset can keep each clip alongside
|
||||
its caption, labels and embeddings, so applications can search and work with
|
||||
those fields together.
|
||||
|
||||
Lance exists because columnar formats derived from Parquet were not designed
|
||||
for random access on vectors: [benchmarks](https://lancedb.com/blog/benchmarking-random-access-in-lance/)
|
||||
show Lance up to 1000x faster than Parquet for that access pattern.
|
||||
## Data access, versioning and column updates {#capabilities}
|
||||
|
||||
<Card
|
||||
title="Lance format documentation"
|
||||
icon="/static/assets/logo/lance-logo-gray.svg"
|
||||
href="https://lance.org/format"
|
||||
>
|
||||
The format specification: design, layout, and how it enables the multimodal lakehouse.
|
||||
</Card>
|
||||
- **Efficient data access.** Read individual rows for retrieval and training,
|
||||
or scan columns for analytics.
|
||||
- **Versioned data.** Read an earlier table version to reproduce an experiment
|
||||
or inspect previous results, while that version is retained.
|
||||
- **Independent column updates.** Add embeddings or other derived features
|
||||
without rewriting the existing image, video or text columns.
|
||||
|
||||
## LanceDB embedded implements the storage half {#embedded}
|
||||
## Learn more {#next}
|
||||
|
||||
Connecting to a directory or a bucket gives you a **storage-only lakehouse**.
|
||||
Your process opens Lance tables directly and does the work itself: it plans and
|
||||
executes queries, builds indexes, and writes new versions.
|
||||
<span id="versioning" />
|
||||
<span id="evolution" />
|
||||
|
||||
```python
|
||||
db = lancedb.connect("./.lancedb")
|
||||
db = lancedb.connect("s3://bucket/path")
|
||||
```
|
||||
|
||||
There is no server, which is what makes it embedded -- and also what bounds it.
|
||||
There is nothing to run work on your behalf, so there are no server-side
|
||||
[jobs](/basics/jobs), no [SQL endpoint](/basics/sql), and no shared catalog
|
||||
beyond what the storage layer itself provides.
|
||||
|
||||
LanceDB OSS is Apache 2.0 licensed. LanceDB Enterprise is the commercial,
|
||||
deployment-only product that implements the catalog specification below.
|
||||
|
||||
## LanceDB Enterprise implements the catalog specification {#enterprise}
|
||||
|
||||
<Badge color="red">Enterprise</Badge> A `db://` connection talks to a deployment
|
||||
that implements the **Lance REST Namespace** -- the REST server defined by the
|
||||
Lance namespace specification, covering the catalog operations in the
|
||||
[REST API reference](/api-reference/rest).
|
||||
|
||||
```python
|
||||
db = lancedb.connect("db://my-database", api_key="...")
|
||||
```
|
||||
|
||||
Implementing the specification rather than inventing an interface is what makes
|
||||
the catalog reachable by anything that speaks it, and what makes
|
||||
[authorization](/governance/authorization), server-side jobs, and
|
||||
[SQL](/basics/sql) possible: there is now a server to enforce and to run them.
|
||||
|
||||
The tables are the same Lance tables either way. What changes is who resolves
|
||||
their names and who does the work.
|
||||
|
||||
## What the format gives you {#capabilities}
|
||||
|
||||
These are properties of Lance, so they hold in both forms:
|
||||
|
||||
| Capability | What it enables |
|
||||
| --- | --- |
|
||||
| Multimodal storage | Images, video, audio, text, embeddings, annotations and metadata in one table. |
|
||||
| First-class blob API | Large binary objects in blob columns, with lazy reads and streaming byte access. |
|
||||
| Fast random access and scans | Sample, shuffle, and fetch individual rows without giving up sequential throughput. |
|
||||
| Zero-copy schema evolution | Add, drop, rename, or alter columns without rewriting existing data files. |
|
||||
| Versioned tables | Reproduce experiments and tie artifacts to the exact table version they used. |
|
||||
| Hybrid search and indexing | Vector search, full-text search, and scalar filters over the same dataset. |
|
||||
| Open interoperability | Object storage underneath, open engines on top. |
|
||||
|
||||
Two of them explain behaviour you will meet elsewhere in these docs.
|
||||
|
||||
### Versioning is why old data lingers {#versioning}
|
||||
|
||||
Every write creates a version, and old versions are not immediately destroyed --
|
||||
another reader may be part-way through one. A hundred versions are not a hundred
|
||||
copies of the data, but they do carry a hundred times the metadata, which
|
||||
eventually slows queries down.
|
||||
|
||||
Deletion works the same way: a deleted row is marked in a deletion file rather
|
||||
than erased, so the data is recoverable for as long as the version is.
|
||||
|
||||
[Compaction](/maintenance/compaction) is what resolves both.
|
||||
|
||||
### Column evolution is why feature engineering is cheap {#evolution}
|
||||
|
||||
Adding a derived column writes only the new column. Existing data -- the
|
||||
expensive part, the images and videos -- is untouched. That is what makes
|
||||
[computed columns](/feature-engineering/add-computed-column) a reasonable thing to
|
||||
do repeatedly on a large table rather than a full rewrite each time.
|
||||
|
||||
<Card
|
||||
title="Learn more about Lance"
|
||||
icon="/static/assets/logo/lance-logo-gray.svg"
|
||||
href="https://lance.org/quickstart"
|
||||
>
|
||||
Lance is a separate open source project with its own documentation.
|
||||
</Card>
|
||||
- [Lance format specification](https://lance.org/format): how the format works.
|
||||
- [Table versioning](/management/table#versioning): read and manage table versions.
|
||||
- [Computed columns](/feature-engineering/add-computed-column): add derived features to your data.
|
||||
- [Architecture](/architecture): how LanceDB OSS and Enterprise build on Lance.
|
||||
|
||||
+51
-86
@@ -1,106 +1,71 @@
|
||||
---
|
||||
title: "Limits"
|
||||
sidebarTitle: "Limits"
|
||||
description: "The bounds you can actually hit, in one place."
|
||||
description: "Find fixed constraints, configurable bounds and what happens when you reach them."
|
||||
icon: ruler
|
||||
keywords: ["limits", "maximum", "quota", "constraints", "bounds"]
|
||||
---
|
||||
|
||||
Limits worth designing around, collected so you do not have to find them by
|
||||
hitting them. Each one is enforced -- where a value is refused rather than
|
||||
clamped, that is stated, because the two fail very differently.
|
||||
<span id="scope" />
|
||||
|
||||
This page covers bounds that are the same for everyone. Deployment-specific
|
||||
quotas -- storage, throughput, concurrency -- come from your deployment.
|
||||
This page lists limits for job events, SQL error reports, sampling and
|
||||
full-text indexing. Each table identifies the affected API and what happens
|
||||
when a request reaches its limit. SQL and server job limits apply to Enterprise.
|
||||
|
||||
## Names {#names}
|
||||
<span id="names" />
|
||||
|
||||
| Object | Maximum | |
|
||||
[Naming Rules](/basics/naming) is the reference for name lengths, characters and
|
||||
namespace depth. Client and server versions can differ in which rules they
|
||||
enforce.
|
||||
|
||||
<span id="jobs" />
|
||||
<span id="sql" />
|
||||
|
||||
## Fixed API limits {#fixed-limits}
|
||||
|
||||
| Operation | Scope | Limit and behavior |
|
||||
| --- | --- | --- |
|
||||
| Database | 64 characters | |
|
||||
| Namespace | 64 characters | |
|
||||
| Table | 128 characters | |
|
||||
| Branch | 64 characters | |
|
||||
| Job event retrieval | Enterprise | At most 10,000 rows per response. Results can be truncated without a truncation indicator. |
|
||||
| `SHOW ERRORS ... LIMIT` | Enterprise SQL | Accepts 1 through 100,000. Values outside the range are rejected. |
|
||||
| `SHOW ERRORS` without `FOR JOB` | Enterprise SQL | Inspects up to the 32 most recent failed jobs. Use `FOR JOB` to inspect a specific job. |
|
||||
| Percentage sampling | Enterprise SQL | `TABLESAMPLE` accepts percentages from 0 through 100. Values outside the range are rejected. |
|
||||
| Row-count sampling | Enterprise SQL | `TABLESAMPLE (... ROWS)` requires a whole number. |
|
||||
|
||||
A name must not be empty, and must not contain `$` or `:` -- both are reserved
|
||||
as path delimiters, so a name holding one could not be addressed unambiguously.
|
||||
Length is counted in characters, not bytes.
|
||||
See [SQL job commands](/sql/ddl/job#errors) and
|
||||
[Sampling](/sql/dql/sampling) for syntax and restrictions on supported table types.
|
||||
|
||||
## Jobs {#jobs}
|
||||
<span id="schema" />
|
||||
|
||||
| | |
|
||||
Data-type and index constraints belong to the operation you choose. Check
|
||||
[Data types](/sql/data-types) and [Vector indexes](/indexing/vector-index)
|
||||
before choosing a schema or index. For materialized views, check
|
||||
[stable row ID requirements](/feature-engineering/materialized-views#stable-row-ids)
|
||||
before creating source tables.
|
||||
|
||||
## Configurable limits {#not-quotas}
|
||||
|
||||
| Setting | What it controls |
|
||||
| --- | --- |
|
||||
| `job.events()` default | 1,000 rows |
|
||||
| `job.events()` maximum | 10,000 rows |
|
||||
| `SHOW ERRORS ... LIMIT` | between 1 and 100,000 |
|
||||
| Failed jobs scanned by one `SHOW ERRORS` | 32 |
|
||||
| Job event `limit` | Requests more or fewer event rows. The default is 1,000; the 10,000-row maximum still applies. |
|
||||
| FTS `max_token_length` | The maximum indexed token length. The default is 40; longer tokens are excluded from the index. |
|
||||
| Enterprise capacity and quotas | Storage, request rate and concurrency are governed by your deployment's resources and configuration. Confirm these with your administrator or LanceDB support. |
|
||||
|
||||
<Warning>
|
||||
`job.events()` **truncates silently**. A job that emits an event per fragment
|
||||
will lose its tail unless you pass `limit` explicitly. This is the limit on this
|
||||
page most likely to mislead you, because nothing in the result says it was cut.
|
||||
</Warning>
|
||||
## Handling rejected or incomplete results {#behavior}
|
||||
|
||||
`SHOW ERRORS` without `FOR JOB` scans the most recent failed jobs and stops at
|
||||
32 of them. Narrow with `FOR JOB` when you need a specific one rather than a
|
||||
survey.
|
||||
- **Rejected requests:** correct the input before retrying, such as a sampling
|
||||
percentage outside the accepted range.
|
||||
- **Truncated responses:** do not assume that a returned event list is the full
|
||||
history. Increasing `limit` cannot exceed the maximum; use filters to narrow
|
||||
the events you inspect.
|
||||
- **Excluded data:** an FTS token-length setting can affect which words are
|
||||
searchable. Choose it for your text and indexing requirements.
|
||||
|
||||
See [Jobs](/basics/jobs).
|
||||
<span id="availability" />
|
||||
|
||||
## Schema and types {#schema}
|
||||
## Learn more {#next}
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| `map` with `keys_sorted = true` | rejected |
|
||||
| `IVF_RQ` vector dimensions | must be divisible by 8 |
|
||||
| FTS `max_token_length` | 40 by default; longer tokens are dropped |
|
||||
|
||||
Through the SDK, a table gets **stable row ids** only at creation, from the
|
||||
`new_table_enable_stable_row_ids` storage option on a local path or an object
|
||||
store; no SDK call adds them later, so decide early. A local
|
||||
[materialized view](/feature-engineering/materialized-views#stable-row-ids) does
|
||||
not need them. On a deployment, whether a view's source needs them and how a
|
||||
table gets them depend on the deployment; see
|
||||
[Against a deployment](/feature-engineering/materialized-views#availability).
|
||||
|
||||
See [Data types](/sql/data-types).
|
||||
|
||||
## SQL {#sql}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| `TABLESAMPLE` percentage | 0 to 100, **refused** outside -- not clamped |
|
||||
| `TABLESAMPLE (n ROWS)` | whole numbers only |
|
||||
| `TABLESAMPLE BUCKET` | not supported |
|
||||
| `TABLESAMPLE` on a WAL-backed table | not supported |
|
||||
| `CREATE INDEX` | one column, server-chosen name |
|
||||
| `REFRESH MATERIALIZED VIEW ... CASCADE` | depends on the deployment's version; see [REFRESH](/sql/dml/refresh#refresh-materialized-view) |
|
||||
|
||||
A sampling percentage over 100 is an error rather than a full scan, because a
|
||||
query asking for 150 percent has a mistake in it and reading it as 100 would
|
||||
hide the mistake. See [Sampling](/sql/dql/sampling).
|
||||
|
||||
## What is unavailable where {#availability}
|
||||
|
||||
Not quotas, but the same class of surprise -- things that exist in one place and
|
||||
not another.
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| SQL on a local database | unavailable; `execute_query` raises |
|
||||
| SQL methods in the TypeScript SDK | none; use a Flight SQL client |
|
||||
| Function methods in the TypeScript SDK | none; use Python or SQL |
|
||||
| Vector search in SQL | no SQL form; use a query builder |
|
||||
| Geospatial `st_*` functions | behind a build feature LanceDB does not ship |
|
||||
| `tableExists` / `namespaceExists` in TypeScript | not present |
|
||||
| Server-side jobs on a local database | run in-process; `events()` raises |
|
||||
|
||||
See [SQL](/basics/sql) for why the first two hold.
|
||||
|
||||
## Nothing here is a quota {#not-quotas}
|
||||
|
||||
None of the above is a billing or capacity limit. Rows per table, table count,
|
||||
storage, and request rate are bounded by your deployment rather than by the
|
||||
format -- ask your deployment, not this page.
|
||||
- [Naming Rules](/basics/naming): choose portable names and paths.
|
||||
- [Jobs](/basics/jobs): track work and inspect events.
|
||||
- [Full-text indexes](/indexing/fts-index): configure tokenization.
|
||||
- [SQL](/basics/sql) and [SDKs](/basics/sdk): check interface support.
|
||||
- [Enterprise support](/deployment/support): discuss deployment capacity.
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
title: "Naming rules"
|
||||
sidebarTitle: "Naming Rules"
|
||||
description: "Choose portable names for your data objects and namespace paths."
|
||||
icon: signature
|
||||
keywords: ["names", "naming", "table name", "namespace", "identifier", "limits"]
|
||||
---
|
||||
|
||||
## Naming rules {#rules}
|
||||
|
||||
<span id="objects" />
|
||||
|
||||
Databases, namespaces, tables, views and branches all have names. Choose names
|
||||
that describe their purpose, such as `documents`, `prod` or `training_2026`.
|
||||
Consistent names make data easier to find and application code easier to read.
|
||||
|
||||
Use the following rules for names that work across LanceDB OSS and Enterprise.
|
||||
|
||||
| Object | Characters | Length |
|
||||
| --- | --- | --- |
|
||||
| Table, materialized view, view | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters |
|
||||
| Namespace component | ASCII letters and digits, `-`, `_`, `.` | 1 to 64 characters; at most 10 components in a path |
|
||||
| Database | ASCII letters and digits, `-`, `_`, `.`, with `/` between components | 1 to 64 characters for the whole name |
|
||||
| Branch | Avoid `$` and `:`; use ASCII letters, digits, `-` and `_` for portability | Keep to 1 to 64 characters |
|
||||
|
||||
- Names and individual namespace or database components must not be empty.
|
||||
- Table, view, namespace and database components cannot be `.` or `..`.
|
||||
- Avoid `$` and `:` because they separate parts of qualified identifiers.
|
||||
- A namespace path beginning with `public` cannot contain another component.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Add these rows to the end of the table above:
|
||||
| Secret | ASCII letters and digits, `-`, `_`, `.` | 1 to 128 characters |
|
||||
| Function | ASCII letters and digits, `_` | 1 to 128 characters |
|
||||
Add this item to the list above, after the rule for `.` and `..`:
|
||||
- Secret names cannot be `.` or `..`.
|
||||
*/}
|
||||
|
||||
## Namespace paths {#paths}
|
||||
|
||||
A namespace path is a list of names. `["prod", "search"]` identifies the
|
||||
`search` namespace within `prod`. A table named `documents` belongs to that
|
||||
namespace; its table name remains `documents`.
|
||||
|
||||
Use the SDK's namespace-path argument to supply the list. A period inside an
|
||||
SDK name is a character in that name, not a separator that creates another
|
||||
namespace. SQL uses its own qualified-name syntax; see
|
||||
[SQL namespace paths](/sql/dql/select#namespaces).
|
||||
|
||||
The empty path `[]` identifies the root namespace.
|
||||
|
||||
## Examples {#examples}
|
||||
|
||||
| Name or path | Guidance |
|
||||
| --- | --- |
|
||||
| `documents`, `user-events.v2`, `train_2026` | Portable table names |
|
||||
| `["prod", "search"]` | A two-component namespace path |
|
||||
| `my table` | Use `my_table` instead of a space |
|
||||
| `orders$2026` | Use `orders_2026`; `$` is reserved |
|
||||
| `résumés` | Use an ASCII name such as `resumes` |
|
||||
| `["public", "search"]` | Invalid nesting under `public`; use a different parent namespace |
|
||||
|
||||
<span id="which-versions-check-them" />
|
||||
|
||||
## Learn more {#next}
|
||||
|
||||
- [Namespaces](/management/namespace): create namespaces and address tables.
|
||||
- [Branches](/tables/branching): name and manage separate lines of table history.
|
||||
- [Limits](/basics/limits): check other constraints before designing your schema.
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
title: "LanceDB OSS or Enterprise?"
|
||||
sidebarTitle: "OSS and Enterprise"
|
||||
description: "Start with an embedded library, or use the LanceDB Enterprise multimodal lakehouse for shared AI workloads."
|
||||
icon: compass
|
||||
keywords: ["oss", "enterprise", "embedded", "deployment", "availability", "comparison"]
|
||||
---
|
||||
|
||||
LanceDB gives you two ways to work with multimodal data, both built on the open
|
||||
[Lance](/basics/lance) format.
|
||||
|
||||
**LanceDB OSS** is an embedded library. Run it inside your application
|
||||
with tables on local disk or in object storage. You control the application,
|
||||
its resources, and table maintenance, without a separate database service.
|
||||
[Try the quickstart](/quickstart) to create a table and run your first search.
|
||||
|
||||
**LanceDB Enterprise** is a multimodal lakehouse for data curation, feature
|
||||
engineering, search, and training data access. It adds query-serving capacity,
|
||||
data caching, and background operations for teams working with shared
|
||||
multimodal datasets.
|
||||
[Connect to Enterprise](/basics/connect#enterprise) when your cluster is ready.
|
||||
|
||||
<span id="labels" />
|
||||
<span id="availability" />
|
||||
|
||||
## Why choose Enterprise? {#why-enterprise}
|
||||
|
||||
### Serve growing workloads {#serving}
|
||||
|
||||
Add query-serving capacity as more applications and users depend on your data.
|
||||
Enterprise spreads query traffic across a fleet and caches frequently read
|
||||
table data near compute, reducing repeated reads from object storage. Query
|
||||
serving and background work use separate resources, so you can size each for
|
||||
your workload. See [Enterprise architecture](/enterprise/architecture).
|
||||
|
||||
### Reduce operational work {#operations}
|
||||
|
||||
Let the service handle routine table maintenance instead of scheduling it in
|
||||
application code. Enterprise handles background [compaction](/maintenance/compaction) and can
|
||||
[clean up old table versions](/maintenance/cleanup) according to your retention
|
||||
policy. Configure these maintenance tasks for your Enterprise cluster so each
|
||||
application does not have to schedule them.
|
||||
|
||||
### Fit your deployment requirements {#deployment}
|
||||
|
||||
Choose a managed deployment or bring LanceDB into your own cloud account.
|
||||
Enterprise integrates with OpenTelemetry for metrics and traces, helping your
|
||||
team monitor the service alongside your applications. Talk to us about your
|
||||
cloud, networking, and operational requirements to plan the right deployment.
|
||||
The [Enterprise overview](/enterprise) explains the available options.
|
||||
|
||||
### Connect your data workflows {#workflows}
|
||||
|
||||
Use the same tables for retrieval, dataset preparation, and training data access.
|
||||
With its [SQL endpoint](/basics/sql) enabled, Enterprise supports analytical
|
||||
[curation](/curation), including joins, grouping, and sampling.
|
||||
[Feature engineering](/feature-engineering) adds derived columns and tables;
|
||||
computed columns and materialized views require the corresponding features to
|
||||
be enabled for your Enterprise cluster.
|
||||
[Geneva](/geneva) runs Python feature backfills as distributed jobs.
|
||||
[Training data loaders](/training/data-loading), available in both offerings,
|
||||
read the resulting datasets for model training.
|
||||
|
||||
## Plan your Enterprise deployment {#get-started}
|
||||
|
||||
[Talk to us about your workload](https://www.lancedb.com/contact) to discuss your
|
||||
data, query traffic, pipelines, and deployment requirements.
|
||||
|
||||
For a closer look at how the service runs, read the
|
||||
[Enterprise architecture](/enterprise/architecture).
|
||||
+37
-76
@@ -1,94 +1,55 @@
|
||||
---
|
||||
title: "Queries"
|
||||
sidebarTitle: "Query"
|
||||
description: "Running a query, and reading its results before it finishes."
|
||||
description: "Choose how to query your data and read the results."
|
||||
icon: magnifying-glass
|
||||
keywords: ["query", "async", "streaming", "arrow", "reader", "execute_query"]
|
||||
keywords: ["query", "search", "SQL", "async", "streaming", "results"]
|
||||
---
|
||||
|
||||
A query returns rows. How you get them depends on two choices that are easy to
|
||||
conflate: whether **your thread** blocks, and whether you take the results all
|
||||
at once or stream them as they arrive.
|
||||
## Querying tables {#queries}
|
||||
|
||||
## Blocking or not is a language concern {#async-client}
|
||||
A query selects data from your tables. You can filter rows, choose columns,
|
||||
search for similar content or combine those operations. For example, a video
|
||||
application might find clips similar to a sample and return only those recorded
|
||||
in the last month.
|
||||
|
||||
A concern of your language, not of LanceDB. The work is identical either way.
|
||||
## SDK searches and SQL queries {#searches}
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
db = lancedb.connect("db://my-db", api_key="...") # blocking
|
||||
tbl = db.open_table("documents")
|
||||
| Interface | When to use it |
|
||||
| --- | --- |
|
||||
| **SDK query builders** | Build vector, full-text and hybrid searches, apply filters and choose the fields to return. Available in OSS and Enterprise. |
|
||||
| **Enterprise SQL** | Write SQL queries for filtering, aggregation, joins and other data curation tasks. Connect through Python or a Flight SQL client. |
|
||||
|
||||
db = await lancedb.connect_async("db://my-db", api_key="...") # coroutines
|
||||
tbl = await db.open_table("documents")
|
||||
```
|
||||
Both interfaces work with LanceDB tables. Choose the one that supports your
|
||||
operation and fits your application. See [SQL](/basics/sql) for connection
|
||||
instructions and SQL search support, and [Search](/search) for the query builders.
|
||||
|
||||
```typescript TypeScript icon="square-js"
|
||||
// TypeScript is async throughout; there is no blocking form.
|
||||
const db = await lancedb.connect("db://my-db", { apiKey: "..." });
|
||||
const tbl = await db.openTable("documents");
|
||||
```
|
||||
</CodeGroup>
|
||||
## Collect or stream results {#results}
|
||||
|
||||
Python offers both. `AsyncConnection` exposes the same surface as coroutines, so
|
||||
choosing between them is about the program you are writing -- a service wants
|
||||
the async form, a script does not care.
|
||||
- **Collect the results** when you want a list, Arrow table or DataFrame that
|
||||
fits in memory. Use a result limit and select only the columns you need.
|
||||
- **Read batches** when you want to process a large result incrementally.
|
||||
Reading batches avoids loading the entire result into application memory.
|
||||
It does not guarantee that the query can return its first batch immediately.
|
||||
|
||||
## Reading before the query finishes {#queries}
|
||||
### Asynchronous clients {#async-client}
|
||||
|
||||
A query is not a job, but it has the same shape of choice: wait for everything,
|
||||
or take a handle and start reading.
|
||||
An async client lets your application do other work while a request is waiting.
|
||||
That is separate from whether you collect results or read them in batches.
|
||||
Python offers synchronous and asynchronous clients; TypeScript uses async APIs.
|
||||
Check the SDK reference for the methods available in your client.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 1000")
|
||||
table = reader.read_all()
|
||||
### Query handles and background jobs {#jobs}
|
||||
|
||||
query = db.execute_query_async("SELECT id, text FROM my_docs")
|
||||
```
|
||||
An Enterprise SQL query handle lets you inspect query execution and request
|
||||
cancellation. Background operations such as index builds return a
|
||||
[job](/basics/jobs) handle instead. Use the API for the returned handle to
|
||||
check status and retrieve results.
|
||||
|
||||
```sql SQL
|
||||
SELECT id, text FROM my_docs LIMIT 1000
|
||||
```
|
||||
</CodeGroup>
|
||||
## Learn more {#next}
|
||||
|
||||
`execute_query` returns a blocking Arrow reader **once results start arriving**
|
||||
-- it does not wait for the whole query to finish. Results stream as record
|
||||
batches, so a large result set does not have to fit in memory before you see its
|
||||
first row.
|
||||
|
||||
`execute_query_async` returns the query handle instead:
|
||||
|
||||
```python
|
||||
query = db.execute_query_async("SELECT id, text FROM my_docs")
|
||||
|
||||
query.id # identifies it on the server
|
||||
query.describe() # what it is doing, without consuming results
|
||||
reader = query.reader() # stream the rows when you want them
|
||||
query.cancel() # stop it
|
||||
```
|
||||
|
||||
`describe_query(id)` reports on one from elsewhere, the way `open_job(id)` does
|
||||
for a job.
|
||||
|
||||
<Note>
|
||||
SQL requires a deployment. See [SQL](/basics/sql#sql-is-remote).
|
||||
</Note>
|
||||
|
||||
Query-builder searches -- vector, full-text, hybrid -- are ordinary calls that
|
||||
return results. They are fast enough not to need a handle. See
|
||||
[Query](/search).
|
||||
|
||||
## Searches are ordinary calls {#searches}
|
||||
|
||||
Query-builder searches -- [vector](/search/vector-search),
|
||||
[full-text](/search/full-text-search), [hybrid](/search/hybrid-search) -- return
|
||||
results directly. They are fast enough not to need a handle, and they have no
|
||||
SQL form. See [Semantic Search](/search).
|
||||
|
||||
## When the work outlives the call {#jobs}
|
||||
|
||||
Some operations take longer than a query should: building an index, refreshing a
|
||||
computed column, registering a function. Those return a **job** rather than
|
||||
rows, and a job is a different handle with a different shape. See
|
||||
[Jobs](/basics/jobs).
|
||||
- [Quickstart](/quickstart): run a complete create, insert and search example.
|
||||
- [Search](/search): choose vector, full-text or hybrid retrieval.
|
||||
- [Filtering](/search/filtering): narrow results with predicates.
|
||||
- [SQL](/basics/sql): connect a client and run SQL queries.
|
||||
- [SDKs](/basics/sdk): choose a language and find its API reference.
|
||||
|
||||
+82
-64
@@ -1,99 +1,117 @@
|
||||
---
|
||||
title: "SDKs"
|
||||
sidebarTitle: "SDKs"
|
||||
description: "The language clients — Python, TypeScript, Rust — and what stays true across all of them."
|
||||
description: "Choose a LanceDB SDK and find the API reference for your language."
|
||||
icon: code-compare
|
||||
keywords: ["python", "typescript", "rust", "java", "sdk", "rest", "api reference"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Python, TypeScript and Rust support both LanceDB OSS and LanceDB Enterprise.
|
||||
Java provides a client for the Enterprise REST API.
|
||||
---
|
||||
|
||||
A language SDK is the primary way to work with LanceDB: create and manage
|
||||
tables, write and query data, build indexes, run feature engineering. The same
|
||||
table API works whether you're connected to an embedded database or an
|
||||
Enterprise deployment -- application code moves between the two by changing
|
||||
the connection URI.
|
||||
Use a LanceDB SDK to create tables, add data, build indexes and run searches
|
||||
from your application. Python, TypeScript and Rust work with both the embedded
|
||||
OSS library and LanceDB Enterprise.
|
||||
|
||||
## Officially supported SDKs {#supported-sdks}
|
||||
The [quickstart](/quickstart) includes installation instructions and complete
|
||||
examples in all three languages. For connection settings, see
|
||||
[Connect to LanceDB](/basics/connect).
|
||||
|
||||
Python, TypeScript and Rust are supported against both embedded and deployed
|
||||
LanceDB.
|
||||
## Official SDKs {#supported-sdks}
|
||||
|
||||
| Reference | Description |
|
||||
|:--------------|-------------------|
|
||||
| <Icon icon="python" /> [Python](https://lancedb.github.io/lancedb/python/python/) | Full-featured Python client with pandas and numpy integration |
|
||||
| <Icon icon="square-js" /> [TypeScript](https://lancedb.github.io/lancedb/js/) | A TypeScript wrapper around the Rust library, built with `napi-rs` |
|
||||
| <Icon icon="rust" /> [Rust](https://docs.rs/lancedb/latest/lancedb/index.html) | Native Rust library with persistent storage and high performance |
|
||||
| SDK | Use it with | API reference |
|
||||
| --- | --- | --- |
|
||||
| Python | Python applications, notebooks, pandas and PyArrow | [Python reference](https://lancedb.github.io/lancedb/python/python/) |
|
||||
| TypeScript | Node.js and TypeScript applications | [TypeScript reference](https://lancedb.github.io/lancedb/js/) |
|
||||
| Rust | Rust applications using the native library | [Rust reference](https://docs.rs/lancedb/latest/lancedb/index.html) |
|
||||
| Java | Applications using the Enterprise REST API | [Java reference](https://lancedb.github.io/lancedb/java/java/) |
|
||||
|
||||
Not every SDK covers the same surface, though:
|
||||
Guides show the available language examples together. Use the API reference for
|
||||
method signatures and options; SDKs do not expose every feature through the same
|
||||
methods. In Rust, enable the `remote` feature to connect to Enterprise.
|
||||
|
||||
| | Python | TypeScript |
|
||||
| --- | :---: | :---: |
|
||||
| Create and manage catalog objects | yes | yes |
|
||||
| Insert, merge, update, delete | yes | yes |
|
||||
| Filter, project, aggregate, join | yes | yes |
|
||||
| Vector search | yes | yes |
|
||||
| Full-text search | yes | yes |
|
||||
| Feature engineering, jobs | yes | partly |
|
||||
| Runs against a local database | yes | yes |
|
||||
|
||||
TypeScript has no methods for [functions](/management/function). Reach the
|
||||
gap with SQL instead -- see [SQL](/basics/sql).
|
||||
For SQL joins, aggregations and window functions, use
|
||||
[SQL](/basics/sql). Python provides a SQL client, and applications in other
|
||||
languages can use a Flight SQL client. TypeScript's table query API does not
|
||||
provide a SQL execution method.
|
||||
|
||||
## REST API {#rest}
|
||||
|
||||
<Badge color="red">Enterprise</Badge> A deployment implements the
|
||||
[Lance REST Namespace](/basics/lance#enterprise), so the catalog is reachable
|
||||
over HTTP by anything that speaks it -- see the
|
||||
[REST API reference](/api-reference/rest).
|
||||
|
||||
| Reference | Description |
|
||||
|:--------------|-------------------|
|
||||
| <Icon icon="java" /> [Java](https://lancedb.github.io/lancedb/java/java/) | REST API Enterprise SDK in Java |
|
||||
LanceDB Enterprise also exposes an HTTP API for catalog and table operations.
|
||||
Use it when integrating a language or tool without a LanceDB SDK. The
|
||||
[REST API reference](/api-reference/rest) describes its endpoints and request
|
||||
formats. For SQL queries, connect through [Flight SQL](/basics/sql).
|
||||
|
||||
## Community SDKs {#community-sdks}
|
||||
|
||||
Contributed by the community rather than supported by LanceDB, so feature parity
|
||||
and release cadence vary.
|
||||
These clients are maintained by the community. Check each project's supported
|
||||
features, installation instructions and releases before choosing one.
|
||||
|
||||
| Reference | Description |
|
||||
|:--------------|-------------------|
|
||||
| <Icon icon="golang" /> [Go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) | Community-contributed Go SDK |
|
||||
| <Icon icon="gem" /> [Ruby](https://github.com/scientist-labs/lancelot) | Community-contributed Ruby bindings |
|
||||
| <Icon icon="swift" /> [Swift](https://github.com/RyanLisse/LanceDbSwiftKit) | Community-contributed Swift SDK |
|
||||
| <Icon icon="R" /> [R](https://github.com/CathalByrneGit/lancedb) | Community-contributed R package |
|
||||
| <Icon icon="flutter" /> [Flutter](https://github.com/Alexcn/flutter_lancedb) | Community-contributed Flutter bindings |
|
||||
|
||||
For any other language, [SQL](/basics/sql) reaches the same tables without a
|
||||
LanceDB-specific SDK at all.
|
||||
| Language | Project |
|
||||
| --- | --- |
|
||||
| Go | [lancedb-go](https://pkg.go.dev/github.com/lancedb/lancedb-go/pkg/lancedb) |
|
||||
| Ruby | [Lancelot](https://github.com/scientist-labs/lancelot) |
|
||||
| Swift | [LanceDbSwiftKit](https://github.com/RyanLisse/LanceDbSwiftKit) |
|
||||
| R | [lancedb](https://github.com/CathalByrneGit/lancedb) |
|
||||
| Flutter | [flutter_lancedb](https://github.com/Alexcn/flutter_lancedb) |
|
||||
|
||||
## Python multiprocessing {#multiprocessing}
|
||||
|
||||
<Warning>
|
||||
Do not use `fork` as the start method with Python's `multiprocessing` module.
|
||||
Lance is multi-threaded internally, and `fork` with alive threads is unsafe --
|
||||
use `spawn` instead. See
|
||||
[this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555)
|
||||
for why.
|
||||
Use `spawn` when starting Python worker processes with `multiprocessing`.
|
||||
LanceDB uses threads internally, and forking a process with active threads can
|
||||
cause deadlocks. Create a connection inside each worker instead of sharing an
|
||||
open connection from the parent process.
|
||||
</Warning>
|
||||
|
||||
## What stays the same {#invariants}
|
||||
For PyTorch worker setup and connection factories, see
|
||||
[PyTorch integration](/training/torch).
|
||||
|
||||
Whichever you pick:
|
||||
## Moving to LanceDB Enterprise {#moving-to-enterprise}
|
||||
|
||||
- **Types are Arrow types.** A vector column is a `FixedSizeList<Float32>`
|
||||
everywhere, not a SQL type mapped onto storage. See
|
||||
[Functions and types](/sql/functions).
|
||||
- **Reads are Arrow.** Nothing is serialized into language objects on the way out
|
||||
unless you convert it.
|
||||
- **The catalog is one catalog.** A table created from TypeScript is visible to
|
||||
Python and to SQL immediately -- there is no per-language namespace.
|
||||
The core table workflow stays familiar: connect, create or open a table, write
|
||||
records, and query them. Enterprise runs those operations on remote compute and
|
||||
manages background maintenance.
|
||||
|
||||
## What to read next {#next}
|
||||
- **Configure the connection.** Replace the local path with your database URI,
|
||||
and supply the endpoint, region and credentials provided by your administrator.
|
||||
[Connect to Enterprise](/basics/connect#enterprise) shows these settings.
|
||||
- **Retrieve query results.** Python's remote tables do not support whole-table
|
||||
`to_arrow()` or `to_pandas()` calls. Use a query and select the columns and
|
||||
number of rows you need, for example `table.search().limit(10).to_arrow()`.
|
||||
- **Handle table listings and namespaces.** Remote table listings are paginated.
|
||||
Use the namespace and pagination options described in
|
||||
[Namespaces](/management/namespace).
|
||||
- **Use server-side jobs.** Index builds and feature refreshes can run as
|
||||
[jobs](/basics/jobs). Each operation's guide explains how to submit work and
|
||||
track completion.
|
||||
- **Let Enterprise manage maintenance.** Use
|
||||
[background maintenance](/maintenance/background-jobs) rather than calling
|
||||
the embedded table's `optimize()` method.
|
||||
|
||||
For SQL connection details, see [SQL](/basics/sql). For the operating model and
|
||||
deployment options, see [LanceDB Enterprise](/enterprise).
|
||||
|
||||
## Shared data across languages {#invariants}
|
||||
|
||||
LanceDB uses Arrow schemas across its SDKs. A fixed-size vector column stores
|
||||
vectors of a consistent dimension; the [data types reference](/sql/data-types)
|
||||
describes the supported element types and other column types.
|
||||
|
||||
Applications in different languages can work with the same tables. Connect to
|
||||
the same database and namespace, use the appropriate credentials, and account
|
||||
for [read consistency](/tables/consistency) when another process writes data.
|
||||
|
||||
## Related guides {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="SQL" icon="table-list" href="/basics/sql">
|
||||
The same tables, reached by any Flight SQL client -- and the one way in for a language with no SDK.
|
||||
Query Enterprise tables with SQL from Python or a Flight SQL client.
|
||||
</Card>
|
||||
<Card title="Agentic Skills" icon="robot" href="/basics/agentic-skills">
|
||||
Give a coding agent a current reference to this same API surface.
|
||||
Use the LanceDB plugin to help a coding agent work with your data.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
+115
-81
@@ -1,125 +1,159 @@
|
||||
---
|
||||
title: "SQL"
|
||||
sidebarTitle: "SQL"
|
||||
description: "The same tables, reached with SQL — the universal access path, and where every operation is formally documented."
|
||||
description: "Query LanceDB Enterprise with SQL from Python or an Arrow Flight SQL client."
|
||||
icon: table-list
|
||||
keywords: ["sql", "flight sql", "datafusion", "reference", "dialect"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Enterprise supports SQL through Arrow Flight SQL. Use the SDK table APIs for embedded OSS tables. See [Connection settings](#sql-is-remote) to connect.
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge> SQL runs on a deployment. A local
|
||||
database has no engine to plan a statement, so `execute_query` raises there.
|
||||
Use SQL in LanceDB Enterprise to filter records, join tables, calculate
|
||||
aggregates, and prepare datasets. SQL queries work with the same Lance tables
|
||||
you create and update through the SDKs, without exporting the data to another
|
||||
system.
|
||||
|
||||
SQL is a second way to reach the same tables the [SDKs](/basics/sdk) do --
|
||||
not a different product, a different interface onto one engine. A table
|
||||
written from Python is queried from SQL without an export step, and a table
|
||||
created from SQL is visible to every SDK immediately.
|
||||
You can run SQL through the Python SDK or an Arrow Flight SQL client. This
|
||||
guide covers connection settings, a first query, and how SQL fits alongside
|
||||
the SDK table APIs.
|
||||
|
||||
## Any language, via Flight SQL {#flight-sql}
|
||||
## Connect a SQL client {#flight-sql}
|
||||
|
||||
SQL is served over an
|
||||
Enterprise exposes an
|
||||
[Arrow Flight SQL](https://arrow.apache.org/docs/format/FlightSql.html)
|
||||
endpoint, so any Flight SQL client reaches it -- which makes SQL the way in
|
||||
for a language LanceDB has no SDK for. You don't need a LanceDB-specific
|
||||
client at all, just a Flight SQL driver for your language.
|
||||
endpoint. A compatible client sends SQL statements to this endpoint and
|
||||
receives results as Arrow record batches.
|
||||
|
||||
The TypeScript SDK itself has no SQL method, for the same reason a Go or C#
|
||||
application would reach for SQL first: connect a Flight SQL client instead of
|
||||
waiting on a native binding. See [SDKs](/basics/sdk) for the officially
|
||||
supported and community clients, and reach for Flight SQL whenever yours
|
||||
isn't among them.
|
||||
- **Python:** use `execute_query()` on an Enterprise connection, as shown below.
|
||||
- **TypeScript:** use a separate Flight SQL client. The LanceDB TypeScript SDK
|
||||
does not expose a SQL execution method.
|
||||
- **Other languages and tools:** use a Flight SQL client or a tool with Flight
|
||||
SQL integration. A LanceDB-specific SDK is not required to submit SQL.
|
||||
|
||||
## Every SDK operation, in SQL {#sdk-parity}
|
||||
The [SQL reference](/sql#connecting) shows a query from Python. See
|
||||
[SDKs](/basics/sdk) for the LanceDB language clients.
|
||||
|
||||
Broadly, a public SDK method has a corresponding SQL statement -- creating a
|
||||
table is `CREATE TABLE`, inserting rows is `INSERT INTO`, building an index is
|
||||
`CREATE INDEX`. The two describe the same operations, so whichever one your
|
||||
code speaks, the other is available too.
|
||||
### Connection settings {#sql-is-remote}
|
||||
|
||||
| | SDK | SQL |
|
||||
| --- | :---: | :---: |
|
||||
| Create and manage catalog objects | yes | partly |
|
||||
| Insert, merge, update, delete | yes | yes |
|
||||
| Filter, project, aggregate, join | yes | yes |
|
||||
| Vector search | yes | version-dependent |
|
||||
| Full-text search | yes | yes |
|
||||
| Feature engineering, jobs | yes | yes |
|
||||
| Runs against a local database | yes | **no** |
|
||||
Get the following values from your Enterprise administrator:
|
||||
|
||||
## Vector search depends on the deployment {#no-vector-sql}
|
||||
- Database URI, such as `db://your-database`.
|
||||
- API key and region.
|
||||
- API endpoint for `host_override`.
|
||||
- Flight SQL endpoint for `sql_host_override`, including its scheme and port.
|
||||
|
||||
The SDK query builder provides vector search across embedded and deployed
|
||||
databases. SQL support depends on the deployment version and supported query
|
||||
shape. Some versions recognize `ORDER BY distance(vector, query_vector) LIMIT k`
|
||||
over a table scan and plan vector search, including indexed and flat search.
|
||||
Confirm support on your deployment and use SQL
|
||||
[`EXPLAIN`](/sql/dql/explain) to check the chosen plan; this is not a guarantee
|
||||
that every deployed release supports that form.
|
||||
The Flight SQL endpoint must be enabled and reachable from your client. Pass
|
||||
it explicitly as `sql_host_override` in Python, as in the example below. The
|
||||
API and SQL endpoints may use different hosts or ports.
|
||||
|
||||
Full-text search is available through the [`fts()`](/sql/functions#fts) table
|
||||
function and the [`contains_tokens`](/sql/scalar-functions#contains-tokens)
|
||||
predicate.
|
||||
See [Connect to Enterprise](/basics/connect#enterprise) for other connection options.
|
||||
|
||||
The reverse gap also exists: a [view](/management/view) is a saved SQL query,
|
||||
so SQL is the only way to create one -- there's nothing for an SDK method to
|
||||
call.
|
||||
SQL runs in Enterprise. Connections to a local directory or an object-storage
|
||||
bucket use the embedded OSS library and cannot execute SQL statements. Use
|
||||
SDK table operations to query those tables.
|
||||
|
||||
## SQL needs a deployment {#sql-is-remote}
|
||||
## Run a query in Python {#sql-from-python}
|
||||
|
||||
A connection to a directory or a bucket has no server to plan and execute a
|
||||
statement, and asking it to raises `NotImplementedError`. This is the largest
|
||||
behavioral difference between the embedded and deployed forms, and the reason
|
||||
the SDKs are the portable choice: SDK code moves between the two by changing
|
||||
the URI, SQL does not.
|
||||
Install the SDK with `pip install lancedb`. Replace the connection placeholders
|
||||
with your Enterprise settings, including the region and both endpoint URLs.
|
||||
This example reads an existing table named `my_docs` with `id` and `text`
|
||||
columns. Change the query to match your table.
|
||||
|
||||
```python
|
||||
db = lancedb.connect("./.lancedb") # no SQL
|
||||
db = lancedb.connect("db://my-database") # SQL
|
||||
```
|
||||
import lancedb
|
||||
|
||||
## Running SQL from Python {#sql-from-python}
|
||||
db = lancedb.connect(
|
||||
"db://your-database",
|
||||
api_key="your-api-key",
|
||||
region="us-east-1",
|
||||
host_override="https://your-api-endpoint",
|
||||
sql_host_override="https://your-sql-endpoint:443",
|
||||
)
|
||||
|
||||
```python
|
||||
reader = db.execute_query("SELECT id, text FROM my_docs LIMIT 10")
|
||||
table = reader.read_all()
|
||||
print(table.to_pylist())
|
||||
```
|
||||
|
||||
`execute_query` returns a blocking Arrow reader once results start arriving; it
|
||||
does not wait for the whole query. `execute_query_async` returns a handle
|
||||
instead -- see [Queries](/basics/query).
|
||||
In application code, load credentials from environment variables or a secrets
|
||||
manager. The [connection guide](/basics/connect#enterprise) shows that setup.
|
||||
|
||||
## Choosing between them {#choosing}
|
||||
`execute_query()` returns a blocking Arrow reader when results start arriving.
|
||||
The example uses `read_all()` to collect the small result in memory. For larger
|
||||
results, consume the reader in batches. To submit a query and check its status
|
||||
separately, use `execute_query_async()`. See [Queries](/basics/query) for query
|
||||
handles, streaming, and cancellation.
|
||||
|
||||
Reach for **the SDKs** when you are writing data, managing the catalog, running
|
||||
vector search, or building an application that should work the same embedded and
|
||||
deployed.
|
||||
## Choose SQL or SDK table operations {#choosing}
|
||||
|
||||
Reach for **SQL** when the question is analytical -- joins across tables,
|
||||
aggregates, window functions -- when the thing asking is a BI tool, a
|
||||
notebook, or a person who already thinks in SQL, or when it's the only client
|
||||
available for your language.
|
||||
Use **SQL** for joins, aggregations, window functions, and analytical queries
|
||||
written in SQL. Use **SDK table operations** to build applications around table
|
||||
management, ingestion, filtering, and search, including applications that run
|
||||
with the embedded OSS library.
|
||||
|
||||
The two compose. A common shape is SDK code to build and maintain tables, and
|
||||
SQL for the analysis on top.
|
||||
You can combine them. For example, create and populate a table with an SDK,
|
||||
then query it with SQL to prepare a training dataset. In Python, both workflows
|
||||
can use the same Enterprise connection.
|
||||
|
||||
## The complete reference {#reference}
|
||||
### Common SQL operations {#sdk-parity}
|
||||
|
||||
This page is the concept; [SQL Reference](/sql) is the specification. It has
|
||||
one page per statement, syntax first -- and since every SDK operation has a
|
||||
SQL form, those pages end up defining the precise behavior of the SDK
|
||||
operations too, not just the SQL dialect.
|
||||
Several tasks are available through both interfaces, but the methods and SQL
|
||||
statements do not have identical coverage. SDK support also differs by
|
||||
language. These SQL reference pages describe the supported statements:
|
||||
|
||||
| Task | SQL reference |
|
||||
| --- | --- |
|
||||
| Create a table | [CREATE TABLE](/sql/ddl/table) |
|
||||
| Insert or change rows | [INSERT](/sql/dml/insert), [UPDATE](/sql/dml/update), [DELETE](/sql/dml/delete) |
|
||||
| Build an index | [CREATE INDEX](/sql/ddl/index) |
|
||||
| Join and aggregate data | [SELECT](/sql/dql/select) |
|
||||
| Save a query as a view | [CREATE VIEW](/sql/ddl/view) |
|
||||
|
||||
[Computed columns](/feature-engineering/add-computed-column) store SQL
|
||||
expressions whose values are populated by a refresh. {/* Draft for the Functions release:
|
||||
They are separate from registered Python Functions. */}
|
||||
See the computed-column and
|
||||
[materialized-view](/management/materialized-view) guides for their SDK and SQL
|
||||
operations and Enterprise setup requirements.
|
||||
|
||||
## Vector and full-text search {#no-vector-sql}
|
||||
|
||||
SQL vector search support depends on the Enterprise server version and query
|
||||
form. Versions that support `ORDER BY distance(vector, query_vector) LIMIT k`
|
||||
over a table scan can plan indexed or flat vector search. Check that your
|
||||
server version supports this form, then use
|
||||
[`EXPLAIN`](/sql/dql/explain) to inspect the query plan.
|
||||
|
||||
The [SDK vector search API](/search/vector-search) is also available in
|
||||
Enterprise and OSS. Use it when you prefer the SDK query builder or need to
|
||||
search embedded tables.
|
||||
|
||||
For full-text search in SQL, use the [`fts()`](/sql/functions#fts) table
|
||||
function or the [`contains_tokens`](/sql/scalar-functions#contains-tokens)
|
||||
predicate.
|
||||
|
||||
## SQL syntax and API references {#reference}
|
||||
|
||||
The [SQL reference](/sql) documents statement syntax, supported expressions,
|
||||
and data types. For SDK methods and their arguments, use the
|
||||
[Python](https://lancedb.github.io/lancedb/python/python/),
|
||||
[TypeScript](https://lancedb.github.io/lancedb/js/), or
|
||||
[Rust](https://docs.rs/lancedb/latest/lancedb/index.html) API reference.
|
||||
|
||||
<Card title="SQL Reference" icon="table-list" href="/sql">
|
||||
DDL by object, then DML, then DQL -- the detailed technical documentation
|
||||
that defines the behavior of every operation and SDK API.
|
||||
Statement syntax, expressions, and data types for LanceDB SQL.
|
||||
</Card>
|
||||
|
||||
## What to read next {#next}
|
||||
## Next steps {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="SDKs" icon="code-compare" href="/basics/sdk">
|
||||
The language clients, and what stays true across all of them.
|
||||
<Card title="Queries" icon="magnifying-glass" href="/basics/query">
|
||||
Read results, track long-running queries, and cancel work.
|
||||
</Card>
|
||||
<Card title="Agentic Skills" icon="robot" href="/basics/agentic-skills">
|
||||
Give a coding agent a current reference to this same API surface.
|
||||
<Card title="SDKs" icon="code-compare" href="/basics/sdk">
|
||||
Compare language clients and find their API references.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
+110
-79
@@ -1,113 +1,144 @@
|
||||
---
|
||||
title: "Profiling a dataset"
|
||||
sidebarTitle: "Aggregate"
|
||||
description: "Counting, grouping and summarizing — finding out what your data actually contains."
|
||||
description: "Measure coverage, missing values and class balance before selecting a dataset."
|
||||
icon: chart-column
|
||||
keywords: ["aggregate", "group by", "class balance", "count", "profiling", "distinct"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Enterprise supports SQL aggregation.
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge> Aggregation is SQL, and
|
||||
[SQL needs a deployment](/basics/sql#sql-is-remote).
|
||||
Use SQL aggregates to summarize a dataset: count records, find missing values,
|
||||
and compare groups. These summaries help you understand the data before
|
||||
selecting records for search, evaluation or training.
|
||||
|
||||
Before curating anything, find out what you have. Most of the surprises in a
|
||||
training set — a class that is 90% of the rows, a column that is half null, a
|
||||
"unique" id that is not — are one `GROUP BY` away from being visible.
|
||||
## Example data {#example-data}
|
||||
|
||||
## How big, and how complete {#size}
|
||||
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
|
||||
document table and a separate label table. Expand the setup below to create them.
|
||||
|
||||
<Accordion title="Create the example tables">
|
||||
|
||||
`curation_documents` has six records describing four documents.
|
||||
`curation_labels` has one label for each of three documents. `row_id` identifies
|
||||
a record; `doc_id` identifies the document it describes.
|
||||
|
||||
Run each statement separately in a scratch database or namespace. Create these
|
||||
tables once for the whole section, using fresh names if they already exist.
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
count(*) AS rows,
|
||||
count(embedding) AS embedded,
|
||||
count(caption) AS captioned
|
||||
FROM images
|
||||
CREATE TABLE curation_documents (
|
||||
row_id BIGINT, doc_id BIGINT, version BIGINT,
|
||||
source VARCHAR, caption VARCHAR, score DOUBLE
|
||||
);
|
||||
INSERT INTO curation_documents VALUES
|
||||
(1, 10, 1, 'web', NULL, 0.80),
|
||||
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
|
||||
(3, 20, 1, 'books', 'A dog in a field', 0.90),
|
||||
(4, 30, 1, 'web', NULL, 0.70),
|
||||
(5, 30, 1, 'web', 'A cat by a window', 0.85),
|
||||
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
|
||||
|
||||
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
|
||||
INSERT INTO curation_labels VALUES
|
||||
(10, 'cat'), (20, 'dog'), (30, 'cat');
|
||||
```
|
||||
|
||||
`count(*)` counts rows; `count(col)` counts **non-null values**. The gap between
|
||||
them is your coverage, which is the fastest way to check how far a
|
||||
[column refresh](/feature-engineering/refresh-column) has got without waiting
|
||||
for the job to report.
|
||||
`score` is a sample quality score, not a search result score. Document 10 has
|
||||
two versions. Document 30 has two records at the same version. Document 40
|
||||
has no label. These cases let you check the effect of each operation.
|
||||
|
||||
## Class balance {#balance}
|
||||
</Accordion>
|
||||
|
||||
The single most useful curation query:
|
||||
## Count records and missing values {#size}
|
||||
|
||||
Use `count(*)` to count rows and `count(column)` to count non-null values.
|
||||
Add `DISTINCT` to count unique values:
|
||||
|
||||
```sql
|
||||
SELECT label, count(*) AS n
|
||||
FROM images
|
||||
SELECT count(*) AS records,
|
||||
count(DISTINCT doc_id) AS documents,
|
||||
count(caption) AS captioned,
|
||||
count(*) FILTER (WHERE caption IS NULL) AS missing_caption
|
||||
FROM curation_documents;
|
||||
```
|
||||
|
||||
The example returns **6 records, 4 documents, 4 populated captions and 2 missing
|
||||
captions**. `count(column)` excludes null values; it does not check whether a
|
||||
string is empty or whether a value is useful for the task.
|
||||
|
||||
## Group records by label {#balance}
|
||||
|
||||
Use `GROUP BY` to calculate a summary for each label:
|
||||
|
||||
```sql
|
||||
SELECT label, count(*) AS documents
|
||||
FROM curation_labels
|
||||
GROUP BY label
|
||||
ORDER BY n DESC
|
||||
ORDER BY documents DESC, label;
|
||||
```
|
||||
|
||||
If the top class is an order of magnitude larger than the rest, sampling
|
||||
uniformly will reproduce that imbalance. Two ways out: draw a fixed number per
|
||||
class with [windowing](/curation/windowing#per-group), or oversample the rare
|
||||
ones and accept the duplication.
|
||||
There are **2 cats and 1 dog** in the label table. Document 40 is absent, so this
|
||||
query does not measure labeling coverage across all source documents. Use a
|
||||
[left join](/curation/join#join-type) to find documents without labels.
|
||||
|
||||
## Distinct values {#distinct}
|
||||
Decide whether the distribution matches your intended use. A uniform random
|
||||
sample tends to preserve the source distribution; it does not balance classes.
|
||||
To cap the number selected per label, use
|
||||
[ranking within a group](/curation/windowing#per-group).
|
||||
|
||||
## Count records that match a condition {#conditional}
|
||||
|
||||
```sql
|
||||
SELECT count(DISTINCT source) FROM images
|
||||
SELECT source,
|
||||
count(*) AS records,
|
||||
count(*) FILTER (WHERE score >= 0.9) AS high_score,
|
||||
count(*) FILTER (WHERE caption IS NULL) AS missing_caption
|
||||
FROM curation_documents
|
||||
GROUP BY source
|
||||
ORDER BY source;
|
||||
```
|
||||
|
||||
On a high-cardinality column this is expensive, because exact distinctness means
|
||||
holding every value seen. When an estimate will do:
|
||||
`FILTER` applies a condition to one aggregate. Here, `books` has 2 records,
|
||||
1 high-score record and no missing captions. `web` has 4 records, 1 high-score
|
||||
record and 2 missing captions. Counts are per record, so repeated document IDs
|
||||
still contribute more than once.
|
||||
|
||||
<span id="distinct"></span>
|
||||
## Group numeric values into ranges {#buckets}
|
||||
|
||||
Use `count(DISTINCT column)` for the number of distinct non-null values.
|
||||
For numeric fields, group by a range or summarize their distribution. For
|
||||
example, this query counts records in score buckets of width 0.1:
|
||||
|
||||
```sql
|
||||
SELECT approx_distinct(source) FROM images
|
||||
SELECT floor(score * 10) / 10 AS score_bucket, count(*) AS records
|
||||
FROM curation_documents
|
||||
GROUP BY score_bucket
|
||||
ORDER BY score_bucket;
|
||||
```
|
||||
|
||||
Cheap enough to run on a table where the exact version is not worth the wait.
|
||||
The other approximate functions — `approx_median`, `approx_percentile_cont` —
|
||||
are useful for the same reason when profiling a numeric column.
|
||||
For time-based data, group by the time interval relevant to the task. The
|
||||
[aggregation reference](/sql/dql/aggregation) and
|
||||
[scalar function reference](/sql/scalar-functions) cover additional summaries,
|
||||
including approximate counts and percentiles. Use an approximate aggregate when an estimate is sufficient; use an exact
|
||||
count when the result must be precise.
|
||||
|
||||
## Conditional counts {#conditional}
|
||||
## Profile a sample {#sampling}
|
||||
|
||||
`FILTER` narrows one aggregate without narrowing the query, which is how you get
|
||||
several slices in one pass rather than one query each:
|
||||
A [sample](/curation/sampling) can help you explore a large dataset before a
|
||||
full scan. Report the sample size and method alongside the result. A sample can miss rare classes, and fragment sampling can reflect the order
|
||||
in which records were written.
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
label,
|
||||
count(*) AS total,
|
||||
count(*) FILTER (WHERE score > 0.9) AS confident,
|
||||
count(*) FILTER (WHERE caption IS NULL) AS uncaptioned
|
||||
FROM images
|
||||
GROUP BY label
|
||||
```
|
||||
Before accepting a curated dataset, check its required fields and key
|
||||
constraints directly rather than inferring them from a small sample.
|
||||
|
||||
## Grouping by a bucket {#buckets}
|
||||
## Continue curating {#reference}
|
||||
|
||||
Raw values are rarely the grouping you want. `date_bin` buckets timestamps, and
|
||||
ordinary arithmetic buckets numbers:
|
||||
|
||||
```sql
|
||||
SELECT date_bin(INTERVAL '1 day', created_at) AS day, count(*)
|
||||
FROM images GROUP BY day ORDER BY day
|
||||
```
|
||||
|
||||
A histogram of a score column is the same trick:
|
||||
|
||||
```sql
|
||||
SELECT floor(score * 10) / 10 AS bucket, count(*)
|
||||
FROM images GROUP BY bucket ORDER BY bucket
|
||||
```
|
||||
|
||||
## Profiling a sample {#sampling}
|
||||
|
||||
On a very large table, profile a [sample](/curation/sampling) and scale:
|
||||
|
||||
```sql
|
||||
SELECT label, count(*) * 100 AS estimated
|
||||
FROM images TABLESAMPLE (1)
|
||||
GROUP BY label
|
||||
```
|
||||
|
||||
Treat the result as an estimate. A 1% sample will miss classes rarer than about
|
||||
1 in 10,000 entirely, which is exactly the tail you are usually looking for — so
|
||||
use this to check the shape of the head, not the existence of the tail.
|
||||
|
||||
## Reference {#reference}
|
||||
|
||||
The complete clause and function list is in
|
||||
[Aggregation](/sql/dql/aggregation).
|
||||
Use these measurements to choose a [sample](/curation/sampling),
|
||||
[attach missing metadata](/curation/join), or
|
||||
[select one record per document](/curation/windowing).
|
||||
See [Aggregation](/sql/dql/aggregation) for complete SQL syntax.
|
||||
|
||||
+104
-33
@@ -1,54 +1,125 @@
|
||||
---
|
||||
title: "Data curation"
|
||||
sidebarTitle: "Overview"
|
||||
description: "Shaping a dataset before you train on it: profiling, sampling, joining, and ranking."
|
||||
description: "Prepare datasets for search, evaluation and training by profiling, selecting and combining records."
|
||||
icon: filter
|
||||
keywords: ["curation", "dataset", "training set", "sampling", "deduplication", "class balance"]
|
||||
---
|
||||
|
||||
Curation is the work between having data and having a *dataset*: finding out
|
||||
what you actually have, drawing a subset you can afford to train on, attaching
|
||||
labels that live elsewhere, and picking one row per thing when you have several.
|
||||
Data curation turns a collection of records into a dataset suited to a task.
|
||||
You might select labeled examples for training, prepare a fixed evaluation
|
||||
set, or remove duplicate documents from a search corpus.
|
||||
|
||||
These pages are task-oriented. Each shows the operation in the context you would
|
||||
reach for it, and links to the [SQL reference](/sql) for the complete syntax.
|
||||
Start with the result you need. Then inspect the source data, choose which
|
||||
records belong, and save the selected data or the rules used to produce it.
|
||||
|
||||
<Note>
|
||||
Most curation is analytical -- grouping, joining, ranking -- which the query
|
||||
builders do not do. It is therefore SQL, and SQL needs a deployment. See
|
||||
[SQL](/basics/sql#sql-is-remote).
|
||||
</Note>
|
||||
## Choose a curation task {#questions}
|
||||
|
||||
## The four questions {#questions}
|
||||
|
||||
| | |
|
||||
| Task | What it helps you decide |
|
||||
| --- | --- |
|
||||
| [What do I have?](/curation/aggregate) | counts, class balance, null coverage, distinct values |
|
||||
| [What can I afford?](/curation/sampling) | a random subset, reproducible when you need it fixed |
|
||||
| [What else do I know?](/curation/join) | labels, metadata and scores that live in another table |
|
||||
| [Which one do I keep?](/curation/windowing) | one row per key, ranking within a group, splits |
|
||||
| [Profile a dataset](/curation/aggregate) | Whether coverage, missing values and class balance suit the task. |
|
||||
| [Sample records](/curation/sampling) | Which subset to inspect or use for an experiment. |
|
||||
| [Join datasets](/curation/join) | How to attach labels, metadata or scores stored elsewhere. |
|
||||
| [Rank and deduplicate](/curation/windowing) | Which records to keep when several describe the same item. |
|
||||
|
||||
## A worked shape {#shape}
|
||||
The analytical examples in this section use **Enterprise SQL**. Connect with
|
||||
the [SQL guide](/basics/sql). For sampling through the Python SDK in OSS or
|
||||
Enterprise, see [Sample without SQL](/curation/sampling#training).
|
||||
|
||||
The four compose into the same pipeline most curation ends up being:
|
||||
## Prepare a labeled document dataset {#shape}
|
||||
|
||||
Suppose you need one labeled record per document for a training experiment.
|
||||
Inspect the source records, choose the latest version of each document, then
|
||||
join its label. Finally, check the class balance of the selected records.
|
||||
|
||||
### Example data {#example-data}
|
||||
|
||||
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
|
||||
document table and a separate label table. Expand the setup below to create them.
|
||||
|
||||
<Accordion title="Create the example tables">
|
||||
|
||||
`curation_documents` has six records describing four documents.
|
||||
`curation_labels` has one label for each of three documents. `row_id` identifies
|
||||
a record; `doc_id` identifies the document it describes.
|
||||
|
||||
Run each statement separately in a scratch database or namespace. Create these
|
||||
tables once for the whole section, using fresh names if they already exist.
|
||||
|
||||
```sql
|
||||
CREATE TABLE curation_documents (
|
||||
row_id BIGINT, doc_id BIGINT, version BIGINT,
|
||||
source VARCHAR, caption VARCHAR, score DOUBLE
|
||||
);
|
||||
INSERT INTO curation_documents VALUES
|
||||
(1, 10, 1, 'web', NULL, 0.80),
|
||||
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
|
||||
(3, 20, 1, 'books', 'A dog in a field', 0.90),
|
||||
(4, 30, 1, 'web', NULL, 0.70),
|
||||
(5, 30, 1, 'web', 'A cat by a window', 0.85),
|
||||
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
|
||||
|
||||
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
|
||||
INSERT INTO curation_labels VALUES
|
||||
(10, 'cat'), (20, 'dog'), (30, 'cat');
|
||||
```
|
||||
|
||||
`score` is a sample quality score, not a search result score. Document 10 has
|
||||
two versions. Document 30 has two records at the same version. Document 40
|
||||
has no label. These cases let you check the effect of each operation.
|
||||
|
||||
</Accordion>
|
||||
|
||||
The following query chooses the latest record for each document, then keeps
|
||||
only documents with a label:
|
||||
|
||||
```sql
|
||||
WITH ranked AS (
|
||||
SELECT d.*, c.label,
|
||||
row_number() OVER (PARTITION BY d.doc_id ORDER BY d.version DESC) AS rn
|
||||
FROM documents AS d
|
||||
JOIN labels AS c ON c.doc_id = d.doc_id
|
||||
SELECT *, row_number() OVER (
|
||||
PARTITION BY doc_id ORDER BY version DESC, row_id DESC
|
||||
) AS rn
|
||||
FROM curation_documents
|
||||
)
|
||||
SELECT * FROM ranked WHERE rn = 1
|
||||
SELECT d.row_id, d.doc_id, d.caption, l.label
|
||||
FROM ranked AS d
|
||||
JOIN curation_labels AS l ON l.doc_id = d.doc_id
|
||||
WHERE d.rn = 1
|
||||
ORDER BY d.doc_id;
|
||||
```
|
||||
|
||||
Join to bring in what you know, window to pick one row per document, filter to
|
||||
keep it. Add [`TABLESAMPLE`](/curation/sampling) when the result is still larger
|
||||
than you want, and [aggregate](/curation/aggregate) to check the balance of what
|
||||
you ended up with.
|
||||
The result contains documents **10, 20 and 30**, with labels **cat, dog and
|
||||
cat**. Document 40 is excluded because this particular task requires a label.
|
||||
The `row_id` tie-breaker makes the choice for document 30 deterministic;
|
||||
choose a tie-breaker that fits your own data.
|
||||
|
||||
## Then what {#next}
|
||||
The example assumes one label per document. Duplicate keys in the label table
|
||||
would multiply the result rows. The [joining guide](/curation/join#labels)
|
||||
shows how to check that assumption.
|
||||
|
||||
A curated result you want to keep is a
|
||||
[materialized view](/management/materialized-view) -- stored, refreshed, and
|
||||
readable by a [data loader](/training/data-loading) like any other table.
|
||||
## Check the selected dataset {#check-result}
|
||||
|
||||
Compare the result with the task's requirements: are the expected documents
|
||||
present, are required fields populated, and is the class distribution suitable?
|
||||
This example has two cats and one dog. That distribution may be appropriate
|
||||
for exploration, but it does not establish a representative evaluation set.
|
||||
|
||||
[Profiling](/curation/aggregate) provides the counts to inspect.
|
||||
[Sampling](/curation/sampling) and [ranking](/curation/windowing) help you
|
||||
adjust the selection without changing the original records.
|
||||
|
||||
## Save and reuse the result {#next}
|
||||
|
||||
Choose the output according to how you will use it:
|
||||
|
||||
- **A fixed dataset:** save the query result in a new table with
|
||||
[`CREATE TABLE AS SELECT`](/sql/ddl/table#create). Keep that output unchanged
|
||||
when comparing experiments.
|
||||
- **A reusable selection:** save the query as a [view](/management/view).
|
||||
A view stores the query definition, so it is not a frozen evaluation set.
|
||||
- **Stored results that you refresh:** use a
|
||||
[materialized view](/management/materialized-view) when its supported query
|
||||
and refresh behavior fit the workflow.
|
||||
|
||||
Record the source versions and selection rules alongside the output. Continue
|
||||
with [data loading](/training/data-loading) to use the curated data in a
|
||||
training pipeline.
|
||||
|
||||
+97
-55
@@ -1,91 +1,133 @@
|
||||
---
|
||||
title: "Joining datasets"
|
||||
sidebarTitle: "Join"
|
||||
description: "Bringing labels, metadata and scores together with the rows they describe."
|
||||
description: "Combine records with labels or metadata, and check for missing or duplicate matches."
|
||||
icon: link
|
||||
keywords: ["join", "labels", "metadata", "enrichment", "namespace"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Enterprise supports SQL joins.
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge> Joins are SQL, and
|
||||
[SQL needs a deployment](/basics/sql#sql-is-remote).
|
||||
A SQL join combines records from two tables using a shared key. Use joins to
|
||||
attach labels, metadata or model scores to your documents. Choose an inner join
|
||||
to keep matching records, or a left join to retain records without a match.
|
||||
|
||||
Curation usually means combining things that were produced separately — images
|
||||
in one table, human labels in another, model scores in a third. A join is how
|
||||
they become one dataset.
|
||||
## Example data {#example-data}
|
||||
|
||||
## Attaching labels {#labels}
|
||||
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
|
||||
document table and a separate label table. Expand the setup below to create them.
|
||||
|
||||
<Accordion title="Create the example tables">
|
||||
|
||||
`curation_documents` has six records describing four documents.
|
||||
`curation_labels` has one label for each of three documents. `row_id` identifies
|
||||
a record; `doc_id` identifies the document it describes.
|
||||
|
||||
Run each statement separately in a scratch database or namespace. Create these
|
||||
tables once for the whole section, using fresh names if they already exist.
|
||||
|
||||
```sql
|
||||
SELECT i.id, i.embedding, l.label
|
||||
FROM images AS i
|
||||
JOIN labels AS l ON l.image_id = i.id
|
||||
CREATE TABLE curation_documents (
|
||||
row_id BIGINT, doc_id BIGINT, version BIGINT,
|
||||
source VARCHAR, caption VARCHAR, score DOUBLE
|
||||
);
|
||||
INSERT INTO curation_documents VALUES
|
||||
(1, 10, 1, 'web', NULL, 0.80),
|
||||
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
|
||||
(3, 20, 1, 'books', 'A dog in a field', 0.90),
|
||||
(4, 30, 1, 'web', NULL, 0.70),
|
||||
(5, 30, 1, 'web', 'A cat by a window', 0.85),
|
||||
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
|
||||
|
||||
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
|
||||
INSERT INTO curation_labels VALUES
|
||||
(10, 'cat'), (20, 'dog'), (30, 'cat');
|
||||
```
|
||||
|
||||
An inner join keeps only rows that matched, which for curation is often exactly
|
||||
what you want: unlabelled rows are not training data.
|
||||
`score` is a sample quality score, not a search result score. Document 10 has
|
||||
two versions. Document 30 has two records at the same version. Document 40
|
||||
has no label. These cases let you check the effect of each operation.
|
||||
|
||||
When you need to *see* what did not match, use a left join and filter:
|
||||
</Accordion>
|
||||
|
||||
## Basic join {#join-type}
|
||||
|
||||
An inner join keeps matching records:
|
||||
|
||||
```sql
|
||||
SELECT i.id
|
||||
FROM images AS i
|
||||
LEFT JOIN labels AS l ON l.image_id = i.id
|
||||
WHERE l.image_id IS NULL
|
||||
SELECT d.row_id, d.doc_id, l.label
|
||||
FROM curation_documents AS d
|
||||
JOIN curation_labels AS l ON l.doc_id = d.doc_id
|
||||
ORDER BY d.row_id;
|
||||
```
|
||||
|
||||
That is your unlabelled backlog, which is the thing worth counting before you
|
||||
plan an annotation round.
|
||||
This returns **5 rows**: both records for document 10, one for document 20,
|
||||
and both records for document 30. To produce one row per document,
|
||||
[deduplicate the source records](/curation/windowing#dedup).
|
||||
|
||||
## Across namespaces {#namespaces}
|
||||
### Find records without a match
|
||||
|
||||
Tables in different [namespaces](/management/namespace) join by their dotted
|
||||
paths — the catalog is one catalog, so there is nothing to federate:
|
||||
A left join keeps every record from the left table and fills missing right-side
|
||||
values with `NULL`. Add `WHERE l.doc_id IS NULL` to return only unlabeled documents:
|
||||
|
||||
```sql
|
||||
SELECT i.id, u.team
|
||||
FROM prod.media.images AS i
|
||||
JOIN prod.reference.uploaders AS u ON u.id = i.uploader_id
|
||||
SELECT DISTINCT d.doc_id
|
||||
FROM curation_documents AS d
|
||||
LEFT JOIN curation_labels AS l ON l.doc_id = d.doc_id
|
||||
WHERE l.doc_id IS NULL
|
||||
ORDER BY d.doc_id;
|
||||
```
|
||||
|
||||
## Joining a search result {#search}
|
||||
The result is **document 40**. You might send it for annotation, retain it for
|
||||
an unlabeled-data workflow, or exclude it from a task that requires labels.
|
||||
|
||||
Full-text search is a table function, so its result is a relation and joins like
|
||||
any other. This is how you ask which categories a query matches, rather than
|
||||
just which rows:
|
||||
## Handle duplicate keys {#labels}
|
||||
|
||||
This example joins `curation_documents.doc_id` to `curation_labels.doc_id`.
|
||||
First check whether the label table contains more than one row per document:
|
||||
|
||||
```sql
|
||||
SELECT c.name, count(*) AS hits
|
||||
FROM fts('documents', 'puppy') AS m
|
||||
JOIN categories AS c ON c.id = m.category_id
|
||||
GROUP BY c.name
|
||||
ORDER BY hits DESC
|
||||
SELECT doc_id, count(*) AS labels
|
||||
FROM curation_labels
|
||||
GROUP BY doc_id
|
||||
HAVING count(*) > 1;
|
||||
```
|
||||
|
||||
<Note>
|
||||
Vector search has no SQL form, so a similarity result cannot be joined this way.
|
||||
Run the search in an SDK, then join over its output — or write the result to a
|
||||
table first.
|
||||
</Note>
|
||||
The example returns no rows. If your data returns duplicates, decide whether
|
||||
they represent valid multiple labels or records that need consolidation.
|
||||
A join does not enforce one-to-one matching.
|
||||
|
||||
## Making it fast {#performance}
|
||||
## Join tables across namespaces {#namespaces}
|
||||
|
||||
Two things dominate join cost on a lakehouse table.
|
||||
Use a table's qualified path to address it in another
|
||||
[namespace](/management/namespace). For example, a table named `images` in
|
||||
`prod.media` is addressed as `prod.media.images`. The same matching-key and
|
||||
join-type rules apply; your identity needs access to the source objects.
|
||||
|
||||
**Index the join key.** A [scalar index](/indexing/scalar-index) on the key lets
|
||||
the planner look rows up instead of scanning for them. Check with
|
||||
[`EXPLAIN`](/sql/dql/explain): an indexed key shows a `ScalarIndexQuery`, an
|
||||
unindexed one shows a `LanceScan` feeding the join.
|
||||
## Combine search results with metadata {#search}
|
||||
|
||||
**Do not carry wide columns through.** Select the columns you need before
|
||||
joining, not after. Embeddings and blobs are the expensive part of a multimodal
|
||||
table, and a join can materialize them many more times than the result has rows.
|
||||
A [SQL full-text search](/sql/dql/fts) returns rows that can be joined to other
|
||||
tables. Join using an identifier retained in the search result, then check
|
||||
whether the metadata has one or several matches per result.
|
||||
|
||||
```sql
|
||||
SELECT i.id, l.label
|
||||
FROM (SELECT id, uploader_id FROM images) AS i
|
||||
JOIN labels AS l ON l.image_id = i.id
|
||||
```
|
||||
For an SDK search, retain the matching IDs and combine them with metadata in
|
||||
your application or save the selected records to a table for further SQL work.
|
||||
See [SQL search interfaces](/basics/sql#no-vector-sql) for supported query forms.
|
||||
|
||||
## Reference {#reference}
|
||||
## Join performance {#performance}
|
||||
|
||||
The join types and their semantics are in [Joins](/sql/dql/join).
|
||||
Check the row count, unmatched keys and duplicate matches before using a
|
||||
joined dataset. Select only the columns needed for the next step, especially
|
||||
when source records include large media or embedding columns.
|
||||
|
||||
Use [Explain a query plan](/search/explain) to see how the query reads and
|
||||
combines data. Check the plan to see whether the query uses indexes and how it performs the
|
||||
join. An index on a key does not guarantee that the join will use it.
|
||||
|
||||
## Save the result or continue curating {#reference}
|
||||
|
||||
Keep the query for reuse or [save the selected data](/curation#next).
|
||||
See [Joins](/sql/dql/join) for join types and exact SQL behavior.
|
||||
|
||||
+132
-58
@@ -1,98 +1,172 @@
|
||||
---
|
||||
title: "Sampling a dataset"
|
||||
sidebarTitle: "Sampling"
|
||||
description: "Drawing a subset you can afford to work with, and fixing it when it has to be reproducible."
|
||||
title: "SQL sampling"
|
||||
sidebarTitle: "SQL Sampling"
|
||||
description: "Choose a sample for exploration or evaluation and preserve the selected records when reproducibility matters."
|
||||
icon: dice
|
||||
keywords: ["sampling", "tablesample", "subset", "eval set", "reproducible", "shuffle"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Enterprise supports SQL `TABLESAMPLE`. For sampling in OSS and Enterprise through the Python SDK, see [Sample without SQL](#training).
|
||||
---
|
||||
|
||||
Most curation starts with a subset. A question you can answer in ten seconds on
|
||||
1% of the data is worth asking twenty times; the same question on the whole
|
||||
table is worth asking once.
|
||||
import { PySampleDataset } from '/snippets/curation_sampling.mdx';
|
||||
|
||||
## A quick look {#quick}
|
||||
Sampling selects records from a larger table. Use it to explore unfamiliar
|
||||
data, run a smaller experiment, or prepare an evaluation dataset. Enterprise
|
||||
SQL provides `TABLESAMPLE` for selecting a percentage or number of rows.
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
This page covers Enterprise SQL sampling and a complete
|
||||
[Python SDK example](#training) that runs locally.
|
||||
|
||||
## Example data {#example-data}
|
||||
|
||||
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
|
||||
document table and a separate label table. Expand the setup below to create them.
|
||||
|
||||
<Accordion title="Create the example tables">
|
||||
|
||||
`curation_documents` has six records describing four documents.
|
||||
`curation_labels` has one label for each of three documents. `row_id` identifies
|
||||
a record; `doc_id` identifies the document it describes.
|
||||
|
||||
Run each statement separately in a scratch database or namespace. Create these
|
||||
tables once for the whole section, using fresh names if they already exist.
|
||||
|
||||
```sql
|
||||
SELECT * FROM images TABLESAMPLE (1) -- 1 percent
|
||||
SELECT * FROM images TABLESAMPLE (500 ROWS) -- 500 rows
|
||||
CREATE TABLE curation_documents (
|
||||
row_id BIGINT, doc_id BIGINT, version BIGINT,
|
||||
source VARCHAR, caption VARCHAR, score DOUBLE
|
||||
);
|
||||
INSERT INTO curation_documents VALUES
|
||||
(1, 10, 1, 'web', NULL, 0.80),
|
||||
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
|
||||
(3, 20, 1, 'books', 'A dog in a field', 0.90),
|
||||
(4, 30, 1, 'web', NULL, 0.70),
|
||||
(5, 30, 1, 'web', 'A cat by a window', 0.85),
|
||||
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
|
||||
|
||||
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
|
||||
INSERT INTO curation_labels VALUES
|
||||
(10, 'cat'), (20, 'dog'), (30, 'cat');
|
||||
```
|
||||
|
||||
A bare number is a percentage; add `ROWS` for a count. Use it while you are
|
||||
exploring — the cost of the query drops with the sample, and so does the cost
|
||||
of being wrong about what you wanted to ask.
|
||||
`score` is a sample quality score, not a search result score. Document 10 has
|
||||
two versions. Document 30 has two records at the same version. Document 40
|
||||
has no label. These cases let you check the effect of each operation.
|
||||
|
||||
## Rows or fragments {#methods}
|
||||
</Accordion>
|
||||
|
||||
## Basic sampling {#quick}
|
||||
|
||||
A percentage selects a fraction of the source; `ROWS` requests a row count:
|
||||
|
||||
```sql
|
||||
SELECT * FROM images TABLESAMPLE BERNOULLI (10) -- rows
|
||||
SELECT * FROM images TABLESAMPLE SYSTEM (10) -- fragments
|
||||
SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (50);
|
||||
SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (3 ROWS);
|
||||
```
|
||||
|
||||
`BERNOULLI` considers each row independently and reads across the whole table.
|
||||
`SYSTEM` picks whole Lance fragments and takes the rows in them — far less I/O,
|
||||
because it skips files rather than rows.
|
||||
A percentage sample does not guarantee an exact number of rows. Use a count
|
||||
when the desired sample size is the main constraint, and inspect the result
|
||||
rather than assuming it contains every category you need.
|
||||
|
||||
The trade is uniformity. A fragment sample is only as representative as your
|
||||
write order. If rows arrived grouped by source, by date, or by label — which is
|
||||
usual — then a fragment sample inherits that grouping.
|
||||
`TABLESAMPLE` requires a table that is not WAL-backed. See the
|
||||
[sampling reference](/sql/dql/sampling#refused) for this restriction; do not
|
||||
change a production table's write configuration just to try an example.
|
||||
|
||||
<Note>
|
||||
Reach for `SYSTEM` when you want a cheap look, and `BERNOULLI` when the number
|
||||
you compute has to be defensible. Omitting the method gives `BERNOULLI`.
|
||||
</Note>
|
||||
## Sampling methods {#methods}
|
||||
|
||||
## A fixed evaluation set {#fixed}
|
||||
Choose a method according to how records should be selected:
|
||||
|
||||
Without a seed, every execution resamples. That is fine for exploring and wrong
|
||||
for evaluation — a metric computed on different rows each run is not comparable
|
||||
with itself.
|
||||
| Method | Selection | When to use it |
|
||||
| --- | --- | --- |
|
||||
| `BERNOULLI` (default) | Select individual rows | When records should be sampled independently of their storage fragments |
|
||||
| `SYSTEM` | Select whole Lance fragments | When reducing reads matters and grouping by fragment is acceptable |
|
||||
|
||||
```sql
|
||||
SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42)
|
||||
SELECT row_id, doc_id
|
||||
FROM curation_documents TABLESAMPLE BERNOULLI (50);
|
||||
|
||||
SELECT row_id, doc_id
|
||||
FROM curation_documents TABLESAMPLE SYSTEM (50);
|
||||
```
|
||||
|
||||
The same seed against the same data gives the same rows, which is what makes a
|
||||
sample usable as a held-out set. If the underlying data changes, so does the
|
||||
draw — when you need the set frozen against that too, materialize it:
|
||||
If write order groups records by label, date or source, a fragment sample can
|
||||
reflect that grouping. On a tiny table stored in one fragment, it may include
|
||||
all records or none. Row sampling also needs an adequate sample size to
|
||||
represent rare groups; check that the resulting sample covers the groups you need.
|
||||
|
||||
## Repeatable samples {#fixed}
|
||||
|
||||
A seed requests a repeatable draw:
|
||||
|
||||
```sql
|
||||
CREATE TABLE eval_set AS
|
||||
SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42)
|
||||
SELECT row_id, doc_id
|
||||
FROM curation_documents TABLESAMPLE (50) REPEATABLE (42);
|
||||
```
|
||||
|
||||
A [materialized view](/management/materialized-view) is the other option when
|
||||
you want it to track the source on a refresh rather than stay frozen.
|
||||
|
||||
## Sampling both sides of a join {#joins}
|
||||
Keep the source data and sampling configuration fixed when comparing draws.
|
||||
For a lasting evaluation set, save the selected records rather than relying
|
||||
on a seed to preserve membership as the source changes:
|
||||
|
||||
```sql
|
||||
SELECT * FROM a TABLESAMPLE (10) JOIN b TABLESAMPLE (10) ON a.id = b.id
|
||||
CREATE TABLE curation_eval AS
|
||||
SELECT * FROM curation_documents TABLESAMPLE (50) REPEATABLE (42);
|
||||
```
|
||||
|
||||
Each side samples independently, so a row survives only if both of its sides
|
||||
were drawn. Sampling 10% of each side does **not** give 10% of the join — it
|
||||
gives roughly 1%. Sample once, after the join, unless you specifically want the
|
||||
smaller result.
|
||||
Create this output once with a fresh name and keep it unchanged. Record the
|
||||
source version and selection policy. Sampling records independently can place
|
||||
revisions of the same document into different splits; decide the
|
||||
[grouping policy](/curation/windowing#splits) before using the result for
|
||||
training or evaluation.
|
||||
|
||||
## Shuffling for training {#training}
|
||||
## Sampling with joins {#joins}
|
||||
|
||||
For a training loop rather than a query, the SDKs read a table in a shuffled
|
||||
order without SQL at all:
|
||||
Sampling both input tables independently changes which pairs can match.
|
||||
For a one-to-one match, independent 10% row samples retain a given pair with
|
||||
about 1% probability, not 10%.
|
||||
|
||||
```python
|
||||
from lancedb.permutation import Permutation
|
||||
To sample documents while retaining their available labels, sample only the
|
||||
document table:
|
||||
|
||||
permutation = Permutation.identity(table)
|
||||
dataloader = torch.utils.data.DataLoader(permutation)
|
||||
```sql
|
||||
SELECT curation_documents.row_id, curation_documents.doc_id, l.label
|
||||
FROM curation_documents TABLESAMPLE (50)
|
||||
JOIN curation_labels AS l ON l.doc_id = curation_documents.doc_id;
|
||||
```
|
||||
|
||||
This is the right tool for epoch-level shuffling, where you want every row
|
||||
eventually and only the order varies. `TABLESAMPLE` is for when you want fewer
|
||||
rows. See [Training](/training/data-loading).
|
||||
This samples source records, not distinct documents. If you need to sample the
|
||||
result of a join or deduplication, first save that result as a table suitable
|
||||
for sampling. `TABLESAMPLE` cannot be applied directly to a subquery result.
|
||||
|
||||
## Reference {#reference}
|
||||
## Sample without SQL {#training}
|
||||
|
||||
The full clause, its options and what it refuses are in
|
||||
[Sampling](/sql/dql/sampling).
|
||||
The Python SDK provides a permutation builder for selecting and ordering rows
|
||||
in OSS and Enterprise. A split selects a subset; a shuffle changes the order
|
||||
in which records are read. `Permutation.identity(table)` retains stored order.
|
||||
|
||||
<Accordion title="Complete Python example: read a random subset">
|
||||
|
||||
Install the Python SDK with `pip install lancedb`, save the program as
|
||||
`sample_dataset.py`, and run `python sample_dataset.py`. It creates a temporary
|
||||
local database with ten rows and reads a random split containing three rows.
|
||||
The temporary database is removed when the program finishes.
|
||||
|
||||
<CodeBlock filename="sample_dataset.py" language="python" icon="python">
|
||||
{PySampleDataset}
|
||||
</CodeBlock>
|
||||
|
||||
The program prints three distinct IDs. It uses a fixed seed, but does not
|
||||
promise the same IDs across SDK versions or changes to the source dataset.
|
||||
|
||||
</Accordion>
|
||||
|
||||
For shuffled training reads, worker configuration and data loaders, see
|
||||
[Loading data for model training](/training/data-loading).
|
||||
|
||||
## Related guides {#reference}
|
||||
|
||||
Count the sampled rows and [inspect their distribution](/curation/aggregate)
|
||||
before using them. Preserve the selected data when the exact membership matters.
|
||||
See [SQL sampling](/sql/dql/sampling) for syntax and restrictions.
|
||||
|
||||
+127
-89
@@ -1,124 +1,162 @@
|
||||
---
|
||||
title: "Ranking and deduplication"
|
||||
sidebarTitle: "Windowing"
|
||||
description: "Picking one row per thing, ranking within a group, and splitting a dataset."
|
||||
description: "Choose records with explicit ranking rules and deterministic tie handling."
|
||||
icon: chart-simple
|
||||
keywords: ["window", "deduplication", "row_number", "rank", "train test split", "per class"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Enterprise supports SQL window functions.
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge> Window functions are SQL, and
|
||||
[SQL needs a deployment](/basics/sql#sql-is-remote).
|
||||
Window functions calculate values across related rows while keeping each row
|
||||
in the result. Use them to rank documents, keep one record per item, or compare
|
||||
a record with the one before it.
|
||||
|
||||
A window function computes across a set of related rows and returns a value
|
||||
**per row** — nothing is collapsed. That is what makes it the tool for "which of
|
||||
these do I keep", where an aggregate would throw away the rows you are choosing
|
||||
between.
|
||||
## Example data {#example-data}
|
||||
|
||||
## One row per thing {#dedup}
|
||||
Connect with an [Enterprise SQL client](/basics/sql). The examples use a small
|
||||
document table and a separate label table. Expand the setup below to create them.
|
||||
|
||||
The most common curation query there is. You have several rows per document —
|
||||
revisions, re-uploads, repeated annotations — and want one.
|
||||
<Accordion title="Create the example tables">
|
||||
|
||||
`curation_documents` has six records describing four documents.
|
||||
`curation_labels` has one label for each of three documents. `row_id` identifies
|
||||
a record; `doc_id` identifies the document it describes.
|
||||
|
||||
Run each statement separately in a scratch database or namespace. Create these
|
||||
tables once for the whole section, using fresh names if they already exist.
|
||||
|
||||
```sql
|
||||
CREATE TABLE curation_documents (
|
||||
row_id BIGINT, doc_id BIGINT, version BIGINT,
|
||||
source VARCHAR, caption VARCHAR, score DOUBLE
|
||||
);
|
||||
INSERT INTO curation_documents VALUES
|
||||
(1, 10, 1, 'web', NULL, 0.80),
|
||||
(2, 10, 2, 'web', 'A cat on a chair', 0.95),
|
||||
(3, 20, 1, 'books', 'A dog in a field', 0.90),
|
||||
(4, 30, 1, 'web', NULL, 0.70),
|
||||
(5, 30, 1, 'web', 'A cat by a window', 0.85),
|
||||
(6, 40, 1, 'books', 'A bird in a tree', 0.60);
|
||||
|
||||
CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR);
|
||||
INSERT INTO curation_labels VALUES
|
||||
(10, 'cat'), (20, 'dog'), (30, 'cat');
|
||||
```
|
||||
|
||||
`score` is a sample quality score, not a search result score. Document 10 has
|
||||
two versions. Document 30 has two records at the same version. Document 40
|
||||
has no label. These cases let you check the effect of each operation.
|
||||
|
||||
</Accordion>
|
||||
|
||||
## Keep one record per document {#dedup}
|
||||
|
||||
`row_number()` numbers rows within each group. `PARTITION BY` defines the group,
|
||||
and `ORDER BY` determines which record comes first. This query selects the
|
||||
latest version of each document, using `row_id` to break ties:
|
||||
|
||||
```sql
|
||||
WITH ranked AS (
|
||||
SELECT *,
|
||||
row_number() OVER (PARTITION BY doc_id ORDER BY version DESC) AS rn
|
||||
FROM revisions
|
||||
SELECT *, row_number() OVER (
|
||||
PARTITION BY doc_id ORDER BY version DESC, row_id DESC
|
||||
) AS rn
|
||||
FROM curation_documents
|
||||
)
|
||||
SELECT * FROM ranked WHERE rn = 1
|
||||
SELECT row_id, doc_id, version, caption
|
||||
FROM ranked
|
||||
WHERE rn = 1
|
||||
ORDER BY doc_id;
|
||||
```
|
||||
|
||||
`PARTITION BY` says what "per thing" means; `ORDER BY` says which one wins. The
|
||||
outer query is not optional — a window function cannot appear in `WHERE`,
|
||||
because windows are computed after filtering, so the predicate would have
|
||||
nothing to read.
|
||||
This returns row IDs **2, 3, 5 and 6**. For document 30, rows 4 and 5 have the
|
||||
same version; the example deliberately chooses the larger `row_id`.
|
||||
Without a complete ordering, the selected row among ties is not determined by
|
||||
the query.
|
||||
|
||||
<Note>
|
||||
Swap `row_number()` for `rank()` when ties should both survive. `row_number()`
|
||||
always picks exactly one; `rank()` gives tied rows the same rank, so `rank = 1`
|
||||
can return several.
|
||||
</Note>
|
||||
Use `rank()` instead of `row_number()` when tied records should all survive,
|
||||
and order only by the fields that define a tie. Adding a unique tie-breaker
|
||||
would make every rank distinct. The outer query filters the calculated rank
|
||||
because window functions are evaluated after the inner query's `WHERE` clause.
|
||||
|
||||
## A fixed number per class {#per-group}
|
||||
## Select the highest-scoring records per class {#per-group}
|
||||
|
||||
Class imbalance found by [profiling](/curation/aggregate#balance) is fixed here:
|
||||
rank within each class, then take the top N.
|
||||
Deduplicate first, then rank the labeled documents within each class:
|
||||
|
||||
```sql
|
||||
WITH ranked AS (
|
||||
SELECT *, row_number() OVER (PARTITION BY label ORDER BY score DESC) AS rn
|
||||
FROM images
|
||||
WITH latest AS (
|
||||
SELECT *, row_number() OVER (
|
||||
PARTITION BY doc_id ORDER BY version DESC, row_id DESC
|
||||
) AS version_rank
|
||||
FROM curation_documents
|
||||
), ranked AS (
|
||||
SELECT d.doc_id, l.label, d.score,
|
||||
row_number() OVER (
|
||||
PARTITION BY l.label ORDER BY d.score DESC, d.doc_id
|
||||
) AS class_rank
|
||||
FROM latest AS d
|
||||
JOIN curation_labels AS l ON l.doc_id = d.doc_id
|
||||
WHERE d.version_rank = 1
|
||||
)
|
||||
SELECT * FROM ranked WHERE rn <= 1000
|
||||
SELECT doc_id, label, score
|
||||
FROM ranked
|
||||
WHERE class_rank <= 1
|
||||
ORDER BY label;
|
||||
```
|
||||
|
||||
That gives at most 1,000 rows per label, taking the highest-scoring ones. Order
|
||||
by something arbitrary but stable if you want an unbiased draw instead, or
|
||||
combine with [sampling](/curation/sampling) to randomize within each class.
|
||||
The result is **document 10 for cat and document 20 for dog**. Change the cap
|
||||
from `1` to the maximum number you want per class. A class with fewer records
|
||||
cannot supply the full cap.
|
||||
|
||||
## Splitting a dataset {#splits}
|
||||
This selects by score, not at random. For a random subset, use
|
||||
[sampling](/curation/sampling) instead of ordering by score or ID.
|
||||
|
||||
`ntile` divides each partition into equal buckets, which is a train/test split
|
||||
that stays balanced per class:
|
||||
## Training and evaluation splits {#splits}
|
||||
|
||||
Ranking can divide ordered records into groups, but ordering by ID does not
|
||||
make those groups random or representative. `ntile()` produces approximately
|
||||
equal-sized buckets within a partition; small classes may not populate every
|
||||
bucket.
|
||||
|
||||
Choose the split policy for the task. Keep related records, such as revisions
|
||||
of one document or frames from one video, in the same split to avoid leakage.
|
||||
A time-based evaluation may instead require a cutoff date. Save the final
|
||||
assignments or resulting datasets so later updates do not silently change the
|
||||
evaluation population.
|
||||
|
||||
## Compare neighboring records {#offset}
|
||||
|
||||
`lag()` and `lead()` read neighboring rows in the specified order. For example,
|
||||
inspect how scores change across a document's records:
|
||||
|
||||
```sql
|
||||
WITH split AS (
|
||||
SELECT *, ntile(10) OVER (PARTITION BY label ORDER BY id) AS bucket
|
||||
FROM images
|
||||
)
|
||||
SELECT * FROM split WHERE bucket <= 8 -- train
|
||||
SELECT doc_id, row_id, score,
|
||||
lag(score) OVER (
|
||||
PARTITION BY doc_id ORDER BY version, row_id
|
||||
) AS previous_score
|
||||
FROM curation_documents
|
||||
ORDER BY doc_id, version, row_id;
|
||||
```
|
||||
|
||||
Buckets 9 and 10 are your validation and test sets. Because the partition is by
|
||||
label, each split keeps the class proportions of the whole.
|
||||
The first record in each document has no previous score. In this dataset,
|
||||
row 2 follows a score of 0.80 and row 5 follows a score of 0.70.
|
||||
|
||||
<Warning>
|
||||
Order by something stable. Ordering by a value that changes — a score you
|
||||
recompute, a timestamp that gets updated — means the split changes with it, and
|
||||
a row can move from train to test between runs.
|
||||
</Warning>
|
||||
<span id="running"></span>
|
||||
## Window query performance {#cost}
|
||||
|
||||
## Comparing a row to its neighbours {#offset}
|
||||
Window operations can require sorting. Filter out records you do not need
|
||||
before ranking, and inspect the
|
||||
[query plan](/search/explain) when cost matters.
|
||||
|
||||
`lag` and `lead` read the previous and next row in the window, which is how you
|
||||
find gaps and changes without a self-join:
|
||||
For cumulative counts, sums and other running summaries, use an aggregate
|
||||
with an explicit window order and frame. The
|
||||
[window reference](/sql/dql/window#window) covers those expressions.
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
id,
|
||||
created_at,
|
||||
created_at - lag(created_at) OVER (ORDER BY created_at) AS gap
|
||||
FROM events
|
||||
```
|
||||
## Continue with the curated data {#reference}
|
||||
|
||||
Useful for spotting ingestion gaps, duplicate timestamps, and runs of identical
|
||||
values that suggest a broken upstream job.
|
||||
|
||||
## Running totals {#running}
|
||||
|
||||
Any [aggregate](/curation/aggregate) becomes a window function with `OVER`,
|
||||
which is how cumulative coverage is expressed:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
label,
|
||||
count(*) AS n,
|
||||
sum(count(*)) OVER (ORDER BY count(*) DESC) AS cumulative
|
||||
FROM images
|
||||
GROUP BY label
|
||||
```
|
||||
|
||||
Reading down the `cumulative` column tells you how many classes cover most of
|
||||
your data — usually far fewer than you expect.
|
||||
|
||||
## Cost {#cost}
|
||||
|
||||
Window functions sort within each partition, so on a large table they are among
|
||||
the more expensive things you can ask for. Filter first, partition on something
|
||||
selective, and check the plan with
|
||||
[Explain Analyze](/search/explain).
|
||||
|
||||
## Reference {#reference}
|
||||
|
||||
The full window syntax, framing, and function list are in
|
||||
[Window functions](/sql/dql/window).
|
||||
[Profile the selected records](/curation/aggregate) and
|
||||
[save the result](/curation#next) for downstream use.
|
||||
See [Window functions](/sql/dql/window) for the full syntax and function list.
|
||||
|
||||
@@ -11,10 +11,10 @@ LanceDB emits internal metrics (currently object store request counts, bytes tra
|
||||
The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper over LanceDB's `metrics` recorder; your application supplies and configures the OpenTelemetry SDK.
|
||||
|
||||
<Note>
|
||||
This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side — see [Enterprise monitoring](#enterprise) below.
|
||||
This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side. See [Enterprise monitoring](#enterprise) below.
|
||||
</Note>
|
||||
|
||||
## What you get {#what-you-get}
|
||||
## Available metrics {#what-you-get}
|
||||
|
||||
Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer:
|
||||
|
||||
@@ -22,8 +22,10 @@ Once instrumented, LanceDB registers one observable instrument per metric on you
|
||||
|--------|------|-------------|
|
||||
| `lance_object_store_requests_total` | Counter | Total object store requests, labelled by `operation` and `base` (store scheme). |
|
||||
| `lance_object_store_request_duration_seconds` | Histogram | Request latency in seconds. |
|
||||
| `lance_object_store_bytes_transferred_total` | Counter | Bytes read from or written to the store. |
|
||||
| `lance_object_store_retryable_responses_total` | Counter | Requests that returned a retryable error (throttles, transient failures). |
|
||||
| `lance_object_store_request_bytes_total` | Counter | Bytes read from or written to the store. |
|
||||
| `lance_object_store_errors_total` | Counter | Object store requests that failed. |
|
||||
| `lance_object_store_retryable_responses_total` | Counter | Retryable HTTP responses (5xx, 429 and 408) from a store reached over HTTP. |
|
||||
| `lance_object_store_throttle_total` | Counter | Throttle responses (429 and 503) from a store reached over HTTP. |
|
||||
| `lance_object_store_in_flight_requests` | Gauge | Currently outstanding object store requests. |
|
||||
|
||||
The recorder is process-global and pull-based: your configured `MetricReader` collects on its own schedule, so there is no hot-path overhead beyond the atomic aggregation that LanceDB does anyway.
|
||||
@@ -37,7 +39,7 @@ The recorder is process-global and pull-based: your configured `MetricReader` co
|
||||
Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects.
|
||||
|
||||
```bash
|
||||
pip install "lancedb[otel]" opentelemetry-sdk
|
||||
pip install "lancedb[otel]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc
|
||||
```
|
||||
|
||||
Call `instrument_lancedb_metrics()` once at startup, before opening any tables. It returns `True` when the recorder is installed and instruments are registered.
|
||||
@@ -70,7 +72,7 @@ If you omit `meter_provider`, LanceDB uses the global provider returned by `open
|
||||
|
||||
## TypeScript {#typescript}
|
||||
|
||||
The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export.
|
||||
Install the OpenTelemetry SDK and exporter in your application:
|
||||
|
||||
```bash
|
||||
npm install @opentelemetry/sdk-metrics @opentelemetry/exporter-metrics-otlp-grpc
|
||||
@@ -93,7 +95,7 @@ const db = await connect("s3://my-bucket/lancedb");
|
||||
|
||||
`instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call.
|
||||
|
||||
## What to watch {#what-to-watch}
|
||||
## Build dashboards and alerts {#what-to-watch}
|
||||
|
||||
A few starting points for dashboards and alerts:
|
||||
|
||||
@@ -116,7 +118,7 @@ export them with data enrichment -- no client-side instrumentation needed.
|
||||
[Contact support](/deployment/support) for the comprehensive metric list and
|
||||
help integrating them with your monitoring dashboard.
|
||||
|
||||
## Where to go next {#where-to-go-next}
|
||||
## Related guides {#where-to-go-next}
|
||||
|
||||
<Columns cols={2}>
|
||||
<Card title="Performance tips" icon="gauge-high" href="/search#performance">
|
||||
|
||||
@@ -1,11 +1,24 @@
|
||||
---
|
||||
title: "Storage"
|
||||
sidebarTitle: "Storage"
|
||||
description: "Choosing and configuring where LanceDB stores its data -- how you deploy an embedded database."
|
||||
title: "Embedded storage configuration"
|
||||
sidebarTitle: "Embedded Storage"
|
||||
description: "Choose and configure local disk or object storage for tables accessed by the embedded LanceDB OSS library."
|
||||
icon: database
|
||||
keywords: ["storage", "s3", "gcs", "azure", "object store", "configuration", "deploy"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: unavailable
|
||||
summary: >-
|
||||
The embedded library chooses and configures its storage backend from the
|
||||
connection URI and `storage_options`. In LanceDB Enterprise, storage is part of
|
||||
the deployment's own configuration instead.
|
||||
---
|
||||
|
||||
This page is about the embedded library, where your connection chooses and
|
||||
configures the storage backend. A LanceDB Enterprise deployment sets up its own
|
||||
storage when it is deployed, so a `db://` connection passes none of these
|
||||
options except an Azure storage account name -- see
|
||||
[Deployment](/enterprise/deployment) for how a deployment is set up.
|
||||
|
||||
LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs).
|
||||
|
||||
Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload.
|
||||
@@ -176,9 +189,9 @@ Table-level `storage_options` inherit every key from the connection and override
|
||||
</CodeGroup>
|
||||
|
||||
<Tip>
|
||||
**Inspect the effective options**
|
||||
**Inspect a table's own options**
|
||||
|
||||
On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release.
|
||||
On `AsyncTable`, `await table.initial_storage_options()` returns the options passed when the table was opened, and `await table.latest_storage_options()` returns them after any provider-driven refresh. Neither includes what the table inherits from its connection's `storage_options`, which still apply. The deprecated `table.storage_options()` method will be removed in a future release.
|
||||
</Tip>
|
||||
|
||||
#### General object store options {#general-object-store-options}
|
||||
|
||||
@@ -9,11 +9,10 @@ keywords: ["support", "help", "discord", "community"]
|
||||
## LanceDB OSS {#oss}
|
||||
|
||||
For a quick question, the
|
||||
[Discord community](https://discord.gg/AUEWnJ7Txb) is fastest -- ask under the
|
||||
relevant language channel for Python, TypeScript, or Rust, and you reach both
|
||||
the community and the engineering team.
|
||||
[Discord community](https://discord.gg/AUEWnJ7Txb) has channels for
|
||||
Python, TypeScript and Rust where you can ask the community for help.
|
||||
|
||||
For anything worth keeping searchable for the next person, use GitHub instead:
|
||||
Use GitHub for discussions and issues that others can search and follow:
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
|
||||
+5
-12
@@ -63,10 +63,6 @@
|
||||
"source": "/feature-engineering/computed-columns",
|
||||
"destination": "/feature-engineering/add-computed-column"
|
||||
},
|
||||
{
|
||||
"source": "/feature-engineering/functions",
|
||||
"destination": "/management/function"
|
||||
},
|
||||
{
|
||||
"source": "/tables/create",
|
||||
"destination": "/ingestion"
|
||||
@@ -163,10 +159,6 @@
|
||||
"source": "/storage/monitoring",
|
||||
"destination": "/deployment/monitoring"
|
||||
},
|
||||
{
|
||||
"source": "/enterprise/deployment",
|
||||
"destination": "/enterprise/deployment/index"
|
||||
},
|
||||
{
|
||||
"source": "/search/sql",
|
||||
"destination": "/sql"
|
||||
@@ -253,6 +245,8 @@
|
||||
"group": "Getting Started",
|
||||
"pages": [
|
||||
"index",
|
||||
"basics/offerings",
|
||||
"basics/connect",
|
||||
"quickstart",
|
||||
"architecture",
|
||||
"basics/sdk",
|
||||
@@ -270,7 +264,8 @@
|
||||
"basics/jobs",
|
||||
"tables/consistency",
|
||||
"basics/guarantees",
|
||||
"basics/limits"
|
||||
"basics/limits",
|
||||
"basics/naming"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -333,8 +328,7 @@
|
||||
"management/table",
|
||||
"tables/branching",
|
||||
"management/materialized-view",
|
||||
"management/view",
|
||||
"management/function"
|
||||
"management/view"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -384,7 +378,6 @@
|
||||
"sql/ddl/view",
|
||||
"sql/ddl/materialized-view",
|
||||
"sql/ddl/index",
|
||||
"sql/ddl/function",
|
||||
"sql/ddl/job"
|
||||
],
|
||||
"icon": "sitemap"
|
||||
|
||||
@@ -3,6 +3,13 @@ title: "Managing Embeddings"
|
||||
sidebarTitle: "Managing Embeddings"
|
||||
description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support."
|
||||
icon: "bars"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Embedding functions run in your process, whether the table is local or on a
|
||||
deployment, which stores their configuration but never runs them. Python
|
||||
and TypeScript also embed query text; in Rust you embed queries yourself.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -30,8 +37,9 @@ the data, so vectors that are close to each other are considered similar.
|
||||
|
||||
LanceDB provides an embedding function registry in OSS as well as its Enterprise versions
|
||||
([see below](#embeddings-in-lancedb-enterprise))
|
||||
that automatically generates vector embeddings during data ingestion. Automatic query-time embedding
|
||||
generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts
|
||||
that automatically generates vector embeddings during data ingestion. Query text is embedded
|
||||
automatically too, in the client, for local and Enterprise tables alike, with SDK-specific query
|
||||
ergonomics. The API abstracts
|
||||
embedding generation, allowing you to focus on your application logic.
|
||||
|
||||
## Embedding Registry {#embedding-registry}
|
||||
@@ -41,6 +49,9 @@ Once configured, the embedding function will automatically generate embeddings w
|
||||
into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly,
|
||||
while Rust examples typically compute query embeddings explicitly before vector search.
|
||||
|
||||
In the Rust examples, replace `your-openai-api-key` with your OpenAI API key. In
|
||||
application code, load credentials from environment variables or a secrets manager.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="python" icon="python">
|
||||
{PyOpenaiEmbeddings}
|
||||
|
||||
@@ -4,14 +4,15 @@ sidebarTitle: "Authentication"
|
||||
description: "Authentication modes for LanceDB Enterprise: API keys and OAuth credentials."
|
||||
icon: key
|
||||
keywords: ["authentication", "oauth", "api key", "azure managed identity", "client credentials"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Requests to a deployment carry an API key or an OAuth token, whichever it is
|
||||
configured to accept. Embedded LanceDB has no server to sign in to; see
|
||||
[Architecture](/architecture).
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. The embedded form
|
||||
of LanceDB has no server to provide it -- see
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
A deployment authenticates every request before
|
||||
[authorization](/governance/authorization) decides what it may do, against a
|
||||
`db://` remote table, in one of two ways:
|
||||
@@ -50,6 +51,10 @@ from an OIDC issuer and attach them to every request. Token acquisition,
|
||||
caching, and refresh are handled inside the client -- your application code
|
||||
only provides the configuration.
|
||||
|
||||
A deployment accepts these tokens only when its administrator has configured
|
||||
it with your identity provider. Ask them for the issuer URL, client ID and
|
||||
scopes to use.
|
||||
|
||||
<Info>
|
||||
In Python, OAuth is supported through `lancedb.connect_async`. The synchronous
|
||||
`connect` entry point continues to use API key authentication for `db://`
|
||||
|
||||
@@ -21,9 +21,13 @@ What it adds over the embedded form:
|
||||
| | |
|
||||
| --- | --- |
|
||||
| [SQL](/sql) | an Arrow Flight SQL endpoint |
|
||||
| [Functions](/management/function) | Python running on workers |
|
||||
| [Jobs](/basics/jobs) | server-side work with ids, status and events |
|
||||
| [Authorization](/governance/authorization) | per-object grants |
|
||||
| Scale | a cluster rather than your process |
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Add this row to the table above, after SQL:
|
||||
| [Functions](/management/function) | Python running on workers |
|
||||
*/}
|
||||
|
||||
See [Architecture](/architecture) for the two shapes side by side.
|
||||
|
||||
@@ -4,13 +4,18 @@ sidebarTitle: "Computed Columns"
|
||||
description: "Declare a column from a SQL expression. Nothing is evaluated until you refresh it."
|
||||
icon: table-columns
|
||||
keywords: ["computed columns", "add_columns", "transforms", "schema"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Both offerings support computed columns. Enterprise requires the deployment to enable them; the embedded library evaluates them in your process.
|
||||
---
|
||||
|
||||
A computed column is declared with an expression and starts with no values.
|
||||
Declaring it is cheap on any table size, because nothing is evaluated until
|
||||
[a refresh](/feature-engineering/refresh-column) asks for it.
|
||||
|
||||
## Declaring one {#declare}
|
||||
## Add a computed column {#declare}
|
||||
|
||||
```python
|
||||
import lancedb
|
||||
@@ -26,14 +31,17 @@ That commits a new table version with the column present and empty. It is an
|
||||
ordinary table version, so the usual [versioning](/management/table#versioning) applies --
|
||||
including that you can travel back to before the column existed.
|
||||
|
||||
## `computed` and `transforms` are different {#computed-vs-transforms}
|
||||
## Compute now or on refresh {#computed-vs-transforms}
|
||||
|
||||
`add_columns` takes either, and the difference matters:
|
||||
|
||||
- **SQL expressions in `transforms`** are evaluated *now*, for every row, as
|
||||
part of the call. Good for a small table, or a column you want populated
|
||||
immediately. A `FunctionApplication` passed through `transforms` instead
|
||||
declares empty Function-bound columns; a refresh fills them.
|
||||
immediately.
|
||||
{/* Draft for the Functions release:
|
||||
A `FunctionApplication` passed through `transforms` instead
|
||||
declares empty Function-bound columns, which a refresh fills; that needs a
|
||||
deployment with Functions enabled, and the embedded library refuses it. */}
|
||||
- **`computed`** stores the expression and evaluates nothing. The column is
|
||||
committed empty and filled later by
|
||||
[`refresh_column`](/feature-engineering/refresh-column).
|
||||
@@ -48,8 +56,8 @@ the table, including another computed column:
|
||||
|
||||
```python
|
||||
table.add_columns(computed={
|
||||
"power_score": "(stats.magic + stats.strength) / 2.0",
|
||||
"is_strong": "power_score > 3.0",
|
||||
"power_score": "cast(x as double) / 2.0",
|
||||
"is_strong": "power_score > 0.5",
|
||||
})
|
||||
```
|
||||
|
||||
@@ -59,12 +67,22 @@ declared earlier in the same mapping, but not one declared after it.
|
||||
The functions available are the ones the
|
||||
[SQL dialect](/sql/functions) documents.
|
||||
|
||||
## Its inputs become load-bearing {#input-constraints}
|
||||
A literal in the expression has to fit the column it meets: `x / 2.0` is
|
||||
refused when `x` is an integer column, so write `cast(x as double) / 2.0`.
|
||||
|
||||
<Warning>
|
||||
In LanceDB Enterprise, keep the expression to top-level columns. A column
|
||||
refresh can fail on an expression that reads a field inside a struct, such as
|
||||
`stats.magic + stats.strength`, which the embedded library computes. Try a new
|
||||
expression on a small table first.
|
||||
</Warning>
|
||||
|
||||
## Change columns used by an expression {#input-constraints}
|
||||
|
||||
<Warning>
|
||||
While a computed column's declaration references another column, that column
|
||||
cannot be renamed, retyped, or dropped. The declaration is part of the schema,
|
||||
and LanceDB will not let you invalidate it out from under itself.
|
||||
cannot be renamed, retyped, or dropped. Drop the computed column first
|
||||
if you need to change one of its inputs.
|
||||
</Warning>
|
||||
|
||||
This is worth knowing before deriving a column from something you expect to
|
||||
@@ -74,18 +92,33 @@ reshape later. Dropping the computed column releases the constraint.
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
When the value cannot be expressed in SQL -- an embedding, a model call, an
|
||||
image decode -- [Geneva](/geneva), the feature engineering package for LanceDB
|
||||
Enterprise, fills the column with a Python function instead.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the paragraph above:
|
||||
|
||||
When the value cannot be expressed in SQL -- an embedding, a model call, an
|
||||
image decode -- the column is filled by a registered
|
||||
[function](/management/function) instead of an expression. Declaring it looks
|
||||
the same; what differs is what fills it.
|
||||
the same; what differs is what fills it. It needs a deployment with Functions
|
||||
enabled.
|
||||
*/}
|
||||
|
||||
## What to read next {#next}
|
||||
## Related guides {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Refresh a column" icon="arrows-rotate" href="/feature-engineering/refresh-column">
|
||||
Fill the rows that lack values.
|
||||
</Card>
|
||||
<Card title="Geneva" icon="layer-group" href="/geneva">
|
||||
Fill a column with Python on LanceDB Enterprise.
|
||||
</Card>
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the Geneva card above:
|
||||
<Card title="Functions" icon="code" href="/management/function">
|
||||
Fill a column with Python.
|
||||
</Card>
|
||||
*/}
|
||||
</CardGroup>
|
||||
|
||||
@@ -4,6 +4,12 @@ sidebarTitle: "Data Evolution"
|
||||
description: "Derive new columns from data already in the table, without rewriting what is already there."
|
||||
icon: wand-magic-sparkles
|
||||
keywords: ["data evolution", "derived columns", "add_columns", "zero copy", "feature engineering"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Adding columns computed from SQL expressions works in the embedded library
|
||||
and in LanceDB Enterprise.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -18,20 +24,23 @@ import {
|
||||
RsAddFeatureColumnsSql as RsAddFeatureColumnsSql,
|
||||
} from '/snippets/tables.mdx';
|
||||
|
||||
Data evolution is adding **values** derived from what a table already holds.
|
||||
[Schema evolution](/management/table#schema) changes the table's shape; this fills the
|
||||
shape with something new.
|
||||
Add derived columns to calculate features from data already in a table.
|
||||
For example, compute a product score from existing fields without reading
|
||||
and rewriting the table in your application.
|
||||
|
||||
The reason it is cheap is the format. Adding a derived column writes only that
|
||||
column -- the existing data, which is the expensive part on a multimodal table,
|
||||
is never rewritten. See [why that works](/basics/lance#evolution).
|
||||
Lance can write the new columns without rewriting the existing columns.
|
||||
This is useful when the table also contains large images, video or embeddings.
|
||||
See [Lance format](/basics/lance) for the storage model.
|
||||
|
||||
<Note>
|
||||
In SQL this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column).
|
||||
In SQL, on an Enterprise cluster that enables computed columns, this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column).
|
||||
</Note>
|
||||
|
||||
## Set up the example table {#set-up-the-example-table}
|
||||
|
||||
The examples use `db`, a connection opened as in
|
||||
[Connect to LanceDB](/basics/connect), and create a small product table in it:
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PySchemaAddSetup}
|
||||
@@ -96,16 +105,24 @@ If your transformation cannot be expressed in SQL, compute the values outside
|
||||
|
||||
## When SQL is not enough {#beyond-sql}
|
||||
|
||||
The expressions above are evaluated once, as part of the call. Two other shapes
|
||||
exist when that does not fit:
|
||||
The expressions above are evaluated once, as part of the call. For other workflows:
|
||||
|
||||
- A [computed column](/feature-engineering/add-computed-column) declares the
|
||||
expression and leaves the column empty, so you choose when the work happens.
|
||||
That is the right default on a large table.
|
||||
Use it when you want to schedule the computation separately.
|
||||
- On LanceDB Enterprise, [Geneva](/geneva) fills a column with Python, for
|
||||
values SQL cannot express -- an embedding, a model call, an image decode.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the Geneva item in the list above:
|
||||
- A [function](/management/function) fills a column with Python, for values SQL
|
||||
cannot express -- an embedding, a model call, an image decode.
|
||||
*/}
|
||||
|
||||
## What to read next {#next}
|
||||
A deployment has to enable computed columns before you use them.
|
||||
{/* Draft for the Functions release: Functions need enabling there too. */}
|
||||
|
||||
## Related guides {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Add a computed column" icon="table-columns" href="/feature-engineering/add-computed-column">
|
||||
|
||||
@@ -3,52 +3,90 @@ title: "Feature engineering"
|
||||
sidebarTitle: "Overview"
|
||||
description: "Derive columns and tables from data already in the lakehouse, without moving it somewhere else to compute."
|
||||
icon: layer-group
|
||||
keywords: ["feature engineering", "computed columns", "materialized views", "functions", "lakehouse", "jobs"]
|
||||
keywords: ["feature engineering", "computed columns", "materialized views", "lakehouse", "jobs"]
|
||||
---
|
||||
|
||||
Feature engineering happens where the data already is. Rather than exporting a
|
||||
table, computing somewhere else and writing the result back, you declare what a
|
||||
column or a table should contain and let LanceDB fill it.
|
||||
Use feature engineering to turn raw data into features for search, analysis
|
||||
and model training. LanceDB stores derived columns and materialized views
|
||||
alongside the source data, so you can reuse the results across those workflows.
|
||||
|
||||
[Materialized views](/feature-engineering/materialized-views) are ordinary
|
||||
[catalog objects](/basics/catalog). Create them through the SDK or SQL,
|
||||
then refresh the derived data when its inputs change.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the paragraph above:
|
||||
|
||||
[Functions](/management/function) and
|
||||
[materialized views](/feature-engineering/materialized-views) are ordinary
|
||||
[catalog objects](/basics/catalog), alongside tables and indexes -- created,
|
||||
versioned, and refreshed like anything else, not a separate system bolted
|
||||
alongside.
|
||||
[catalog objects](/basics/catalog). Create them through the SDK or SQL,
|
||||
then refresh the derived data when its inputs change.
|
||||
*/}
|
||||
|
||||
<Note>
|
||||
Every operation here has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it.
|
||||
In LanceDB Enterprise, every operation here also has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it.
|
||||
</Note>
|
||||
|
||||
## The shape everything shares {#the-shape}
|
||||
## Computed columns and materialized views {#the-shape}
|
||||
|
||||
Every mechanism here separates **declaring** from **filling**, and that
|
||||
separation is the point:
|
||||
Computed columns and materialized views both keep derived data in the
|
||||
lakehouse, but they fill it differently, and the difference decides what a
|
||||
refresh costs:
|
||||
|
||||
- Declaring is cheap and immediate. It records intent in the schema and commits
|
||||
a new table version, with no data written. Declaring a column on a
|
||||
billion-row table costs what declaring one on an empty table costs.
|
||||
- Filling is a separate call you control, can retry, and -- against a deployment
|
||||
-- can hand to the server as a [job](/basics/jobs) rather than waiting on.
|
||||
| | Computed column | Materialized view |
|
||||
| --- | --- | --- |
|
||||
| Declaring or creating it | records the expression; no values are written | runs the query and stores its rows, unless created empty |
|
||||
| A refresh | fills rows with no value and recomputes rows whose inputs changed | applies source changes or rebuilds the view, depending on the query and SDK version |
|
||||
| A refresh when nothing changed | fills nothing | can be a no-op when incremental refresh is supported |
|
||||
| In LanceDB Enterprise | runs as a [job](/basics/jobs) | runs as a server-side materialization job |
|
||||
|
||||
**Filling does not redo current work.** A refresh gives values to rows that
|
||||
lack them and recomputes rows whose inputs changed since they were computed. It
|
||||
### Computed columns fill incrementally {#incremental}
|
||||
|
||||
A [computed column](/feature-engineering/add-computed-column) separates
|
||||
**declaring** from **filling**:
|
||||
|
||||
- Declaring is cheap and immediate. It records the expression in the schema
|
||||
and commits a new table version, with no data written. It does not evaluate the expression over existing rows.
|
||||
- Fill the column with a separate refresh call that you can retry. In LanceDB
|
||||
Enterprise, submit the refresh as a [job](/basics/jobs) and track its progress.
|
||||
|
||||
**Refresh new and changed rows.** A
|
||||
[refresh](/feature-engineering/refresh-column) gives values to rows that lack
|
||||
them and recomputes rows whose inputs changed since they were computed. It
|
||||
leaves the rest alone, so calling it twice is not twice the work, and appending
|
||||
rows means the next refresh handles only the new ones.
|
||||
|
||||
The corollary is worth stating plainly: **a filled value catches up with its
|
||||
inputs only when you refresh.** If a column derives from `price` and a row's
|
||||
**Computed values update when you refresh.** If a column derives from `price` and a row's
|
||||
`price` is later updated, the derived value keeps its old value until the next
|
||||
refresh recomputes it. That is a deliberate trade -- it is what makes refreshes
|
||||
idempotent and cheap -- but it will surprise you if you expect a spreadsheet.
|
||||
refresh recomputes it. Schedule refreshes after changing the source data when consumers need
|
||||
up-to-date features.
|
||||
|
||||
## Which one to reach for {#which-one}
|
||||
### Materialized views store query results {#views}
|
||||
|
||||
A [materialized view](/feature-engineering/materialized-views) has no separate
|
||||
declaration step: creating one runs its query and stores the result, so the view
|
||||
is ready to read. A refresh updates those stored results. Python SDK 0.40.0 can apply
|
||||
changes incrementally for supported queries; other queries require a full rebuild.
|
||||
See [refresh behavior](/feature-engineering/materialized-views#incremental) for
|
||||
version-specific details.
|
||||
|
||||
Enterprise runs materialization on remote compute. Use the SDK and refresh
|
||||
options supported by your server version; see
|
||||
[Enterprise refreshes](/feature-engineering/materialized-views#availability).
|
||||
|
||||
## Choose a feature engineering workflow {#which-one}
|
||||
|
||||
| | What it produces | Filled by | Available |
|
||||
| --- | --- | --- | --- |
|
||||
| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | everywhere |
|
||||
| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | everywhere |
|
||||
| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | OSS and Enterprise |
|
||||
| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | OSS and Enterprise |
|
||||
| [Geneva](/geneva) | columns filled by Python | the `geneva` package | <Badge color="red">Enterprise</Badge> |
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replace the Geneva row of the table above with:
|
||||
| [Function](/management/function) | a computed column filled by Python | registered code | <Badge color="red">Enterprise</Badge> |
|
||||
| [Geneva](/geneva) | columns filled by Python, for existing pipelines | the `geneva` package | <Badge color="red">Enterprise</Badge> |
|
||||
*/}
|
||||
|
||||
**Computed columns** are the common case: a column declared with a SQL
|
||||
expression, filled by [a refresh](/feature-engineering/refresh-column).
|
||||
@@ -57,23 +95,34 @@ expression, filled by [a refresh](/feature-engineering/refresh-column).
|
||||
aggregated view of a source -- and bring it up to date on each refresh. The
|
||||
result is an ordinary table: queryable, indexable, searchable.
|
||||
|
||||
**Geneva** is LanceDB Enterprise's Python package for work SQL cannot express:
|
||||
embedding a column, calling a model, decoding an image. It defines features as
|
||||
Python functions and backfills them as distributed jobs.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the Geneva paragraph above:
|
||||
|
||||
**Functions** run Python where the data is, for work SQL cannot express:
|
||||
embedding a column, calling a model, decoding an image. Registered once,
|
||||
versioned, then applied as a column.
|
||||
versioned, then applied as a column. They are the way to do that on a
|
||||
deployment; [Geneva](/geneva) remains available for existing pipelines.
|
||||
*/}
|
||||
|
||||
They compose. A materialized view can select computed columns, and a computed
|
||||
column can be filled by a registered function.
|
||||
A materialized view can select computed columns.
|
||||
{/* Draft for the Functions release: A computed column can be filled by a registered function. */}
|
||||
|
||||
## Where the work runs {#where}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
Locally, a refresh runs in your process. Against a deployment it runs across the
|
||||
Locally, a refresh runs in your process. With LanceDB Enterprise it runs across the
|
||||
cluster, and the API reflects that: `refresh_column` raises, and
|
||||
`refresh_column_async` returns a [job](/basics/jobs) handle instead.
|
||||
`refresh_column_async` returns a [job](/basics/jobs) handle instead. Computed
|
||||
columns are off by default in LanceDB Enterprise, and its administrator has to enable
|
||||
them before a column can be declared there.
|
||||
|
||||
```python
|
||||
job = table.refresh_column_async("embedding")
|
||||
job = table.refresh_column_async("power_score")
|
||||
result = job.wait()
|
||||
result.rows_assigned
|
||||
```
|
||||
@@ -81,24 +130,28 @@ result.rows_assigned
|
||||
A backfill over a large table can therefore be started, left, and checked on
|
||||
from a different process -- the connection does not have to stay open.
|
||||
|
||||
That relocation is the main thing to understand about Enterprise feature
|
||||
engineering. The API is the same; what changes is that the code runs somewhere
|
||||
else, which is also why a function must declare the environment it needs.
|
||||
Enterprise manages the compute for the refresh. You still control when to
|
||||
submit it; declaring a computed column does not schedule recurring refreshes.
|
||||
{/* Draft for the Functions release: That is also why a function must declare the environment it needs. */}
|
||||
|
||||
## Recovering from a failure {#failures}
|
||||
|
||||
{/*
|
||||
Draft for the Functions release.
|
||||
|
||||
A function that raises fails the job. There is no per-row retry or skip policy
|
||||
in the function definition, so a function calling a flaky external service
|
||||
should handle that itself, inside the function body.
|
||||
*/}
|
||||
|
||||
The alternative is to let the job fail, fix the cause, and run the refresh
|
||||
When a refresh fails, fix the cause and run the refresh
|
||||
again. The new refresh fills what is still empty or stale. How much of the
|
||||
failed run was kept depends on where it ran (see
|
||||
[Refresh a column](/feature-engineering/refresh-column#failures)), and a retry
|
||||
can recompute values that were already written, so do not count on it costing
|
||||
only the rows that did not complete.
|
||||
|
||||
## Where to go next {#next}
|
||||
## Related guides {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Add a computed column" icon="table-columns" href="/feature-engineering/add-computed-column">
|
||||
@@ -110,7 +163,10 @@ only the rows that did not complete.
|
||||
<Card title="Refresh a materialized view" icon="table" href="/feature-engineering/materialized-views">
|
||||
Bring a derived table up to date with its source.
|
||||
</Card>
|
||||
{/*
|
||||
Draft for the Functions release:
|
||||
<Card title="Functions" icon="code" href="/management/function">
|
||||
Register Python to fill a column with.
|
||||
</Card>
|
||||
*/}
|
||||
</CardGroup>
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
---
|
||||
title: "Refresh a materialized view"
|
||||
sidebarTitle: "Materialized View Refresh"
|
||||
description: "A lakehouse object: a table defined by a query over another table, brought up to date by each refresh."
|
||||
description: "Create a materialized view and refresh it when the source data changes."
|
||||
icon: table
|
||||
keywords: ["materialized view", "create_materialized_view", "refresh", "full refresh"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Both offerings support materialized views. OSS refreshes run in your process; Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -11,12 +16,9 @@ import {
|
||||
PyMaterializedViewRefresh as MaterializedViewRefresh,
|
||||
} from '/snippets/materialized_view.mdx';
|
||||
|
||||
A materialized view is a lakehouse object: a table whose contents are defined by
|
||||
a query over another table. The query is recorded in the view's schema;
|
||||
creating the view computes its rows, and a refresh brings them up to date.
|
||||
|
||||
The result is an ordinary table. It can be queried, indexed, and searched, and
|
||||
it appears in `table_names()` — nothing about reading it is special.
|
||||
A materialized view stores the result of a query as a table. Use it to keep a
|
||||
filtered or transformed dataset that you can query, index and search repeatedly.
|
||||
Refresh the view after its source data changes.
|
||||
|
||||
<Note>
|
||||
In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-view).
|
||||
@@ -36,89 +38,80 @@ reach the view through a refresh:
|
||||
{MaterializedViewRefresh}
|
||||
</CodeBlock>
|
||||
|
||||
<Note>
|
||||
These results were verified with the development SDK 0.40.0b13, built from
|
||||
lancedb main. The published 0.39.0 differs: the source must be created with
|
||||
stable row ids (the `new_table_enable_stable_row_ids` storage option),
|
||||
`create_materialized_view` creates the view empty and has no `with_no_data`,
|
||||
and `refresh()` still accepts `full`. Its first `refresh()` writes every
|
||||
matching row, two in this example.
|
||||
</Note>
|
||||
The example works with Python SDK 0.40.0. It creates the source with stable
|
||||
row IDs, which that release requires for ordinary materialized-view sources.
|
||||
|
||||
`select` takes column names, `(alias, expression)` pairs, or a dict of the same.
|
||||
Omitting it selects every source column, expanded against the schema as it was
|
||||
at creation time. `where` filters source rows; `limit` caps the view.
|
||||
|
||||
## Local sources no longer need stable row ids {#stable-row-ids}
|
||||
## Prepare the source table {#stable-row-ids}
|
||||
|
||||
A local refresh does not track which source rows the view came from, so a local
|
||||
view's source does not need stable row ids (`new_table_enable_stable_row_ids`):
|
||||
the example above creates its source with the defaults. Earlier development
|
||||
builds and the published 0.39.0 required them, and a deployment may still
|
||||
require them; see [Against a deployment](#availability).
|
||||
In Python SDK 0.40.0, create the source with
|
||||
`storage_options={"new_table_enable_stable_row_ids": "true"}`, as the example
|
||||
above does. Stable row IDs let the view track source rows across updates and
|
||||
compaction. Set this option when creating the table; it cannot be added later.
|
||||
|
||||
The source is still checked, when the view is created and on every refresh: one
|
||||
with an LSM write spec, or with retained write-ahead-log rows not yet compacted,
|
||||
is refused.
|
||||
A source with an LSM write specification or retained, uncompacted write-ahead-log
|
||||
rows is not accepted by the local materialized-view implementation.
|
||||
|
||||
## Every local refresh rebuilds the view {#incremental}
|
||||
## Refresh behavior {#incremental}
|
||||
|
||||
Locally, `refresh()` runs the view's query against the source and replaces
|
||||
everything the view holds with the result, in one commit. It does not reconcile
|
||||
changes: `rows_written` is the size of the whole result — two after adding Bea,
|
||||
not one — and a refresh with nothing new to read still rewrites the view and
|
||||
commits a new table version.
|
||||
With Python SDK 0.40.0, a refresh applies source changes incrementally when the
|
||||
query supports it, and rebuilds the view otherwise. In the example, adding Bea
|
||||
writes one new view row. Refreshing again without source changes writes none.
|
||||
Use `refresh(full=True)` to force a rebuild.
|
||||
|
||||
`source_version=N` refreshes against that source version instead of the latest,
|
||||
which is how a view reproduces a known point in time. There is no `full`
|
||||
argument; every local refresh is a full one.
|
||||
`rows_written` counts rows written by that refresh, not all rows in the view.
|
||||
Use `view.table.count_rows()` to get the total. Pass `source_version=N` to
|
||||
refresh against a particular source version instead of the latest.
|
||||
|
||||
## Against a deployment {#availability}
|
||||
<Note>
|
||||
Refresh behavior has changed between SDK versions. The earlier 0.40.0b13
|
||||
development build rebuilt every local view on refresh and did not accept
|
||||
`full`; 0.39.0 created views empty. Use the reference for your installed SDK
|
||||
when maintaining an existing pipeline.
|
||||
</Note>
|
||||
|
||||
## Enterprise refreshes {#availability}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
`create_materialized_view` and `refresh` also work on a `db://` connection.
|
||||
There the SDK sends them to the deployment as SQL -- `CREATE MATERIALIZED VIEW`
|
||||
and `REFRESH MATERIALIZED VIEW` -- and the work runs on the cluster rather than
|
||||
in your process. `source_version` is for local views only; the SDK refuses it
|
||||
on a remote refresh.
|
||||
Enterprise runs materialized-view creation and refresh on remote compute.
|
||||
For SQL, see [`CREATE MATERIALIZED VIEW`](/sql/ddl/materialized-view) and
|
||||
[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view).
|
||||
|
||||
Materialized views are off by default on a deployment and have to be enabled
|
||||
there. How a deployment refreshes a view -- rebuilding the whole view, as above,
|
||||
or reconciling what changed in the source -- depends on its LanceDB Enterprise
|
||||
version and configuration, and so do the options
|
||||
[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view)
|
||||
accepts and whether the source table needs stable row ids.
|
||||
Use an SDK version supported by your Enterprise deployment. Materialized-view
|
||||
APIs and stored definitions have changed across releases; a local SDK example
|
||||
alone does not establish compatibility with an older server. Before upgrading
|
||||
an existing pipeline, confirm the supported refresh options and whether its
|
||||
views need to be recreated with your administrator.
|
||||
|
||||
Stable row ids are set up on the deployment, not by the SDK: a `db://`
|
||||
connection does not send `new_table_enable_stable_row_ids`, and no SDK call adds
|
||||
them to an existing table. Depending on its version and configuration, a
|
||||
deployment may give new tables stable row ids, may refuse a view created through
|
||||
SQL -- as the SDK creates one -- over a source without them, or may convert an
|
||||
existing source when a view is created through its REST API. Check with the
|
||||
deployment's administrator what it requires before relying on a source.
|
||||
Stable row IDs and feature configuration for Enterprise sources are managed on
|
||||
the server. The source-table storage option in the local example does not
|
||||
configure the Enterprise deployment, and Python SDK 0.40.0's `create_table` on a
|
||||
`db://` connection does not accept `storage_options`: leave it out, and confirm
|
||||
with your administrator that the deployment gives new tables stable row IDs.
|
||||
|
||||
`SHOW MATERIALIZED VIEWS` lists them, which is cheaper than
|
||||
`list_materialized_views` -- that method finds views by reading every table's
|
||||
schema, so it costs one open per table.
|
||||
`list_materialized_views` requests the server's view listing. On a local
|
||||
database it reads table schemas to find views.
|
||||
|
||||
## Working with the view {#working-with-the-view}
|
||||
|
||||
`view.table` is the underlying table, for querying, indexing, or search.
|
||||
`view.definition` is the recorded query — the select list, predicate, and
|
||||
limit — which is how you check what a view actually contains without inferring
|
||||
it from the rows. Both are properties; on an async view, the definition is read
|
||||
`view.definition` is the recorded query: the select list, predicate, and
|
||||
limit. Use it to inspect the query without inferring it from the rows. Both are properties; on an async view, the definition is read
|
||||
with `await view.definition()`.
|
||||
|
||||
An existing view is reopened with `db.open_materialized_view(name)`.
|
||||
|
||||
## What to read next {#next}
|
||||
## Related guides {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Refresh a column" icon="arrows-rotate" href="/feature-engineering/refresh-column">
|
||||
Derive a column instead of a whole table.
|
||||
</Card>
|
||||
<Card title="Versioning" icon="clock-rotate-left" href="/management/table#versioning">
|
||||
Every refresh commits a table version.
|
||||
Inspect the table history and read a particular version.
|
||||
</Card>
|
||||
</CardGroup>
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
---
|
||||
title: "Refresh a column"
|
||||
sidebarTitle: "Column Refresh"
|
||||
description: "Fill the rows of a computed column that lack values — in process, or as a server-side job."
|
||||
description: "Populate a computed column and update it when its inputs change."
|
||||
icon: arrows-rotate
|
||||
keywords: ["refresh_column", "backfill", "job", "idempotent", "REFRESH COLUMN"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Both offerings support column refresh. The embedded library runs it in your process; Enterprise runs it as a server job that must be enabled for your Enterprise cluster.
|
||||
---
|
||||
|
||||
A [declared column](/feature-engineering/add-computed-column) starts empty. A
|
||||
refresh gives values to the rows that lack them.
|
||||
Refresh a [computed column](/feature-engineering/add-computed-column) to fill
|
||||
new rows and recompute values whose inputs changed. The example below continues
|
||||
with the `doubled` column from the computed-column guide.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
@@ -37,8 +43,7 @@ The SQL form is [`REFRESH COLUMN`](/sql/refresh#refresh-column), with the same r
|
||||
|
||||
A refresh fills rows that have no value and recomputes rows whose inputs
|
||||
changed since they were computed. Every other row is left alone, so a second
|
||||
refresh with nothing changed does nothing. Two consequences follow, and both
|
||||
are load-bearing.
|
||||
refresh with nothing changed does nothing. This supports two common workflows.
|
||||
|
||||
**Appending is cheap.** Rows added since the last refresh are filled by the
|
||||
next one; filled rows whose inputs are unchanged are skipped. Backfilling a
|
||||
@@ -63,16 +68,16 @@ REFRESH COLUMN embedding ON documents WHERE created_at > '2026-01-01'
|
||||
```
|
||||
|
||||
<Note>
|
||||
**Enterprise job type:** `computed_refresh` for a SQL-expression column,
|
||||
`function_refresh` for one filled by a registered
|
||||
[function](/management/function).
|
||||
**Enterprise job type:** `computed_refresh` for a SQL-expression column.
|
||||
{/* Draft for the Functions release: `function_refresh` for one filled by a registered [function](/management/function). */}
|
||||
</Note>
|
||||
|
||||
## As a job {#async}
|
||||
## Run a refresh on Enterprise {#async}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
Against a deployment the refresh runs on the cluster. `refresh_column` raises
|
||||
With an Enterprise cluster that enables computed columns, the refresh runs on the
|
||||
cluster. `refresh_column` raises
|
||||
there; use `refresh_column_async`, which returns a [job](/basics/jobs) you can
|
||||
wait on or poll instead of holding a connection open:
|
||||
|
||||
@@ -92,7 +97,7 @@ does not raise, which is what makes it safe in a loop; `wait()` raises
|
||||
A long backfill can therefore be started, left, and checked on from a different
|
||||
process.
|
||||
|
||||
### Resourcing the job {#resources}
|
||||
### Configure workers and resources {#resources}
|
||||
|
||||
SQL exposes the worker budget directly, which is the lever when a backfill is
|
||||
too slow or too expensive:
|
||||
@@ -111,7 +116,7 @@ rather than failing the job.
|
||||
|
||||
What a failure keeps depends on where the refresh ran. `refresh_column` in your
|
||||
process writes the column in one commit, so a failure leaves the column as it
|
||||
was. A refresh job on a deployment commits finished batches as it goes, so some
|
||||
was. A refresh job in LanceDB Enterprise commits finished batches as it goes, so some
|
||||
values can remain after a failure.
|
||||
|
||||
Either way, recover by fixing the cause and refreshing again. The new refresh
|
||||
@@ -121,12 +126,16 @@ that were already written -- for example when the record of the inputs they
|
||||
were computed from was not saved -- so do not count on a retry costing only the
|
||||
rows that did not complete.
|
||||
|
||||
{/*
|
||||
Draft for the Functions release.
|
||||
|
||||
For a column filled by a [function](/management/function), the usual cause is
|
||||
the worker environment rather than the data. See
|
||||
[environment mismatches](/management/function#mismatch-symptoms) for the
|
||||
symptoms, which rarely name their cause.
|
||||
*/}
|
||||
|
||||
## What to read next {#next}
|
||||
## Related guides {#next}
|
||||
|
||||
<CardGroup cols={2}>
|
||||
<Card title="Jobs" icon="hourglass-half" href="/basics/jobs">
|
||||
|
||||
@@ -4,15 +4,13 @@ sidebarTitle: "Authorization"
|
||||
description: "Who may do what: privileges granted on catalog objects to principals, groups, roles, and API keys."
|
||||
icon: shield-halved
|
||||
keywords: ["authorization", "authz", "privileges", "rbac", "access control"]
|
||||
availability:
|
||||
oss: unavailable
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Enterprise supports catalog privileges when its administrator configures enforcement. Embedded LanceDB has no catalog authorization layer: access is whatever the object store allows.
|
||||
---
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This page describes a LanceDB Enterprise capability. Embedded LanceDB has no
|
||||
such layer -- access is whatever the object store enforces on the bucket or
|
||||
directory, which is coarser and has no concept of a table. See
|
||||
[Architecture](/architecture) for what differs between the two.
|
||||
|
||||
[Authentication](/enterprise/authentication) establishes who is calling.
|
||||
Authorization decides what they may do.
|
||||
|
||||
@@ -28,15 +26,16 @@ A grant is held by one of four kinds of subject, distinguished by a prefix:
|
||||
| `p:` | a principal, by ID -- `P:` by name |
|
||||
| `g:` | a group, by ID -- `G:` by name |
|
||||
| `r:` | a role |
|
||||
| `a:` | an API key |
|
||||
| `a:` | the principal holding an API key, named by the key |
|
||||
|
||||
Granting to a group or role rather than a principal is what keeps the entry
|
||||
count manageable: a principal's effective privileges are the union of what it
|
||||
holds directly and what it inherits.
|
||||
|
||||
Granting to an API key is narrower than granting to the principal that owns
|
||||
it -- useful when a key is issued for one job and should not carry that
|
||||
principal's full authority.
|
||||
`P:`, `G:` and `a:` are ways of naming a subject when you make a request. The
|
||||
entry is stored against the ID they resolve to, so a grant to `a:` belongs to
|
||||
the principal holding that key and carries that principal's full authority,
|
||||
not a narrower one.
|
||||
|
||||
## Objects {#objects}
|
||||
|
||||
@@ -45,10 +44,28 @@ Objects are addressed by kind and path:
|
||||
```
|
||||
system
|
||||
database:db
|
||||
namespace:db/ns
|
||||
table:db/ns/table
|
||||
namespace:db:ns
|
||||
table:db:ns$table
|
||||
view:db:ns$view
|
||||
```
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Replaces the object kinds above:
|
||||
|
||||
```
|
||||
system
|
||||
database:db
|
||||
namespace:db:ns
|
||||
table:db:ns$table
|
||||
view:db:ns$view
|
||||
function:db:ns$function
|
||||
secret:db:ns$secret
|
||||
```
|
||||
*/}
|
||||
|
||||
A nested namespace joins its path with `$`, as in `namespace:db:parent$child`
|
||||
or `table:db:parent$child$table`.
|
||||
|
||||
The hierarchy is not decorative. Reaching a table requires `USAGE` on the
|
||||
namespace containing it and on the database containing that -- a grant on the
|
||||
table alone is not enough to traverse to it.
|
||||
@@ -57,10 +74,11 @@ table alone is not enough to traverse to it.
|
||||
|
||||
| Privilege | Granted on | Allows |
|
||||
| --- | --- | --- |
|
||||
| `OWNERSHIP` | any object | ownership, conveying full authority |
|
||||
| `OWNERSHIP` | any object but `system` | ownership, conveying full authority |
|
||||
| `USAGE` | system | listing jobs and their events, and looking up other identities |
|
||||
| `USAGE` | database | accessing the namespaces inside |
|
||||
| `USAGE` | namespace | accessing the tables inside |
|
||||
| `SELECT` | table | reading rows |
|
||||
| `SELECT` | table, view | reading rows |
|
||||
| `INSERT` | table | appending rows |
|
||||
| `UPDATE` | table | modifying existing rows |
|
||||
| `DELETE` | table | deleting rows |
|
||||
@@ -69,7 +87,19 @@ table alone is not enough to traverse to it.
|
||||
| `CREATE_TABLE` | namespace | creating a table in it |
|
||||
| `CREATE_MATERIALIZED_VIEW` | namespace | creating a materialized view in it |
|
||||
| `CREATE_VIEW` | namespace | creating a non-materialized view in it |
|
||||
| `OPERATE` | system | reading server configuration |
|
||||
| `OPERATE` | system | describing and controlling jobs, managing caches, and reading server configuration |
|
||||
|
||||
{/*
|
||||
Draft for the Functions release. Add these rows to the table above.
|
||||
After the `USAGE` row for namespace:
|
||||
| `USAGE` | function | binding it to a column, and refreshing a column it computes |
|
||||
| `USAGE` | secret | binding and resolving it |
|
||||
After the `UPDATE` row for table:
|
||||
| `UPDATE` | secret | rotating it, without the right to bind or read it |
|
||||
After the `CREATE_VIEW` row:
|
||||
| `CREATE_FUNCTION` | namespace | registering a function in it |
|
||||
| `CREATE_SECRET` | namespace | creating a secret in it |
|
||||
*/}
|
||||
|
||||
The four data privileges map onto the four
|
||||
[ingestion operations](/ingestion), so a principal that may append but not
|
||||
@@ -87,28 +117,32 @@ grant the other.
|
||||
A subtlety worth knowing, because it explains grants that appear to work on
|
||||
objects that do not exist yet.
|
||||
|
||||
When `USAGE` is checked on a database and **no entries exist for it at all**,
|
||||
the database is assumed not to exist yet, and `CREATE_DATABASE` on `system` is
|
||||
checked instead. The same applies to the default namespace: absent any
|
||||
entries, the check falls back to `CREATE_NAMESPACE` on the containing
|
||||
database.
|
||||
When you create an object, the deployment checks the database and each
|
||||
namespace that will contain it. One that **has no owner** -- because it does
|
||||
not exist yet, or predates per-object authorization -- is treated as one you are
|
||||
creating: the check falls back to `CREATE_DATABASE` on `system` for the
|
||||
database, or to `CREATE_NAMESPACE` on its parent for a namespace, and you become
|
||||
its owner. One that has an owner needs `USAGE` as usual.
|
||||
|
||||
So a principal holding `CREATE_DATABASE` can act on a database that has not
|
||||
been created -- which is what makes creation possible at all, and why an empty
|
||||
ACL set is not the same as a deny.
|
||||
So a principal holding `CREATE_DATABASE` can create a table in a database that
|
||||
has not been created -- which is what makes creation possible at all, and why
|
||||
an empty ACL set is not the same as a deny. The fallback applies only when
|
||||
creating; other operations on a database or namespace need `USAGE` on it.
|
||||
|
||||
## Listing grants {#listing}
|
||||
|
||||
Entries can be listed filtered by object, by subject, or both:
|
||||
|
||||
```
|
||||
object = table:db/ns/documents
|
||||
object = table:db:ns$documents
|
||||
subject = G:data-science
|
||||
```
|
||||
|
||||
Results paginate through a continuation token. One stored `(object, subject)`
|
||||
tuple carrying several privileges expands into several entries in the
|
||||
response, so the count reflects privileges rather than tuples.
|
||||
Results paginate through a continuation token. A page holds at most 100 stored
|
||||
`(object, subject)` tuples, and a tuple carrying several privileges expands
|
||||
into several entries in the response, so a page can return more than 100
|
||||
entries: the count reflects privileges rather than tuples. Keep requesting with
|
||||
the returned token until a page comes back without one.
|
||||
|
||||
Listing by subject answers "what can this team reach"; listing by object
|
||||
answers "who can reach this table". Both are worth running before changing a
|
||||
|
||||
@@ -3,6 +3,13 @@ title: "Full-Text Search (FTS) Index"
|
||||
sidebarTitle: "FTS Index"
|
||||
description: "Create and tune BM25-based full-text search indexes in LanceDB."
|
||||
icon: "book"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Full-text indexes are built in the embedded library and in LanceDB Enterprise.
|
||||
The embedded library builds the index before the call returns; a
|
||||
deployment builds it in the background.
|
||||
---
|
||||
import { PyFtsIndexAsync as FtsIndexAsync, PyFtsIndexCreate as FtsIndexCreate, PyFtsIndexNested as FtsIndexNested, PyFtsIndexWait as FtsIndexWait } from '/snippets/indexing.mdx';
|
||||
|
||||
@@ -114,7 +121,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True))
|
||||
- `jieba/*` is for Chinese word segmentation with Jieba.
|
||||
- `lindera/*` loads a compiled Lindera dictionary, such as `lindera/ipadic` for Japanese or `lindera/ko-dic` for Korean.
|
||||
|
||||
Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `<model-home>/jieba/default/...`.
|
||||
Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `<model-home>/jieba/default/...`. For a table in LanceDB Enterprise, the files must be present where the deployment builds and queries the index, so ask its administrator whether they are installed.
|
||||
|
||||
`language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`.
|
||||
|
||||
|
||||
+26
-10
@@ -3,20 +3,26 @@ title: "Indexing Data"
|
||||
sidebarTitle: "Overview"
|
||||
description: "Optimize search performance with LanceDB using vector indexes, full-text search, scalar indexes, and more."
|
||||
icon: "list"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Indexes are built in the embedded library and in LanceDB Enterprise. Enterprise builds them in the background, and builds fewer vector index types than the
|
||||
embedded library.
|
||||
---
|
||||
|
||||
An **index** is a data structure that facilitates efficient scans and lookups on an embedded dataset.
|
||||
LanceDB provides a comprehensive suite of indexes to optimize performance across different use cases and data types:
|
||||
Indexes speed up searches and filters by organizing data for specific query
|
||||
patterns. LanceDB supports three main index families:
|
||||
|
||||
- **Vector Index**: Efficiently searches for similar vectors across high-dimensional data (e.g. images, audio, or text embeddings)
|
||||
- **Full-Text Search Index**: Enables fast keyword-based searches by indexing words and phrases
|
||||
- **Scalar Index**: Accelerates filtering and sorting of structured numeric or categorical data
|
||||
- **Scalar Index**: Accelerates filtering of structured numeric or categorical data
|
||||
|
||||
## Supported Indexes
|
||||
|
||||
| Index | Use Case | Description |
|
||||
| :--------- | :------- | :---------- |
|
||||
| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index—a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search. <br />**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`|
|
||||
| `IVF` (Vector) | Large-scale vector search with configurable accuracy/speed trade-offs. | Inverted File Index: a partition-based approximate nearest neighbor algorithm that groups similar vectors into partitions for efficient search. <br />**Quantizations**: `None/Flat` `PQ` `SQ` `RQ`|
|
||||
| `IVF_HNSW` (Vector) | Large-scale vector search requiring both high recall and efficient partitioning. Combines the scalability of IVF with the search quality of HNSW. | Hybrid index combining IVF partitioning with HNSW graphs in each partition. Provides improved search quality over pure IVF while maintaining scalability.<br />**Quantizations**: `None/Flat` `SQ` `PQ`|
|
||||
| `FTS` (Full-text search) | String columns (e.g., title, description, content) requiring keyword-based search with BM25 ranking. | Full-text search index using BM25 ranking algorithm. Tokenizes text with configurable tokenization, stemming, stop word removal, and language-specific processing. |
|
||||
| `BTree` (Scalar) | Numeric, temporal, and string columns with mostly distinct values. Best for selective equality, inequality, and range predicates. | Sorted index storing sorted copies of scalar columns with block headers in a btree cache. Header entries map to blocks of rows (4096 rows per block) for efficient disk reads. |
|
||||
@@ -24,6 +30,10 @@ LanceDB provides a comprehensive suite of indexes to optimize performance across
|
||||
| `LabelList` (Scalar) | List columns (e.g., tags, categories, keywords) requiring `array_contains_all` or `array_contains_any` filters. | Scalar index for `List<T>` and `LargeList<T>` columns of primitive values, using an underlying bitmap index structure to enable fast array membership lookups. |
|
||||
| `FM` (Scalar) | String or binary columns that need raw substring search. | FM-Index over `Utf8`, `LargeUtf8`, `Binary`, or `LargeBinary` data for filters such as `contains(path, 'needle')`. Use FTS instead for tokenized word search and BM25 ranking. |
|
||||
|
||||
A deployment builds the vector indexes `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and
|
||||
`IVF_HNSW_SQ`. `IVF_SQ`, `IVF_HNSW_FLAT` and `IVF_HNSW_PQ` are built only by the
|
||||
embedded library.
|
||||
|
||||
## Quantization
|
||||
LanceDB also supports several [quantization](/indexing/vector-index) methods, used by vector indexes to compress vectors and reduce storage requirements:
|
||||
|
||||
@@ -32,26 +42,30 @@ LanceDB also supports several [quantization](/indexing/vector-index) methods, us
|
||||
| `PQ` (Product Quantization) | Default choice for most vector search scenarios. Use when you need to balance index size and recall. | Divides vectors into subvectors and quantizes each subvector independently. Provides a good balance between compression ratio and search accuracy. |
|
||||
| `RQ` (RabitQ Quantization) | Use when you need maximum compression or have specific per-dimension requirements. | Per-dimension quantization using a RabitQ codebook. Provides fine-grained control over compression per dimension. For `IVF_RQ`, vector dimensions must be divisible by `8`. |
|
||||
| `SQ` (Scalar Quantization) | Use when you need faster indexing or when vector dimensions have consistent value ranges. | Quantizes each dimension independently. Simpler than PQ but typically provides less compression. |
|
||||
| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization—stores raw vectors. Provides the highest accuracy but requires more storage and memory. |
|
||||
| `None/Flat` | Use for binary vectors (with `hamming` distance) or when you need maximum recall and have sufficient storage. | No quantization; stores raw vectors. Provides the highest accuracy but requires more storage and memory. |
|
||||
|
||||
## Performance {#performance}
|
||||
|
||||
#### Vector indexes {#vector-indexes}
|
||||
|
||||
If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), the cause is the default brute-force scan over every vector. That works fine below ~100K vectors, but past that you should build a dedicated vector index. Pick the type by your data shape:
|
||||
If vector search latency climbs with table size (i.e., queries that ran in milliseconds on a small table take seconds as it grows to millions of rows), check whether the query is scanning every vector without an index. The point at which an index helps depends on vector dimensions, hardware and
|
||||
your latency target. Measure your workload before choosing an index. Pick the type by your data shape:
|
||||
|
||||
| Index | When to use |
|
||||
|-------|-------------|
|
||||
| `IVF_PQ` | General-purpose default; what Enterprise builds automatically. Higher accuracy than RQ at small dimensions (≤ 256). |
|
||||
| `IVF_PQ` | General-purpose default, and what a deployment builds when it is set to index a column automatically. Higher accuracy than RQ at small dimensions (≤ 256). |
|
||||
| `IVF_RQ` | Maximum compression on high-dim vectors, faster builds than PQ. |
|
||||
| `IVF_HNSW_SQ` | Best recall/latency for unfiltered search; higher latency variance under selective filters. |
|
||||
| `IVF_FLAT` | Required for binary vectors with `hamming`. |
|
||||
|
||||
The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; LanceDB Enterprise may reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index).
|
||||
The distance metric is fixed once the index is built. Pick the distance metric based on how the embedding model was trained: `cosine` (unnormalized), `dot` (already-normalized, best performance), `l2` (general-purpose, default), `hamming` (binary). If a query explicitly requests a different metric, the index cannot be used and the query falls back to a brute-force scan; a deployment can be configured to reject that scan on a large table. For parameter tuning, see [Vector Indexing](/indexing/vector-index).
|
||||
|
||||
#### Scalar indexes {#scalar-indexes}
|
||||
|
||||
If filtered queries slow down as the table grows — even when the filter is selective — the cause is a full column scan: without a scalar index, LanceDB evaluates the `where(...)` predicate on every row, and the same applies to `merge_insert` join keys. The best practice is to build a scalar index on every column you filter or join on, picking the type by the column's shape:
|
||||
Scalar indexes can reduce the rows scanned for selective filters and speed up
|
||||
`merge_insert` key lookups. Start with columns used frequently in those operations,
|
||||
then use [query plans](/search/explain) to check whether the index helps. Choose
|
||||
the index type based on the column's values:
|
||||
|
||||
| Index | Best for |
|
||||
|-------|----------|
|
||||
@@ -63,4 +77,6 @@ See [Scalar Indexing](/indexing/scalar-index).
|
||||
|
||||
#### Full-text search {#full-text-search}
|
||||
|
||||
If your full-text index is much larger than expected, or takes longer than expected to build, the cause is usually phrase-query flags being enabled when they aren't needed: `with_position=True` and `remove_stop_words=False` both significantly inflate index size and build time. The best practice is to keep the defaults for most workloads, and only enable those flags when you actually need to search for phrases. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options.
|
||||
Token positions and retained stop words increase the information stored in a
|
||||
full-text index. Keep the defaults for basic keyword search; enable
|
||||
`with_position=True` and retain the needed stop words for phrase queries. See [FTS Indexing](/indexing/fts-index) configuration options for the full set of options.
|
||||
|
||||
@@ -4,6 +4,14 @@ sidebarTitle: "Build & Repair"
|
||||
description: "Learn how to keep your indexes up-to-date in LanceDB using incremental indexing, including best practices for adding new records without full reindexing."
|
||||
icon: "refresh"
|
||||
keywords: ["reindexing", "re-index", "incremental indexing", "reindex"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Indexes are kept up to date in the embedded library and in LanceDB Enterprise. In
|
||||
the embedded library you add new rows to them with `optimize()`; a
|
||||
deployment does it in the background, and `optimize()` is not available
|
||||
there.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -20,16 +28,19 @@ As data is being added and a reindex operation is running, LanceDB will combine
|
||||
Rather than dropping an existing index entirely and reindexing from scratch, LanceDB supports **incremental indexing**.
|
||||
|
||||
<Note>
|
||||
**Enterprise job types:** building an index runs as `create_ivf_pq`, `create_btree`,
|
||||
`create_fts`, and one more per index algorithm -- the type names the algorithm,
|
||||
not a separate feature. Folding new rows into an existing one runs as the
|
||||
matching `reindex_*` type instead.
|
||||
**Enterprise job types:** an index build runs as `reindex_` followed by the
|
||||
algorithm, such as `reindex_ivf_pq`, `reindex_btree` or `reindex_fts`, and so
|
||||
does folding new rows into an existing index. A create-only build, one that must
|
||||
not replace an existing index, runs as `create_` followed by the algorithm
|
||||
instead; Python's `create_scalar_index` and `create_fts_index` ask for that by
|
||||
default. The name says which algorithm is building, not a separate feature.
|
||||
</Note>
|
||||
|
||||
## Incremental Reindexing {#incremental-reindexing}
|
||||
|
||||
You can manually trigger an incremental indexing operation on updated data
|
||||
using the `optimize()` method on a table.
|
||||
In the embedded library, you trigger an incremental indexing operation on
|
||||
updated data with the `optimize()` method on a table. A deployment does this
|
||||
for you and does not support `optimize()`, as described below.
|
||||
|
||||
Table optimization performs three maintenance operations:
|
||||
|
||||
@@ -45,10 +56,12 @@ Table optimization performs three maintenance operations:
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
LanceDB Enterprise support incremental reindexing through an automated background process. When new data is added to a table, the system automatically triggers a new index build. As the dataset grows, indexes are asynchronously updated in the background.
|
||||
LanceDB Enterprise supports incremental reindexing through an automated background process. Once enough new rows have been added to a table, the system automatically adds them to its indexes. As the dataset grows, indexes are asynchronously updated in the background.
|
||||
|
||||
- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. To avoid this, set `fast_search=True` to search only indexed data.
|
||||
- While indexes are being rebuilt, queries use brute force methods on unindexed rows, which may temporarily increase latency. If you can omit unindexed rows, set `fast_search=True` to search only indexed data.
|
||||
This can return incomplete results while indexing catches up.
|
||||
- Use `index_stats()` to view the number of unindexed rows. This will be zero when indexes are fully up-to-date. If you call `wait_for_index(...)`, it polls the same status and can time out while continuous writes keep adding unindexed rows.
|
||||
- Rows below the deployment's threshold stay unindexed until more arrive, so the count can stay above zero without further writes. `FM` indexes are not updated in the background.
|
||||
|
||||
The benefit of using LanceDB Enterprise is that it automates the reindexing process
|
||||
and operates continuously in the background, minimizing the impact on latency under high loads.
|
||||
@@ -71,9 +84,9 @@ If you need to reclaim space more aggressively in OSS, use a shorter retention w
|
||||
|
||||
## Building on GPUs {#gpu}
|
||||
|
||||
With LanceDB's GPU-powered vector indexing you can index very large datasets in far less time
|
||||
than you could with the default CPU-based indexing. In our tests, LanceDB
|
||||
is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster.
|
||||
GPU acceleration can reduce the time needed to build vector indexes on large
|
||||
datasets. Actual build time depends on the index configuration, vector
|
||||
dimensions, storage throughput and available GPU resources.
|
||||
|
||||
<Info>
|
||||
**Automatic GPU indexing**
|
||||
@@ -82,9 +95,8 @@ is capable of indexing billions of rows in under four hours on a 1-8 GPU cluster
|
||||
Automatic GPU Indexing is currently only available in [LanceDB Enterprise](/enterprise/).
|
||||
Please [contact us](https://www.lancedb.com/contact) to enable this feature for your deployment.
|
||||
|
||||
The vector index is created when you call `create_index`. The backend will use GPU resources
|
||||
to build either the IVF or HNSW indexes. The system automatically selects the optimal GPU
|
||||
configuration based on your data size and available hardware.
|
||||
The vector index is created when you call `create_index`. The backend uses GPU resources
|
||||
to build `IVF_PQ` indexes, with the number of GPUs set by the deployment's configuration.
|
||||
|
||||
This process is also asynchronous by default, but you can use `wait_for_index` to convert it
|
||||
into a synchronous process by waiting until the index is built.
|
||||
@@ -93,9 +105,9 @@ into a synchronous process by waiting until the index is built.
|
||||
|
||||
`wait_for_index(...)` waits for the index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`. It can time out if the table is receiving continuous writes while the build is trying to catch up.
|
||||
|
||||
GPU acceleration changes how the vector index is built, not the lifecycle after the build. Rows
|
||||
appended later still need `optimize()` before they are part of the index, and `fast_search()` only
|
||||
searches indexed rows.
|
||||
GPU acceleration changes how the vector index is built, not the lifecycle after the build. In the
|
||||
embedded library, rows appended later still need `optimize()` before they are part of the index; a
|
||||
deployment adds them in the background. `fast_search()` only searches indexed rows.
|
||||
|
||||
### Manual GPU indexing in LanceDB OSS {#gpu-manual-gpu-indexing-in-lancedb-oss}
|
||||
|
||||
@@ -125,9 +137,9 @@ to enable GPU training on your device.
|
||||
### Performance considerations {#gpu-performance-considerations}
|
||||
|
||||
- GPU memory usage scales with `num_partitions` and vector dimensions
|
||||
- For optimal performance, ensure GPU memory exceeds dataset size
|
||||
- Batch size is automatically tuned based on available GPU memory
|
||||
- Indexing speed improves with larger batch sizes
|
||||
- Size GPU memory for the training batches and index configuration
|
||||
- Monitor memory use while choosing training batch sizes
|
||||
- Measure build time and recall before changing training settings
|
||||
|
||||
### Troubleshooting {#gpu-troubleshooting}
|
||||
|
||||
|
||||
@@ -3,6 +3,13 @@ title: "Scalar Indexes"
|
||||
sidebarTitle: "Scalar Index"
|
||||
description: "Learn how to use scalar indexes in LanceDB for efficient metadata filtering and query optimization."
|
||||
icon: "tree"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Scalar indexes are built in the embedded library and in LanceDB Enterprise. A
|
||||
deployment builds them in the background and keeps most of them up to date
|
||||
itself; in the embedded library you update them with `optimize()`.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -19,7 +26,9 @@ import {
|
||||
PyScalarIndexNestedFields as ScalarIndexNestedFields,
|
||||
} from '/snippets/indexing.mdx';
|
||||
|
||||
Scalar indexes organize data by scalar attributes (e.g., numbers, categories) and enable fast filtering of vector data. They accelerate retrieval of scalar data associated with vectors, thus enhancing query performance.
|
||||
Scalar indexes accelerate filters on values such as numbers, categories and
|
||||
strings. Use them for selective table queries or metadata filters combined
|
||||
with vector and full-text search.
|
||||
|
||||
LanceDB supports four types of scalar indexes:
|
||||
|
||||
@@ -29,7 +38,7 @@ LanceDB supports four types of scalar indexes:
|
||||
- `FM`: FM-Index over string or binary columns that accelerates substring search via `contains(col, 'needle')`.
|
||||
|
||||
<Tip title="Working with scalar indices">
|
||||
Scalar indices serve as a foundational optimization layer, accelerating filtering across diverse search workloads. They can be combined with:
|
||||
Use scalar indexes with:
|
||||
|
||||
- Vector search (prefilter results using metadata)
|
||||
- Full-text search (combining keyword matching with structured filters)
|
||||
@@ -70,11 +79,11 @@ If you are using LanceDB Enterprise, the `create_scalar_index` API returns immed
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout.
|
||||
`wait_for_index(...)` waits until the named scalar indexes exist and `index_stats(...)` reports `num_unindexed_rows == 0`. If a table is receiving steady writes, that fully indexed state may not stabilize before the timeout. In LanceDB Enterprise, rows added after the index was built are indexed only once enough have accumulated, so a small remainder can also keep the wait from finishing.
|
||||
|
||||
### 3. Update the Index
|
||||
|
||||
Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
|
||||
Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. In the embedded library, call `optimize`, which will trigger an update to the existing scalar index.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -83,7 +92,7 @@ Updating the table data (adding, deleting, or modifying records) requires that y
|
||||
</CodeGroup>
|
||||
|
||||
<Note title="LanceDB Enterprise">
|
||||
New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise automates the optimize process, minimizing the impact on search speed.
|
||||
New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Enterprise does not support `optimize()`; instead it adds new rows to `BTREE`, `BITMAP` and `LABEL_LIST` indexes in the background once enough have accumulated. It does not update `FM` indexes automatically.
|
||||
</Note>
|
||||
|
||||
### 4. Run Indexed Searches
|
||||
@@ -125,7 +134,7 @@ Nested paths follow Lance field-path semantics: dot-separate each struct field f
|
||||
The `FM` index is a scalar index built over string or binary columns that
|
||||
accelerates substring lookups expressed as `contains(col, 'needle')`. Unlike the
|
||||
tokenized [FTS index](/indexing/fts-index), which matches whole words after
|
||||
tokenization, the FM-Index matches arbitrary substrings of the raw bytes — so it
|
||||
tokenization, the FM-Index matches arbitrary substrings of the raw bytes. It
|
||||
works well for URLs, file paths, identifiers, log lines, or any column where you
|
||||
search for a fragment rather than a word.
|
||||
|
||||
@@ -143,6 +152,8 @@ or language-aware tokenization.
|
||||
|
||||
Build an FM-Index with the async `create_index` API by passing the `Fm` config in
|
||||
Python or `Index.fm()` in TypeScript. In Rust, use `Index::Fm(FmIndexBuilder::default())`.
|
||||
A deployment builds FM-Indexes too, but does not add new rows to them in the background as it
|
||||
does for the other scalar indexes.
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
|
||||
@@ -3,6 +3,14 @@ title: "Vector Indexes"
|
||||
sidebarTitle: "Vector Index"
|
||||
description: "Build and manage LanceDB vector indexes."
|
||||
icon: "arrow-up-right-dots"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Vector indexes are built in the embedded library and in LanceDB Enterprise. The
|
||||
embedded library builds an index before `create_index()` returns; a
|
||||
deployment builds it in the background, keeps it up to date, and builds
|
||||
only `IVF_FLAT`, `IVF_PQ`, `IVF_RQ` and `IVF_HNSW_SQ`.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -53,7 +61,7 @@ Lance with `index_type`.
|
||||
| Maximum compression | `IVF_RQ` | IVF indexing with RaBitQ quantization | Around `1/32` of raw size | `IvfRq` |
|
||||
| Higher accuracy at small dimensions (`dimension <= 256`) | `IVF_PQ` | IVF indexing with product quantization | Usually `1/64` to `1/16` of raw size (depends on `num_sub_vectors`) | `IvfPq` |
|
||||
| Best recall/latency trade-off | `IVF_HNSW_SQ` | IVF-HNSW indexing with scalar | Typically a little larger than `1/4` of raw size | `IvfHnswSq` |
|
||||
| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` |
|
||||
| Highest recall / no quantization | `IVF_HNSW_FLAT` | IVF-HNSW indexing with no quantization; embedded library only | Around raw vector size plus HNSW graph overhead | `IvfHnswFlat` |
|
||||
|
||||
<Warning>
|
||||
If your vector search frequently includes metadata filters (`where(...)`), use `IVF_RQ` or `IVF_PQ`. In filtered workloads, HNSW-backed IVF indexes such as `IVF_HNSW_FLAT` and `IVF_HNSW_SQ` can show higher latency variance.
|
||||
@@ -114,7 +122,7 @@ your destination. This iterative series of road networks mimics the layered grap
|
||||
In LanceDB, HNSW is not exposed as a top-level vector index. Instead, it's available as a substructure which
|
||||
further indexes the selected vectors inside each IVF partition.
|
||||
This combines the scalability of IVF with the high recall of HNSW.
|
||||
LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`.
|
||||
LanceDB supports IVF-HNSW-based quantized indexes `IVF_HNSW_FLAT`, `IVF_HNSW_PQ`, and `IVF_HNSW_SQ`; a deployment builds only `IVF_HNSW_SQ`.
|
||||
</Info>
|
||||
|
||||
|
||||
@@ -141,7 +149,7 @@ Learn how to configure, build, and search LanceDB vector indexes, including buil
|
||||
| `limit` | Number of results to return (the `k` in `k-ANN`). |
|
||||
| `nprobes` | Sets both `minimum_nprobes` and `maximum_nprobes` to the same value in Lance. Leave unset to use Lance's adaptive defaults. At very high scales you will likely need to experiment to find your preferred recall / latency balance. |
|
||||
| `minimum_nprobes` | Minimum number of partitions scanned. Leave unset to use Lance's default. |
|
||||
| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results — useful for narrow filters. Leave unset, or set to `0`, to remove the cap. |
|
||||
| `maximum_nprobes` | Maximum number of partitions scanned. Only scans more than `minimum_nprobes` if an initial pass does not return enough results, which is useful for narrow filters. Leave unset, or set to `0`, to remove the cap. |
|
||||
| `ef` | HNSW search-time exploration factor. Start around `1.5 * k` and increase up to `10 * k` for higher recall. |
|
||||
| `refine_factor` | Reads and reranks additional candidates in memory to recover recall lost to quantization. Start between `5` and `50` (a factor of `25` reads and reranks the top `25 * k` candidates) and measure the latency cost, typically a few milliseconds, against the recall gained. |
|
||||
| `distance_range(lower_bound, upper_bound)` | Return only rows whose distance falls within `[lower_bound, upper_bound)`. Either bound is optional. Useful for near-duplicate detection or "close-enough" matching. |
|
||||
@@ -152,8 +160,8 @@ The `distance_type` can be specified at search time in addition to build time. I
|
||||
you do this, set it to the same value used to build the vector index. If the search
|
||||
metric does not match the index metric, LanceDB cannot use the index and falls
|
||||
back to a costly flat search. On LanceDB Cloud, the brute-force KNN safety limit
|
||||
can reject that search on a large table instead of executing it. LanceDB
|
||||
Enterprise doesn't have the limit.
|
||||
can reject that search on a large table instead of executing it. An Enterprise
|
||||
deployment applies the same limit only when it is configured to.
|
||||
|
||||
When the search metric is omitted, current SDKs use the index metric if an index
|
||||
is available. Some older SDK versions sent `l2` when it was omitted, so explicitly
|
||||
@@ -173,6 +181,8 @@ older client.
|
||||
**Filtered queries and adaptive `nprobes`.** When a `where(...)` filter is active, LanceDB initially scans `minimum_nprobes`
|
||||
partitions and uses a wider scan if too few rows are found.
|
||||
Set `minimum_nprobes == maximum_nprobes` or call `nprobes(n)` to instead fix the partition count.
|
||||
Whether a deployment widens the scan this way depends on its version; one that does not scans exactly
|
||||
`minimum_nprobes` partitions.
|
||||
</Note>
|
||||
|
||||
Here is an example of a vector search exercising several of the above parameters.
|
||||
@@ -211,7 +221,7 @@ Multivector indexing currently requires `distance_type="cosine"`. Use `bypass_ve
|
||||
#### Async API and Config Objects
|
||||
|
||||
Create vector indexes asynchronously with `await table.create_index("vector", config=...)`. The `config` object
|
||||
admits the same build-time parameters described above — pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index):
|
||||
admits the same build-time parameters described above. Pass an instance of a Python config class from the table in [Choosing the Right Index](#choosing-the-right-index):
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -234,7 +244,7 @@ Specify `vector_column_name` if you have multiple vector columns or non-default
|
||||
TypeScript currently doesn't support `IvfSq` (IVF with Scalar Quantization).
|
||||
</Note>
|
||||
|
||||
For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`) — the same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax.
|
||||
For a vector field nested inside a struct, pass its full dotted path as `vector_column_name` (e.g. `image.embedding`). The same path is used at query time and is what `list_indices()` reports under `columns`. See [Selecting the vector column](/search/vector-search#selecting-the-vector-column) for the full path syntax.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -265,17 +275,18 @@ The snippet below builds and queries an `IVF_HNSW_SQ` index.
|
||||
<Badge color="red">Enterprise-only</Badge>
|
||||
In LanceDB Enterprise, vector indexes are managed **automatically**. The system asynchronously updates and optimizes indexes as a background process:
|
||||
- Automatically manages indexing parameters and storage
|
||||
- Infers vector columns from the schema
|
||||
- GPU-based indexing is available on request -- [contact support](/deployment/support) to enable it for your deployment. A few billion vectors index in under 4 hours on GPU.
|
||||
- Can build an `IVF_PQ` index automatically on a vector column the deployment is configured to index
|
||||
- GPU-based `IVF_PQ` builds are available on request -- [contact support](/deployment/support) to enable them for your deployment. Build time depends on dataset size, dimensions and GPU resources.
|
||||
|
||||
<Badge color="gray">Open-Source</Badge>
|
||||
LanceDB OSS users can manually create vector indexes by calling `table.create_index()`.
|
||||
See the above sections for guidance on manually tuning index parameters as data changes.
|
||||
|
||||
<Note>
|
||||
`create_index()` returns immediately, but the vector index builds asynchronously.
|
||||
To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward —
|
||||
it waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`.
|
||||
In LanceDB Enterprise, `create_index()` returns immediately and the vector index builds asynchronously.
|
||||
To wait until all data is indexed, specify the `wait_timeout` parameter, or call `wait_for_index(...)` afterward.
|
||||
It waits for the named index to exist and for `index_stats(...)` to report `num_unindexed_rows == 0`.
|
||||
In the embedded library the index is built before `create_index()` returns.
|
||||
</Note>
|
||||
|
||||
<Note>
|
||||
@@ -287,11 +298,12 @@ fallback and searches only indexed rows.
|
||||
<Info>
|
||||
**Operational checks**
|
||||
|
||||
After appends or other writes, use `optimize()` to fold new rows into existing indexes.
|
||||
In the embedded library, use `optimize()` after appends or other writes to fold new rows into existing indexes.
|
||||
A deployment does this in the background, and `optimize()` is not available there.
|
||||
</Info>
|
||||
#### Check Index Status
|
||||
|
||||
Vector index creation runs in the background and may take some time to complete.
|
||||
In LanceDB Enterprise, vector index creation runs in the background and may take some time to complete.
|
||||
While it is ongoing, you can check its status through the API or the **LanceDB Enterprise UI**.
|
||||
|
||||
In the LanceDB Enterprise UI, navigate to your table page - the "Index" column reflects each column's index status: it is blank when no index exists, shows an "in progress" label while the index is being built, and shows the index type once the build completes.
|
||||
@@ -318,7 +330,7 @@ Each entry returned by `list_indices()` also carries detailed per-index metadata
|
||||
|
||||
|
||||
<Note>
|
||||
These fields are populated for local and embedded tables. On LanceDB Enterprise remote tables they are returned as `None` / `undefined` until the server response surfaces them.
|
||||
These fields are populated for local tables. In LanceDB Enterprise, `list_indices()` returns them except `num_unindexed_rows`; use `index_stats()` for index coverage.
|
||||
</Note>
|
||||
#### Custom Index Names
|
||||
|
||||
@@ -343,9 +355,10 @@ compactly store an index.
|
||||
|
||||
#### Product Quantization (PQ)
|
||||
To visualize PQ, assume a vector dataset has $d$ dimensions, with a **chunk** of a vector denoting a contiguous block of entries. Imagine that each vector has
|
||||
$m$ disjoint chunks of $d/m$ entries each, where the first chunk represents entries $0$ through $d-1$, the second contains entries $d$ through $2d - 1$, and so on.
|
||||
$m$ disjoint chunks of $d/m$ entries each, where the first chunk contains entries $0$ through $d/m-1$, the second contains
|
||||
entries $d/m$ through $2d/m-1$, and so on.
|
||||
|
||||
Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d$ to $di - 1$).
|
||||
Now, for each $i$, let $S_i$ represent the set containing chunk $i$ of each vector (entries $(i-1)d/m$ to $id/m-1$).
|
||||
For each $S_i$ independently, a small set of **centroids** is computed corresponding to an approximate solution to the $k$-means clustering problem.
|
||||
|
||||
For each original vector in the dataset, every chunk is associated to its nearest centroid; thus, the vector itself is associated with the concatenation of $m$ centroids.
|
||||
@@ -397,7 +410,7 @@ among all integral multiples $\{j \times 10^5: -128 \leq j \leq 127 \}$.
|
||||
|
||||
| Parameter | Description |
|
||||
| :--- | :--- |
|
||||
| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ) — not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. |
|
||||
| `num_bits` | Bits per dimension for quantization. Only applies to `IVF_PQ`/`IVF_HNSW_PQ` (default `8`) and `IVF_RQ` (default `1`, RaBitQ); not used by `IVF_FLAT`/`IVF_SQ`/`IVF_HNSW_FLAT`/`IVF_HNSW_SQ`. Higher values improve accuracy at the cost of additional storage. |
|
||||
|
||||
`max_iterations` and `sample_rate` also affect quantizer training, but since they apply to every IVF/HNSW index type (not just quantized ones), they're documented as general [Build-time Parameters](/indexing/vector-index#build-time-parameters) instead. All three are passed as keyword arguments to `create_index`, alongside `index_type`:
|
||||
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
---
|
||||
title: "Creating a table"
|
||||
sidebarTitle: "Create Table"
|
||||
description: "Start here: turn the data you already have into a LanceDB table."
|
||||
description: "Create a table from records or a schema and choose input formats."
|
||||
icon: table
|
||||
keywords: ["create table", "ingest", "arrow", "pandas", "polars", "pydantic", "schema"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Create and open tables in OSS and Enterprise.
|
||||
---
|
||||
|
||||
import {
|
||||
@@ -11,65 +16,60 @@ import {
|
||||
RsConnect,
|
||||
} from '/snippets/connection.mdx';
|
||||
import {
|
||||
PyCreateTableFromDicts as CreateTableFromDicts,
|
||||
TsCreateTableFromDicts as TsCreateTableFromDicts,
|
||||
RsCreateTableFromDicts as RsCreateTableFromDicts,
|
||||
PyCreateTableConflictHandling as CreateTableConflictHandling,
|
||||
TsCreateTableConflictHandling as TsCreateTableConflictHandling,
|
||||
RsCreateTableConflictHandling as RsCreateTableConflictHandling,
|
||||
PyCreateTableFromPandas as CreateTableFromPandas,
|
||||
PyCreateTableCustomSchema as CreateTableCustomSchema,
|
||||
TsCreateTableCustomSchema as TsCreateTableCustomSchema,
|
||||
RsCreateTableCustomSchema as RsCreateTableCustomSchema,
|
||||
PyCreateTableFromPolars as CreateTableFromPolars,
|
||||
PyCreateTableFromArrow as CreateTableFromArrow,
|
||||
TsCreateTableFromArrow as TsCreateTableFromArrow,
|
||||
RsCreateTableFromArrow as RsCreateTableFromArrow,
|
||||
PyCreateTableFromPydantic as CreateTableFromPydantic,
|
||||
PyCreateTableNestedSchema as CreateTableNestedSchema,
|
||||
PyOpenExistingTable as OpenExistingTable,
|
||||
TsOpenExistingTable as TsOpenExistingTable,
|
||||
RsOpenExistingTable as RsOpenExistingTable,
|
||||
PyCreateEmptyTable as CreateEmptyTable,
|
||||
TsCreateEmptyTable as TsCreateEmptyTable,
|
||||
RsCreateEmptyTable as RsCreateEmptyTable,
|
||||
PyCreateEmptyTablePydantic as CreateEmptyTablePydantic,
|
||||
PyDropTable as DropTable,
|
||||
TsDropTable as TsDropTable,
|
||||
RsDropTable as RsDropTable,
|
||||
PyTablesBasicConnect as TablesBasicConnect,
|
||||
PyTablesDocumentModel as TablesDocumentModel,
|
||||
PyTablesTzValidator as TablesTzValidator,
|
||||
} from '/snippets/tables.mdx';
|
||||
import {
|
||||
PyCreateTableFromDicts as CreateTableFromDicts,
|
||||
PyCreateTableConflictHandling as CreateTableConflictHandling,
|
||||
PyCreateTableFromPandas as CreateTableFromPandas,
|
||||
PyCreateTableCustomSchema as CreateTableCustomSchema,
|
||||
PyCreateTableFromPolars as CreateTableFromPolars,
|
||||
PyCreateTableFromArrow as CreateTableFromArrow,
|
||||
PyCreateTableFromPydantic as CreateTableFromPydantic,
|
||||
PyCreateTableNestedSchema as CreateTableNestedSchema,
|
||||
PyOpenExistingTable as OpenExistingTable,
|
||||
PyCreateEmptyTable as CreateEmptyTable,
|
||||
PyCreateEmptyTablePydantic as CreateEmptyTablePydantic,
|
||||
PyDropTable as DropTable,
|
||||
} from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
A table is created from data you already have -- a list of objects, an Arrow
|
||||
table, a pandas or polars frame, Pydantic models -- or from a schema alone when
|
||||
the rows come later.
|
||||
import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
Create a table from existing records, or define an empty table before data
|
||||
arrives. Define a schema to specify column types, allow or disallow missing
|
||||
values, and set the size of your vectors.
|
||||
|
||||
## Create a table with data {#create-a-table-with-data}
|
||||
|
||||
Initialize a LanceDB connection and create a table
|
||||
The Python input examples below are complete programs. Install `lancedb` and
|
||||
any additional libraries imported by the example, save one as a Python file,
|
||||
and run it. Each uses its own temporary local database.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{TablesBasicConnect}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsConnect}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsConnect}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Depending on the SDK, LanceDB can ingest arrays of records, Arrow tables or record batches, and Arrow batch iterators or readers. Let's take a look at some of the common patterns.
|
||||
The TypeScript and Rust blocks illustrate the corresponding input formats.
|
||||
For a complete connection-to-result program in either language, expand
|
||||
[the write workflow](#complete-example). Table names follow the
|
||||
[naming rules](/basics/naming).
|
||||
|
||||
### From list of objects {#from-list-of-objects}
|
||||
|
||||
You can provide a list of objects to create a table. The Python and TypeScript SDKs
|
||||
support lists/arrays of dictionaries, while the Rust SDK supports lists of structs.
|
||||
support lists/arrays of records. The Rust example converts structs to Arrow batches.
|
||||
In Python, pass a list or other batch-like object; a single bare `dict` or single
|
||||
`LanceModel` is rejected.
|
||||
|
||||
@@ -87,37 +87,42 @@ In Python, pass a list or other batch-like object; a single bare `dict` or singl
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
### Handle existing tables {#handle-existing-tables}
|
||||
### If the table already exists {#handle-existing-tables}
|
||||
|
||||
By default, `create_table` raises an error if a table with the same name already exists.
|
||||
You can change this behavior with two parameters that resolve the conflict in different ways:
|
||||
By default, creating a table fails if its name is already in use. Choose whether
|
||||
to open that table or overwrite it.
|
||||
|
||||
- **Idempotent open**: return the existing table without modifying it. Use when your
|
||||
code may run more than once (notebooks, retries, init scripts) and you want to reuse
|
||||
the table on subsequent runs. The provided data is ignored, but the schema is
|
||||
validated against the existing table and a mismatch raises an error.
|
||||
- **Overwrite**: drop the existing table and create a new one with the provided data.
|
||||
Use this for test fixtures or when you intentionally want to replace prior contents.
|
||||
This permanently discards the old table's data.
|
||||
#### Open the existing table
|
||||
|
||||
Use `exist_ok=True` in Python or `existOk: true` in TypeScript to return the
|
||||
existing table without changing its contents. This is useful for setup code
|
||||
that may run more than once. Any data supplied to the create call is ignored
|
||||
when the table already exists.
|
||||
|
||||
Inspect the returned table's schema if your code depends on specific columns
|
||||
or types. Opening an existing table does not change its schema to match the
|
||||
data supplied in the create call.
|
||||
|
||||
#### Overwrite the table
|
||||
|
||||
Use `mode="overwrite"` in Python or `mode: "overwrite"` in TypeScript to replace
|
||||
the existing table with the supplied data and schema. Choose this only when
|
||||
you intend to replace its previous contents.
|
||||
|
||||
The following Python program demonstrates both choices:
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{CreateTableConflictHandling}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsCreateTableConflictHandling}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsCreateTableConflictHandling}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
|
||||
<Note>
|
||||
`exist_ok` / `existOk` does not append the provided data to an existing table. Use
|
||||
[`table.add()`](/ingestion/insert) for that. If you need to ensure a table exists *and*
|
||||
contains specific rows, prefer the [empty-table-then-add pattern](#create-empty-table).
|
||||
contains specific rows, choose an explicit [merge policy](/ingestion/merge-insert). Creating an empty table
|
||||
and repeatedly appending the same batch does not deduplicate it.
|
||||
</Note>
|
||||
|
||||
### From a custom schema {#from-a-custom-schema}
|
||||
@@ -138,7 +143,7 @@ You can define a custom Arrow schema for the table. This is useful when you want
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
An explicit schema is also where you control nullability. If later writes omit a
|
||||
An explicit schema also determines which columns accept missing values (`null`). If later writes omit a
|
||||
non-nullable column, or provide actual nulls for it, ingestion fails; nullable columns can be
|
||||
omitted or written with null values. Without an explicit schema, Python infers list-like vector
|
||||
values as fixed-size `float32` vector fields from the observed dimension.
|
||||
@@ -186,10 +191,8 @@ The **`vector`** column needs to be a [Vector](/integrations/data/pydantic#vecto
|
||||
### From a Polars DataFrame {#from-a-polars-dataframe}
|
||||
<Badge color="green">Python Only</Badge>
|
||||
|
||||
LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
|
||||
written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
|
||||
under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
|
||||
is on the way.
|
||||
Pass a Polars DataFrame as input. Install `polars` for this example; the
|
||||
conversion preserves the table structure through Arrow.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -263,12 +266,14 @@ for a `created_at` field.
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
When you run this code it, should raise the `ValidationError`.
|
||||
The example catches a `ValidationError` for the timestamp with a different
|
||||
timezone. A validator runs when the Pydantic model is constructed; declaring a
|
||||
model as the table schema does not make every later write run that validator.
|
||||
|
||||
<Note>
|
||||
Datasets too large to pass to `create_table` at once are covered separately --
|
||||
see [Insert data](/ingestion/insert#large-datasets), which explains why
|
||||
`table.add()` parallelizes where `create_table` does not.
|
||||
Datasets too large to pass to `create_table` at once are covered separately;
|
||||
see [Insert data](/ingestion/insert#large-datasets) for batches, file-backed
|
||||
input and write parallelism.
|
||||
</Note>
|
||||
|
||||
## Create empty table {#create-empty-table}
|
||||
@@ -292,9 +297,8 @@ An empty table can be initialized via an Arrow schema.
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not
|
||||
directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel`
|
||||
that has been extended to support LanceDB specific types like `Vector`.
|
||||
Alternatively, define the schema with `LanceModel`, a Pydantic model class
|
||||
provided by `lancedb.pydantic`. Its `Vector` type declares a fixed vector dimension.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -342,6 +346,46 @@ Use the `drop_table()` method on the database to remove a table.
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
This permanently removes the table and is not recoverable, unlike deleting rows.
|
||||
By default, if the table does not exist an exception is raised. To suppress this,
|
||||
you can pass in `ignore_missing=True`.
|
||||
Dropping a table removes the table object. Deleting rows keeps the table and
|
||||
changes its contents; retained versions and cleanup are described in
|
||||
[Cleanup](/maintenance/cleanup).
|
||||
On a local database, dropping a table that does not exist raises an error; in
|
||||
Python, pass `ignore_missing=True` to suppress it. A deployment may report
|
||||
success instead, depending on how it stores its catalog, so do not rely on the
|
||||
error to tell you a table was missing. Python's deployment connection does not
|
||||
take `ignore_missing`.
|
||||
|
||||
<span id="verify"></span>
|
||||
|
||||
## Complete write workflow {#complete-example}
|
||||
|
||||
<Accordion title="Complete examples: create, append, update, upsert and delete">
|
||||
|
||||
Each program creates a temporary local database, performs the five operations,
|
||||
prints the result, and removes its temporary data when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `ingestion.py`, and run
|
||||
`python ingestion.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`,
|
||||
and run `node ingestion.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip),
|
||||
extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo
|
||||
manifest and a lockfile for the example's dependencies. The code is shown below.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="ingestion.py" language="python" icon="python">
|
||||
{PyWriteWorkflow}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.mts" language="typescript" icon="square-js">
|
||||
{TsWriteWorkflowBody}
|
||||
{TsWriteWorkflowTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="src/main.rs" language="rust" icon="rust">
|
||||
{RsWriteWorkflow}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6.
|
||||
Dana is selected for deletion and is absent from the final result.
|
||||
|
||||
</Accordion>
|
||||
|
||||
@@ -1,77 +1,110 @@
|
||||
---
|
||||
title: "Delete rows"
|
||||
sidebarTitle: "Delete"
|
||||
description: "Remove rows by predicate, and reclaim their space."
|
||||
description: "Select and delete records, verify the remaining rows, and understand storage retention."
|
||||
icon: trash
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Delete rows through the SDKs in OSS and Enterprise. Storage reclamation follows compaction and retention policy.
|
||||
---
|
||||
|
||||
These examples use the table set up in
|
||||
[Ingesting data](/ingestion), and the connection made there.
|
||||
import { RsExample as RsDeleteRows } from '/snippets/ingestion_delete.mdx';
|
||||
import { TsDeleteRowsBody, TsDeleteRowsTail } from '/snippets/ingestion_examples.mdx';
|
||||
import { PyDeleteRows } from '/snippets/ingestion_examples.mdx';
|
||||
import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
import {
|
||||
PyDeleteOperation as DeleteOperation,
|
||||
PyUpdateOptimizeCleanup as UpdateOptimizeCleanup,
|
||||
TsDeleteOperation,
|
||||
TsUpdateOptimizeCleanup,
|
||||
RsDeleteOperation,
|
||||
RsUpdateOptimizeCleanup,
|
||||
} from '/snippets/tables.mdx';
|
||||
Deleting rows removes matching records from the current table contents. It
|
||||
keeps the table and schema so you can continue reading and writing. To remove
|
||||
the table itself, see [Drop a table](/ingestion/create-table#drop-a-table).
|
||||
|
||||
<Note>
|
||||
In SQL this is [`DELETE`](/sql/dml/delete).
|
||||
</Note>
|
||||
## Inspect the rows you intend to delete {#select-rows}
|
||||
|
||||
## Delete rows {#delete-rows}
|
||||
Use the same predicate for inspection and deletion. The example below creates
|
||||
Alice, Bob and Charlie, prints Charlie's row, and deletes it by `id = 3`.
|
||||
Inspection and deletion are separate operations; concurrent writes can change
|
||||
which rows match between them.
|
||||
|
||||
Delete operations **soft delete** rows that match a given condition.
|
||||
The underlying data is not immediately removed, but is marked
|
||||
for deletion (in the [deletion files](https://lance.org/format/table/#deletion-files) at the Lance format level) and excluded from query results.
|
||||
## Delete matching records {#delete-rows}
|
||||
|
||||
Each program creates Alice, Bob and Charlie in a temporary local database,
|
||||
deletes Charlie, and prints the remaining rows: **Alice and Bob**. The temporary
|
||||
database is removed when the program finishes.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `delete_rows.py`, and run
|
||||
`python delete_rows.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `delete_rows.mts`,
|
||||
and run `node delete_rows.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip),
|
||||
extract the project, and run `cargo run --locked --bin delete`.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{DeleteOperation}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsDeleteOperation}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsDeleteOperation}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Python" language="python" icon="python">
|
||||
{PyDeleteRows}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="TypeScript" language="typescript" icon="square-js">
|
||||
{TsDeleteRowsBody}
|
||||
{TsDeleteRowsTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="rust" icon="rust">
|
||||
{RsDeleteRows}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Expected table contents:
|
||||
Enterprise also supports [`DELETE`](/sql/dml/delete).
|
||||
|
||||
| id | name | login_count |
|
||||
| --- | --- | --- |
|
||||
| 1 | Alice | 10 |
|
||||
| 2 | Bob | 20 |
|
||||
## Verify that the rows are absent {#verify}
|
||||
|
||||
Query the same predicate after deletion and check the remaining row count.
|
||||
A reader still using an older snapshot may see the earlier contents; see
|
||||
[Consistency](/tables/consistency).
|
||||
|
||||
<Warning>
|
||||
**Deleting rows removes them from the index**
|
||||
Deleted rows are excluded from current query results, including indexed
|
||||
searches. Deletion does not imply immediate physical erasure of every stored
|
||||
copy of the data.
|
||||
|
||||
When rows are deleted, those rows are also excluded from the index segments, so indexed queries will not return them either. If ALL the rows are deleted (i.e., the table is emptied), ensure that you recreate the index after ingesting new data.
|
||||
</Warning>
|
||||
## Understand storage reclamation {#storage}
|
||||
|
||||
To permanently remove deleted rows, you can optimize the table, which will run compaction and cleans up the soft-deleted rows, which frees up storage space.
|
||||
Compaction rewrites the live data layout; cleanup removes eligible retained
|
||||
versions and files. Retained history can continue to reference data that no
|
||||
longer appears in the latest table version.
|
||||
|
||||
- In LanceDB OSS, compaction and cleanup are manual. Run `table.optimize()` regularly to free up disk space.
|
||||
- In LanceDB Enterprise, files aren't cleaned up by default. You can configure automatic compaction and cleanup behavior at cluster setup time to suit your organization's retention policy.
|
||||
In OSS, `optimize()` provides maintenance operations. In Enterprise, managed
|
||||
maintenance and retention follow the service configuration. Choose retention
|
||||
according to the history you need, rather than shortening it as part of a
|
||||
routine delete example. See [Compaction](/maintenance/compaction) and
|
||||
[Cleanup](/maintenance/cleanup) for the separate maintenance workflow.
|
||||
|
||||
By default, table cleanup removes data up to 7 days ago. If you need to reclaim space from deleted rows more aggressively, manually call `table.optimize()` use a shorter retention window as follows:
|
||||
## Complete write workflow {#complete-example}
|
||||
|
||||
<Accordion title="Complete examples: create, append, update, upsert and delete">
|
||||
|
||||
Each program creates a temporary local database, performs the five operations,
|
||||
prints the result, and removes its temporary data when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `ingestion.py`, and run
|
||||
`python ingestion.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`,
|
||||
and run `node ingestion.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip),
|
||||
extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo
|
||||
manifest and a lockfile for the example's dependencies. The code is shown below.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{UpdateOptimizeCleanup}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsUpdateOptimizeCleanup}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateOptimizeCleanup}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.py" language="python" icon="python">
|
||||
{PyWriteWorkflow}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.mts" language="typescript" icon="square-js">
|
||||
{TsWriteWorkflowBody}
|
||||
{TsWriteWorkflowTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="src/main.rs" language="rust" icon="rust">
|
||||
{RsWriteWorkflow}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6.
|
||||
Dana is selected for deletion and is absent from the final result.
|
||||
|
||||
</Accordion>
|
||||
|
||||
+85
-113
@@ -1,125 +1,97 @@
|
||||
---
|
||||
title: "Ingesting data"
|
||||
sidebarTitle: "Overview"
|
||||
description: "Getting data into a table: creating one from your data, and the four write operations that follow."
|
||||
description: "Choose how to create a table, append records or change existing data."
|
||||
icon: inbox
|
||||
keywords: ["ingest", "create table", "add", "insert", "arrow", "pandas", "polars", "pydantic"]
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Create tables and write records through the SDKs in OSS and Enterprise. Enterprise also provides SQL write statements.
|
||||
---
|
||||
|
||||
import {
|
||||
PyUpdateConnectEnterprise as UpdateConnectEnterprise,
|
||||
PyUpdateConnectLocal as UpdateConnectLocal,
|
||||
PyUpdateExampleTableSetup as UpdateExampleTableSetup,
|
||||
TsUpdateConnectEnterprise,
|
||||
TsUpdateConnectLocal,
|
||||
TsUpdateExampleTableSetup,
|
||||
RsUpdateConnectEnterprise,
|
||||
RsUpdateConnectLocal,
|
||||
RsUpdateExampleTableSetup,
|
||||
RsUpdateMakeUsersReader,
|
||||
} from '/snippets/tables.mdx';
|
||||
import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
In LanceDB, tables store records with a defined schema that specifies column
|
||||
names and types. Across the SDKs, you can create tables from row-oriented data
|
||||
and Apache Arrow data structures. The Python SDK additionally supports PyArrow
|
||||
schemas for explicit schema control, and `LanceModel` for Pydantic-based
|
||||
validation.
|
||||
Ingestion brings data into a LanceDB table. Start by choosing a schema and
|
||||
creating the table, then choose a write operation according to how incoming
|
||||
records relate to data already stored.
|
||||
|
||||
Getting data in has two halves: creating a table from data you already have, and
|
||||
writing to a table that exists. This page covers the first and introduces the
|
||||
second.
|
||||
## Choose a write operation {#choose-a-write-method}
|
||||
|
||||
<Note>
|
||||
Every operation here has a SQL form -- see [DML](/sql#dml).
|
||||
</Note>
|
||||
|
||||
Start by [creating a table](/ingestion/create-table); the rest of this
|
||||
section writes to one that exists.
|
||||
|
||||
## Choose a write method {#choose-a-write-method}
|
||||
|
||||
| Family | Method | Use this when... |
|
||||
| --------------- | --------------- | ---------------- |
|
||||
| [`add`](/ingestion/insert) | `add(data)` | You are appending new rows and do not need to match against existing ones. |
|
||||
| `update` | `update(where=..., values=...)` | You want to edit rows that already exist, using a SQL filter. |
|
||||
| `merge_insert` | `.when_matched_update_all()` | You have incoming rows and want to update keys that already exist in the table. |
|
||||
| `merge_insert` | `.when_not_matched_insert_all()` | You have incoming rows and want to insert keys that do not exist yet. |
|
||||
| `merge_insert` | `.when_matched_update_all()` + `.when_not_matched_insert_all()` | You want both behaviors together (often called **upsert**: update existing keys **and** insert missing keys in the same operation). |
|
||||
| `merge_insert` | `.when_not_matched_by_source_delete(...)` | You want to remove target rows that are missing from the incoming source set. |
|
||||
|
||||
Write operations return status metadata. For example, `update` reports updated row count and
|
||||
committed `version`, `delete` reports deleted row count and `version`, and `merge_insert` reports
|
||||
inserted, updated, deleted, retry-attempt, and `version` fields. Writes require a mutable table
|
||||
handle; after checking out an older version for reads, call `checkout_latest` / `checkoutLatest`
|
||||
before modifying data.
|
||||
The committed `version` advances even for writes that affect zero rows, such as a delete predicate
|
||||
that matches nothing.
|
||||
|
||||
## Connect to LanceDB {#connect-to-lancedb}
|
||||
|
||||
Connect to your local LanceDB instance:
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{UpdateConnectLocal}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsUpdateConnectLocal}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateConnectLocal}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Or, connect to LanceDB Enterprise:
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{UpdateConnectEnterprise}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsUpdateConnectEnterprise}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateConnectEnterprise}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
In the Rust snippets, a `make_users_reader` helper is used to build Arrow input data.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateMakeUsersReader}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
## Create the example table {#create-the-example-table}
|
||||
|
||||
We'll start by creating a simple table with `id`, `name`, and `login_count` columns. All examples below use the same table.
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{UpdateExampleTableSetup}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsUpdateExampleTableSetup}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateExampleTableSetup}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Expected table contents:
|
||||
|
||||
| id | name | login_count |
|
||||
| Task | Operation | What happens |
|
||||
| --- | --- | --- |
|
||||
| 1 | Alice | 10 |
|
||||
| 2 | Bob | 20 |
|
||||
| [Create a table](/ingestion/create-table) | `create_table` / `createTable` | Create a table from data or an explicit schema. |
|
||||
| [Append records](/ingestion/insert) | `add` | Add rows without matching existing keys. |
|
||||
| [Upsert by key](/ingestion/merge-insert) | `merge_insert` / `mergeInsert` | Choose whether matching rows are updated and new keys inserted. |
|
||||
| [Update selected rows](/ingestion/update) | `update` | Change values in rows selected by a predicate. |
|
||||
| [Delete selected rows](/ingestion/delete) | `delete` | Remove matching rows from the current table contents. |
|
||||
|
||||
The example above shows a PyArrow schema. You can just as well create the table using other
|
||||
table creation patterns (Pandas, Polars, Pydantic, iterators, etc.) -- see [Create a table](/ingestion/create-table#create-a-table-with-data).
|
||||
Appending the same input twice can create duplicate records. Use merge insert
|
||||
with a stable key when the incoming data should update existing records.
|
||||
|
||||
## Connect and prepare the data {#connect-to-lancedb}
|
||||
|
||||
Use the [connection guide](/basics/connect) to connect to OSS or Enterprise.
|
||||
The SDK write operations in this section work with both. Enterprise also
|
||||
supports [SQL write statements](/sql#dml); an SDK filter expression does not
|
||||
require a separate SQL connection.
|
||||
|
||||
Choose the column types, required fields and vector dimensions before a large
|
||||
load. SDKs support different input representations, including row objects and
|
||||
Arrow batches. The [table creation guide](/ingestion/create-table) covers
|
||||
those choices.
|
||||
|
||||
## Try a complete write workflow {#create-the-example-table}
|
||||
|
||||
This example starts with Alice and Bob, appends Charlie, updates Bob's name
|
||||
and login count, then merges a batch containing Charlie and Dana. Finally, it
|
||||
selects Dana's row for inspection and deletes it.
|
||||
|
||||
<Accordion title="Complete examples: create, append, update, upsert and delete">
|
||||
|
||||
Each program creates a temporary local database, performs the five operations,
|
||||
prints the result, and removes its temporary data when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `ingestion.py`, and run
|
||||
`python ingestion.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`,
|
||||
and run `node ingestion.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip),
|
||||
extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo
|
||||
manifest and a lockfile for the example's dependencies. The code is shown below.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="ingestion.py" language="python" icon="python">
|
||||
{PyWriteWorkflow}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.mts" language="typescript" icon="square-js">
|
||||
{TsWriteWorkflowBody}
|
||||
{TsWriteWorkflowTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="src/main.rs" language="rust" icon="rust">
|
||||
{RsWriteWorkflow}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6.
|
||||
Dana is selected for deletion and is absent from the final result.
|
||||
|
||||
</Accordion>
|
||||
|
||||
## Verify the result {#verify}
|
||||
|
||||
After a write completes, inspect the affected keys and fields and check the row
|
||||
count. SDK return values differ by operation and language; use the relevant
|
||||
API reference when relying on returned counts or version metadata.
|
||||
|
||||
Readers with older snapshots may not see another handle's writes immediately.
|
||||
See [Consistency](/tables/consistency). A table checked out at a historical
|
||||
version must return to the latest version before it can be modified.
|
||||
|
||||
## Load larger datasets {#large-loads}
|
||||
|
||||
Use batches or file-backed input when the dataset is too large to hold in
|
||||
memory. The [insert guide](/ingestion/insert#large-datasets) covers input
|
||||
formats, progress reporting and write parallelism. Keep retry behavior in mind:
|
||||
repeating an append is not the same as retrying a key-based upsert.
|
||||
|
||||
+181
-161
@@ -1,217 +1,237 @@
|
||||
---
|
||||
title: "Insert data"
|
||||
sidebarTitle: "Insert"
|
||||
description: "Append rows to a table, including datasets too large to hold in memory."
|
||||
description: "Append records, load file-backed data or batches, and verify ingestion."
|
||||
icon: plus
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Append records through the SDKs in OSS and Enterprise. For large Enterprise writes, see [large upload settings](#bulk-ingests-into-a-remote-table).
|
||||
---
|
||||
|
||||
Rows are appended with `table.add()`. It accepts the same shapes
|
||||
[table creation](/ingestion) does — lists of objects, Arrow tables, pandas
|
||||
and polars frames, Pydantic models — and commits a new table version.
|
||||
|
||||
These examples use the table set up in [Ingesting data](/ingestion).
|
||||
|
||||
import {
|
||||
TsCreateTableFromIterator as TsCreateTableFromIterator,
|
||||
RsCreateTableFromIterator as RsCreateTableFromIterator,
|
||||
} from '/snippets/tables.mdx';
|
||||
import {
|
||||
PyAddFromDataset as AddFromDataset,
|
||||
PyCreateTableFromIterator as CreateTableFromIterator,
|
||||
TsCreateTableFromIterator as TsCreateTableFromIterator,
|
||||
RsCreateTableFromIterator as RsCreateTableFromIterator,
|
||||
TsAddProgress,
|
||||
} from '/snippets/tables.mdx';
|
||||
} from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
<Note>
|
||||
In SQL this is [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT` to move rows server-side.
|
||||
</Note>
|
||||
import { RsExample as RsAddProgress } from '/snippets/ingestion_progress.mdx';
|
||||
import { PyAddProgress, TsAddProgressBody, TsAddProgressTail } from '/snippets/ingestion_examples.mdx';
|
||||
import { PyAppendRows } from '/snippets/ingestion_examples.mdx';
|
||||
import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
## Ingesting large datasets {#large-datasets}
|
||||
Use `add()` to append records to an existing table. It does not compare incoming
|
||||
keys with stored keys. Choose [merge insert](/ingestion/merge-insert) when the
|
||||
same record may arrive again and should update an existing row.
|
||||
|
||||
When ingesting large datasets, use `table.add()` on an existing table rather than
|
||||
passing all data to `create_table()`. The `add()` method auto-parallelizes large
|
||||
writes, while `create_table(name, data)` does not.
|
||||
## Append a first batch {#append}
|
||||
|
||||
<Tip>
|
||||
For best performance with large datasets, create an empty table first and then call
|
||||
`table.add()`. This enables automatic write parallelism for materialized data sources.
|
||||
</Tip>
|
||||
Install the Python SDK with `pip install lancedb`. This complete example starts
|
||||
with Alice, appends Bob and Charlie, and prints all three records.
|
||||
|
||||
### From files (Parquet, CSV, etc.) {#from-files-parquet-csv-etc}
|
||||
<Badge color="green">Python Only</Badge>
|
||||
<CodeBlock filename="append_rows.py" language="python" icon="python">
|
||||
{PyAppendRows}
|
||||
</CodeBlock>
|
||||
|
||||
For file-based data, pass a `pyarrow.dataset.Dataset` to `table.add()`. This streams
|
||||
data from disk without loading the entire dataset into memory.
|
||||
The program uses temporary local storage. For a complete TypeScript or Rust
|
||||
example, see [Complete write workflow](#complete-example).
|
||||
Enterprise SQL offers [`INSERT`](/sql/dml/insert), including `INSERT ... SELECT`.
|
||||
|
||||
## Prepare the input {#prepare}
|
||||
|
||||
Match the table's schema, including required fields and vector dimensions.
|
||||
Supported input formats differ by SDK; see
|
||||
[Create a table](/ingestion/create-table). Inspect conversion errors before
|
||||
retrying, and verify the final row count and representative values.
|
||||
|
||||
## Choose input for a large load {#large-datasets}
|
||||
|
||||
Avoid building one large in-memory list when records already live in files
|
||||
or arrive as batches. Create a table with the intended schema, then append
|
||||
through the appropriate input path.
|
||||
|
||||
<span id="bulk-ingestion-for-data-you-already-have"></span>
|
||||
### Read from files {#from-files-parquet-csv-etc}
|
||||
|
||||
Python accepts a `pyarrow.dataset.Dataset`. This example writes a small Parquet
|
||||
file locally, creates an empty table with its schema, and appends the two records.
|
||||
For your own data, point the dataset reader at the input files instead.
|
||||
|
||||
<CodeBlock filename="load_parquet.py" language="python" icon="python">
|
||||
{AddFromDataset}
|
||||
</CodeBlock>
|
||||
|
||||
A file-backed source can be scanned without first loading the whole dataset
|
||||
into application memory. This Python input API is distinct from converting
|
||||
files to Arrow batches yourself in another language.
|
||||
|
||||
<span id="iterator-ingestion-for-data-you-transform-on-the-fly"></span>
|
||||
### Produce batches during ingestion {#from-iterators-custom-batch-generation}
|
||||
|
||||
Use batches when processing the input as it arrives. The Python example creates
|
||||
an empty table and appends ten records produced by a batch iterator.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{AddFromDataset}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Python" language="python" icon="python">
|
||||
{CreateTableFromIterator}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="TypeScript" language="typescript" icon="square-js">
|
||||
{TsCreateTableFromIterator}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="rust" icon="rust">
|
||||
{RsCreateTableFromIterator}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
<Note>
|
||||
`pa.dataset()` input is currently Python-only. TypeScript and Rust support for
|
||||
file-based dataset ingestion is tracked in
|
||||
[lancedb#3173](https://github.com/lancedb/lancedb/issues/3173).
|
||||
</Note>
|
||||
The language APIs differ: the TypeScript example appends each constructed Arrow
|
||||
batch, while the Rust example supplies an Arrow reader when creating the table.
|
||||
These are input-format examples, not equivalent parallel-write benchmarks.
|
||||
|
||||
### From iterators (custom batch generation) {#from-iterators-custom-batch-generation}
|
||||
### Control write parallelism {#write-parallelism}
|
||||
|
||||
When you need custom batch logic — generating embeddings on the fly, transforming
|
||||
rows from an external source, etc. — use an iterator of `RecordBatch` objects.
|
||||
Python's `add()` can estimate parallelism for materialized inputs. A streaming
|
||||
reader may not expose its total size or support replay. Use `write_parallelism`
|
||||
when controlling the number of concurrent partitions; increasing it can also
|
||||
increase memory use. TypeScript does not expose that manual option.
|
||||
|
||||
Measure with realistic row sizes and storage. Avoid one write per row when a
|
||||
batch will do, and avoid a large worker count for a small input.
|
||||
|
||||
## Track completion and progress {#tracking-ingestion-progress}
|
||||
|
||||
Python, TypeScript and Rust support progress callbacks when adding data.
|
||||
These examples start with Alice, append Bob and Charlie, and print the number
|
||||
of rows written as progress updates arrive. After the write succeeds, they
|
||||
query the table to show all three records.
|
||||
|
||||
Each program uses a temporary local database and removes it when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `insert_progress.py`, and run
|
||||
`python insert_progress.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `insert_progress.mts`,
|
||||
and run `node insert_progress.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [Rust examples](/static/code/ingestion-feedback-rust.zip),
|
||||
extract the project, and run `cargo run --locked --bin progress`.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{CreateTableFromIterator}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsCreateTableFromIterator}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsCreateTableFromIterator}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Python" language="python" icon="python">
|
||||
{PyAddProgress}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="TypeScript" language="typescript" icon="square-js">
|
||||
{TsAddProgressBody}
|
||||
{TsAddProgressTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="rust" icon="rust">
|
||||
{RsAddProgress}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Use this pattern when:
|
||||
For a progress bar in Python, install `tqdm` and pass `progress=True` instead
|
||||
of a callback.
|
||||
|
||||
- Your source data already arrives in Arrow batches, readers, datasets, or streams.
|
||||
- Materializing the entire ingest as one giant in-memory list or array would be too expensive.
|
||||
- You want to control chunk size explicitly during ingestion.
|
||||
### Read progress updates
|
||||
|
||||
Python can also consume iterators of other supported types like Pandas DataFrames or Python lists.
|
||||
Updates include row and byte counts, elapsed time, and parallel write activity.
|
||||
The field names differ by language:
|
||||
|
||||
### Write parallelism {#write-parallelism}
|
||||
| Meaning | Python dictionary key | TypeScript property | Rust method |
|
||||
|:--------|:----------------------|:--------------------|:------------|
|
||||
| Rows written so far | `output_rows` | `outputRows` | `output_rows()` |
|
||||
| Bytes written so far | `output_bytes` | `outputBytes` | `output_bytes()` |
|
||||
| Expected row count, when known | `total_rows` | `totalRows` | `total_rows()` |
|
||||
| Elapsed time | `elapsed_seconds` | `elapsedSeconds` | `elapsed()` |
|
||||
| Active write tasks | `active_tasks` | `activeTasks` | `active_tasks()` |
|
||||
| Total write tasks | `total_tasks` | `totalTasks` | `total_tasks()` |
|
||||
| Final update | `done` | `done` | `done()` |
|
||||
|
||||
<Note title="Automatic parallelism">
|
||||
For materialized data (`pa.Table`, `pd.DataFrame`, `pa.dataset()`), LanceDB
|
||||
automatically parallelizes large writes — no configuration needed. Auto-parallelism
|
||||
targets approximately 1M rows or 2GB per write partition.
|
||||
Python and TypeScript report elapsed time in seconds; Rust returns a `Duration`.
|
||||
The expected row count may be unknown for streaming input. On the final update,
|
||||
it falls back to the number of rows written.
|
||||
|
||||
For streaming sources (iterators, `RecordBatchReader`), LanceDB cannot determine
|
||||
total size upfront. Python exposes `write_parallelism` on both synchronous and
|
||||
asynchronous `table.add()` calls; see the [iterator example](#iterator-ingestion-for-data-you-transform-on-the-fly).
|
||||
TypeScript does not expose a manual write-parallelism option.
|
||||
</Note>
|
||||
Keep callbacks short and use the write operation's result to determine success.
|
||||
The final `done` update means progress reporting has ended; it can also occur
|
||||
after a write fails or is cancelled. Input errors before writing starts may
|
||||
produce no updates.
|
||||
|
||||
### Tracking ingestion progress {#tracking-ingestion-progress}
|
||||
<Badge color="green">TypeScript example</Badge>
|
||||
TypeScript delivers callbacks asynchronously. Slow callbacks can queue up,
|
||||
and the `add()` promise may resolve before every callback has run. Exceptions
|
||||
from a TypeScript callback are logged and do not abort the write.
|
||||
|
||||
In TypeScript, for long-running writes, pass a `progress` callback to `table.add()` to surface
|
||||
per-batch progress in your UI, logs, or metrics pipeline. Progress updates are dispatched
|
||||
asynchronously to the JavaScript event loop, including a final `done: true`
|
||||
update when an active write finishes.
|
||||
## Verify writes and handle retries {#performance}
|
||||
|
||||
Each invocation receives a `WriteProgress` object:
|
||||
Wait for the write operation's result, then check the expected records.
|
||||
Progress callbacks describe activity; they do not establish that a write
|
||||
succeeded. If a request fails after the server may have committed it, blindly
|
||||
repeating an append can duplicate rows. The client's automatic retries can do
|
||||
the same: on a remote table, the Python and TypeScript 0.40.0 clients resend an
|
||||
append after a retryable status such as `504`, even when the server had already
|
||||
committed it, and `add()` still returns success. Reconcile using your record
|
||||
IDs, or choose an upsert policy designed for replay.
|
||||
|
||||
| Field | Description |
|
||||
|:------|:------------|
|
||||
| `outputRows` | Rows written so far. |
|
||||
| `outputBytes` | Bytes written so far. |
|
||||
| `totalRows` | Expected total rows when the input source reports one. Always set on the final callback. |
|
||||
| `elapsedSeconds` | Wall-clock seconds since the write started. |
|
||||
| `activeTasks` | Parallel write tasks currently in flight. |
|
||||
| `totalTasks` | Total parallel write tasks (the write parallelism). |
|
||||
| `done` | `true` only for the final callback. |
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsAddProgress}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
A few things to know before you wire this up:
|
||||
|
||||
- Asynchronous delivery: slow callbacks queue updates without back-pressuring the writer. Keep the callback cheap to avoid accumulating queued updates.
|
||||
- Completion: an active write emits a final `done: true` update when it finishes, including execution errors or cancellation. Errors before writing starts, such as invalid input, may produce no progress callbacks. Use the `table.add()` promise to determine whether the write succeeded; it does not wait for JavaScript to process every queued callback.
|
||||
- Errors swallowed: anything your callback throws is logged with `console.warn` and won't abort the write, so keep the callback side-effect-only and don't rely on it for control flow.
|
||||
- Row totals: `totalRows` is only populated when the input source can report it up front (for example, a materialized `arrow.Table`). For streaming sources it stays `undefined` until the final callback, where it falls back to the actual rows written.
|
||||
|
||||
## Performance {#performance}
|
||||
|
||||
If ingestion is taking longer than expected on a large dataset, the cause is almost always how `add()` is called: each call commits a new version and a new fragment, so a per-row loop pays that per-call overhead at every row. The best practice is to pick the ingestion mode that matches your data shape — bulk ingestion when the data is already materialized, or iterator ingestion when it's streamed or computed on the fly.
|
||||
|
||||
<Note>
|
||||
**Why `merge_insert()` is significantly slower than `add()`**
|
||||
|
||||
A merge has to scan existing data to find matches on the join key (or look them up via a scalar index, if one exists), and then delete-and-reinsert any updated rows in a single transaction; `add()` simply appends new fragments.
|
||||
|
||||
Use `add()` for pure appends, and reach for `merge_insert()` only when you need upsert or conditional-insert logic. When you do use it, build a scalar index on the join column first — otherwise the match step falls back to a full column scan, which is the dominant cost at scale.
|
||||
</Note>
|
||||
|
||||
#### Bulk ingestion: for data you already have {#bulk-ingestion-for-data-you-already-have}
|
||||
|
||||
For materialized inputs (Arrow Tables, DataFrames) and file-backed sources (`pyarrow.dataset(...)`), LanceDB auto-parallelizes the write across workers, estimating the partition count from the data size — more partitions means more concurrent writes and higher throughput, up to the CPU core count.
|
||||
|
||||
```python Python icon="python"
|
||||
table.add(arrow_table) # in-memory
|
||||
table.add(df) # pandas
|
||||
table.add(ds.dataset("data/", format="parquet")) # streams from disk, still parallelized
|
||||
```
|
||||
|
||||
Pass `progress=True` to watch it happen: LanceDB shows a live tqdm bar with rows written, throughput in MB/s, and active worker count.
|
||||
|
||||
```python Python icon="python"
|
||||
table.add(ds.dataset("data/", format="parquet"), progress=True)
|
||||
```
|
||||
|
||||
```text
|
||||
71%|███████▏ | 710000/1000000 [00:12<00:05, 58.8kit/s, 42.3 MB/s | 8/8 workers]
|
||||
```
|
||||
|
||||
For larger-than-memory data, prefer scanning a file-backed dataset (`ds.dataset(...)`) over a hand-built `pyarrow.RecordBatchReader`: a `Dataset` can be counted and rescanned, so LanceDB knows the row count upfront (better auto-parallelism) and can retry a failed write from the start — a reader can only be consumed once, so neither is possible. Reach for the iterator path below only when the data genuinely can't be backed by files, e.g. to apply custom data transformations as you ingest.
|
||||
|
||||
For very large initial loads, create the table empty first; passing data directly to `create_table(name, data)` skips the auto-parallel path.
|
||||
|
||||
#### Iterator ingestion: for data you transform on the fly {#iterator-ingestion-for-data-you-transform-on-the-fly}
|
||||
|
||||
If each row needs work before it can be written — applying a custom transformation as you ingest, for example — the data doesn't exist as a file you can point `ds.dataset(...)` at. The best practice is to pass an iterator of `pyarrow.RecordBatch` instead; LanceDB consumes one batch at a time as you produce them.
|
||||
|
||||
```python Python icon="python"
|
||||
def stream():
|
||||
for raw in source:
|
||||
vectors = model.encode(raw["text"])
|
||||
yield pa.RecordBatch.from_pydict({**raw, "vector": vectors})
|
||||
|
||||
table.add(stream())
|
||||
```
|
||||
|
||||
Use decently large chunks of several thousand rows or more, rather than yielding single-row batches.
|
||||
|
||||
<Note>
|
||||
**Set `write_parallelism` yourself for large inputs**
|
||||
|
||||
A reader can't be counted or rescanned the way a `Dataset` can (see above), so LanceDB can't auto-size parallelism for it — set `write_parallelism` explicitly for large inputs:
|
||||
|
||||
```python Python icon="python"
|
||||
table.add(stream(), write_parallelism=4)
|
||||
```
|
||||
|
||||
Each partition becomes its own fragment, so don't over-allocate on a small input — budget one unit of parallelism per ~100K rows or ~1 GB of data as a rule of thumb.
|
||||
</Note>
|
||||
|
||||
#### Bulk ingests into a remote table {#bulk-ingests-into-a-remote-table}
|
||||
## Configure large uploads {#bulk-ingests-into-a-remote-table}
|
||||
|
||||
<Badge color="red">Enterprise</Badge>
|
||||
|
||||
This section applies only to remote tables (`db://` connections). Embedded (local) writes are unaffected.
|
||||
|
||||
When you write to a remote LanceDB Enterprise table, the client splits each write partition into one or more HTTP `insert` parts and uploads them under a single upload id. Each part is streamed rather than buffered, so peak memory stays bounded. But each part still has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as:
|
||||
When you write to a remote LanceDB Enterprise table, the client sends the write as one HTTP `insert` request unless it splits the data into more than one partition, which it does for a large input it can measure or when you set [`write_parallelism`](#write-parallelism). A partitioned write goes up as a multipart write: the client cuts each partition into one or more `insert` parts and uploads them under a single upload id. Each part is streamed. Its size and duration still affect resource use and
|
||||
timeouts: it has to finish within the client read timeout while the server writes it to object storage, so on very large ingests an oversized part can run past that timeout and surface as:
|
||||
|
||||
```text
|
||||
lancedb.remote.errors.HttpError: operation timed out
|
||||
```
|
||||
|
||||
Two environment variables control how parts are cut. Both are picked up automatically by the Python and TypeScript clients, and are the only way to tune this from those SDKs:
|
||||
Two environment variables control how a multipart write's parts are cut. Both are picked up automatically by the Python and TypeScript clients, for clients that support these settings:
|
||||
|
||||
| Variable | Default | What it controls |
|
||||
|----------|---------|------------------|
|
||||
| `LANCE_CLIENT_MAX_BYTES_PER_REQUEST` | 8 GiB | Maximum size of a single insert part, in LZ4-compressed Arrow IPC bytes. Set to `0` to disable splitting and send one request per partition. |
|
||||
| `LANCE_CLIENT_MAX_REQUEST_DURATION` | Half the read timeout | Maximum wall-clock time, in whole seconds, that any one insert part may stay open. Set to `0` to disable the time-based cut. |
|
||||
|
||||
A part is cut when it hits either limit, whichever comes first. Lower the byte budget when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor.
|
||||
A part is cut after the input batch that reaches either limit, so cuts fall between batches. Lower the byte budget, with smaller input batches, when large ingests hit the read timeout; lower the duration when uploads are slow or throttled and the byte budget isn't the limiting factor.
|
||||
|
||||
```bash
|
||||
export LANCE_CLIENT_MAX_BYTES_PER_REQUEST=1073741824 # 1 GiB
|
||||
export LANCE_CLIENT_MAX_REQUEST_DURATION=120 # 2 minutes
|
||||
```
|
||||
|
||||
Splitting into more parts does not change the final table. The server stages every part under the shared upload id and merges them atomically when the write completes.
|
||||
Splitting into more parts does not change the final table. The server stages the parts under their shared upload ID before committing
|
||||
the write. This upload protocol is separate from repeating an application-level
|
||||
`add()` after a timeout.
|
||||
|
||||
## Complete write workflow {#complete-example}
|
||||
|
||||
<Accordion title="Complete examples: create, append, update, upsert and delete">
|
||||
|
||||
Each program creates a temporary local database, performs the five operations,
|
||||
prints the result, and removes its temporary data when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `ingestion.py`, and run
|
||||
`python ingestion.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`,
|
||||
and run `node ingestion.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip),
|
||||
extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo
|
||||
manifest and a lockfile for the example's dependencies. The code is shown below.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="ingestion.py" language="python" icon="python">
|
||||
{PyWriteWorkflow}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.mts" language="typescript" icon="square-js">
|
||||
{TsWriteWorkflowBody}
|
||||
{TsWriteWorkflowTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="src/main.rs" language="rust" icon="rust">
|
||||
{RsWriteWorkflow}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6.
|
||||
Dana is selected for deletion and is absent from the final result.
|
||||
|
||||
</Accordion>
|
||||
|
||||
@@ -1,19 +1,16 @@
|
||||
---
|
||||
title: "Merge insert"
|
||||
sidebarTitle: "Merge Insert"
|
||||
description: "Reconcile incoming rows against a key: update what matches, insert what does not."
|
||||
description: "Match incoming records by key and choose which rows to update, insert or remove."
|
||||
icon: code-merge
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Merge insert is available through the SDKs in OSS and Enterprise. Use it for key-based updates and inserts.
|
||||
---
|
||||
|
||||
These examples use the table set up in
|
||||
[Ingesting data](/ingestion), and the connection made there.
|
||||
|
||||
import {
|
||||
PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly,
|
||||
PyInsertIfNotExists as InsertIfNotExists,
|
||||
PyMergeUpdateInsert as MergeUpdateInsert,
|
||||
PyMergeDeleteMissingBySource as MergeDeleteMissingBySource,
|
||||
PyMergePartialColumns as MergePartialColumns,
|
||||
TsMergeMatchedUpdateOnly,
|
||||
TsInsertIfNotExists,
|
||||
TsMergeUpdateInsert,
|
||||
@@ -25,15 +22,24 @@ import {
|
||||
RsMergeDeleteMissingBySource,
|
||||
RsMergePartialColumns,
|
||||
} from '/snippets/tables.mdx';
|
||||
import {
|
||||
PyMergeMatchedUpdateOnly as MergeMatchedUpdateOnly,
|
||||
PyInsertIfNotExists as InsertIfNotExists,
|
||||
PyMergeUpdateInsert as MergeUpdateInsert,
|
||||
PyMergeDeleteMissingBySource as MergeDeleteMissingBySource,
|
||||
PyMergePartialColumns as MergePartialColumns,
|
||||
} from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
<Note>
|
||||
Merge insert has **no SQL form** -- it needs a source relation and a per-match policy that one statement cannot express. See [DML](/sql/dml/insert#next).
|
||||
</Note>
|
||||
import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
Use merge insert when incoming records should be compared with existing rows.
|
||||
Combine update and insert policies for an **upsert**. Use [append](/ingestion/insert)
|
||||
when every incoming row should simply be added.
|
||||
|
||||
## Merge incoming rows by key {#merge-incoming-rows-by-key}
|
||||
|
||||
Merging is different from updating because it involves comparing incoming rows to existing rows by key, and then choosing what to do based on whether the key exists in the target table or not.
|
||||
The `merge_insert(""..."")` method lets you do this.
|
||||
Use `merge_insert("id")` in Python or the corresponding SDK builder.
|
||||
|
||||
In merge operations, rows are split into three groups:
|
||||
|
||||
@@ -45,37 +51,28 @@ Conditional merge clauses can compare old and new values. Use the `target.` pref
|
||||
existing table row and `source.` for the incoming row, for example
|
||||
`target.last_update < source.last_update`.
|
||||
|
||||
<Tip>
|
||||
**Use scalar indexes to speed up merge insert**
|
||||
Choose a key that identifies the same record in the source and target. Check
|
||||
for duplicate and null keys before merging. A primary-key declaration does not
|
||||
enforce uniqueness on ordinary writes, and append operations do not deduplicate
|
||||
records for you.
|
||||
|
||||
The merge insert command performs a join between the input data and the target table `on` the key you provide. This requires scanning that entire column, which can be expensive for large tables. To speed up this operation, create a scalar index on the join column, which will allow LanceDB to find matches without scanning the whole table.
|
||||
A scalar index on the key can help matching. Keep it current and measure the
|
||||
actual workload rather than assuming every merge uses an index. Enterprise may
|
||||
reject a merge when the unindexed scan exceeds its configured limit; inspect
|
||||
that error and the index state before retrying.
|
||||
|
||||
Read more about scalar indices in the [Scalar Index](/indexing/scalar-index/) guide.
|
||||
</Tip>
|
||||
## Choose a merge policy {#policies}
|
||||
|
||||
<Info>
|
||||
If you see this HTTP 400 error from `merge_insert`: `Bad request: Merge insert cannot be performed because the number of unindexed rows exceeds the maximum of 10000`. Verify that the scalar index on the join column is up to date before retrying.
|
||||
</Info>
|
||||
| Source and target relationship | Available action |
|
||||
| --- | --- |
|
||||
| Key appears in both | Update matching target rows, optionally with a condition. |
|
||||
| Key appears only in the source | Insert it, or leave it out. |
|
||||
| Key appears only in the target | Keep it, or explicitly delete it under a synchronization policy. |
|
||||
|
||||
<Note>
|
||||
**Rust: build merge predicates with DataFusion expressions**
|
||||
|
||||
In the Rust SDK, you can pass a `datafusion_expr::Expr` directly instead of a SQL string by using
|
||||
`when_matched_update_all_expr` and `when_not_matched_by_source_delete_expr` on `MergeInsertBuilder`.
|
||||
This is useful when you are constructing predicates programmatically and want to avoid building a SQL string.
|
||||
|
||||
These methods are only supported on local tables. Calling them against a remote table returns a
|
||||
`NotSupported` error — use the SQL string variants (`when_matched_update_all` /
|
||||
`when_not_matched_by_source_delete`) for remote tables.
|
||||
</Note>
|
||||
|
||||
Like the create table and add APIs, the merge insert API will automatically compute embeddings based on the [embedding registry](/embedding/index#embedding-registry) if the table has an embedding definition in its schema.
|
||||
|
||||
During `merge_insert`, if the input data doesn't contain the source column (i.e., the original field used to generate embeddings, such as text for a text embedding model or `image_uri` for an image model), or if a vector value is already provided, LanceDB skips embedding generation for that row. Embeddings are only auto-generated when that source field is present in the incoming data, **and** the vector field is empty.
|
||||
|
||||
Primary keys in LanceDB are metadata used by operations such as `merge_insert`; they are not
|
||||
enforced as uniqueness constraints on ordinary writes. Keep using `merge_insert` or your own
|
||||
deduplication logic when you need key-based upsert semantics.
|
||||
The following examples are independent. Each starts from the table shown in
|
||||
its code, so the expected results do not depend on running the previous example.
|
||||
Python blocks include their own temporary connection. For a full TypeScript or
|
||||
Rust program, see [Complete write workflow](#complete-example).
|
||||
|
||||
### Update matched rows only {#update-matched-rows-only}
|
||||
|
||||
@@ -160,7 +157,10 @@ Expected table contents:
|
||||
|
||||
### Delete target rows that are missing from source {#delete-target-rows-that-are-missing-from-source}
|
||||
|
||||
Use `when_not_matched_by_source_delete()` when you want to remove any target row that does not appear in the incoming source data.
|
||||
Use `when_not_matched_by_source_delete()` only when the incoming data is the
|
||||
complete set for the target scope you intend to synchronize. An incremental
|
||||
batch is usually not a complete source: deleting everything absent from it
|
||||
could remove unrelated target records. Inspect the unmatched target keys first.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
@@ -183,7 +183,8 @@ Expected table contents:
|
||||
| 2 | Bobby | 21 |
|
||||
| 3 | Charlie | 5 |
|
||||
|
||||
In the example above, LanceDB matches rows by `id`. Rows with `id=2` and `id=3` exist in both the table and incoming data, so they are updated. Row `id=1` exists only in the target, so it is deleted.
|
||||
Here, IDs 2 and 3 are updated and ID 1 is deleted. This policy synchronizes
|
||||
the target to the supplied source set.
|
||||
|
||||
### Use partial columns in merge updates {#use-partial-columns-in-merge-updates}
|
||||
|
||||
@@ -212,3 +213,54 @@ Expected table contents:
|
||||
| 3 | Charlie | null |
|
||||
|
||||
Note that in the example above, when `merge_insert` creates a new row, any missing columns are written as `null`. If a missing column is non-nullable in your schema, the insert will fail.
|
||||
|
||||
## Check the merged records {#verify}
|
||||
|
||||
Inspect the keys that should have changed, confirm that unrelated rows remain,
|
||||
and check inserted or deleted counts when the SDK returns them. Repeating a
|
||||
merge is only safe when its keys, conditions and update values produce the
|
||||
intended result on retry.
|
||||
|
||||
### Embeddings and expression predicates {#advanced}
|
||||
|
||||
When a table uses an [embedding definition](/embedding), behavior depends on
|
||||
which source and vector fields the incoming records provide. Do not assume a
|
||||
partial metadata update recomputes embeddings. Inspect the resulting vector
|
||||
when changing the input used to generate it.
|
||||
|
||||
Rust's `when_matched_update_all_expr` and
|
||||
`when_not_matched_by_source_delete_expr` accept DataFusion expressions for
|
||||
local tables. Use the string predicate variants for Enterprise tables.
|
||||
|
||||
## Complete write workflow {#complete-example}
|
||||
|
||||
<Accordion title="Complete examples: create, append, update, upsert and delete">
|
||||
|
||||
Each program creates a temporary local database, performs the five operations,
|
||||
prints the result, and removes its temporary data when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `ingestion.py`, and run
|
||||
`python ingestion.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`,
|
||||
and run `node ingestion.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip),
|
||||
extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo
|
||||
manifest and a lockfile for the example's dependencies. The code is shown below.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="ingestion.py" language="python" icon="python">
|
||||
{PyWriteWorkflow}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.mts" language="typescript" icon="square-js">
|
||||
{TsWriteWorkflowBody}
|
||||
{TsWriteWorkflowTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="src/main.rs" language="rust" icon="rust">
|
||||
{RsWriteWorkflow}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6.
|
||||
Dana is selected for deletion and is absent from the final result.
|
||||
|
||||
</Accordion>
|
||||
|
||||
@@ -1,82 +1,123 @@
|
||||
---
|
||||
title: "Update rows"
|
||||
sidebarTitle: "Update"
|
||||
description: "Change values in rows that already exist, by predicate or SQL expression."
|
||||
description: "Select existing rows, change values or expressions, and verify the changes."
|
||||
icon: pen
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Update rows through the SDKs in OSS and Enterprise. Enterprise also supports the SQL UPDATE statement.
|
||||
---
|
||||
|
||||
These examples use the table set up in
|
||||
[Ingesting data](/ingestion), and the connection made there.
|
||||
|
||||
import {
|
||||
PyUpdateOperation as UpdateOperation,
|
||||
PyUpdateUsingSql as UpdateUsingSql,
|
||||
TsUpdateOperation,
|
||||
TsUpdateUsingSql,
|
||||
RsUpdateOperation,
|
||||
RsUpdateUsingSql,
|
||||
} from '/snippets/tables.mdx';
|
||||
import {
|
||||
PyUpdateOperation as UpdateOperation,
|
||||
PyUpdateUsingSql as UpdateUsingSql,
|
||||
} from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
<Note>
|
||||
In SQL this is [`UPDATE`](/sql/dml/update).
|
||||
</Note>
|
||||
import { PyWriteWorkflow, TsWriteWorkflowBody, TsWriteWorkflowTail, RsWriteWorkflow } from '/snippets/ingestion_examples.mdx';
|
||||
|
||||
## Update rows {#update-rows}
|
||||
Use `update` to change rows selected by a predicate. Use
|
||||
[merge insert](/ingestion/merge-insert) instead when matching an incoming dataset
|
||||
against existing keys.
|
||||
|
||||
Use `update` when you already know which target rows to modify and you do not need to compare against an incoming dataset.
|
||||
## Select the rows to change {#select-rows}
|
||||
|
||||
Check the predicate with a [filtered query](/search/filtering) before applying
|
||||
the update. These examples target the row with `id = 2`. Review the scope
|
||||
carefully when omitting a predicate, since that can update the entire table.
|
||||
|
||||
Each example below starts from Alice with 10 logins and Bob with 20. The
|
||||
Python programs create and remove their own temporary database. TypeScript
|
||||
and Rust operation blocks use `db`; a complete program is provided below.
|
||||
|
||||
## Assign a new value {#update-rows}
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{UpdateOperation}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsUpdateOperation}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateOperation}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Python" language="python" icon="python">
|
||||
{UpdateOperation}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="TypeScript" language="typescript" icon="square-js">
|
||||
{TsUpdateOperation}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="rust" icon="rust">
|
||||
{RsUpdateOperation}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Expected table contents:
|
||||
The result contains Alice and **Bobby**, with their login counts unchanged.
|
||||
The Python `values` and TypeScript `values` arguments take literal values.
|
||||
Rust's `column` argument is an expression, so the string value includes SQL quotes.
|
||||
|
||||
| id | name | login_count |
|
||||
| --- | --- | --- |
|
||||
| 1 | Alice | 10 |
|
||||
| 2 | Bobby | 20 |
|
||||
|
||||
<Warning title="Warning">
|
||||
Updating nested columns is not yet supported.
|
||||
</Warning>
|
||||
|
||||
## Update rows with SQL expressions {#update-rows-with-sql-expressions}
|
||||
|
||||
Use `values_sql` when you want to use SQL-like expressions to update rows. This is useful for operations like incrementing a counter, or setting a column value based on another column.
|
||||
## Calculate a value from the existing row {#update-rows-with-sql-expressions}
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{UpdateUsingSql}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="TypeScript" language="TypeScript" icon="square-js">
|
||||
{TsUpdateUsingSql}
|
||||
</CodeBlock>
|
||||
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsUpdateUsingSql}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Python" language="python" icon="python">
|
||||
{UpdateUsingSql}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="TypeScript" language="typescript" icon="square-js">
|
||||
{TsUpdateUsingSql}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="rust" icon="rust">
|
||||
{RsUpdateUsingSql}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
Expected table contents:
|
||||
In this separate example, Bob keeps his name and his login count becomes **21**.
|
||||
Python's `values_sql` and TypeScript's `valuesSql` accept expressions.
|
||||
These SDK expressions work with embedded OSS tables; they do not require an
|
||||
Enterprise SQL endpoint. Enterprise also offers [`UPDATE`](/sql/dml/update).
|
||||
|
||||
| id | name | login_count |
|
||||
| --- | --- | --- |
|
||||
| 1 | Alice | 10 |
|
||||
| 2 | Bob | 21 |
|
||||
## Verify the affected data {#verify}
|
||||
|
||||
<Note title="SQL syntax">
|
||||
See the [SQL queries](/sql#connecting) page for more information on the supported SQL syntax.
|
||||
</Note>
|
||||
Read the selected rows again and inspect the changed fields. For another
|
||||
connection or table handle, account for its
|
||||
[read consistency](/tables/consistency) settings.
|
||||
|
||||
When rows are updated, they are moved out of any existing index. The row will still show up in search queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider triggering an index rebuild afterwards.
|
||||
Nested-field updates are not supported by these operations. For schema changes,
|
||||
see [schema evolution](/management/table#schema). Treat derived values separately:
|
||||
changing an input field is not a guarantee that every stored feature has been
|
||||
recomputed. See [Refresh a column](/feature-engineering/refresh-column).
|
||||
|
||||
Updates can change index coverage and query cost. After a large update,
|
||||
inspect the workload and follow [index maintenance](/indexing/reindexing)
|
||||
guidance for the backend you use.
|
||||
|
||||
## Complete write workflow {#complete-example}
|
||||
|
||||
<Accordion title="Complete examples: create, append, update, upsert and delete">
|
||||
|
||||
Each program creates a temporary local database, performs the five operations,
|
||||
prints the result, and removes its temporary data when finished.
|
||||
|
||||
- **Python:** install `lancedb`, save the code as `ingestion.py`, and run
|
||||
`python ingestion.py`.
|
||||
- **TypeScript:** install `@lancedb/lancedb`, save the code as `ingestion.mts`,
|
||||
and run `node ingestion.mts` with Node.js 22.18 or later.
|
||||
- **Rust:** download the [complete Rust project](/static/code/ingestion-rust.zip),
|
||||
extract it, and run `cargo run --locked`. It includes `src/main.rs`, the Cargo
|
||||
manifest and a lockfile for the example's dependencies. The code is shown below.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="ingestion.py" language="python" icon="python">
|
||||
{PyWriteWorkflow}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="ingestion.mts" language="typescript" icon="square-js">
|
||||
{TsWriteWorkflowBody}
|
||||
{TsWriteWorkflowTail}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="src/main.rs" language="rust" icon="rust">
|
||||
{RsWriteWorkflow}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
After the operations, Alice has 10 logins, Bobby has 21, and Charlie has 6.
|
||||
Dana is selected for deletion and is absent from the final result.
|
||||
|
||||
</Accordion>
|
||||
|
||||
@@ -29,7 +29,7 @@ prompt, plus lexical (keyword) search over past sessions. Both are useful, but k
|
||||
misses paraphrases of what you originally typed — the exact thing you need when recalling a
|
||||
fact you phrased differently months ago.
|
||||
|
||||
LanceDB is an embedded retrieval library, which makes it a natural fit here:
|
||||
LanceDB OSS is an embedded retrieval library, which makes it a natural fit here:
|
||||
|
||||
- **No server to stand up** — it reads and writes a table on local disk, so the plugin ships
|
||||
as a dependency rather than a service to operate.
|
||||
|
||||
@@ -2,6 +2,12 @@
|
||||
title: "Hugging Face Hub"
|
||||
sidebarTitle: "Hugging Face"
|
||||
description: "Use LanceDB directly on Lance datasets hosted on the Hugging Face Hub for multimodal search and retrieval."
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: unavailable
|
||||
summary: >-
|
||||
The embedded library opens Lance datasets on the Hugging Face Hub in place
|
||||
through `hf://` URIs. A deployment does not read `hf://` datasets.
|
||||
---
|
||||
|
||||
[Hugging Face Hub](https://huggingface.co/datasets?format=format:lance&sort=trending) is a popular platform for sharing machine learning datasets, models, and other resources.
|
||||
|
||||
@@ -80,6 +80,9 @@ It returns list of ids of the added texts.
|
||||
{PyFrameworksLangchainAddTexts}
|
||||
</CodeBlock>
|
||||
|
||||
With LanceDB Enterprise, `tbl.to_pandas()` raises in the synchronous Python client;
|
||||
read the rows with a query instead, such as `tbl.search().limit(100).to_pandas()`.
|
||||
|
||||
------
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ description: "Learn how to use the DuckDB-Lance extension to query Lance tables
|
||||
|
||||
LanceDB integrates with [DuckDB](https://duckdb.org/) through the [Lance extension](https://github.com/lance-format/lance-duckdb) for DuckDB. In this page, we'll show how LanceDB manages table lifecycle, and DuckDB provides SQL analytics (including joins) and search over those tables.
|
||||
|
||||
Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer.
|
||||
Note that earlier versions of LanceDB used to recommend converting Lance tables to Arrow tables via `table.to_arrow()`. Although this method is still available for local tables (because DuckDB [natively scans Arrow tables](https://duckdb.org/2021/12/03/duck-arrow)), it is no longer the recommended workflow for working with Lance tables in DuckDB. This page shows how to use the Lance extension with namespace-attached LanceDB tables, allowing you to pushdown SQL queries directly to the Lance layer.
|
||||
|
||||
|
||||
## Install {#install}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
---
|
||||
title: "Pandas and PyArrow"
|
||||
sidebarTitle: "Pandas & PyArrow"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
DataFrames go into `create_table` and come back from query results the same
|
||||
way for local and deployment tables.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
---
|
||||
title: "Polars"
|
||||
sidebarTitle: "Polars & Arrow"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
Polars data goes into `create_table`, and query results come back as Polars
|
||||
DataFrames, for local and deployment tables alike; a LazyFrame over a whole
|
||||
table is local only.
|
||||
|
||||
---
|
||||
|
||||
@@ -36,7 +43,7 @@ Run vector search and keep the results as a Polars `DataFrame` for further proce
|
||||
|
||||
## Work with LazyFrames {#work-with-lazyframes}
|
||||
|
||||
When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once.
|
||||
When you want to operate on the entire table (potentially larger than RAM), convert to a Polars `LazyFrame` so you can chain transformations without loading everything at once. This works for local tables only; with LanceDB Enterprise, `search(...).to_polars()` returns an eager DataFrame of query results.
|
||||
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyPlatformsPolarsLazyframe}
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
---
|
||||
title: "Pydantic"
|
||||
sidebarTitle: "Pydantic"
|
||||
availability:
|
||||
oss: available
|
||||
enterprise: available
|
||||
summary: >-
|
||||
The SDK converts `LanceModel` schemas to Arrow, and query rows back to
|
||||
models, in your process, so they work the same with local and deployment
|
||||
tables.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -8,6 +8,10 @@ import { PyEmbeddingCohereUsage } from '/snippets/integrations.mdx';
|
||||
Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
|
||||
You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
|
||||
|
||||
```shell
|
||||
export COHERE_API_KEY="your-api-key"
|
||||
```
|
||||
|
||||
Supported models are:
|
||||
|
||||
- embed-english-v3.0
|
||||
|
||||
@@ -3,7 +3,7 @@ title: IBM watsonx
|
||||
sidebarTitle: IBM WatsonX
|
||||
---
|
||||
|
||||
import { PyEmbeddingIbmUsage } from '/snippets/integrations.mdx';
|
||||
import { PyEmbeddingIbmUsage } from '/snippets/provider_examples.mdx';
|
||||
|
||||
Generate text embeddings using IBM's watsonx.ai platform.
|
||||
|
||||
@@ -54,9 +54,13 @@ Optionally set environment variables (if not passing credentials to `create` dir
|
||||
export WATSONX_API_KEY="YOUR_WATSONX_API_KEY"
|
||||
# Provide exactly one of the following:
|
||||
export WATSONX_PROJECT_ID="YOUR_WATSONX_PROJECT_ID"
|
||||
export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID"
|
||||
# export WATSONX_SPACE_ID="YOUR_WATSONX_SPACE_ID"
|
||||
```
|
||||
|
||||
Replace the credential placeholders with your provider credentials. The registry
|
||||
variable keeps the API key out of stored embedding configuration. In application
|
||||
code, load its value from environment variables or a secrets manager.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyEmbeddingIbmUsage}
|
||||
|
||||
@@ -6,7 +6,7 @@ sidebarTitle: Jina
|
||||
import {
|
||||
PyEmbeddingJinaText,
|
||||
PyEmbeddingJinaMultimodal,
|
||||
} from '/snippets/integrations.mdx';
|
||||
} from '/snippets/provider_examples.mdx';
|
||||
|
||||
## Text Embedding Models {#text-embedding-models}
|
||||
|
||||
@@ -23,6 +23,8 @@ Supported parameters (to be passed in `create` method) are:
|
||||
|
||||
Usage Example:
|
||||
|
||||
Replace `your-api-key` with your Jina API key. This provider reads it from `JINA_API_KEY`; the example sets it explicitly. In application code, supply it through your environment or a secrets manager.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyEmbeddingJinaText}
|
||||
|
||||
@@ -10,9 +10,15 @@ LanceDB registers the OpenAI embeddings function in the registry by default, as
|
||||
| Parameter | Type | Default Value | Description |
|
||||
|---|---|---|---|
|
||||
| `name` | `str` | `"text-embedding-ada-002"` | The name of the model. |
|
||||
| `dim` | `int` | Model default | For OpenAI's newer text-embedding-3 model, we can specify a dimensionality that is smaller than the 1536 size. This feature supports it |
|
||||
| `dim` | `int` | Model default | Requested output dimensions for models that support shortening embeddings. Use `func.ndims()` when defining the vector field. |
|
||||
| `use_azure` | bool | `False` | Set true to use Azure OpenAI SDK |
|
||||
|
||||
Set your OpenAI API key before running the example:
|
||||
|
||||
```shell
|
||||
export OPENAI_API_KEY="your-api-key"
|
||||
```
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyEmbeddingOpenaiBasic}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user